From 3e746e7b208b812c87a807d60e1be329a841188e Mon Sep 17 00:00:00 2001 From: ruthwikdasyam Date: Thu, 19 Feb 2026 00:49:34 -0800 Subject: [PATCH 01/13] initial commit --- dimos/teleop/phone/web/fastapi_server.py | 118 +++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 dimos/teleop/phone/web/fastapi_server.py diff --git a/dimos/teleop/phone/web/fastapi_server.py b/dimos/teleop/phone/web/fastapi_server.py new file mode 100644 index 0000000000..83f2a1bf4b --- /dev/null +++ b/dimos/teleop/phone/web/fastapi_server.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Phone Teleop FastAPI Server. + +Replaces the Deno WebSocket-to-LCM bridge with a Python FastAPI server. +Extends RobotWebInterface to serve the phone teleop web app and forward +WebSocket binary LCM packets to the LCM network via UDP multicast. +""" + +from __future__ import annotations + +from pathlib import Path +import socket +import struct + +from fastapi import WebSocket, WebSocketDisconnect +from fastapi.responses import HTMLResponse +from fastapi.staticfiles import StaticFiles + +from dimos.utils.logging_config import setup_logger +from dimos.web.robot_web_interface import RobotWebInterface + +logger = setup_logger() + +# LCM default multicast group and port +LCM_MULTICAST_GROUP = "239.255.76.67" +LCM_MULTICAST_PORT = 7667 + +STATIC_DIR = Path(__file__).parent / "static" + + +class PhoneTeleopServer(RobotWebInterface): + """Phone teleoperation server built on RobotWebInterface. + + Adds a WebSocket endpoint that bridges binary LCM packets from the + phone browser to the LCM UDP multicast network. + """ + + def __init__( + self, + port: int = 8444, + lcm_multicast_group: str = LCM_MULTICAST_GROUP, + lcm_multicast_port: int = LCM_MULTICAST_PORT, + ) -> None: + super().__init__(port=port) + self.lcm_multicast_group = lcm_multicast_group + self.lcm_multicast_port = lcm_multicast_port + self._udp_sock: socket.socket | None = None + + self._setup_teleop_routes() + + # ------------------------------------------------------------------ + # LCM UDP socket + # ------------------------------------------------------------------ + + def _ensure_udp_socket(self) -> socket.socket: + """Create (or reuse) a UDP socket for LCM multicast publishing.""" + if self._udp_sock is None: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, struct.pack("b", 1)) + self._udp_sock = sock + logger.info("LCM UDP multicast socket created") + return self._udp_sock + + def _publish_lcm_packet(self, data: bytes) -> None: + """Send a raw LCM packet to the multicast group.""" + sock = self._ensure_udp_socket() + sock.sendto(data, (self.lcm_multicast_group, self.lcm_multicast_port)) + + # ------------------------------------------------------------------ + # Routes + # ------------------------------------------------------------------ + + def _setup_teleop_routes(self) -> None: + """Add phone-teleop-specific routes on top of the base interface.""" + + @self.app.get("/teleop", response_class=HTMLResponse) + async def teleop_index() -> HTMLResponse: + index_path = STATIC_DIR / "index.html" + return HTMLResponse(content=index_path.read_text()) + + if STATIC_DIR.is_dir(): + self.app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="teleop_static") + + @self.app.websocket("/ws") + async def websocket_endpoint(ws: WebSocket) -> None: + await ws.accept() + logger.info("Phone client connected") + try: + while True: + data = await ws.receive_bytes() + try: + self._publish_lcm_packet(data) + except Exception: + logger.exception("Failed to forward LCM packet") + except WebSocketDisconnect: + logger.info("Phone client disconnected") + except Exception: + logger.exception("WebSocket error") + + +if __name__ == "__main__": + server = PhoneTeleopServer() + server.run() From 35c8dd53f56e922b777a3e67d4013ebf201d1c86 Mon Sep 17 00:00:00 2001 From: Ruthwik Date: Fri, 27 Feb 2026 13:43:15 -0800 Subject: [PATCH 02/13] feat: chhange from deno to python server - using fastAPI --- dimos/teleop/phone/phone_teleop_module.py | 157 ++++++++++++---------- dimos/teleop/phone/web/fastapi_server.py | 118 ---------------- dimos/teleop/phone/web/teleop_server.ts | 85 ------------ 3 files changed, 85 insertions(+), 275 deletions(-) delete mode 100644 dimos/teleop/phone/web/fastapi_server.py delete mode 100755 dimos/teleop/phone/web/teleop_server.ts diff --git a/dimos/teleop/phone/phone_teleop_module.py b/dimos/teleop/phone/phone_teleop_module.py index c0da85c27c..48a3919807 100644 --- a/dimos/teleop/phone/phone_teleop_module.py +++ b/dimos/teleop/phone/phone_teleop_module.py @@ -17,42 +17,46 @@ Phone Teleoperation Module. Receives raw sensor data (TwistStamped) and button state (Bool) from the -phone web app via the Deno LCM bridge. Computes orientation deltas from -a initial orientation captured on engage, converts to TwistStamped velocity -commands via configurable gains, and publishes. - +phone web app via an embedded FastAPI WebSocket server. Computes orientation +deltas from an initial orientation captured on engage, converts to TwistStamped +velocity commands via configurable gains, and publishes. """ from dataclasses import dataclass from pathlib import Path -import shutil -import signal -import subprocess import threading import time from typing import Any -from reactivex.disposable import Disposable +from dimos_lcm.geometry_msgs import TwistStamped as LCMTwistStamped +from dimos_lcm.std_msgs import Bool as LCMBool +from fastapi import WebSocket, WebSocketDisconnect +from fastapi.responses import HTMLResponse +from fastapi.staticfiles import StaticFiles -from dimos.core import In, Module, Out, rpc +from dimos.core import Module, Out, rpc from dimos.core.module import ModuleConfig from dimos.msgs.geometry_msgs import Twist, TwistStamped, Vector3 from dimos.msgs.std_msgs.Bool import Bool from dimos.utils.logging_config import setup_logger +from dimos.web.robot_web_interface import RobotWebInterface logger = setup_logger() +STATIC_DIR = Path(__file__).parent / "web" / "static" + @dataclass class PhoneTeleopConfig(ModuleConfig): control_loop_hz: float = 50.0 linear_gain: float = 1.0 / 30.0 # Gain: maps degrees of tilt to m/s. 30 deg -> 1.0 m/s angular_gain: float = 1.0 / 30.0 # Gain: maps gyro deg/s to rad/s. 30 deg/s -> 1.0 rad/s + server_port: int = 8444 class PhoneTeleopModule(Module[PhoneTeleopConfig]): """ - Receives raw sensor data from the phone web app: + Receives raw sensor data from the phone web app via an embedded WebSocket server: - TwistStamped: linear=(roll, pitch, yaw) deg, angular=(gyro) deg/s - Bool: teleop button state (True = held) @@ -62,9 +66,6 @@ class PhoneTeleopModule(Module[PhoneTeleopConfig]): default_config = PhoneTeleopConfig - # Inputs from Deno bridge - phone_sensors: In[TwistStamped] - phone_button: In[Bool] # Output: velocity command to robot twist_output: Out[TwistStamped] @@ -85,9 +86,52 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self._control_loop_thread: threading.Thread | None = None self._stop_event = threading.Event() - # Deno bridge server - self._server_process: subprocess.Popen[bytes] | None = None - self._server_script = Path(__file__).parent / "web" / "teleop_server.ts" + # Embedded web server — RobotWebInterface provides FastAPI app + run()/shutdown() + self._web_server = RobotWebInterface(port=self.config.server_port) + self._web_server_thread: threading.Thread | None = None + + # Fingerprint-based message dispatch table + self._decoders: dict[bytes, Any] = { + LCMTwistStamped._get_packed_fingerprint(): self._on_sensors_bytes, + LCMBool._get_packed_fingerprint(): self._on_button_bytes, + } + + self._setup_routes() + + # ------------------------------------------------------------------------- + # Web Server Routes + # ------------------------------------------------------------------------- + + def _setup_routes(self) -> None: + """Register teleop routes on the embedded web server.""" + + @self._web_server.app.get("/teleop", response_class=HTMLResponse) + async def teleop_index() -> HTMLResponse: + index_path = STATIC_DIR / "index.html" + return HTMLResponse(content=index_path.read_text()) + + if STATIC_DIR.is_dir(): + self._web_server.app.mount( + "/static", StaticFiles(directory=str(STATIC_DIR)), name="teleop_static" + ) + + @self._web_server.app.websocket("/ws") + async def websocket_endpoint(ws: WebSocket) -> None: + await ws.accept() + logger.info("Phone client connected") + try: + while True: + data = await ws.receive_bytes() + fingerprint = data[:8] + decoder = self._decoders.get(fingerprint) + if decoder: + decoder(data) + else: + logger.warning(f"Unknown message fingerprint: {fingerprint.hex()}") + except WebSocketDisconnect: + logger.info("Phone client disconnected") + except Exception: + logger.exception("WebSocket error") # ------------------------------------------------------------------------- # Lifecycle @@ -96,11 +140,6 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: @rpc def start(self) -> None: super().start() - for stream, handler in ( - (self.phone_sensors, self._on_sensors), - (self.phone_button, self._on_button), - ): - self._disposables.add(Disposable(stream.subscribe(handler))) # type: ignore[attr-defined] self._start_server() self._start_control_loop() @@ -131,73 +170,47 @@ def _disengage(self) -> None: logger.info("Phone teleop disengaged") # ------------------------------------------------------------------------- - # Callbacks + # WebSocket Message Decoders # ------------------------------------------------------------------------- - def _on_sensors(self, msg: TwistStamped) -> None: - """Callback for raw sensor TwistStamped from the phone""" + def _on_sensors_bytes(self, data: bytes) -> None: + """Decode raw LCM bytes into TwistStamped and update sensor state.""" + msg = TwistStamped.lcm_decode(data) with self._lock: self._current_sensors = msg - def _on_button(self, msg: Bool) -> None: - """Callback for teleop button state.""" + def _on_button_bytes(self, data: bytes) -> None: + """Decode raw LCM bytes into Bool and update button state.""" + msg = Bool.lcm_decode(data) with self._lock: self._teleop_button = bool(msg.data) # ------------------------------------------------------------------------- - # Deno Bridge Server + # Embedded Web Server # ------------------------------------------------------------------------- def _start_server(self) -> None: - """Launch the Deno WebSocket-to-LCM bridge server as a subprocess.""" - if self._server_process is not None and self._server_process.poll() is None: - logger.warning("Deno bridge already running", pid=self._server_process.pid) - return - - if shutil.which("deno") is None: - logger.error( - "Deno is not installed. Install it with: curl -fsSL https://deno.land/install.sh | sh" - ) + """Start the embedded FastAPI server with HTTPS in a daemon thread.""" + if self._web_server_thread is not None and self._web_server_thread.is_alive(): + logger.warning("Web server already running") return - script = str(self._server_script) - cmd = [ - "deno", - "run", - "--allow-net", - "--allow-read", - "--allow-run", - "--allow-write", - "--unstable-net", - script, - ] - try: - self._server_process = subprocess.Popen(cmd) - logger.info(f"Deno bridge server started (pid {self._server_process.pid})") - except OSError as e: - logger.error(f"Failed to start Deno bridge: {e}") + self._web_server_thread = threading.Thread( + target=self._web_server.run, + kwargs={"ssl": True}, + daemon=True, + name="PhoneTeleopWebServer", + ) + self._web_server_thread.start() + logger.info(f"Phone teleop web server started on https://0.0.0.0:{self.config.server_port}") def _stop_server(self) -> None: - """Terminate the Deno bridge server subprocess.""" - if self._server_process is None or self._server_process.poll() is not None: - self._server_process = None - return - - logger.info("Stopping Deno bridge server", pid=self._server_process.pid) - self._server_process.send_signal(signal.SIGTERM) - try: - self._server_process.wait(timeout=3) - except subprocess.TimeoutExpired: - logger.warning( - "Deno bridge did not exit, sending SIGKILL", pid=self._server_process.pid - ) - self._server_process.kill() - try: - self._server_process.wait(timeout=5) - except subprocess.TimeoutExpired: - logger.error("Deno bridge did not exit after SIGKILL") - logger.info("Deno bridge server stopped") - self._server_process = None + """Shutdown the embedded web server.""" + self._web_server.shutdown() + if self._web_server_thread is not None: + self._web_server_thread.join(timeout=3) + self._web_server_thread = None + logger.info("Phone teleop web server stopped") # ------------------------------------------------------------------------- # Control Loop diff --git a/dimos/teleop/phone/web/fastapi_server.py b/dimos/teleop/phone/web/fastapi_server.py deleted file mode 100644 index 83f2a1bf4b..0000000000 --- a/dimos/teleop/phone/web/fastapi_server.py +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2025-2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Phone Teleop FastAPI Server. - -Replaces the Deno WebSocket-to-LCM bridge with a Python FastAPI server. -Extends RobotWebInterface to serve the phone teleop web app and forward -WebSocket binary LCM packets to the LCM network via UDP multicast. -""" - -from __future__ import annotations - -from pathlib import Path -import socket -import struct - -from fastapi import WebSocket, WebSocketDisconnect -from fastapi.responses import HTMLResponse -from fastapi.staticfiles import StaticFiles - -from dimos.utils.logging_config import setup_logger -from dimos.web.robot_web_interface import RobotWebInterface - -logger = setup_logger() - -# LCM default multicast group and port -LCM_MULTICAST_GROUP = "239.255.76.67" -LCM_MULTICAST_PORT = 7667 - -STATIC_DIR = Path(__file__).parent / "static" - - -class PhoneTeleopServer(RobotWebInterface): - """Phone teleoperation server built on RobotWebInterface. - - Adds a WebSocket endpoint that bridges binary LCM packets from the - phone browser to the LCM UDP multicast network. - """ - - def __init__( - self, - port: int = 8444, - lcm_multicast_group: str = LCM_MULTICAST_GROUP, - lcm_multicast_port: int = LCM_MULTICAST_PORT, - ) -> None: - super().__init__(port=port) - self.lcm_multicast_group = lcm_multicast_group - self.lcm_multicast_port = lcm_multicast_port - self._udp_sock: socket.socket | None = None - - self._setup_teleop_routes() - - # ------------------------------------------------------------------ - # LCM UDP socket - # ------------------------------------------------------------------ - - def _ensure_udp_socket(self) -> socket.socket: - """Create (or reuse) a UDP socket for LCM multicast publishing.""" - if self._udp_sock is None: - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) - sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, struct.pack("b", 1)) - self._udp_sock = sock - logger.info("LCM UDP multicast socket created") - return self._udp_sock - - def _publish_lcm_packet(self, data: bytes) -> None: - """Send a raw LCM packet to the multicast group.""" - sock = self._ensure_udp_socket() - sock.sendto(data, (self.lcm_multicast_group, self.lcm_multicast_port)) - - # ------------------------------------------------------------------ - # Routes - # ------------------------------------------------------------------ - - def _setup_teleop_routes(self) -> None: - """Add phone-teleop-specific routes on top of the base interface.""" - - @self.app.get("/teleop", response_class=HTMLResponse) - async def teleop_index() -> HTMLResponse: - index_path = STATIC_DIR / "index.html" - return HTMLResponse(content=index_path.read_text()) - - if STATIC_DIR.is_dir(): - self.app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="teleop_static") - - @self.app.websocket("/ws") - async def websocket_endpoint(ws: WebSocket) -> None: - await ws.accept() - logger.info("Phone client connected") - try: - while True: - data = await ws.receive_bytes() - try: - self._publish_lcm_packet(data) - except Exception: - logger.exception("Failed to forward LCM packet") - except WebSocketDisconnect: - logger.info("Phone client disconnected") - except Exception: - logger.exception("WebSocket error") - - -if __name__ == "__main__": - server = PhoneTeleopServer() - server.run() diff --git a/dimos/teleop/phone/web/teleop_server.ts b/dimos/teleop/phone/web/teleop_server.ts deleted file mode 100755 index 26202cf166..0000000000 --- a/dimos/teleop/phone/web/teleop_server.ts +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env -S deno run --allow-net --allow-read --allow-run --allow-write --unstable-net - -// WebSocket to LCM Bridge for Phone Teleop -// Forwards twist data from Phone browser to LCM - -import { LCM } from "jsr:@dimos/lcm"; -import { dirname, fromFileUrl, join } from "jsr:@std/path"; - -const PORT = 8444; - -// Resolve paths relative to script location -const scriptDir = dirname(fromFileUrl(import.meta.url)); -const certsDir = join(scriptDir, "../../../../assets/teleop_certs"); -const certPath = join(certsDir, "cert.pem"); -const keyPath = join(certsDir, "key.pem"); - -// Auto-generate self-signed certificates if they don't exist -async function ensureCerts(): Promise<{ cert: string; key: string }> { - try { - const cert = await Deno.readTextFile(certPath); - const key = await Deno.readTextFile(keyPath); - return { cert, key }; - } catch { - console.log("Generating self-signed certificates..."); - await Deno.mkdir(certsDir, { recursive: true }); - const cmd = new Deno.Command("openssl", { - args: [ - "req", "-x509", "-newkey", "rsa:2048", - "-keyout", keyPath, "-out", certPath, - "-days", "365", "-nodes", "-subj", "/CN=localhost" - ], - }); - const { code } = await cmd.output(); - if (code !== 0) { - throw new Error("Failed to generate certificates. Is openssl installed?"); - } - console.log("Certificates generated in assets/teleop_certs/"); - return { - cert: await Deno.readTextFile(certPath), - key: await Deno.readTextFile(keyPath), - }; - } -} - -const { cert, key } = await ensureCerts(); - -const lcm = new LCM(); -await lcm.start(); - -// Binds to all interfaces so the phone can reach the server over LAN. -Deno.serve({ port: PORT, cert, key }, async (req) => { - const url = new URL(req.url); - - if (req.headers.get("upgrade") === "websocket") { - const { socket, response } = Deno.upgradeWebSocket(req); - socket.onopen = () => console.log("Phone client connected"); - socket.onclose = () => console.log("Phone client disconnected"); - - // Forward binary LCM packets from browser directly to UDP - socket.binaryType = "arraybuffer"; - socket.onmessage = async (event) => { - if (event.data instanceof ArrayBuffer) { - const packet = new Uint8Array(event.data); - try { - await lcm.publishPacket(packet); - } catch (e) { - console.error("Forward error:", e); - } - } - }; - - return response; - } - - if (url.pathname === "/" || url.pathname === "/index.html") { - const html = await Deno.readTextFile(new URL("./static/index.html", import.meta.url)); - return new Response(html, { headers: { "content-type": "text/html" } }); - } - - return new Response("Not found", { status: 404 }); -}); - -console.log(`Phone Teleop Server: https://localhost:${PORT}`); - -await lcm.run(); From 0edcc14ed311923a68ce173cfc0487344e57cd92 Mon Sep 17 00:00:00 2001 From: Ruthwik Date: Fri, 27 Feb 2026 13:43:51 -0800 Subject: [PATCH 03/13] feat: sending encoded msgs directly - no topic --- dimos/teleop/phone/web/static/index.html | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/dimos/teleop/phone/web/static/index.html b/dimos/teleop/phone/web/static/index.html index 6fad23b6c8..5bad9c727e 100644 --- a/dimos/teleop/phone/web/static/index.html +++ b/dimos/teleop/phone/web/static/index.html @@ -165,7 +165,7 @@

Sensors