diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..bebf26e --- /dev/null +++ b/package-lock.json @@ -0,0 +1,999 @@ +{ + "name": "codedash-app", + "version": "6.15.11", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "codedash-app", + "version": "6.15.11", + "license": "MIT", + "dependencies": { + "@huggingface/transformers": "^4.0.1" + }, + "bin": { + "codedash": "bin/cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.2.tgz", + "integrity": "sha512-3U4+MIWHImeyu1wnmVygh5WlgfYDtyf0k8AbLhMFxOipihf6nrWC4syIm/SwEeec0mNSafiiNnMJwbza/Is6Lw==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@huggingface/jinja": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.5.6.tgz", + "integrity": "sha512-MyMWyLnjqo+KRJYSH7oWNbsOn5onuIvfXYPcc0WOGxU0eHUV7oAYUoQTl2BMdu7ml+ea/bu11UM+EshbeHwtIA==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@huggingface/tokenizers": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@huggingface/tokenizers/-/tokenizers-0.1.3.tgz", + "integrity": "sha512-8rF/RRT10u+kn7YuUbUg0OF30K8rjTc78aHpxT+qJ1uWSqxT1MHi8+9ltwYfkFYJzT/oS+qw3JVfHtNMGAdqyA==", + "license": "Apache-2.0" + }, + "node_modules/@huggingface/transformers": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-4.0.1.tgz", + "integrity": "sha512-tAQYEy+cnW0ku/NxBSjFXCymi+DZa1/JkoGf4McxjzO36CZZIL/J4TF6X7i/tzs75yTjshUDgsvSz03s2xym2A==", + "license": "Apache-2.0", + "dependencies": { + "@huggingface/jinja": "^0.5.6", + "@huggingface/tokenizers": "^0.1.3", + "onnxruntime-node": "1.24.3", + "onnxruntime-web": "1.25.0-dev.20260327-722743c0e2", + "sharp": "^0.34.5" + } + }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "cpu": [ + "arm" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "cpu": [ + "ppc64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "cpu": [ + "riscv64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "cpu": [ + "s390x" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "cpu": [ + "riscv64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "cpu": [ + "s390x" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.7.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", + "license": "BSD-3-Clause" + }, + "node_modules/@types/node": { + "version": "25.5.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.5.2.tgz", + "integrity": "sha512-tO4ZIRKNC+MDWV4qKVZe3Ql/woTnmHDr5JD8UI5hn2pwBrHEwOEMZK7WlNb5RKB6EoJ02gwmQS9OrjuFnZYdpg==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/adm-zip": { + "version": "0.5.17", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.17.tgz", + "integrity": "sha512-+Ut8d9LLqwEvHHJl1+PIHqoyDxFgVN847JTVM3Izi3xHDWPE4UtzzXysMZQs64DMcrJfBeS/uoEP4AD3HQHnQQ==", + "license": "MIT", + "engines": { + "node": ">=12.0" + } + }, + "node_modules/boolean": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", + "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "MIT" + }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/define-properties": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", + "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", + "license": "MIT", + "dependencies": { + "define-data-property": "^1.0.1", + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", + "license": "MIT" + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", + "license": "MIT" + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", + "license": "Apache-2.0" + }, + "node_modules/global-agent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", + "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==", + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "es6-error": "^4.1.1", + "matcher": "^3.0.0", + "roarr": "^2.15.3", + "semver": "^7.3.2", + "serialize-error": "^7.0.1" + }, + "engines": { + "node": ">=10.0" + } + }, + "node_modules/globalthis": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", + "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==", + "license": "MIT", + "dependencies": { + "define-properties": "^1.2.1", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==", + "license": "ISC" + }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", + "license": "ISC" + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/matcher": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", + "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/onnxruntime-common": { + "version": "1.24.3", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.3.tgz", + "integrity": "sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA==", + "license": "MIT" + }, + "node_modules/onnxruntime-node": { + "version": "1.24.3", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.24.3.tgz", + "integrity": "sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg==", + "hasInstallScript": true, + "license": "MIT", + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "adm-zip": "^0.5.16", + "global-agent": "^3.0.0", + "onnxruntime-common": "1.24.3" + } + }, + "node_modules/onnxruntime-web": { + "version": "1.25.0-dev.20260327-722743c0e2", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.25.0-dev.20260327-722743c0e2.tgz", + "integrity": "sha512-8PXdZy4Ekhg10CLg+cFFt39b4tFDGMRJB6lGjnQL6eA+2boUQYDymZ0gtxiS+H6oIWoCjQp/ziyirvFbaFKfiw==", + "license": "MIT", + "dependencies": { + "flatbuffers": "^25.1.24", + "guid-typescript": "^1.0.9", + "long": "^5.2.3", + "onnxruntime-common": "1.24.0-dev.20251116-b39e144322", + "platform": "^1.3.6", + "protobufjs": "^7.2.4" + } + }, + "node_modules/onnxruntime-web/node_modules/onnxruntime-common": { + "version": "1.24.0-dev.20251116-b39e144322", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.0-dev.20251116-b39e144322.tgz", + "integrity": "sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw==", + "license": "MIT" + }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "license": "MIT" + }, + "node_modules/protobufjs": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", + "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/roarr": { + "version": "2.15.4", + "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", + "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "detect-node": "^2.0.4", + "globalthis": "^1.0.1", + "json-stringify-safe": "^5.0.1", + "semver-compare": "^1.0.0", + "sprintf-js": "^1.1.2" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/semver-compare": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", + "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", + "license": "MIT" + }, + "node_modules/serialize-error": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", + "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.13.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/sharp": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" + } + }, + "node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "license": "BSD-3-Clause" + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD", + "optional": true + }, + "node_modules/type-fest": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", + "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "license": "MIT" + } + } +} diff --git a/package.json b/package.json index f872d88..dd45de6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codedash-app", - "version": "6.15.10", + "version": "6.15.11", "description": "Dashboard + CLI for Claude Code, Codex & OpenCode sessions. View, search, resume, convert, handoff between agents.", "bin": { "codedash": "./bin/cli.js" @@ -32,5 +32,8 @@ "license": "MIT", "engines": { "node": ">=18" + }, + "dependencies": { + "@huggingface/transformers": "^4.0.1" } } diff --git a/src/copilot-client.js b/src/copilot-client.js new file mode 100644 index 0000000..1909233 --- /dev/null +++ b/src/copilot-client.js @@ -0,0 +1,414 @@ +/** + * @module copilot-client + * + * Auto-discovers GitHub Copilot OAuth tokens and exchanges them for session + * tokens to call LLM APIs via the Copilot chat/completions endpoint. + * + * Ported from the Rust implementation at codex-rs/github-copilot/src/ + * (auth.rs, token_exchange.rs). + * + * Flow: + * 1. Load OAuth token (gho_...) from ~/.copilot/auth/credential.json + * Fallback: ~/.config/github-copilot/apps.json + * 2. Exchange for short-lived session token via GitHub internal endpoint + * 3. Cache session token until expiry (refresh 60s before) + * 4. Call chat/completions on the dynamic Copilot API base + * + * Zero dependencies — uses only Node.js stdlib (https, fs, path, os). + */ + +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const os = require('os'); +const https = require('https'); + +// ── Constants ──────────────────────────────────────────────────── + +/** Copilot internal token exchange endpoint. */ +const COPILOT_TOKEN_ENDPOINT = 'https://api.github.com/copilot_internal/v2/token'; + +/** Default Copilot API base (individual accounts). */ +const DEFAULT_API_BASE = 'https://api.individual.githubcopilot.com'; + +/** Default model for chat completions. */ +const DEFAULT_MODEL = 'gpt-4.1'; + +/** User-Agent header matching the official Copilot CLI. */ +const USER_AGENT = `copilot/1.0.14 (client/github/cli ${process.platform} v24.11.1) term/${process.env.TERM_PROGRAM || 'xterm'}`; + +/** Refresh session token 60 seconds before actual expiry. */ +const TOKEN_REFRESH_MARGIN_SEC = 60; + +// ── Credential file paths ──────────────────────────────────────── + +/** + * Returns the path to ~/.copilot/auth/credential.json + * @returns {string} + */ +function copilotCliCredentialPath() { + return path.join(os.homedir(), '.copilot', 'auth', 'credential.json'); +} + +/** + * Returns the path to ~/.config/github-copilot/apps.json (legacy VS Code / Copilot extension) + * @returns {string} + */ +function copilotAppsJsonPath() { + return path.join(os.homedir(), '.config', 'github-copilot', 'apps.json'); +} + +// ── Cached session state ───────────────────────────────────────── + +/** @type {string|null} Cached OAuth token (gho_...) */ +let _oauthToken = null; + +/** @type {string|null} Cached session token (short-lived) */ +let _sessionToken = null; + +/** @type {number} Unix seconds when session token expires */ +let _sessionExpiresAt = 0; + +/** @type {string} Dynamic API base URL from token exchange */ +let _apiBase = DEFAULT_API_BASE; + +// ── HTTP helper ────────────────────────────────────────────────── + +/** + * Makes an HTTPS request using Node.js stdlib. + * @param {string} url - Full URL + * @param {Object} opts - Options + * @param {string} [opts.method='GET'] - HTTP method + * @param {Object} [opts.headers={}] - Request headers + * @param {string|null} [opts.body=null] - Request body (for POST) + * @returns {Promise<{statusCode: number, body: string, json: function}>} + */ +function httpsRequest(url, opts = {}) { + return new Promise((resolve, reject) => { + const parsed = new URL(url); + const reqOpts = { + hostname: parsed.hostname, + port: parsed.port || 443, + path: parsed.pathname + parsed.search, + method: opts.method || 'GET', + headers: { + 'User-Agent': USER_AGENT, + Accept: 'application/json', + ...(opts.headers || {}), + }, + }; + + const req = https.request(reqOpts, (res) => { + const chunks = []; + res.on('data', (chunk) => chunks.push(chunk)); + res.on('end', () => { + const body = Buffer.concat(chunks).toString('utf8'); + resolve({ + statusCode: res.statusCode, + body, + json() { + return JSON.parse(body); + }, + }); + }); + }); + + req.on('error', reject); + req.setTimeout(30000, () => { + req.destroy(new Error('Request timed out')); + }); + + if (opts.body) { + req.write(opts.body); + } + req.end(); + }); +} + +// ── OAuth token discovery ──────────────────────────────────────── + +/** + * Attempts to load the OAuth token from known credential locations. + * + * Priority: + * 1. ~/.copilot/auth/credential.json (field: "token") + * 2. ~/.config/github-copilot/apps.json (field: oauth_token under github.com key) + * + * @returns {string|null} The gho_... OAuth token, or null if not found. + */ +function loadOAuthToken() { + // Collect ALL candidate tokens, return as array (caller tries each) + return loadAllOAuthTokens()[0] || null; +} + +/** + * Loads all available OAuth tokens from known credential locations. + * Returns tokens most-likely-to-work first: apps.json (actively refreshed + * by VS Code/Copilot extension) before credential.json (can go stale). + * @returns {string[]} + */ +function loadAllOAuthTokens() { + const tokens = []; + + // 1. apps.json (preferred — refreshed by VS Code Copilot extension) + try { + const appsPath = copilotAppsJsonPath(); + if (fs.existsSync(appsPath)) { + const data = JSON.parse(fs.readFileSync(appsPath, 'utf8')); + for (const key of Object.keys(data)) { + if (key.startsWith('github.com') && data[key] && data[key].oauth_token) { + tokens.push(data[key].oauth_token); + } + } + } + } catch (_) {} + + // 2. Copilot CLI credential file (can be stale) + try { + const credPath = copilotCliCredentialPath(); + if (fs.existsSync(credPath)) { + const data = JSON.parse(fs.readFileSync(credPath, 'utf8')); + if (data.token && typeof data.token === 'string' && data.token.length > 0) { + if (!tokens.includes(data.token)) tokens.push(data.token); + } + } + } catch (_) {} + + return tokens; +} + +// ── Token exchange ─────────────────────────────────────────────── + +/** + * Exchange a persistent OAuth token (gho_...) for a short-lived Copilot API + * session token via the internal GitHub endpoint. + * + * The response includes: + * - token: session token for Bearer auth + * - expires_at: unix seconds + * - endpoints.api: dynamic API base URL + * + * @param {string} oauthToken - The gho_... OAuth token + * @returns {Promise<{token: string, expires_at: number, endpoints?: {api?: string}}>} + * @throws {Error} If the exchange fails + */ +async function exchangeToken(oauthToken) { + const res = await httpsRequest(COPILOT_TOKEN_ENDPOINT, { + method: 'GET', + headers: { + Authorization: `token ${oauthToken}`, + }, + }); + + if (res.statusCode !== 200) { + throw new Error( + `Copilot token exchange failed (HTTP ${res.statusCode}): ${res.body}` + ); + } + + const data = res.json(); + if (!data.token) { + throw new Error('Copilot token exchange returned empty token'); + } + + return data; +} + +/** + * Ensures we have a valid session token, exchanging/refreshing as needed. + * Caches the token and refreshes 60 seconds before expiry. + * + * @returns {Promise} The session token + * @throws {Error} If no OAuth token is available or exchange fails + */ +async function ensureSessionToken() { + const now = Math.floor(Date.now() / 1000); + + // Return cached token if still valid + if (_sessionToken && _sessionExpiresAt > now + TOKEN_REFRESH_MARGIN_SEC) { + return _sessionToken; + } + + // Try all available OAuth tokens until one exchanges successfully. + // apps.json tokens (refreshed by VS Code) are tried first; credential.json + // (Copilot CLI, can go stale) is tried last. + const allTokens = loadAllOAuthTokens(); + if (allTokens.length === 0) { + throw new Error( + 'No GitHub Copilot OAuth token found. ' + + 'Expected at ~/.copilot/auth/credential.json or ~/.config/github-copilot/apps.json' + ); + } + + let lastErr = null; + for (const token of allTokens) { + try { + const result = await exchangeToken(token); + _oauthToken = token; + _sessionToken = result.token; + _sessionExpiresAt = result.expires_at || 0; + if (result.endpoints && result.endpoints.api) { + _apiBase = result.endpoints.api; + } else { + _apiBase = DEFAULT_API_BASE; + } + return _sessionToken; + } catch (e) { + lastErr = e; + // Try next token + } + } + throw lastErr || new Error('All OAuth tokens failed exchange'); +} + +// ── Public API ─────────────────────────────────────────────────── + +/** + * Synchronous check whether a Copilot credential file exists on disk. + * Does NOT validate the token — just checks file presence. + * + * @returns {boolean} True if a credential file exists + */ +function isAvailable() { + try { + if (fs.existsSync(copilotCliCredentialPath())) return true; + } catch (_) { /* ignore */ } + + try { + if (fs.existsSync(copilotAppsJsonPath())) return true; + } catch (_) { /* ignore */ } + + return false; +} + +/** + * Call the Copilot chat/completions endpoint. + * + * @param {Array<{role: string, content: string}>} messages - Chat messages + * @param {Object} [opts={}] - Options + * @param {string} [opts.model='gpt-4.1'] - Model ID + * @param {number} [opts.max_tokens=4000] - Max tokens in response + * @param {string} [opts.reasoning_effort] - Reasoning effort level (e.g. 'xhigh' for gpt-5-mini) + * @returns {Promise<{content: string, model: string, usage: Object}>} + * @throws {Error} If authentication fails or the API returns an error + */ +async function chatCompletion(messages, opts = {}) { + const token = await ensureSessionToken(); + const model = opts.model || DEFAULT_MODEL; + const maxTokens = opts.max_tokens || 4000; + + const body = { + model, + messages, + max_tokens: maxTokens, + }; + + // Add reasoning_effort for models that support it (e.g. gpt-5-mini) + if (opts.reasoning_effort) { + body.reasoning_effort = opts.reasoning_effort; + } + + const url = `${_apiBase}/chat/completions`; + const res = await httpsRequest(url, { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + 'Editor-Version': 'vscode/1.96.0', + 'Copilot-Integration-Id': 'vscode-chat', + 'Editor-Plugin-Version': 'copilot-chat/0.24.2', + }, + body: JSON.stringify(body), + }); + + if (res.statusCode === 401 || res.statusCode === 403) { + // Session token may have expired despite our margin check — force refresh + _sessionToken = null; + _sessionExpiresAt = 0; + throw new Error( + `Copilot API auth error (HTTP ${res.statusCode}): ${res.body}. ` + + 'Token has been cleared; retry will attempt re-authentication.' + ); + } + + if (res.statusCode !== 200) { + throw new Error( + `Copilot API error (HTTP ${res.statusCode}): ${res.body}` + ); + } + + const data = res.json(); + + // Extract the assistant message content + const choice = data.choices && data.choices[0]; + const content = choice && choice.message ? choice.message.content : ''; + + return { + content: content || '', + model: data.model || model, + usage: data.usage || {}, + }; +} + +/** + * Summarize a coding session conversation using the Copilot LLM. + * + * @param {Array<{role: string, content: string}>} messages - Session messages + * @returns {Promise} A concise summary of the session + * @throws {Error} If the API call fails + */ +async function summarizeSession(messages) { + // Truncate very long conversations to stay within context limits. + // Keep the first 2 and last 8 messages for large sessions. + let truncated = messages; + if (messages.length > 20) { + truncated = [ + ...messages.slice(0, 2), + { role: 'system', content: `[... ${messages.length - 10} messages omitted for brevity ...]` }, + ...messages.slice(-8), + ]; + } + + const systemPrompt = { + role: 'system', + content: + 'You are a helpful assistant that summarizes coding sessions. ' + + 'Given the conversation below, produce a concise 2-4 sentence summary ' + + 'describing what was accomplished, key decisions made, and any outstanding issues. ' + + 'Be specific about file names, features, and technologies mentioned.', + }; + + const result = await chatCompletion( + [systemPrompt, ...truncated], + { model: DEFAULT_MODEL, max_tokens: 500 } + ); + + return result.content.trim(); +} + +/** + * Returns the current authentication and connection status. + * + * @returns {{authenticated: boolean, model: string, api_base: string, token_expires_at: number}} + */ +function getStatus() { + const hasToken = !!_oauthToken || isAvailable(); + const hasSession = !!_sessionToken && _sessionExpiresAt > Math.floor(Date.now() / 1000); + + return { + authenticated: hasSession, + model: DEFAULT_MODEL, + api_base: _apiBase, + token_expires_at: _sessionExpiresAt, + }; +} + +// ── Exports ────────────────────────────────────────────────────── + +module.exports = { + isAvailable, + chatCompletion, + summarizeSession, + getStatus, +}; diff --git a/src/data.js b/src/data.js index 55b1eee..660ebbe 100644 --- a/src/data.js +++ b/src/data.js @@ -98,10 +98,18 @@ function parseOpenCodeMcpServer(toolName) { return toolName.slice(0, idx); } +// Persistent cache dir (survives tmpdir cleanup and process kills) +const CODEDASH_CACHE_DIR = path.join(os.homedir(), '.codedash', 'cache'); +try { fs.mkdirSync(CODEDASH_CACHE_DIR, { recursive: true }); } catch {} + // Disk cache for parsed Claude session files (keyed by path + mtime + size) -const PARSED_CACHE_FILE = path.join(os.tmpdir(), 'codedash-parsed-cache.json'); +const PARSED_CACHE_FILE = path.join(CODEDASH_CACHE_DIR, 'parsed-cache.json'); +// Legacy tmpdir path (migration: read once, then ignore) +const LEGACY_PARSED_CACHE_FILE = path.join(os.tmpdir(), 'codedash-parsed-cache.json'); let _parsedDiskCache = null; let _parsedDiskCacheDirty = false; +let _parsedDiskCacheEntriesSinceFlush = 0; +const PARSED_CACHE_FLUSH_EVERY = 50; // flush after every N new entries // Reverse index: file path -> cache key (avoids repeated fs.statSync) const _fileCacheKeyIndex = {}; @@ -110,20 +118,548 @@ function _loadParsedDiskCache() { try { if (fs.existsSync(PARSED_CACHE_FILE)) { _parsedDiskCache = JSON.parse(fs.readFileSync(PARSED_CACHE_FILE, 'utf8')); + } else if (fs.existsSync(LEGACY_PARSED_CACHE_FILE)) { + // Migrate from tmpdir once + _parsedDiskCache = JSON.parse(fs.readFileSync(LEGACY_PARSED_CACHE_FILE, 'utf8')); + _parsedDiskCacheDirty = true; } } catch {} if (!_parsedDiskCache) _parsedDiskCache = {}; } -function _saveParsedDiskCache() { +function _saveParsedDiskCache(force) { if (!_parsedDiskCacheDirty || !_parsedDiskCache) return; + if (!force && _parsedDiskCacheEntriesSinceFlush < PARSED_CACHE_FLUSH_EVERY) return; try { - fs.writeFileSync(PARSED_CACHE_FILE, JSON.stringify(_parsedDiskCache)); + // Atomic write: write to .tmp then rename + const tmp = PARSED_CACHE_FILE + '.tmp'; + fs.writeFileSync(tmp, JSON.stringify(_parsedDiskCache)); + fs.renameSync(tmp, PARSED_CACHE_FILE); _parsedDiskCacheDirty = false; + _parsedDiskCacheEntriesSinceFlush = 0; } catch {} } -function parseClaudeSessionFile(sessionFile) { +// ── Disk cache for computed session cost (path+mtime+size → cost) ───────── +const COST_CACHE_FILE = path.join(CODEDASH_CACHE_DIR, 'cost-cache.json'); +let _costDiskCache = null; +let _costDiskCacheDirty = false; +let _costDiskCacheEntriesSinceFlush = 0; +const COST_CACHE_FLUSH_EVERY = 50; + +function _loadCostDiskCache() { + if (_costDiskCache) return; + try { + if (fs.existsSync(COST_CACHE_FILE)) { + _costDiskCache = JSON.parse(fs.readFileSync(COST_CACHE_FILE, 'utf8')); + } + } catch {} + if (!_costDiskCache) _costDiskCache = {}; +} + +function _saveCostDiskCache(force) { + if (!_costDiskCacheDirty || !_costDiskCache) return; + if (!force && _costDiskCacheEntriesSinceFlush < COST_CACHE_FLUSH_EVERY) return; + try { + const tmp = COST_CACHE_FILE + '.tmp'; + fs.writeFileSync(tmp, JSON.stringify(_costDiskCache)); + fs.renameSync(tmp, COST_CACHE_FILE); + _costDiskCacheDirty = false; + _costDiskCacheEntriesSinceFlush = 0; + } catch {} +} + +// ── Background parse warming state ──────────────────────── +// Populated by sync loadSessions() with file paths whose parse cache is stale. +// A singleton background task drains this set with setImmediate yielding so +// the HTTP event loop stays responsive. UI can read _warmingStatus for progress. +const _pendingParseFiles = new Set(); +let _warmingRunning = false; +const _warmingStatus = { + running: false, + done: 0, + total: 0, + phase: 'idle', + startedAt: 0, + finishedAt: 0, +}; + +// ── SQLite ingest helper ──────────────────────────────────── +// Parse a single Claude session file fully (messages + metadata) and push +// a batch row into the SQLite index. Used by the background warmer after +// the parsed-cache entry has been created. Cheap when already indexed. +let _sqliteIngestBatch = []; +const _SQLITE_INGEST_FLUSH_EVERY = 5; + +function _ingestClaudeFileToSqlite(filePath) { + let sqliteIndex; + try { sqliteIndex = require('./sqlite-index'); } catch { return; } + try { + if (sqliteIndex.isFileCurrent(filePath)) return; // already indexed + } catch { return; } + + let stat; + try { stat = fs.statSync(filePath); } catch { return; } + + // Full read — at ingest time we need messages for FTS. Streaming for big + // files would be nicer but the warmer yields between files already. + let lines; + try { lines = readLines(filePath); } catch { return; } + + const sid = path.basename(filePath, '.jsonl'); + let projectPath = ''; + let tool = 'claude'; + let firstTs = stat.mtimeMs; + let lastTs = stat.mtimeMs; + let firstMsg = ''; + let userMsgCount = 0; + let totalMsgCount = 0; + const msgs = []; + const dayMsgs = {}; // day → {messages, first_ts, last_ts} + let seq = 0; + + for (const line of lines) { + try { + const entry = JSON.parse(line); + if (entry.type !== 'user' && entry.type !== 'assistant') continue; + const role = entry.type; + const content = extractContent((entry.message || {}).content); + if (!content) continue; + const ts = entry.timestamp ? (typeof entry.timestamp === 'number' ? entry.timestamp : new Date(entry.timestamp).getTime()) : 0; + if (ts) { + if (ts < firstTs) firstTs = ts; + if (ts > lastTs) lastTs = ts; + } + if (!projectPath && entry.cwd) projectPath = entry.cwd; + totalMsgCount++; + if (role === 'user') { + userMsgCount++; + if (!firstMsg && content) firstMsg = content.slice(0, 200); + } + msgs.push({ seq: seq++, role, ts, content: content.slice(0, 8000) }); + // Day breakdown for user messages only + if (role === 'user' && ts > 1000000000000) { + const d = new Date(ts); + const day = d.getFullYear() + '-' + String(d.getMonth()+1).padStart(2,'0') + '-' + String(d.getDate()).padStart(2,'0'); + if (!dayMsgs[day]) dayMsgs[day] = { messages: 0, first_ts: ts, last_ts: ts }; + dayMsgs[day].messages++; + if (ts < dayMsgs[day].first_ts) dayMsgs[day].first_ts = ts; + if (ts > dayMsgs[day].last_ts) dayMsgs[day].last_ts = ts; + } + } catch {} + } + + // Convert dayMsgs to daily_stats rows + const daily = {}; + for (const day in dayMsgs) { + const dm = dayMsgs[day]; + daily[day] = { + messages: dm.messages, + hours: Math.min((dm.last_ts - dm.first_ts) / 3600000, 16), + }; + } + + _sqliteIngestBatch.push({ + filePath, + session: { + id: sid, + tool, + project: projectPath, + project_short: projectPath.replace(os.homedir(), '~'), + first_ts: firstTs, + last_ts: lastTs, + messages: totalMsgCount, + user_messages: userMsgCount, + file_size: stat.size, + first_message: firstMsg, + source_mtime: stat.mtimeMs, + source_size: stat.size, + }, + messages: msgs, + daily, + }); + + // Don't flush here — backfill loop awaits the flush between files. +} + +// Ingest a single Codex rollout JSONL into SQLite. +function _ingestCodexFileToSqlite(filePath) { + let sqliteIndex; + try { sqliteIndex = require('./sqlite-index'); } catch { return; } + try { if (sqliteIndex.isFileCurrent(filePath)) return; } catch { return; } + + let stat; + try { stat = fs.statSync(filePath); } catch { return; } + + // Extract session id from filename + const basename = path.basename(filePath, '.jsonl'); + const uuidMatch = basename.match(/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/); + if (!uuidMatch) return; + const sid = uuidMatch[1]; + + let lines; + try { lines = readLines(filePath); } catch { return; } + + let projectPath = ''; + let firstTs = stat.mtimeMs; + let lastTs = stat.mtimeMs; + let firstMsg = ''; + let userMsgCount = 0; + let totalMsgCount = 0; + const msgs = []; + const dayMsgs = {}; + let seq = 0; + + for (const line of lines) { + try { + const entry = JSON.parse(line); + if (entry.type === 'session_meta' && entry.payload && entry.payload.cwd && !projectPath) { + projectPath = entry.payload.cwd; + continue; + } + if (entry.type !== 'response_item' || !entry.payload) continue; + const role = entry.payload.role; + if (role !== 'user' && role !== 'assistant') continue; + const content = extractContent(entry.payload.content); + if (!content || isSystemMessage(content)) continue; + let ts = 0; + const tsVal = entry.timestamp || entry.ts; + if (typeof tsVal === 'number') ts = tsVal; + else if (typeof tsVal === 'string') ts = Date.parse(tsVal) || 0; + if (ts && ts < firstTs) firstTs = ts; + if (ts && ts > lastTs) lastTs = ts; + + totalMsgCount++; + if (role === 'user') { + userMsgCount++; + if (!firstMsg) firstMsg = content.slice(0, 200); + } + msgs.push({ seq: seq++, role, ts, content: content.slice(0, 8000) }); + if (role === 'user' && ts > 1000000000000) { + const d = new Date(ts); + const day = d.getFullYear() + '-' + String(d.getMonth()+1).padStart(2,'0') + '-' + String(d.getDate()).padStart(2,'0'); + if (!dayMsgs[day]) dayMsgs[day] = { messages: 0, first_ts: ts, last_ts: ts }; + dayMsgs[day].messages++; + if (ts < dayMsgs[day].first_ts) dayMsgs[day].first_ts = ts; + if (ts > dayMsgs[day].last_ts) dayMsgs[day].last_ts = ts; + } + } catch {} + } + + const daily = {}; + for (const day in dayMsgs) { + const dm = dayMsgs[day]; + daily[day] = { + messages: dm.messages, + hours: Math.min((dm.last_ts - dm.first_ts) / 3600000, 16), + }; + } + + _sqliteIngestBatch.push({ + filePath, + session: { + id: sid, + tool: 'codex', + project: projectPath, + project_short: projectPath.replace(os.homedir(), '~'), + first_ts: firstTs, + last_ts: lastTs, + messages: totalMsgCount, + user_messages: userMsgCount, + file_size: stat.size, + first_message: firstMsg, + source_mtime: stat.mtimeMs, + source_size: stat.size, + }, + messages: msgs, + daily, + }); + + // Don't flush here — backfill loop awaits the flush between files. +} + +async function _flushSqliteIngestBatch() { + if (_sqliteIngestBatch.length === 0) return; + const batch = _sqliteIngestBatch; + _sqliteIngestBatch = []; // take ownership before the async call + try { + const sqliteIndex = require('./sqlite-index'); + await sqliteIndex.indexBatchAsync(batch); + } catch (e) { + try { console.error('sqlite ingest flush failed:', e.message); } catch {} + } +} + +// ── SQLite backfill ──────────────────────────────────────── +// One-shot background task that iterates all existing Claude session files +// and ingests them into SQLite (if not already there, matched by mtime+size +// via files_seen). Yields between files. +let _sqliteBackfillRunning = false; +const _sqliteBackfillStatus = { + running: false, + done: 0, + total: 0, + phase: 'idle', + startedAt: 0, + finishedAt: 0, +}; + +function getSqliteBackfillStatus() { + return Object.assign({}, _sqliteBackfillStatus); +} + +function _ensureSqliteBackfillRunning() { + if (_sqliteBackfillRunning) return; + // Don't re-scan after the initial backfill completed this process lifetime. + // New/changed files are ingested incrementally by the warmer; this full + // scan is only needed once on cold start. + if (_sqliteBackfillStatus.phase === 'done') return; + let sqliteIndex; + try { sqliteIndex = require('./sqlite-index'); } catch { return; } + + _sqliteBackfillRunning = true; + _sqliteBackfillStatus.running = true; + _sqliteBackfillStatus.startedAt = Date.now(); + _sqliteBackfillStatus.phase = 'scanning'; + _sqliteBackfillStatus.done = 0; + + setImmediate(async () => { + try { + sqliteIndex.ensureSchema(); + + // Enumerate all Claude JSONL files + Codex session files + const allFiles = []; // array of {file, kind} + const walkClaude = (dir) => { + try { + for (const proj of fs.readdirSync(dir)) { + const pDir = path.join(dir, proj); + try { + if (!fs.statSync(pDir).isDirectory()) continue; + for (const f of fs.readdirSync(pDir)) { + if (f.endsWith('.jsonl')) allFiles.push({ file: path.join(pDir, f), kind: 'claude' }); + } + } catch {} + } + } catch {} + }; + if (fs.existsSync(PROJECTS_DIR)) walkClaude(PROJECTS_DIR); + for (const extra of EXTRA_CLAUDE_DIRS) { + const ep = path.join(extra, 'projects'); + if (fs.existsSync(ep)) walkClaude(ep); + } + // Codex session files (~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl) + const codexSessDir = path.join(CODEX_DIR, 'sessions'); + if (fs.existsSync(codexSessDir)) { + const walkCodex = (dir) => { + try { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) walkCodex(full); + else if (entry.name.endsWith('.jsonl')) allFiles.push({ file: full, kind: 'codex' }); + } + } catch {} + }; + walkCodex(codexSessDir); + } + + _sqliteBackfillStatus.total = allFiles.length; + _sqliteBackfillStatus.phase = 'ingesting'; + + // Sort newest first so fresh sessions hit the index sooner + allFiles.sort((a, b) => { + try { return fs.statSync(b.file).mtimeMs - fs.statSync(a.file).mtimeMs; } catch { return 0; } + }); + + // Single bulk read of files_seen — avoids N sync SQL calls + const filesSeen = sqliteIndex.loadAllFilesSeen(); + + for (const item of allFiles) { + try { + let stat; + try { stat = fs.statSync(item.file); } catch { _sqliteBackfillStatus.done++; continue; } + const seen = filesSeen.get(item.file); + const isCurrent = seen && seen.mtime === stat.mtimeMs && seen.size === stat.size; + if (!isCurrent) { + if (item.kind === 'codex') { + _ingestCodexFileToSqlite(item.file); + } else { + _ingestClaudeFileToSqlite(item.file); + } + if (_sqliteIngestBatch.length >= _SQLITE_INGEST_FLUSH_EVERY) { + await _flushSqliteIngestBatch(); + } + } + } catch {} + _sqliteBackfillStatus.done++; + // Yield every 10 files so HTTP requests can slot in + if (_sqliteBackfillStatus.done % 10 === 0) { + await new Promise(r => setImmediate(r)); + } + } + await _flushSqliteIngestBatch(); + + // Phase 2: compute embeddings for sessions that don't have them yet + // (requires optional @huggingface/transformers npm package) + try { + const embeddings = require('./embeddings'); + if (embeddings.isAvailable()) { + _sqliteBackfillStatus.phase = 'computing embeddings'; + embeddings.ensureEmbeddingTable(); + const existingCount = embeddings.getEmbeddingCount(); + // Get sessions without embeddings + const allSessionRows = sqliteIndex._execJson(`SELECT id, first_message FROM sessions WHERE id NOT IN (SELECT session_id FROM session_embeddings) AND first_message != '' LIMIT 5000`); + if (allSessionRows.length > 0) { + _sqliteBackfillStatus.total = allSessionRows.length; + _sqliteBackfillStatus.done = 0; + const BATCH = 32; + for (let i = 0; i < allSessionRows.length; i += BATCH) { + const batch = allSessionRows.slice(i, i + BATCH); + const texts = batch.map(r => (r.first_message || '').slice(0, 512)); + try { + const embs = await embeddings.embedBatch(texts); + const rows = batch.map((r, j) => ({ + session_id: r.id, + embedding: embs[j], + model: embeddings.MODEL_ID, + })); + embeddings.storeEmbeddings(rows); + } catch (e) { + // Model download/init error — skip embeddings + break; + } + _sqliteBackfillStatus.done = Math.min(i + BATCH, allSessionRows.length); + await new Promise(r => setImmediate(r)); + } + } + } + } catch {} + + _sqliteBackfillStatus.phase = 'done'; + _sqliteBackfillStatus.finishedAt = Date.now(); + } catch (e) { + _sqliteBackfillStatus.phase = 'error: ' + (e && e.message || 'unknown'); + _sqliteBackfillStatus.finishedAt = Date.now(); + } finally { + _sqliteBackfillStatus.running = false; + _sqliteBackfillRunning = false; + } + }); +} + +function _ensureWarmingRunning() { + if (_warmingRunning) return; + if (_pendingParseFiles.size === 0 && _pendingCodexFiles.size === 0) return; + _warmingRunning = true; + _warmingStatus.running = true; + _warmingStatus.startedAt = Date.now(); + _warmingStatus.done = 0; + _warmingStatus.total = _pendingParseFiles.size + _pendingCodexFiles.size; + _warmingStatus.phase = 'parsing session files'; + setImmediate(async () => { + try { + while (_pendingParseFiles.size > 0 || _pendingCodexFiles.size > 0) { + // Claude batch (newest-first) + if (_pendingParseFiles.size > 0) { + const batch = Array.from(_pendingParseFiles); + batch.sort((a, b) => { + try { return fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs; } catch { return 0; } + }); + _warmingStatus.total = _warmingStatus.done + batch.length + _pendingCodexFiles.size; + for (const file of batch) { + _pendingParseFiles.delete(file); + try { + let stat; + try { stat = fs.statSync(file); } catch { _warmingStatus.done++; continue; } + if (stat.size >= 5 * 1024 * 1024) { + await parseClaudeSessionFileAsync(file); + } else { + parseClaudeSessionFile(file); + await new Promise(r => setImmediate(r)); + } + // Also push into persistent SQLite index (batched) + try { _ingestClaudeFileToSqlite(file); } catch {} + } catch {} + _warmingStatus.done++; + } + _flushSqliteIngestBatch(); + } + // Codex batch + if (_pendingCodexFiles.size > 0) { + const batch = Array.from(_pendingCodexFiles); + batch.sort((a, b) => { + try { return fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs; } catch { return 0; } + }); + _warmingStatus.total = _warmingStatus.done + batch.length + _pendingParseFiles.size; + for (const file of batch) { + _pendingCodexFiles.delete(file); + try { + parseCodexSessionFile(file); + await new Promise(r => setImmediate(r)); + } catch {} + _warmingStatus.done++; + } + } + // Invalidate session cache so next /api/sessions picks up new detail + _sessionsCache = null; + _sessionsCacheTs = 0; + } + _saveParsedDiskCache(true); + _flushSqliteIngestBatch(); + _warmingStatus.phase = 'done'; + _warmingStatus.finishedAt = Date.now(); + } catch (e) { + _warmingStatus.phase = 'error: ' + (e && e.message || 'unknown'); + _warmingStatus.finishedAt = Date.now(); + } finally { + _warmingStatus.running = false; + _warmingRunning = false; + } + }); +} + +function getWarmingStatus() { + return Object.assign({}, _warmingStatus, { + pending: _pendingParseFiles.size, + }); +} + +// Flush all caches on process exit / signals so partial progress isn't lost +let _flushHandlersInstalled = false; +let _flushInProgress = null; +function _installFlushHandlers() { + if (_flushHandlersInstalled) return; + _flushHandlersInstalled = true; + const flushAllSync = () => { + try { _saveParsedDiskCache(true); } catch {} + try { _saveCostDiskCache(true); } catch {} + try { if (typeof _saveDailyStatsDiskCache === 'function') _saveDailyStatsDiskCache(); } catch {} + try { if (typeof _saveGitRootDiskCache === 'function') _saveGitRootDiskCache(); } catch {} + }; + const flushAll = async () => { + flushAllSync(); + try { + if (typeof _flushSqliteIngestBatch === 'function') { + await _flushSqliteIngestBatch(); + } + } catch {} + }; + const handleSignal = async () => { + if (!_flushInProgress) { + _flushInProgress = flushAll(); + } + try { + await _flushInProgress; + } finally { + process.exit(0); + } + }; + process.on('exit', flushAllSync); + process.on('SIGINT', handleSignal); + process.on('SIGTERM', handleSignal); +} +_installFlushHandlers(); + +function parseClaudeSessionFile(sessionFile, opts) { if (!fs.existsSync(sessionFile)) return null; let stat; @@ -139,6 +675,14 @@ function parseClaudeSessionFile(sessionFile) { _fileCacheKeyIndex[sessionFile] = cacheKey; if (_parsedDiskCache[cacheKey]) return _parsedDiskCache[cacheKey]; + // Cache-only mode: skip actual file read when not cached. Caller gets + // null-ish placeholder and the real parse happens in a background job. + // This is what keeps the sync loadSessions() fast on cold caches. + if (opts && opts.cacheOnly) { + _pendingParseFiles.add(sessionFile); + return null; + } + let lines; try { lines = readLines(sessionFile); @@ -158,11 +702,27 @@ function parseClaudeSessionFile(sessionFile) { const mcpSet = new Set(); const skillSet = new Set(); + // Helper: is this `type=user` entry a REAL user prompt, not a tool_result? + // In Claude Code JSONL, tool_result messages are stored as type='user' with + // content=[{type:'tool_result', ...}]. We want to count only messages whose + // content contains real text from the human. + const isRealUserPrompt = (entry) => { + const c = (entry.message || {}).content; + if (typeof c === 'string') return c.trim().length > 0; + if (Array.isArray(c)) { + for (const p of c) { + if (p && p.type === 'text' && p.text && p.text.trim()) return true; + } + return false; + } + return false; + }; + for (const line of lines) { try { const entry = JSON.parse(line); if (entry.type === 'user' || entry.type === 'assistant') msgCount++; - if (entry.type === 'user') userMsgCount++; + if (entry.type === 'user' && isRealUserPrompt(entry)) userMsgCount++; if (entry.timestamp) { if (entry.timestamp < firstTs) firstTs = entry.timestamp; if (entry.timestamp > lastTs) lastTs = entry.timestamp; @@ -225,6 +785,128 @@ function parseClaudeSessionFile(sessionFile) { // Cache to disk _parsedDiskCache[cacheKey] = result; _parsedDiskCacheDirty = true; + _parsedDiskCacheEntriesSinceFlush++; + // Periodic flush so long operations don't lose progress on kill + if (_parsedDiskCacheEntriesSinceFlush >= PARSED_CACHE_FLUSH_EVERY) { + _saveParsedDiskCache(true); + } + return result; +} + +// Async variant: for large files (>5 MB) reads in chunks with setImmediate +// between chunks so the Node event loop can handle other requests. Same +// result shape and cache as sync version. +async function parseClaudeSessionFileAsync(sessionFile) { + if (!fs.existsSync(sessionFile)) return null; + let stat; + try { stat = fs.statSync(sessionFile); } catch { return null; } + + _loadParsedDiskCache(); + const cacheKey = sessionFile + '|' + stat.mtimeMs + '|' + stat.size; + _fileCacheKeyIndex[sessionFile] = cacheKey; + if (_parsedDiskCache[cacheKey]) return _parsedDiskCache[cacheKey]; + + // Small files: use sync path (faster, avoids Promise overhead) + const BIG_THRESHOLD = 5 * 1024 * 1024; // 5 MB + if (stat.size < BIG_THRESHOLD) return parseClaudeSessionFile(sessionFile); + + // Big file: stream read + parse line by line with periodic yielding. + let projectPath = ''; + let tool = 'claude'; + let msgCount = 0; + let firstMsg = ''; + let customTitle = ''; + let firstTs = stat.mtimeMs; + let lastTs = stat.mtimeMs; + let userMsgCount = 0; + let entrypointFound = false; + let worktreeOriginalCwd = ''; + const mcpSet = new Set(); + const skillSet = new Set(); + + const readline = require('readline'); + const stream = fs.createReadStream(sessionFile, { encoding: 'utf8', highWaterMark: 1 << 20 }); + const rl = readline.createInterface({ input: stream, crlfDelay: Infinity }); + + const isRealUserPromptAsync = (entry) => { + const c = (entry.message || {}).content; + if (typeof c === 'string') return c.trim().length > 0; + if (Array.isArray(c)) { + for (const p of c) { + if (p && p.type === 'text' && p.text && p.text.trim()) return true; + } + return false; + } + return false; + }; + + let linesSinceYield = 0; + for await (const line of rl) { + if (!line) continue; + try { + const entry = JSON.parse(line); + if (entry.type === 'user' || entry.type === 'assistant') msgCount++; + if (entry.type === 'user' && isRealUserPromptAsync(entry)) userMsgCount++; + const rawTimestamp = entry.timestamp; + const normalizedTimestamp = + typeof rawTimestamp === 'number' + ? rawTimestamp + : (typeof rawTimestamp === 'string' ? Date.parse(rawTimestamp) : NaN); + if (Number.isFinite(normalizedTimestamp)) { + if (normalizedTimestamp < firstTs) firstTs = normalizedTimestamp; + if (normalizedTimestamp > lastTs) lastTs = normalizedTimestamp; + } + if (!projectPath && entry.type === 'user' && entry.cwd) projectPath = entry.cwd; + if (!worktreeOriginalCwd && entry.type === 'worktree-state' && entry.worktreeSession && entry.worktreeSession.originalCwd) { + worktreeOriginalCwd = entry.worktreeSession.originalCwd; + } + if (!entrypointFound && entry.type === 'user' && entry.entrypoint) { + entrypointFound = true; + if (entry.entrypoint !== 'cli') tool = 'claude-ext'; + } + if (entry.type === 'custom-title' && typeof entry.customTitle === 'string') { + const title = entry.customTitle.trim(); + if (title) customTitle = title.slice(0, 200); + } + if (!firstMsg && entry.type === 'user' && entry.message && entry.message.content) { + const content = extractContent(entry.message.content).trim(); + if (content) firstMsg = content.slice(0, 200); + } + if (entry.type === 'assistant') { + const aContent = (entry.message || {}).content; + if (Array.isArray(aContent)) { + for (const block of aContent) { + if (!block || block.type !== 'tool_use') continue; + const name = block.name || ''; + if (name.startsWith('mcp__')) { + const parts = name.split('__'); + if (parts.length >= 3) mcpSet.add(parts[1]); + } else if (name === 'Skill') { + const sk = (block.input || {}).skill; + if (sk) skillSet.add(sk.includes(':') ? sk.split(':')[0] : sk); + } + } + } + } + } catch {} + if (++linesSinceYield >= 2000) { + linesSinceYield = 0; + await new Promise(r => setImmediate(r)); + } + } + + const result = { + projectPath, tool, msgCount, userMsgCount, + firstMsg, customTitle, firstTs, lastTs, + fileSize: stat.size, worktreeOriginalCwd, + mcpServers: Array.from(mcpSet), skills: Array.from(skillSet), + }; + _parsedDiskCache[cacheKey] = result; + _parsedDiskCacheDirty = true; + _parsedDiskCacheEntriesSinceFlush++; + if (_parsedDiskCacheEntriesSinceFlush >= PARSED_CACHE_FLUSH_EVERY) { + _saveParsedDiskCache(true); + } return result; } @@ -329,10 +1011,11 @@ function scanOpenCodeSessions() { const sessionMcp = {}; const sessionSkills = {}; try { - const toolRows = execSync( - `sqlite3 -separator $'\\t' "${OPENCODE_DB}" "SELECT session_id, json_extract(data, '\\$.tool'), json_extract(data, '\\$.state.input.name') FROM part WHERE json_extract(data, '\\$.type') = 'tool'"`, - { encoding: 'utf8', timeout: 10000, maxBuffer: 50 * 1024 * 1024 } - ).trim(); + const toolRows = execFileSync('sqlite3', [ + '-separator', '\t', + OPENCODE_DB, + "SELECT session_id, json_extract(data, '$.tool'), json_extract(data, '$.state.input.name') FROM part WHERE json_extract(data, '$.type') = 'tool'" + ], { encoding: 'utf8', timeout: 10000, maxBuffer: 50 * 1024 * 1024, windowsHide: true }).trim(); if (toolRows) { for (const tr of toolRows.split('\n')) { const cols = tr.split('\t'); @@ -926,13 +1609,25 @@ function loadCursorVscdbDetail(sessionId) { return { messages: messages.slice(0, 200) }; } -function parseCodexSessionFile(sessionFile) { +function parseCodexSessionFile(sessionFile, opts) { if (!fs.existsSync(sessionFile)) return null; let stat; + try { stat = fs.statSync(sessionFile); } catch { return null; } + + // Reuse the same parsed-cache keyed by path+mtime+size + _loadParsedDiskCache(); + const cacheKey = 'codex:' + sessionFile + '|' + stat.mtimeMs + '|' + stat.size; + if (_parsedDiskCache[cacheKey]) return _parsedDiskCache[cacheKey]; + + // Cache-only mode: queue for background parsing, don't block + if (opts && opts.cacheOnly) { + _pendingCodexFiles.add(sessionFile); + return null; + } + let lines; try { - stat = fs.statSync(sessionFile); lines = readLines(sessionFile); } catch { return null; @@ -955,6 +1650,25 @@ function parseCodexSessionFile(sessionFile) { let firstMsg = ''; let firstTs = stat.mtimeMs; let lastTs = stat.mtimeMs; + // Detect sub-agent/scripted codex sessions. Multiple signals: + // 1. originator='codex_exec' or source='exec' (standard `codex exec`) + // 2. first user prompt matches known auto-script patterns (team scripts + // that spawn codex without the exec flag). Covers: "You are in /path. + // Task: X", "Read-only task. Inspect ...", "Work in /path", "Pair-local + // lane", "## Memory Writing Agent", etc. + let isHelper = false; + // Regexes checked against the first user prompt (see after the loop) + const AUTO_SCRIPT_PATTERNS = [ + /^You are in \//, + /^Read-only task\./, + /^Work (only )?in \//, + /^Pair-local [^\n]{1,60} lane/, + /^## [A-Z][a-z]+ [A-Z][a-z]+ Agent/, // "## Memory Writing Agent:..." + /^Read \/[^\s]+\/AGENTS\.md/, + /^Read \$[A-Z_]+\//, + /^\[Sub-agent results\]/, + /^You are OMX /, + ]; const mcpSet = new Set(); for (const line of lines) { @@ -966,8 +1680,11 @@ function parseCodexSessionFile(sessionFile) { if (ts > lastTs) lastTs = ts; } - if (entry.type === 'session_meta' && entry.payload && entry.payload.cwd && !projectPath) { - projectPath = entry.payload.cwd; + if (entry.type === 'session_meta' && entry.payload) { + if (entry.payload.cwd && !projectPath) projectPath = entry.payload.cwd; + const originator = entry.payload.originator || ''; + const source = entry.payload.source || ''; + if (originator === 'codex_exec' || source === 'exec') isHelper = true; continue; } @@ -995,7 +1712,14 @@ function parseCodexSessionFile(sessionFile) { } catch {} } - return { + // Secondary helper detection by first user prompt pattern + if (!isHelper && firstMsg) { + for (const re of AUTO_SCRIPT_PATTERNS) { + if (re.test(firstMsg)) { isHelper = true; break; } + } + } + + const result = { projectPath, msgCount, userMsgCount, @@ -1004,11 +1728,23 @@ function parseCodexSessionFile(sessionFile) { lastTs, fileSize: stat.size, mcpServers: Array.from(mcpSet), + isHelper, }; + _parsedDiskCache[cacheKey] = result; + _parsedDiskCacheDirty = true; + _parsedDiskCacheEntriesSinceFlush++; + if (_parsedDiskCacheEntriesSinceFlush >= PARSED_CACHE_FLUSH_EVERY) { + _saveParsedDiskCache(true); + } + return result; } +// Queue for background codex file parsing (drained by ensureWarmingRunning) +const _pendingCodexFiles = new Set(); + function scanCodexSessions() { - const sessions = []; + // Map for O(1) session lookups (was O(n²) with .find) + const sessionsById = new Map(); const codexTitles = parseCodexSessionIndex(CODEX_DIR); const codexHistory = path.join(CODEX_DIR, 'history.jsonl'); if (fs.existsSync(codexHistory)) { @@ -1016,12 +1752,11 @@ function scanCodexSessions() { for (const line of lines) { try { const d = JSON.parse(line); - // Codex uses session_id, ts (seconds), text const sid = d.session_id || d.sessionId || d.id; if (!sid) continue; const ts = d.ts ? d.ts * 1000 : (d.timestamp || Date.now()); - if (!sessions.find(s => s.id === sid)) { - sessions.push({ + if (!sessionsById.has(sid)) { + sessionsById.set(sid, { id: sid, tool: 'codex', project: d.project || d.cwd || '', @@ -1040,10 +1775,10 @@ function scanCodexSessions() { } // Enrich with session files from ~/.codex/sessions/ + // Cache-only: uses the parse cache; uncached files go to background queue. const codexSessionsDir = path.join(CODEX_DIR, 'sessions'); if (fs.existsSync(codexSessionsDir)) { try { - // Walk year/month/day directories const files = []; const walkDir = (dir) => { for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { @@ -1055,37 +1790,61 @@ function scanCodexSessions() { walkDir(codexSessionsDir); for (const f of files) { - // Extract session ID from filename (rollout-DATE-UUID.jsonl) const basename = path.basename(f, '.jsonl'); const uuidMatch = basename.match(/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/); if (!uuidMatch) continue; const sid = uuidMatch[1]; - const summary = parseCodexSessionFile(f); - if (!summary) continue; - - const existing = sessions.find(s => s.id === sid); + const summary = parseCodexSessionFile(f, { cacheOnly: true }); + if (!summary) { + // No cached summary yet — create a minimal placeholder from stat + // so the session still shows in the list. Background job will enrich. + let size = 0, mtimeMs = 0; + try { const st = fs.statSync(f); size = st.size; mtimeMs = st.mtimeMs; } catch {} + const existing = sessionsById.get(sid); + if (existing) { + existing.has_detail = true; + existing.file_size = size; + existing._detail_pending = true; + } else { + sessionsById.set(sid, { + id: sid, + tool: 'codex', + project: '', + project_short: '', + first_ts: mtimeMs, + last_ts: mtimeMs, + messages: 0, + first_message: codexTitles[sid] || '', + has_detail: true, + file_size: size, + detail_messages: 0, + user_messages: 0, + mcp_servers: [], + skills: [], + _detail_pending: true, + }); + } + continue; + } + const existing = sessionsById.get(sid); if (existing) { existing.has_detail = true; existing.file_size = summary.fileSize; existing.messages = summary.msgCount; existing.detail_messages = summary.msgCount; existing.user_messages = summary.userMsgCount || 0; - if (codexTitles[sid]) { - existing.first_message = codexTitles[sid]; - } else if (summary.firstMsg && !existing.first_message) { - existing.first_message = summary.firstMsg; - } + if (summary.isHelper) existing.is_helper = true; + if (codexTitles[sid]) existing.first_message = codexTitles[sid]; + else if (summary.firstMsg && !existing.first_message) existing.first_message = summary.firstMsg; if (summary.projectPath && !existing.project) { existing.project = summary.projectPath; existing.project_short = summary.projectPath.replace(os.homedir(), '~'); } existing.first_ts = Math.min(existing.first_ts, summary.firstTs); existing.last_ts = Math.max(existing.last_ts, summary.lastTs); - if (summary.mcpServers && summary.mcpServers.length > 0) { - existing.mcp_servers = summary.mcpServers; - } + if (summary.mcpServers && summary.mcpServers.length > 0) existing.mcp_servers = summary.mcpServers; } else { - sessions.push({ + sessionsById.set(sid, { id: sid, tool: 'codex', project: summary.projectPath, @@ -1100,13 +1859,14 @@ function scanCodexSessions() { user_messages: summary.userMsgCount || 0, mcp_servers: summary.mcpServers || [], skills: [], + is_helper: summary.isHelper || false, }); } } } catch {} } - return sessions; + return Array.from(sessionsById.values()); } // ── Git root resolver ─────────────────────────────────────── @@ -1121,46 +1881,105 @@ function scanCodexSessions() { // from the session cwd string. Works without git for standard worktree layouts. const _gitRootCache = {}; -const GIT_ROOT_CACHE_FILE = path.join(os.tmpdir(), 'codedash-gitroot-cache.json'); +// Persistent dir (survives /tmp cleanup). Legacy file in tmpdir migrated once. +const GIT_ROOT_CACHE_FILE = path.join(CODEDASH_CACHE_DIR, 'git-root-cache.json'); +const LEGACY_GIT_ROOT_CACHE_FILE = path.join(os.tmpdir(), 'codedash-gitroot-cache.json'); let _gitRootDiskCache = null; +let _gitRootDirty = false; +// Queue of project paths that still need git resolution. Filled by sync +// loadSessions() when a path is uncached, drained in background by +// _ensureGitRootResolverRunning() so the sync path returns immediately. +const _pendingGitRoots = new Set(); +let _gitRootResolverRunning = false; function _loadGitRootDiskCache() { if (_gitRootDiskCache) return; try { if (fs.existsSync(GIT_ROOT_CACHE_FILE)) { _gitRootDiskCache = JSON.parse(fs.readFileSync(GIT_ROOT_CACHE_FILE, 'utf8')); - // Pre-fill memory cache from disk - Object.assign(_gitRootCache, _gitRootDiskCache); + } else if (fs.existsSync(LEGACY_GIT_ROOT_CACHE_FILE)) { + _gitRootDiskCache = JSON.parse(fs.readFileSync(LEGACY_GIT_ROOT_CACHE_FILE, 'utf8')); + _gitRootDirty = true; } + if (_gitRootDiskCache) Object.assign(_gitRootCache, _gitRootDiskCache); } catch {} if (!_gitRootDiskCache) _gitRootDiskCache = {}; } function _saveGitRootDiskCache() { + if (!_gitRootDirty) return; try { - fs.writeFileSync(GIT_ROOT_CACHE_FILE, JSON.stringify(_gitRootCache)); + const tmp = GIT_ROOT_CACHE_FILE + '.tmp'; + fs.writeFileSync(tmp, JSON.stringify(_gitRootCache)); + fs.renameSync(tmp, GIT_ROOT_CACHE_FILE); + _gitRootDirty = false; } catch {} } +// Sync resolver: never shells out from the hot path. Uses cache; queues +// unknown paths for background resolution. +// ── Conversation group key ───────────────────────────────── +// Two sessions belong to the same conversation group if they share the same +// project AND the same initial user prompt (first ~200 chars). This dedupes +// `codex exec` retries, `codex resume` chains, and sub-agent re-runs that +// would otherwise show as N separate cards in the UI. Helper sessions +// (is_helper=true) all collapse into their own single bucket per project. +function computeSessionGroupKey(s) { + const proj = (s.project || 'unknown').trim(); + // Normalize first message: collapse whitespace, trim, truncate + let msg = (s.first_message || '').replace(/\s+/g, ' ').trim().slice(0, 200); + if (s.is_helper) { + // All helpers in the same project collapse into one group. + return 'helper::' + proj; + } + if (!msg) return s.tool + '::' + proj + '::' + s.id; // no prompt — unique + return s.tool + '::' + proj + '::' + msg; +} + function resolveGitRoot(projectPath) { if (!projectPath) return ''; _loadGitRootDiskCache(); if (_gitRootCache[projectPath] !== undefined) return _gitRootCache[projectPath]; - // Skip remote/non-existent paths + // Fast skip: non-existent paths can never be git roots, record & done. if (!fs.existsSync(projectPath)) { _gitRootCache[projectPath] = ''; + _gitRootDirty = true; return ''; } - try { - const root = execFileSync('git', ['-C', projectPath, 'rev-parse', '--show-toplevel'], { - encoding: 'utf8', timeout: 2000, windowsHide: true, stdio: ['pipe', 'pipe', 'pipe'] - }).trim(); - _gitRootCache[projectPath] = root; - return root; - } catch { - _gitRootCache[projectPath] = ''; - return ''; - } + // Queue for background resolver — sync path returns '' for now + _pendingGitRoots.add(projectPath); + _ensureGitRootResolverRunning(); + return ''; +} + +function _ensureGitRootResolverRunning() { + if (_gitRootResolverRunning) return; + if (_pendingGitRoots.size === 0) return; + _gitRootResolverRunning = true; + setImmediate(async () => { + try { + while (_pendingGitRoots.size > 0) { + const snapshot = Array.from(_pendingGitRoots); + for (const p of snapshot) { + _pendingGitRoots.delete(p); + if (_gitRootCache[p] !== undefined) continue; + let root = ''; + try { + root = execFileSync('git', ['-C', p, 'rev-parse', '--show-toplevel'], { + encoding: 'utf8', timeout: 2000, windowsHide: true, stdio: ['pipe', 'pipe', 'pipe'], + }).trim(); + } catch {} + _gitRootCache[p] = root; + _gitRootDirty = true; + // Yield between shell-outs so HTTP stays responsive + await new Promise(r => setImmediate(r)); + } + _saveGitRootDiskCache(); + } + } finally { + _gitRootResolverRunning = false; + } + }); } const _gitInfoCache = {}; @@ -1205,15 +2024,15 @@ function getProjectGitInfo(projectPath) { let _sessionsCache = null; let _sessionsCacheTs = 0; -const SESSIONS_CACHE_TTL = 60000; // 60 seconds — hot cache, invalidated by file changes +const SESSIONS_CACHE_TTL = 60000; // 60 seconds — hot cache, extended if no file changes -// Track file mtimes for smart invalidation +// Track history/projects mtime so we only rescan when files actually changed. +// Lets loadSessions() extend the cache window past TTL when nothing happened. let _historyMtime = 0; let _historySize = 0; let _projectsDirMtime = 0; function _sessionsNeedRescan() { - // Check if history.jsonl or projects dir changed since last scan try { if (fs.existsSync(HISTORY_FILE)) { const st = fs.statSync(HISTORY_FILE); @@ -1374,11 +2193,11 @@ function _loadCursorVscdbInBackground() { function loadSessions() { const now = Date.now(); if (_sessionsCache) { - // Hot cache: return immediately if within TTL and no file changes + // Hot cache: return immediately within TTL window if ((now - _sessionsCacheTs) < SESSIONS_CACHE_TTL) return _sessionsCache; - // Extended cache: even after TTL, only rescan if files actually changed + // Extended cache: after TTL, only rescan if files actually changed if (!_sessionsNeedRescan()) { - _sessionsCacheTs = now; // extend TTL + _sessionsCacheTs = now; return _sessionsCache; } } @@ -1539,12 +2358,19 @@ function loadSessions() { } if (sessionFile) { - const summary = parseClaudeSessionFile(sessionFile); + // Cache-only: never read disk from the sync path. Uncached files are + // queued for background warming — subsequent calls will see full data. + const summary = parseClaudeSessionFile(sessionFile, { cacheOnly: true }); if (summary) mergeClaudeSessionDetail(s, summary, sessionFile); else { + // Placeholder: session exists on disk, actual details pending s.has_detail = true; - try { s.file_size = fs.statSync(sessionFile).size; } catch { s.file_size = 0; } s._session_file = sessionFile; + s._detail_pending = true; + try { s.file_size = fs.statSync(sessionFile).size; } catch { s.file_size = 0; } + if (s.detail_messages === undefined) s.detail_messages = 0; + if (!s.mcp_servers) s.mcp_servers = []; + if (!s.skills) s.skills = []; } } else if (!s.has_detail) { s.has_detail = false; @@ -1556,6 +2382,7 @@ function loadSessions() { } // Scan project dirs for orphan sessions (e.g. Claude Extension sessions not in history.jsonl) + // Cache-only in sync path; uncached files get queued and background job parses them. if (fs.existsSync(PROJECTS_DIR)) { try { for (const proj of fs.readdirSync(PROJECTS_DIR)) { @@ -1566,12 +2393,46 @@ function loadSessions() { const sid = file.replace('.jsonl', ''); const filePath = path.join(projDir, file); if (sessions[sid]) { - const summary = parseClaudeSessionFile(filePath); + const summary = parseClaudeSessionFile(filePath, { cacheOnly: true }); if (summary) mergeClaudeSessionDetail(sessions[sid], summary, filePath); + else sessions[sid]._detail_pending = true; + continue; + } + const summary = parseClaudeSessionFile(filePath, { cacheOnly: true }); + if (!summary) { + // Create a placeholder from stat + history of same sid (none here) + let size = 0; + let placeholderTs = 0; + try { + const stat = fs.statSync(filePath); + size = stat.size; + if (Number.isFinite(stat.mtimeMs) && stat.mtimeMs > 0) { + placeholderTs = Math.floor(stat.mtimeMs); + } else if (Number.isFinite(stat.ctimeMs) && stat.ctimeMs > 0) { + placeholderTs = Math.floor(stat.ctimeMs); + } + } catch {} + sessions[sid] = { + id: sid, + tool: 'claude', + project: '', + project_short: '', + first_ts: placeholderTs, + last_ts: placeholderTs, + messages: 0, + first_message: '', + has_detail: true, + file_size: size, + detail_messages: 0, + mcp_servers: [], + skills: [], + _claude_dir: CLAUDE_DIR, + _session_file: filePath, + _detail_pending: true, + worktree_original_cwd: '', + }; continue; } - const summary = parseClaudeSessionFile(filePath); - if (!summary) continue; sessions[sid] = { id: sid, tool: summary.tool, @@ -1595,6 +2456,12 @@ function loadSessions() { } catch {} } + // If any Claude files were uncached, _pendingParseFiles has been populated + // by parseClaudeSessionFile(cacheOnly: true). Kick off the warmer. + _ensureWarmingRunning(); + // Kick off one-shot SQLite backfill (noop if already running/done) + _ensureSqliteBackfillRunning(); + // Ensure all sessions have mcp_servers/skills (defaults for non-Claude) for (const s of Object.values(sessions)) { if (!s.mcp_servers) s.mcp_servers = []; @@ -1625,12 +2492,27 @@ function loadSessions() { s.date = dt.getFullYear() + '-' + String(dt.getMonth()+1).padStart(2,'0') + '-' + String(dt.getDate()).padStart(2,'0'); // Priority: worktree-state.originalCwd (container-safe) > git rev-parse > path heuristic (frontend) s.git_root = s.worktree_original_cwd || (s.project ? (_gitRootCache[s.project] || '') : ''); + // Conversation group key — sessions with the same project + initial + // prompt are treated as retries/resumes of the same conversation. Used + // by the shared groupSessions() helper in all views (Timeline, All + // Sessions, Cloud Sync) so we don't render 50 duplicate cards. + s.group_key = computeSessionGroupKey(s); } // Flag for frontend: true = cursor vscdb still loading, will have more data soon result._loading = !_cursorVscdbSessions && _cursorVscdbLoading; + // Warming state: true when background parse of session files is in progress + result._warming = _warmingStatus.running; + if (_warmingStatus.running || _warmingStatus.pending > 0) { + result._warmingProgress = { + done: _warmingStatus.done, + total: _warmingStatus.total, + phase: _warmingStatus.phase, + pending: _pendingParseFiles.size, + }; + } - // Flush disk caches + // Flush disk caches (non-force: only writes if dirty beyond threshold) _saveParsedDiskCache(); _saveGitRootDiskCache(); _updateScanMarkers(); @@ -1640,23 +2522,111 @@ function loadSessions() { return result; } -function loadSessionDetail(sessionId, project) { +// ── Async pre-warm of parse cache + incremental change detection ───────── +// Lists all Claude session JSONL files and parses those whose cache entry +// is stale (mtime or size changed) or missing. Yields between files so the +// HTTP event loop stays responsive. Reports progress via callback. +async function loadSessionsAsync(progressCb) { + _loadParsedDiskCache(); + _loadCostDiskCache(); + + const report = (phase, done, total, extra) => { + if (typeof progressCb === 'function') { + try { progressCb({ phase, done, total, ...(extra || {}) }); } catch {} + } + }; + + // Phase 1: enumerate all Claude JSONL files (fast, just readdir) + report('scanning files', 0, 0); + const allClaudeFiles = []; + const walkClaude = (dir) => { + try { + for (const proj of fs.readdirSync(dir)) { + const projDir = path.join(dir, proj); + try { + if (!fs.statSync(projDir).isDirectory()) continue; + for (const file of fs.readdirSync(projDir)) { + if (!file.endsWith('.jsonl')) continue; + allClaudeFiles.push(path.join(projDir, file)); + } + } catch {} + } + } catch {} + }; + if (fs.existsSync(PROJECTS_DIR)) walkClaude(PROJECTS_DIR); + for (const extra of EXTRA_CLAUDE_DIRS) { + const ep = path.join(extra, 'projects'); + if (fs.existsSync(ep)) walkClaude(ep); + } + + // Phase 2: figure out which files need (re)parsing — incremental path + const toParse = []; + let cachedCount = 0; + for (const f of allClaudeFiles) { + let stat; + try { stat = fs.statSync(f); } catch { continue; } + const key = f + '|' + stat.mtimeMs + '|' + stat.size; + if (_parsedDiskCache[key]) { + _fileCacheKeyIndex[f] = key; + cachedCount++; + } else { + toParse.push({ file: f, size: stat.size }); + } + } + // Parse newest (largest mtime) first so dashboards reflect fresh data fastest + toParse.sort((a, b) => { + try { return fs.statSync(b.file).mtimeMs - fs.statSync(a.file).mtimeMs; } catch { return 0; } + }); + + const total = allClaudeFiles.length; + report('parsing session files', cachedCount, total, { cached: cachedCount, toParse: toParse.length }); + + // Phase 3: parse uncached files with yielding + let done = cachedCount; + for (const { file, size } of toParse) { + try { + if (size >= 5 * 1024 * 1024) { + await parseClaudeSessionFileAsync(file); + } else { + parseClaudeSessionFile(file); + // Tiny yield so poll requests can slot in between files + await new Promise(r => setImmediate(r)); + } + } catch {} + done++; + report('parsing session files', done, total, { cached: cachedCount, toParse: toParse.length }); + } + _saveParsedDiskCache(true); + + // Phase 4: invalidate any stale in-memory session cache, then build full + // sessions list from now-warm parse cache (sync call, fast) + _sessionsCache = null; + _sessionsCacheTs = 0; + report('aggregating sessions', total, total); + const sessions = loadSessions(); + return sessions; +} + +function loadSessionDetail(sessionId, project, opts) { const found = findSessionFile(sessionId, project); - if (!found) return { error: 'Session file not found', messages: [] }; + if (!found) return { error: 'Session file not found', messages: [], total: 0 }; + + var offset = (opts && typeof opts.offset === 'number') ? opts.offset : 0; + var limit = (opts && typeof opts.limit === 'number') ? opts.limit : 0; // 0 = all (legacy) // OpenCode uses SQLite if (found.format === 'opencode') { - return loadOpenCodeDetail(sessionId); + return _paginateDetailResult(loadOpenCodeDetail(sessionId), offset, limit); } // Cursor if (found.format === 'cursor') { - return loadCursorDetail(sessionId); + return _paginateDetailResult(loadCursorDetail(sessionId), offset, limit); } // Kiro uses SQLite if (found.format === 'kiro') { - return loadKiroDetail(sessionId); + return _paginateDetailResult(loadKiroDetail(sessionId), offset, limit); } const messages = []; @@ -1722,7 +2692,19 @@ function loadSessionDetail(sessionId, project) { if (m._toolSeen) delete m._toolSeen; } - return { messages: messages.slice(0, 200) }; + return _paginateDetailResult({ messages }, offset, limit); +} + +function _paginateDetailResult(result, offset, limit) { + if (!result || !result.messages) return { messages: [], total: 0, offset: offset, hasMore: false }; + const all = result.messages; + const total = all.length; + if (!limit || limit <= 0) { + // Legacy behavior: return all (capped at 500) + return { messages: all.slice(0, 500), total, offset: 0, hasMore: false }; + } + const sliced = all.slice(offset, offset + limit); + return { messages: sliced, total, offset, hasMore: offset + limit < total }; } function deleteSession(sessionId, project) { @@ -1852,7 +2834,7 @@ function exportSessionMarkdown(sessionId, project) { // Session file index: sessionId -> file path (built once, avoids O(sessions*projects) scans) let _sessionFileIndex = null; let _sessionFileIndexTs = 0; -const SESSION_FILE_INDEX_TTL = 120000; // 2 minutes — dirs rarely change +const SESSION_FILE_INDEX_TTL = 30000; // 30 seconds function _buildSessionFileIndex() { const now = Date.now(); @@ -1996,14 +2978,27 @@ function isSystemMessage(text) { if (!text) return true; var t = text.trim(); if (t === 'exit' || t === 'quit' || t === '/exit') return true; + // XML-wrapped injected context (Claude Code + Codex) if (t.startsWith('')) return true; - // Codex developer role system prompts + if (t.startsWith('')) return true; + if (t.startsWith('')) return true; + if (t.startsWith('')) return true; + if (t.startsWith('')) return true; + // Agent instruction docs and skill metadata + if (t.startsWith('# AGENTS.md')) return true; + if (t.startsWith('# CLAUDE.md')) return true; + // Codex developer/system prompts if (t.startsWith('You are Codex')) return true; if (t.startsWith('Filesystem sandboxing')) return true; + // Codex runtime nudges / auto-steering (not real user prompts) + if (t.startsWith('Warning: The maximum number of unified exec')) return true; + if (t.indexOf('AUTOSTEERING:') >= 0 && t.length < 400) return true; + // Sub-agent delegate result injection + if (t.startsWith('[Sub-agent results]')) return true; + if (t.startsWith('[sub-agent result]')) return true; return false; } @@ -2188,14 +3183,34 @@ function getSearchIndex(sessions) { function searchFullText(query, sessions) { if (!query || query.length < 2) return []; + + // Prefer the persistent SQLite FTS5 index — O(log n), no RAM bloat. + try { + const sqliteIndex = require('./sqlite-index'); + const rows = sqliteIndex.search(query, 200); + if (rows && rows.length > 0) { + // Group by session_id, keep up to 3 snippets per session + const bySession = {}; + for (const r of rows) { + if (!bySession[r.session_id]) bySession[r.session_id] = []; + if (bySession[r.session_id].length >= 3) continue; + bySession[r.session_id].push({ + role: r.role, + snippet: (r.snippet || '').replace(/<>/g, ''), + }); + } + return Object.keys(bySession).map(sid => ({ sessionId: sid, matches: bySession[sid] })); + } + } catch (e) { + // Fall through to in-memory fallback + } + + // Fallback: in-memory scan (pre-SQLite ingest completion) const q = query.toLowerCase(); const index = getSearchIndex(sessions); const results = []; - for (const entry of index) { if (entry.fullText.indexOf(q) === -1) continue; - - // Find matching messages with snippets const matches = []; for (const t of entry.texts) { if (matches.length >= 3) break; @@ -2209,12 +3224,8 @@ function searchFullText(query, sessions) { }); } } - - if (matches.length > 0) { - results.push({ sessionId: entry.sessionId, matches }); - } + if (matches.length > 0) results.push({ sessionId: entry.sessionId, matches }); } - return results; } @@ -2298,27 +3309,7 @@ function getModelPricing(model) { } // ── Compute real cost from session file token usage ──────── - -// Disk cache for computed session costs -const COST_CACHE_FILE = path.join(os.tmpdir(), 'codedash-cost-cache.json'); -let _costDiskCache = null; - -function _loadCostDiskCache() { - if (_costDiskCache) return; - try { - if (fs.existsSync(COST_CACHE_FILE)) { - _costDiskCache = JSON.parse(fs.readFileSync(COST_CACHE_FILE, 'utf8')); - } - } catch {} - if (!_costDiskCache) _costDiskCache = {}; -} - -function _saveCostDiskCache() { - if (!_costDiskCache) return; - try { - fs.writeFileSync(COST_CACHE_FILE, JSON.stringify(_costDiskCache)); - } catch {} -} +// (COST_CACHE_FILE/_costDiskCache/_loadCostDiskCache/_saveCostDiskCache defined at top) const EMPTY_COST = { cost: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreateTokens: 0, contextPctSum: 0, contextTurnCount: 0, model: '' }; @@ -2458,57 +3449,217 @@ function computeSessionCost(sessionId, project) { } const result = { cost: totalCost, inputTokens: totalInput, outputTokens: totalOutput, cacheReadTokens: totalCacheRead, cacheCreateTokens: totalCacheCreate, contextPctSum, contextTurnCount, model }; - if (cacheKey) _costDiskCache[cacheKey] = result; + if (cacheKey) { + _costDiskCache[cacheKey] = result; + _costDiskCacheDirty = true; + _costDiskCacheEntriesSinceFlush++; + if (_costDiskCacheEntriesSinceFlush >= COST_CACHE_FLUSH_EVERY) { + _saveCostDiskCache(true); + } + } _costMemCache[sessionId] = result; return result; } // ── Cost analytics ──────────────────────────────────────── -// Analytics result cache — avoids recomputing 31k sessions every request -const ANALYTICS_CACHE_FILE = path.join(os.tmpdir(), 'codedash-analytics-cache.json'); -let _analyticsCacheResult = null; -let _analyticsCacheKey = null; +// ── Incremental cost aggregator ─────────────────────────────── +// Streaming version of getCostAnalytics: create an aggregator, feed sessions +// one at a time via merge(), read a snapshot at any point via finalize(). +// Lets the background job expose live partial results to the UI. +function createCostAggregator() { + const state = { + byDay: {}, byProject: {}, byWeek: {}, byAgent: {}, + totalCost: 0, totalTokens: 0, + totalInputTokens: 0, totalOutputTokens: 0, + totalCacheReadTokens: 0, totalCacheCreateTokens: 0, + globalContextPctSum: 0, globalContextTurnCount: 0, + firstDate: null, lastDate: null, + sessionsWithData: 0, + sessionCosts: [], + agentNoCostData: {}, + seenAgents: new Set(), + processedCount: 0, + }; + return { + state, + merge(session, costData) { + state.processedCount++; + if (!state.seenAgents.has(session.tool)) { + state.seenAgents.add(session.tool); + if (!state.byAgent[session.tool]) state.byAgent[session.tool] = { cost: 0, sessions: 0, tokens: 0, estimated: false }; + } + const cost = costData.cost; + const tokens = costData.inputTokens + costData.outputTokens + costData.cacheReadTokens + costData.cacheCreateTokens; + if (cost === 0 && tokens === 0) { + if (!state.agentNoCostData[session.tool]) state.agentNoCostData[session.tool] = 0; + state.agentNoCostData[session.tool]++; + return; + } + state.sessionsWithData++; + state.totalCost += cost; + state.totalTokens += tokens; + state.totalInputTokens += costData.inputTokens; + state.totalOutputTokens += costData.outputTokens; + state.totalCacheReadTokens += costData.cacheReadTokens; + state.totalCacheCreateTokens += costData.cacheCreateTokens; + + const agent = session.tool || 'unknown'; + if (!state.byAgent[agent]) state.byAgent[agent] = { cost: 0, sessions: 0, tokens: 0, estimated: false }; + state.byAgent[agent].cost += cost; + state.byAgent[agent].sessions++; + state.byAgent[agent].tokens += tokens; + if (agent === 'codex') state.byAgent[agent].estimated = true; + if (agent === 'cursor' && costData.model && costData.model.includes('-estimated')) state.byAgent[agent].estimated = true; + if (agent === 'opencode' && !costData.model) state.byAgent[agent].estimated = true; + + state.globalContextPctSum += costData.contextPctSum; + state.globalContextTurnCount += costData.contextTurnCount; + + const day = session.date || 'unknown'; + if (session.date) { + if (!state.firstDate || session.date < state.firstDate) state.firstDate = session.date; + if (!state.lastDate || session.date > state.lastDate) state.lastDate = session.date; + } + if (!state.byDay[day]) state.byDay[day] = { cost: 0, sessions: 0, tokens: 0 }; + state.byDay[day].cost += cost; + state.byDay[day].sessions++; + state.byDay[day].tokens += tokens; + + if (session.date) { + const d = new Date(session.date); + const weekStart = new Date(d); + weekStart.setDate(d.getDate() - d.getDay()); + const weekKey = weekStart.toISOString().slice(0, 10); + if (!state.byWeek[weekKey]) state.byWeek[weekKey] = { cost: 0, sessions: 0 }; + state.byWeek[weekKey].cost += cost; + state.byWeek[weekKey].sessions++; + } -function _analyticsKey(sessions) { - // Key: session count + newest session mtime - let newest = 0; - for (const s of sessions) { - if (s.last_ts > newest) newest = s.last_ts; - } - return sessions.length + ':' + newest; + const proj = session.project_short || session.project || 'unknown'; + if (!state.byProject[proj]) state.byProject[proj] = { cost: 0, sessions: 0, tokens: 0 }; + state.byProject[proj].cost += cost; + state.byProject[proj].sessions++; + state.byProject[proj].tokens += tokens; + + state.sessionCosts.push({ id: session.id, cost, project: proj, date: session.date, last_ts: session.last_ts || 0 }); + }, + finalize() { + // Sort top sessions by cost (snapshot a shallow copy so partials stay consistent) + const topCopy = state.sessionCosts.slice().sort((a, b) => b.cost - a.cost); + const days = state.firstDate && state.lastDate + ? Math.max(1, Math.round((new Date(state.lastDate) - new Date(state.firstDate)) / 86400000) + 1) + : 1; + const now = Date.now(); + const todayStr = new Date().toISOString().slice(0, 10); + const hoursElapsedToday = (now - new Date(todayStr).getTime()) / 3600000; + let last1hCost = 0; + let todayCost = 0; + for (const sc of topCopy) { + if (sc.last_ts >= now - 3600000) last1hCost += sc.cost; + if (sc.date === todayStr) todayCost += sc.cost; + } + return { + totalCost: state.totalCost, + totalTokens: state.totalTokens, + totalInputTokens: state.totalInputTokens, + totalOutputTokens: state.totalOutputTokens, + totalCacheReadTokens: state.totalCacheReadTokens, + totalCacheCreateTokens: state.totalCacheCreateTokens, + avgContextPct: state.globalContextTurnCount > 0 ? Math.round(state.globalContextPctSum / state.globalContextTurnCount) : 0, + dailyRate: state.totalCost / days, + firstDate: state.firstDate, + lastDate: state.lastDate, + days, + totalSessions: state.sessionsWithData, + totalSessionsAll: state.totalSessionsAll || state.processedCount, + byDay: state.byDay, + byWeek: state.byWeek, + byProject: state.byProject, + topSessions: topCopy.slice(0, 10), + byAgent: state.byAgent, + agentNoCostData: state.agentNoCostData, + last1hCost, + todayCost, + hoursElapsedToday: Math.max(1, hoursElapsedToday), + processedCount: state.processedCount, + }; + }, + setTotalSessionsAll(n) { state.totalSessionsAll = n; }, + }; } -function getCostAnalytics(sessions) { - // Fast cache check — if sessions haven't changed, return cached result - const key = _analyticsKey(sessions); - if (_analyticsCacheResult && _analyticsCacheKey === key) return _analyticsCacheResult; - - // Try disk cache - if (!_analyticsCacheResult) { - try { - if (fs.existsSync(ANALYTICS_CACHE_FILE)) { - const cached = JSON.parse(fs.readFileSync(ANALYTICS_CACHE_FILE, 'utf8')); - if (cached._key === key) { - _analyticsCacheResult = cached.data; - _analyticsCacheKey = key; - return cached.data; - } - } - } catch {} +// Compute per-session cost respecting special per-tool paths used by +// getCostAnalytics (opencode batch, cursor vscdb tokens, ...). +function computeSessionCostForAnalytics(session, opencodeCostCache) { + if (session.tool === 'opencode' && opencodeCostCache && opencodeCostCache[session.id]) { + return opencodeCostCache[session.id]; } + if (session.tool === 'cursor') { + const inp = session._cursor_input_tokens || 0; + const out = session._cursor_output_tokens || 0; + if (inp > 0 || out > 0) { + const model = session._cursor_model || ''; + const pricing = getModelPricing(model); + return { cost: inp * pricing.input + out * pricing.output, inputTokens: inp, outputTokens: out, cacheReadTokens: 0, cacheCreateTokens: 0, contextPctSum: 0, contextTurnCount: 0, model: model }; + } + if (session.user_messages > 0 || session.messages > 0) { + const userMsgs = session.user_messages || Math.ceil((session.messages || 0) * 0.07); + const model = session._cursor_model || 'claude-sonnet'; + const pricing = getModelPricing(model); + const estInput = userMsgs * 2000; + const estOutput = userMsgs * 1000; + return { cost: estInput * pricing.input + estOutput * pricing.output, inputTokens: estInput, outputTokens: estOutput, cacheReadTokens: 0, cacheCreateTokens: 0, contextPctSum: 0, contextTurnCount: 0, model: model + '-estimated' }; + } + return EMPTY_COST; + } + return computeSessionCost(session.id, session.project); +} - const result = _computeCostAnalytics(sessions); - - // Save to cache - _analyticsCacheResult = result; - _analyticsCacheKey = key; - try { fs.writeFileSync(ANALYTICS_CACHE_FILE, JSON.stringify({ _key: key, data: result })); } catch {} - - return result; +// Pre-compute OpenCode costs in one batch SQL — used by the streaming path +function buildOpencodeCostCache(sessions) { + const cache = {}; + const opencodeSessions = sessions.filter(s => s.tool === 'opencode'); + if (opencodeSessions.length === 0 || !fs.existsSync(OPENCODE_DB)) return cache; + try { + const batchRows = execFileSync('sqlite3', [ + '-json', + OPENCODE_DB, + `SELECT session_id, data FROM message WHERE json_extract(data, '$.role') = 'assistant' ORDER BY time_created` + ], { encoding: 'utf8', timeout: 30000, windowsHide: true }).trim(); + if (batchRows) { + const rows = JSON.parse(batchRows); + for (const row of rows) { + const sessId = row.session_id; + const jsonStr = row.data; + if (!sessId || typeof jsonStr !== 'string') continue; + try { + const msgData = JSON.parse(jsonStr); + const t = msgData.tokens || {}; + const inp = t.input || 0; + const out = (t.output || 0) + (t.reasoning || 0); + const cacheRead = (t.cache && t.cache.read) || 0; + const cacheCreate = (t.cache && t.cache.write) || 0; + if (inp === 0 && out === 0) continue; + if (!cache[sessId]) cache[sessId] = { cost: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreateTokens: 0, contextPctSum: 0, contextTurnCount: 0, model: '' }; + const c = cache[sessId]; + if (!c.model && msgData.modelID) c.model = msgData.modelID; + const pricing = getModelPricing(msgData.modelID || c.model); + c.inputTokens += inp; + c.outputTokens += out; + c.cacheReadTokens += cacheRead; + c.cacheCreateTokens += cacheCreate; + c.cost += inp * pricing.input + cacheCreate * pricing.cache_create + cacheRead * pricing.cache_read + out * pricing.output; + const ctx = inp + cacheCreate + cacheRead; + if (ctx > 0) { c.contextPctSum += (ctx / CONTEXT_WINDOW) * 100; c.contextTurnCount++; } + } catch {} + } + } + } catch {} + return cache; } -function _computeCostAnalytics(sessions) { +function getCostAnalytics(sessions) { const byDay = {}; const byProject = {}; const byWeek = {}; @@ -2675,7 +3826,8 @@ function _computeCostAnalytics(sessions) { if (sc.date === todayStr) todayCost += sc.cost; } - _saveCostDiskCache(); + _saveCostDiskCache(true); + _saveParsedDiskCache(true); return { totalCost, @@ -2690,6 +3842,7 @@ function _computeCostAnalytics(sessions) { lastDate, days, totalSessions: sessionsWithData, + totalSessionsAll: sessions.length, byDay, byWeek, byProject, @@ -2704,13 +3857,63 @@ function _computeCostAnalytics(sessions) { // ── Active sessions detection ───────────────────────────── +// ── Active sessions (cached, non-blocking) ────────────────── +// getActiveSessions() is called from /api/active which the browser polls on +// a timer. The previous version shelled out to `ps` + `lsof` synchronously +// once per matching process, doing an O(N) sync scan that blocked the event +// loop for tens of seconds when many processes had "codex"/"claude" in their +// cmdline (e.g. codex-up-exec wrapper spawned by Claude Code tooling). +// +// New design: +// 1. Cached result served for up to 3 seconds. +// 2. Tighter cmd matching — must be a real agent CLI, not a substring hit. +// 3. pid→cwd is remembered across calls (pids are stable for the process +// lifetime). We only look up cwd for pids we've never seen. +// 4. lsof is run ONCE as a single batch call for all unknown pids, with +// a hard 2-second total timeout. +// 5. No inner loadSessions() calls; session matching uses the in-memory +// sessions cache if it already exists, otherwise cwd-match is skipped +// and a background refresh fires. +let _activeCache = null; +let _activeCacheTs = 0; +const ACTIVE_CACHE_TTL = 3000; // 3 seconds +const _pidCwdCache = new Map(); // pid → cwd (stable for process lifetime) + +// Real agent CLI invocations — tighter than a substring match. +// Matches the binary name at the START of cmd (or after a /path/). +// Includes codex-up variants (user wrapper that spawns codex exec). +const AGENT_CLI_MATCHERS = [ + { tool: 'claude', re: /(^|\/)claude(\s|$)/ }, + { tool: 'codex', re: /(^|\/)codex(-up(-exec)?)?(\s|$)/ }, + { tool: 'opencode',re: /(^|\/)opencode(\s|$)/ }, + { tool: 'kiro', re: /(^|\/)kiro-cli(\s|$)/ }, + { tool: 'cursor', re: /(^|\/)cursor-agent(\s|$)/ }, +]; + +function _matchAgentCli(cmd) { + for (const m of AGENT_CLI_MATCHERS) { + if (m.re.test(cmd)) return m.tool; + } + return ''; +} + function getActiveSessions() { + const now = Date.now(); + if (_activeCache && (now - _activeCacheTs) < ACTIVE_CACHE_TTL) { + return _activeCache; + } const active = []; - const seenPids = new Set(); - // 1. Claude Code — read PID files for session ID mapping + // Skip on Windows + if (process.platform === 'win32') { + _activeCache = active; + _activeCacheTs = now; + return active; + } + + // 1. Read Claude Code PID files (provides cwd + sessionId directly) const sessionsDir = path.join(CLAUDE_DIR, 'sessions'); - const claudePidMap = {}; // pid → {sessionId, cwd, startedAt} + const claudePidMap = {}; if (fs.existsSync(sessionsDir)) { for (const file of fs.readdirSync(sessionsDir)) { if (!file.endsWith('.json')) continue; @@ -2721,104 +3924,133 @@ function getActiveSessions() { } } - // 2. Scan ALL agent processes via ps - const agentPatterns = [ - { pattern: 'claude', tool: 'claude', match: /\bclaude\b/ }, - { pattern: 'codex', tool: 'codex', match: /\bcodex\b/ }, - { pattern: 'opencode', tool: 'opencode', match: /\bopencode\b/ }, - { pattern: 'kiro', tool: 'kiro', match: /kiro-cli/ }, - { pattern: 'cursor-agent', tool: 'cursor', match: /cursor-agent/ }, - ]; - - // Skip process scanning on Windows (no ps/grep) - if (process.platform === 'win32') return active; - + // 2. Single `ps` call + let psOut = ''; try { - const psOut = execSync( - 'ps aux 2>/dev/null | grep -E "claude|codex|opencode|kiro-cli|cursor-agent" | grep -v grep || true', - { encoding: 'utf8', timeout: 3000, stdio: ['pipe', 'pipe', 'pipe'] } + psOut = execSync( + 'ps -eo pid=,pcpu=,rss=,stat=,command= 2>/dev/null', + { encoding: 'utf8', timeout: 2000, stdio: ['pipe', 'pipe', 'pipe'], maxBuffer: 4 * 1024 * 1024 } ); + } catch { + _activeCache = active; + _activeCacheTs = now; + return active; + } - for (const line of psOut.split('\n').filter(Boolean)) { - const parts = line.trim().split(/\s+/); - if (parts.length < 11) continue; - - const pid = parseInt(parts[1]); - if (seenPids.has(pid)) continue; + // 3. Parse + filter with strict agent CLI matching + const matches = []; + const livePids = new Set(); + for (const line of psOut.split('\n')) { + if (!line) continue; + const m = line.match(/^\s*(\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(.*)$/); + if (!m) continue; + const pid = parseInt(m[1]); + if (!pid) continue; + livePids.add(pid); + const cpu = parseFloat(m[2]) || 0; + const rss = parseInt(m[3]) || 0; + const stat = m[4] || ''; + const cmd = m[5] || ''; + + // Skip wrappers, MCP servers, plugins — only main agent processes + if (cmd.includes('node bin/cli') || cmd.includes('/codedash') || cmd.includes('npm ')) continue; + if (cmd.startsWith('grep ') || cmd.includes(' grep ')) continue; + if (cmd.includes('mcp-server') || cmd.includes('mcp_server') || cmd.includes('/mcp/') || cmd.includes('/mcp-servers/')) continue; + if (cmd.includes('/plugins/') || cmd.includes('plugin-') || cmd.includes('app-server-broker')) continue; + + const tool = _matchAgentCli(cmd); + if (!tool) continue; + if (cmd.includes('.claude/') && !cmd.includes('claude ') && tool === 'claude') continue; + if (cmd.includes('.codex/') && !cmd.includes('codex ') && tool === 'codex') continue; + + matches.push({ pid, cpu, rss, stat, cmd, tool }); + } - const cpu = parseFloat(parts[2]) || 0; - const rss = parseInt(parts[5]) || 0; - const stat = parts[7] || ''; - const cmd = parts.slice(10).join(' '); + // Drop stale pids from cwd cache + for (const pid of Array.from(_pidCwdCache.keys())) { + if (!livePids.has(pid)) _pidCwdCache.delete(pid); + } - // Determine tool - let tool = ''; - for (const ap of agentPatterns) { - if (ap.match.test(cmd)) { tool = ap.tool; break; } + // 4. Collect unknown pids (not in PID files, not in cache). Look up cwd + // via a SINGLE batch lsof call with a hard 1-second timeout. + const unknownPids = []; + for (const m of matches) { + if (claudePidMap[m.pid] && claudePidMap[m.pid].cwd) continue; + if (_pidCwdCache.has(m.pid)) continue; + unknownPids.push(m.pid); + } + if (unknownPids.length > 0) { + try { + // -a ANDs the -d/-p conditions (without it, lsof ORs and returns cwd for + // all processes, not just the requested pids). + const out = execSync( + `lsof -a -d cwd -Fpn -p ${unknownPids.join(',')} 2>/dev/null`, + { encoding: 'utf8', timeout: 1500, stdio: ['pipe', 'pipe', 'pipe'], maxBuffer: 1 * 1024 * 1024 } + ); + // lsof -Fpn output: p\nn\np\nn... + let curPid = 0; + for (const line of out.split('\n')) { + if (line.startsWith('p')) { + curPid = parseInt(line.slice(1)) || 0; + } else if (line.startsWith('n') && curPid) { + _pidCwdCache.set(curPid, line.slice(1)); + } } - if (!tool) continue; - - // Skip node/npm/shell wrappers — only main processes - if (cmd.includes('node bin/cli') || cmd.includes('npm') || cmd.includes('grep')) continue; - - seenPids.add(pid); - - // Get session ID from Claude PID files - let sessionId = ''; - let cwd = ''; - let startedAt = 0; - let sessionSource = ''; - if (claudePidMap[pid]) { - sessionId = claudePidMap[pid].sessionId || ''; - cwd = claudePidMap[pid].cwd || ''; - startedAt = claudePidMap[pid].startedAt || 0; - if (sessionId) sessionSource = 'pid-file'; + // Mark pids we asked about but didn't get cwd for — empty string so we + // don't keep asking on every poll. + for (const pid of unknownPids) { + if (!_pidCwdCache.has(pid)) _pidCwdCache.set(pid, ''); } + } catch { + // Timeout / error: mark all as unknown so we don't retry each poll + for (const pid of unknownPids) _pidCwdCache.set(pid, ''); + } + } - // Try to get cwd from lsof if not from PID file - if (!cwd) { - try { - const lsofOut = execSync(`lsof -d cwd -p ${pid} -Fn 2>/dev/null`, { encoding: 'utf8', timeout: 2000, stdio: ['pipe', 'pipe', 'pipe'] }); - const match = lsofOut.match(/\nn(\/[^\n]+)/); - if (match) cwd = match[1]; - } catch {} - } + // 5. Assemble results using only already-loaded sessions cache for cwd-match. + // Never call loadSessions() here — it may invalidate and re-parse. + const cachedSessions = _sessionsCache || []; + + for (const m of matches) { + let sessionId = ''; + let cwd = ''; + let startedAt = 0; + let sessionSource = ''; + + if (claudePidMap[m.pid]) { + sessionId = claudePidMap[m.pid].sessionId || ''; + cwd = claudePidMap[m.pid].cwd || ''; + startedAt = claudePidMap[m.pid].startedAt || 0; + if (sessionId) sessionSource = 'pid-file'; + } + if (!cwd) cwd = _pidCwdCache.get(m.pid) || ''; - // Try to find session ID by matching cwd + tool to loaded sessions - if (!sessionId) { - const allS = loadSessions(); - const match = allS.find(s => s.tool === tool && s.project === cwd); - if (match) { - sessionId = match.id; - sessionSource = 'cwd-match'; - } - // If still no match, find latest session of this tool - if (!sessionId) { - const latest = allS.filter(s => s.tool === tool).sort((a,b) => b.last_ts - a.last_ts)[0]; - if (latest) { - sessionId = latest.id; - sessionSource = 'fallback-latest'; - } - } + if (!sessionId && cachedSessions.length > 0) { + const match = cachedSessions.find(s => s.tool === m.tool && s.project === cwd); + if (match) { + sessionId = match.id; + sessionSource = 'cwd-match'; } - - const status = cpu < 1 && (stat.includes('S') || stat.includes('T')) ? 'waiting' : 'active'; - - active.push({ - pid: pid, - sessionId: sessionId, - cwd: cwd, - startedAt: startedAt, - kind: tool, - entrypoint: tool, - status: status, - cpu: cpu, - memoryMB: Math.round(rss / 1024), - _sessionSource: sessionSource, - }); } - } catch {} + const status = m.cpu < 1 && (m.stat.includes('S') || m.stat.includes('T')) ? 'waiting' : 'active'; + + active.push({ + pid: m.pid, + sessionId, + cwd, + startedAt, + kind: m.tool, + entrypoint: m.tool, + status, + cpu: m.cpu, + memoryMB: Math.round(m.rss / 1024), + _sessionSource: sessionSource, + }); + } + + _activeCache = active; + _activeCacheTs = Date.now(); return active; } @@ -2852,7 +4084,8 @@ const fmtLocalDay = (ts) => { }; // Disk cache for per-session daily message breakdown -const DAILY_STATS_CACHE_FILE = path.join(os.tmpdir(), 'codedash-daily-stats-cache.json'); +const DAILY_STATS_CACHE_FILE = path.join(CODEDASH_CACHE_DIR, 'daily-stats-cache.json'); +const LEGACY_DAILY_STATS_CACHE_FILE = path.join(os.tmpdir(), 'codedash-daily-stats-cache.json'); let _dailyStatsDiskCache = null; function _loadDailyStatsDiskCache() { @@ -2860,6 +4093,8 @@ function _loadDailyStatsDiskCache() { try { if (fs.existsSync(DAILY_STATS_CACHE_FILE)) { _dailyStatsDiskCache = JSON.parse(fs.readFileSync(DAILY_STATS_CACHE_FILE, 'utf8')); + } else if (fs.existsSync(LEGACY_DAILY_STATS_CACHE_FILE)) { + _dailyStatsDiskCache = JSON.parse(fs.readFileSync(LEGACY_DAILY_STATS_CACHE_FILE, 'utf8')); } } catch {} if (!_dailyStatsDiskCache) _dailyStatsDiskCache = {}; @@ -2868,7 +4103,9 @@ function _loadDailyStatsDiskCache() { function _saveDailyStatsDiskCache() { if (!_dailyStatsDiskCache) return; try { - fs.writeFileSync(DAILY_STATS_CACHE_FILE, JSON.stringify(_dailyStatsDiskCache)); + const tmp = DAILY_STATS_CACHE_FILE + '.tmp'; + fs.writeFileSync(tmp, JSON.stringify(_dailyStatsDiskCache)); + fs.renameSync(tmp, DAILY_STATS_CACHE_FILE); } catch {} } @@ -2920,37 +4157,7 @@ function _computeSessionDailyBreakdown(s, found) { return { msgsByDay, tsByDay }; } -// Daily stats result cache -const DAILY_RESULT_CACHE_FILE = path.join(os.tmpdir(), 'codedash-daily-result-cache.json'); -let _dailyResultCache = null; -let _dailyResultCacheKey = null; - function getDailyStats(sessions) { - const key = _analyticsKey(sessions); - if (_dailyResultCache && _dailyResultCacheKey === key) return _dailyResultCache; - - // Try disk cache - if (!_dailyResultCache) { - try { - if (fs.existsSync(DAILY_RESULT_CACHE_FILE)) { - const cached = JSON.parse(fs.readFileSync(DAILY_RESULT_CACHE_FILE, 'utf8')); - if (cached._key === key) { - _dailyResultCache = cached.data; - _dailyResultCacheKey = key; - return cached.data; - } - } - } catch {} - } - - const result = _computeDailyStats(sessions); - _dailyResultCache = result; - _dailyResultCacheKey = key; - try { fs.writeFileSync(DAILY_RESULT_CACHE_FILE, JSON.stringify({ _key: key, data: result })); } catch {} - return result; -} - -function _computeDailyStats(sessions) { const byDay = {}; const ensureDay = (date) => { if (!byDay[date]) byDay[date] = { date, sessions: 0, messages: 0, hours: 0, cost: 0, agents: {} }; @@ -3005,12 +4212,10 @@ function _computeDailyStats(sessions) { const day = s.date || fmtLocalDay(s.last_ts); const d = ensureDay(day); d.sessions++; - // Use exact user_messages count if available, otherwise estimate + // Only count EXACT user_messages. The 0.5 estimate was wildly wrong + // because Claude type=user entries include tool_results (up to 28x inflation). if (s.user_messages > 0) { d.messages += s.user_messages; - } else { - const totalMsgEst = s.detail_messages || s.messages || 0; - d.messages += Math.ceil(totalMsgEst * 0.5); } d.hours += Math.min((s.last_ts - s.first_ts) / 3600000, 16); d.cost += sessionCost; @@ -3023,6 +4228,8 @@ function _computeDailyStats(sessions) { d.cost = Math.round(d.cost * 100) / 100; } _saveDailyStatsDiskCache(); + _saveCostDiskCache(true); + _saveParsedDiskCache(true); return Object.values(byDay).sort((a, b) => b.date.localeCompare(a.date)); } @@ -3083,6 +4290,12 @@ function getLeaderboardStats() { module.exports = { loadSessions, + loadSessionsAsync, + getWarmingStatus, + getSqliteBackfillStatus, + createCostAggregator, + computeSessionCostForAnalytics, + buildOpencodeCostCache, loadSessionDetail, getProjectGitInfo, getLeaderboardStats, diff --git a/src/embeddings.js b/src/embeddings.js new file mode 100644 index 0000000..0add562 --- /dev/null +++ b/src/embeddings.js @@ -0,0 +1,607 @@ +// Memento-style 6-stage hybrid RAG search pipeline. +// +// Based on the Memento paper (arXiv 2603.18743) retrieval pipeline (Figure 8): +// Stage 1: FTS5 sparse recall (top-20) +// Stage 2: Dense embedding recall (top-20) +// Stage 3: Reciprocal Rank Fusion (k=60) +// Stage 4: Utility reranking (per-entry success/failure rate) +// Stage 5: Threshold filter +// Stage 6: Top-k +// +// Provider chain (matches codex-git kb_embedding_store.rs): +// 1. Local: MiniLM-L6-v2 (default, 384d, 23MB) or Qwen3-Embedding-0.6B (1024d) +// 2. API: OpenAI-compatible /embeddings endpoint (GitHub Models, Copilot proxy, etc) +// 3. TF-IDF fallback: bag-of-words 256-dim hashed vectors (always available) +// +// Weights from Memento paper results: +// BM25 Recall@1 = 0.32 → weight 0.3 +// Embedding Recall@1 = 0.54 → weight 0.7 +// Utility reranking: final = rrf * (0.7 + 0.3 * utility_rate) + +const https = require('https'); +const http = require('http'); +const path = require('path'); +const os = require('os'); +const fs = require('fs'); + +// ── Constants (Memento paper) ───────────────────────────────── +const RRF_K = 60; // Standard RRF smoothing (Cormack et al. 2009) +const STAGE_CANDIDATE_K = 20; // Candidates per retrieval stage +const BM25_WEIGHT = 0.3; // Weaker lexical signal +const EMBEDDING_WEIGHT = 0.7; // Stronger dense signal +const UTILITY_BASE = 0.7; // Minimum influence even for zero-utility +const UTILITY_SCALE = 0.3; // Scale factor for utility rate +const TFIDF_DIM = 256; // TF-IDF fallback bucket count + +// ── Model configuration ────────────────────────────────────── +const MODELS = { + 'minilm': { + id: 'Xenova/all-MiniLM-L6-v2', + dim: 384, + description: 'Fast, English-optimized (23MB)', + }, + 'qwen3': { + id: 'onnx-community/Qwen3-Embedding-0.6B-ONNX', + dim: 1024, + description: 'Best quality, multilingual (600MB)', + }, +}; + +// Config file at ~/.codedash/embedding-config.json +const CONFIG_FILE = path.join(os.homedir(), '.codedash', 'embedding-config.json'); + +function loadConfig() { + try { + if (fs.existsSync(CONFIG_FILE)) return JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf8')); + } catch {} + return {}; +} + +function getModelId() { + const cfg = loadConfig(); + const key = cfg.model || 'minilm'; + return (MODELS[key] || MODELS.minilm).id; +} + +function getModelDim() { + const cfg = loadConfig(); + const key = cfg.model || 'minilm'; + return (MODELS[key] || MODELS.minilm).dim; +} + +// ── Provider 1: Local (transformers.js ONNX) ───────────────── +let _extractor = null; +let _loadPromise = null; +let _localAvailable = null; + +async function _ensureLocalModel() { + if (_extractor) return _extractor; + if (_loadPromise) return _loadPromise; + _loadPromise = (async () => { + const { pipeline } = await import('@huggingface/transformers'); + _extractor = await pipeline('feature-extraction', getModelId(), { + device: 'cpu', dtype: 'fp32', + }); + _localAvailable = true; + return _extractor; + })(); + return _loadPromise; +} + +async function embedLocal(texts) { + const extractor = await _ensureLocalModel(); + const dim = getModelDim(); + if (!Array.isArray(texts)) texts = [texts]; + const results = []; + // Process in batches of 32 + for (let i = 0; i < texts.length; i += 32) { + const batch = texts.slice(i, i + 32); + const out = await extractor(batch, { pooling: 'mean', normalize: true }); + for (let j = 0; j < batch.length; j++) { + const start = j * dim; + results.push(Array.from(out.data.slice(start, start + dim))); + } + } + return results; +} + +// ── Copilot token exchange (for automatic API embeddings) ──── +// Reads GitHub Copilot OAuth credentials and exchanges for a +// short-lived session token, exactly like copilot-client.js does +// for chat completions. + +const COPILOT_TOKEN_ENDPOINT = 'https://api.github.com/copilot_internal/v2/token'; +const COPILOT_DEFAULT_API_BASE = 'https://api.individual.githubcopilot.com'; +const COPILOT_USER_AGENT = `copilot/1.0.14 (client/github/cli ${process.platform} v24.11.1) term/${process.env.TERM_PROGRAM || 'xterm'}`; +const COPILOT_TOKEN_REFRESH_MARGIN_SEC = 60; + +let _copilotSessionToken = null; +let _copilotSessionExpiresAt = 0; +let _copilotApiBase = COPILOT_DEFAULT_API_BASE; + +/** + * Load the Copilot OAuth token from known credential locations. + * @returns {string|null} The gho_... token, or null + */ +function _loadCopilotOAuthToken() { + // 1. ~/.copilot/auth/credential.json + try { + const credPath = path.join(os.homedir(), '.copilot', 'auth', 'credential.json'); + if (fs.existsSync(credPath)) { + const data = JSON.parse(fs.readFileSync(credPath, 'utf8')); + if (data.token && typeof data.token === 'string' && data.token.length > 0) { + return data.token; + } + } + } catch (_) {} + // 2. ~/.config/github-copilot/apps.json (legacy VS Code extension) + try { + const appsPath = path.join(os.homedir(), '.config', 'github-copilot', 'apps.json'); + if (fs.existsSync(appsPath)) { + const data = JSON.parse(fs.readFileSync(appsPath, 'utf8')); + for (const key of Object.keys(data)) { + if (key.startsWith('github.com') && data[key] && data[key].oauth_token) { + return data[key].oauth_token; + } + } + } + } catch (_) {} + return null; +} + +/** + * Exchange OAuth token for a short-lived Copilot session token. + * Caches result and refreshes 60s before expiry. + * @returns {Promise<{token: string, api_base: string}>} + */ +async function _ensureCopilotSession() { + const now = Math.floor(Date.now() / 1000); + if (_copilotSessionToken && _copilotSessionExpiresAt > now + COPILOT_TOKEN_REFRESH_MARGIN_SEC) { + return { token: _copilotSessionToken, api_base: _copilotApiBase }; + } + + const oauthToken = _loadCopilotOAuthToken(); + if (!oauthToken) throw new Error('No Copilot OAuth token found'); + + const result = await new Promise((resolve, reject) => { + const parsed = new URL(COPILOT_TOKEN_ENDPOINT); + const req = https.request({ + hostname: parsed.hostname, + port: 443, + path: parsed.pathname + parsed.search, + method: 'GET', + headers: { + 'User-Agent': COPILOT_USER_AGENT, + 'Accept': 'application/json', + 'Authorization': 'token ' + oauthToken, + }, + }, (res) => { + const chunks = []; + res.on('data', c => chunks.push(c)); + res.on('end', () => { + const body = Buffer.concat(chunks).toString('utf8'); + if (res.statusCode !== 200) { + return reject(new Error('Copilot token exchange failed (HTTP ' + res.statusCode + '): ' + body)); + } + try { resolve(JSON.parse(body)); } catch (e) { reject(e); } + }); + }); + req.on('error', reject); + req.setTimeout(15000, () => { req.destroy(); reject(new Error('Copilot token exchange timeout')); }); + req.end(); + }); + + if (!result.token) throw new Error('Copilot token exchange returned empty token'); + + _copilotSessionToken = result.token; + _copilotSessionExpiresAt = result.expires_at || 0; + _copilotApiBase = (result.endpoints && result.endpoints.api) || COPILOT_DEFAULT_API_BASE; + + return { token: _copilotSessionToken, api_base: _copilotApiBase }; +} + +/** + * Check if Copilot credentials exist on disk (sync, no network). + * @returns {boolean} + */ +function _isCopilotAvailable() { + return _loadCopilotOAuthToken() !== null; +} + +// ── Provider 2: API (OpenAI-compatible) ────────────────────── +// Priority: 1) Copilot auto-discovery 2) Manual config fallback +async function embedAPI(texts) { + if (!Array.isArray(texts)) texts = [texts]; + + // ── Try Copilot first ── + if (_isCopilotAvailable()) { + try { + const session = await _ensureCopilotSession(); + return await _callEmbeddingsEndpoint( + session.api_base + '/embeddings', + session.token, + 'text-embedding-3-small', + texts, + { 'Copilot-Integration-Id': 'codedash', 'Editor-Version': 'codedash/1.0' } + ); + } catch (copilotErr) { + // Copilot failed — fall through to manual config + const cfg = loadConfig(); + if (!cfg.api_base_url || !cfg.api_key) { + throw new Error('Copilot embeddings failed: ' + copilotErr.message); + } + } + } + + // ── Fallback: manual config (api_base_url + api_key) ── + const cfg = loadConfig(); + if (!cfg.api_base_url || !cfg.api_key) throw new Error('API embedding not configured (no Copilot credentials and no manual config)'); + + return await _callEmbeddingsEndpoint( + cfg.api_base_url, + cfg.api_key, + cfg.api_model || 'text-embedding-3-small', + texts, + {} + ); +} + +/** + * Call an OpenAI-compatible /embeddings endpoint. + * @param {string} endpointUrl - Full URL (e.g. "https://api.../embeddings") + * @param {string} token - Bearer token + * @param {string} model - Model name + * @param {string[]} texts - Input texts + * @param {Object} extraHeaders - Additional headers + * @returns {Promise} + */ +function _callEmbeddingsEndpoint(endpointUrl, token, model, texts, extraHeaders) { + const url = new URL(endpointUrl); + const body = JSON.stringify({ + model, + input: texts, + encoding_format: 'float', + }); + + return new Promise((resolve, reject) => { + const mod = url.protocol === 'https:' ? https : http; + const req = mod.request(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': 'Bearer ' + token, + 'Accept': 'application/json', + ...extraHeaders, + }, + timeout: 30000, + }, (res) => { + let data = ''; + res.on('data', d => data += d); + res.on('end', () => { + try { + const parsed = JSON.parse(data); + if (parsed.data && Array.isArray(parsed.data)) { + resolve(parsed.data.map(d => d.embedding)); + } else { + reject(new Error('Unexpected API response: ' + (data.slice(0, 200)))); + } + } catch (e) { reject(e); } + }); + }); + req.on('error', reject); + req.on('timeout', () => { req.destroy(); reject(new Error('API timeout')); }); + req.write(body); + req.end(); + }); +} + +// ── Provider 3: TF-IDF fallback (always available) ─────────── +function tokenize(text) { + return (text || '').toLowerCase() + .replace(/[^a-zа-яёüöäß0-9\s]/g, ' ') + .split(/\s+/) + .filter(t => t.length > 1); +} + +function embedTFIDF(texts) { + if (!Array.isArray(texts)) texts = [texts]; + return texts.map(text => { + const tokens = tokenize(text); + const freq = {}; + for (const t of tokens) { + const bucket = hashStr(t) % TFIDF_DIM; + freq[bucket] = (freq[bucket] || 0) + 1; + } + // TF vector + L2 normalize + const vec = new Float64Array(TFIDF_DIM); + for (const [b, f] of Object.entries(freq)) { + vec[parseInt(b)] = f; // simple TF, no IDF (single-doc context) + } + let norm = 0; + for (let i = 0; i < TFIDF_DIM; i++) norm += vec[i] * vec[i]; + norm = Math.sqrt(norm) || 1; + return Array.from(vec.map(v => v / norm)); + }); +} + +function hashStr(s) { + let h = 5381; + for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) >>> 0; + return h; +} + +// ── Provider chain ─────────────────────────────────────────── +async function embed(texts) { + if (!Array.isArray(texts)) texts = [texts]; + // Try local first + try { + if (_localAvailable !== false) { + return await embedLocal(texts); + } + } catch {} + // Try API (Copilot auto-discovery or manual config) + try { + const cfg = loadConfig(); + if (_isCopilotAvailable() || (cfg.api_base_url && cfg.api_key)) { + return await embedAPI(texts); + } + } catch {} + // TF-IDF fallback + return embedTFIDF(texts); +} + +function isAvailable() { + try { + require.resolve('@huggingface/transformers'); + return true; + } catch { + if (_isCopilotAvailable()) return true; + const cfg = loadConfig(); + return !!(cfg.api_base_url && cfg.api_key); + } +} + +// ── Cosine similarity ──────────────────────────────────────── +function cosineSimilarity(a, b) { + if (a.length !== b.length) return 0; + let dot = 0; + for (let i = 0; i < a.length; i++) dot += a[i] * b[i]; + return dot; // Already L2-normalized +} + +// ── Reciprocal Rank Fusion (Cormack et al. 2009) ───────────── +function reciprocalRankFusion(bm25Ranked, embeddingRanked, bm25Weight, embWeight) { + const scores = new Map(); + // Graceful degradation + const hasBM25 = bm25Ranked.length > 0; + const hasEmb = embeddingRanked.length > 0; + if (!hasBM25 && !hasEmb) return []; + const effBM25 = hasBM25 && hasEmb ? bm25Weight : hasBM25 ? 1.0 : 0.0; + const effEmb = hasBM25 && hasEmb ? embWeight : hasEmb ? 1.0 : 0.0; + + for (let rank = 0; rank < bm25Ranked.length; rank++) { + const id = bm25Ranked[rank].id; + scores.set(id, (scores.get(id) || 0) + effBM25 / (RRF_K + rank + 1)); + } + for (let rank = 0; rank < embeddingRanked.length; rank++) { + const id = embeddingRanked[rank].id; + scores.set(id, (scores.get(id) || 0) + effEmb / (RRF_K + rank + 1)); + } + return [...scores.entries()] + .map(([id, score]) => ({ id, rrf_score: score })) + .sort((a, b) => b.rrf_score - a.rrf_score); +} + +// ── Utility tracker (SQLite-backed) ────────────────────────── +function _sqliteIndex() { return require('./sqlite-index'); } + +function ensureTables() { + const sq = _sqliteIndex(); + sq._exec(` + CREATE TABLE IF NOT EXISTS session_embeddings ( + session_id TEXT PRIMARY KEY, + embedding TEXT NOT NULL, + model TEXT NOT NULL, + provider TEXT DEFAULT 'local', + computed_at INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_emb_model ON session_embeddings(model); + + CREATE TABLE IF NOT EXISTS search_utility ( + session_id TEXT NOT NULL, + query_hash TEXT NOT NULL, + outcome TEXT NOT NULL, + ts INTEGER NOT NULL, + PRIMARY KEY (session_id, query_hash) + ); + `); +} + +function recordUtility(sessionId, queryHash, outcome) { + // outcome: 'click' | 'expand' | 'ignore' + try { + const sq = _sqliteIndex(); + const esc = (s) => "'" + String(s).replace(/'/g, "''") + "'"; + sq._exec(`INSERT OR REPLACE INTO search_utility (session_id, query_hash, outcome, ts) VALUES (${esc(sessionId)}, ${esc(queryHash)}, ${esc(outcome)}, ${Date.now()});`); + } catch {} +} + +function getUtilityRate(sessionId) { + try { + const sq = _sqliteIndex(); + const esc = (s) => "'" + String(s).replace(/'/g, "''") + "'"; + const rows = sq._execJson(`SELECT outcome, COUNT(*) AS n FROM search_utility WHERE session_id = ${esc(sessionId)} GROUP BY outcome`); + let positive = 0, total = 0; + for (const r of rows) { + total += r.n; + if (r.outcome === 'click' || r.outcome === 'expand') positive += r.n; + } + return total > 0 ? positive / total : 0.5; // default 0.5 for unknown + } catch { return 0.5; } +} + +// ── Embedding storage ──────────────────────────────────────── +function storeEmbeddings(rows) { + if (!rows || rows.length === 0) return; + const sq = _sqliteIndex(); + ensureTables(); + const now = Date.now(); + const parts = ['BEGIN;']; + for (const r of rows) { + const sid = r.session_id.replace(/'/g, "''"); + const emb = JSON.stringify(r.embedding); + const model = (r.model || getModelId()).replace(/'/g, "''"); + const provider = (r.provider || 'local').replace(/'/g, "''"); + parts.push(`INSERT OR REPLACE INTO session_embeddings (session_id, embedding, model, provider, computed_at) VALUES ('${sid}', '${emb}', '${model}', '${provider}', ${now});`); + } + parts.push('COMMIT;'); + sq._exec(parts.join('\n'), { timeout: 60000 }); +} + +function loadAllEmbeddings() { + const sq = _sqliteIndex(); + ensureTables(); + const rows = sq._execJson(`SELECT session_id, embedding FROM session_embeddings`); + return rows.map(r => ({ + session_id: r.session_id, + embedding: JSON.parse(r.embedding), + })); +} + +function getEmbeddingCount() { + const sq = _sqliteIndex(); + ensureTables(); + const rows = sq._execJson(`SELECT COUNT(*) AS n FROM session_embeddings`); + return (rows[0] || {}).n || 0; +} + +// ── 6-Stage Memento Search Pipeline ────────────────────────── + +async function hybridSearch(query, limit) { + limit = limit || 20; + const sq = _sqliteIndex(); + + // ── Stage 1: FTS5 sparse recall (top-20) ── + const ftsResults = sq.search(query, STAGE_CANDIDATE_K); + const ftsRanked = ftsResults.map(r => ({ + id: r.session_id, + bm25_score: 1.0, // FTS5 doesn't expose raw BM25, use rank position + ...r, + })); + + // ── Stage 2: Dense embedding recall (top-20) ── + let embRanked = []; + try { + const queryEmb = (await embed(query))[0]; + const all = loadAllEmbeddings(); + if (all.length > 0) { + const scored = all.map(r => ({ + id: r.session_id, + emb_score: cosineSimilarity(queryEmb, r.embedding), + })); + scored.sort((a, b) => b.emb_score - a.emb_score); + embRanked = scored.slice(0, STAGE_CANDIDATE_K); + } + } catch {} + + // ── Stage 3: Reciprocal Rank Fusion (k=60) ── + const fused = reciprocalRankFusion(ftsRanked, embRanked, BM25_WEIGHT, EMBEDDING_WEIGHT); + + // Build lookup maps + const ftsMap = new Map(ftsResults.map(r => [r.session_id, r])); + const embMap = new Map(embRanked.map(r => [r.id, r.emb_score])); + + // ── Stage 4: Utility reranking ── + const reranked = fused.map(item => { + const utilityRate = getUtilityRate(item.id); + const multiplier = UTILITY_BASE + UTILITY_SCALE * utilityRate; + return { + ...item, + utility_rate: utilityRate, + fused_score: item.rrf_score * multiplier, + }; + }); + reranked.sort((a, b) => b.fused_score - a.fused_score); + + // ── Stage 5: Threshold filter ── + const MIN_SCORE = 0.0001; + + // ── Stage 6: Top-k with enrichment ── + const results = []; + for (const item of reranked) { + if (item.fused_score < MIN_SCORE) continue; + if (results.length >= limit) break; + + const fts = ftsMap.get(item.id); + const embScore = embMap.get(item.id) || 0; + + results.push({ + session_id: item.id, + fused_score: item.fused_score, + rrf_score: item.rrf_score, + bm25_rank: ftsRanked.findIndex(r => r.id === item.id), + embedding_score: embScore, + utility_rate: item.utility_rate, + search_type: fts && embScore > 0 ? 'hybrid' : fts ? 'text' : 'semantic', + matches: fts ? [{ + role: fts.role || 'unknown', + snippet: (fts.snippet || '').replace(/<>/g, ''), + }] : [], + }); + } + + return results; +} + +// Pure semantic search (no FTS5) +async function semanticSearch(query, limit) { + limit = limit || 20; + const queryEmb = (await embed(query))[0]; + const all = loadAllEmbeddings(); + if (all.length === 0) return []; + const scored = all.map(r => ({ + session_id: r.session_id, + score: cosineSimilarity(queryEmb, r.embedding), + })); + scored.sort((a, b) => b.score - a.score); + return scored.slice(0, limit); +} + +// Batch embed for backfill +async function embedBatch(texts, batchSize) { + return embed(Array.isArray(texts) ? texts : [texts]); +} + +module.exports = { + // Config + MODELS, + loadConfig, + getModelId, + getModelDim, + // Provider chain + embed, + embedLocal, + embedAPI, + embedTFIDF, + isAvailable, + // Search pipeline + hybridSearch, + semanticSearch, + reciprocalRankFusion, + cosineSimilarity, + // Storage + ensureTables, + storeEmbeddings, + loadAllEmbeddings, + getEmbeddingCount, + embedBatch, + // Utility tracker + recordUtility, + getUtilityRate, + // Constants + EMBEDDING_DIM: 384, // default, actual may vary by model + MODEL_ID: 'configurable', // use getModelId() + RRF_K, + BM25_WEIGHT, + EMBEDDING_WEIGHT, +}; diff --git a/src/frontend/analytics.js b/src/frontend/analytics.js index d414be9..eded4d0 100644 --- a/src/frontend/analytics.js +++ b/src/frontend/analytics.js @@ -9,9 +9,43 @@ async function renderAnalytics(container) { if (dateFrom) params.push('from=' + dateFrom); if (dateTo) params.push('to=' + dateTo); if (params.length) url += '?' + params.join('&'); - var resp = await fetch(url); - var data = await resp.json(); + // Poll loop — server runs a background job, we render live partial snapshots + var pollStart = Date.now(); + var data = null; + while (true) { + var resp = await fetch(url); + var payload = await resp.json(); + if (payload.status === 'done') { data = payload; break; } + if (payload.status === 'error') { + container.innerHTML = '
Analytics failed: ' + escHtml(payload.error || 'unknown') + '
'; + return; + } + var p = (payload.progress || {}); + var done = p.done || 0, total = p.total || 0; + var pct = total > 0 ? Math.round(done / total * 100) : 0; + var phase = p.phase || 'working'; + var elapsed = Math.round((payload.elapsedMs || (Date.now() - pollStart)) / 1000); + if (payload.partialResult && payload.partialResult.totalSessions > 0) { + renderAnalyticsUI(container, payload.partialResult, { + live: true, done: done, total: total, pct: pct, phase: phase, elapsed: elapsed, + }); + } else { + container.innerHTML = + '
' + + '
Computing cost analytics…
' + + '
' + escHtml(phase) + ' — ' + done + ' / ' + total + ' sessions (' + pct + '%)
' + + '
' + + '
elapsed ' + elapsed + 's · cached for next time
' + + '
'; + } + await new Promise(function(r){ setTimeout(r, 500); }); + } + + renderAnalyticsUI(container, data, { live: false }); + return; + /* === below is the original inline render block; kept so the upstream + split structure is preserved and renderAnalyticsUI uses the same code path === */ var html = '
'; html += '

Cost Analytics

'; @@ -227,6 +261,145 @@ async function renderAnalytics(container) { } } +// Renders the full Cost Analytics UI from either partial (live) or final data. +// When opts.live is true, prepends a sticky progress banner. +function renderAnalyticsUI(container, data, opts) { + opts = opts || {}; + var html = '
'; + + if (opts.live) { + var pct = opts.pct || 0; + html += '
'; + html += '
Computing cost analytics — live' + (opts.elapsed || 0) + 's
'; + html += '
' + escHtml(opts.phase || '') + ' — ' + (opts.done || 0) + ' / ' + (opts.total || 0) + ' sessions (' + pct + '%) · numbers update as sessions are processed
'; + html += '
'; + html += '
'; + } + + html += '

Cost Analytics' + (opts.live ? ' (live)' : '') + '

'; + + // Summary cards + html += '
'; + html += '
$' + (data.totalCost || 0).toFixed(2) + 'Total cost (API-equivalent)
'; + html += '
' + formatTokens(data.totalTokens || 0) + 'Total tokens
'; + html += '
$' + (data.dailyRate || 0).toFixed(2) + 'Avg per day (' + (data.days || 1) + ' days)
'; + html += '
' + (data.totalSessions || 0) + 'Sessions with cost data' + (data.totalSessionsAll > data.totalSessions ? ' / ' + data.totalSessionsAll + ' total' : '') + '
'; + html += '
'; + + // Burn rate + var todayCost = data.todayCost || 0; + var last1hCost = data.last1hCost || 0; + var dailyRate = data.dailyRate || 0; + var hoursElapsed = data.hoursElapsedToday || 1; + var projectedDaily = todayCost / (hoursElapsed / 24); + var paceRatio = dailyRate > 0 ? projectedDaily / dailyRate : 0; + var burnClass = paceRatio >= 2 ? 'burn-high' : paceRatio >= 1.3 ? 'burn-medium' : 'burn-low'; + var paceLabel = paceRatio >= 2 ? '🔥 ' + Math.round(paceRatio) + 'x avg' : paceRatio >= 1.3 ? '↑ ' + paceRatio.toFixed(1) + 'x avg' : dailyRate > 0 ? '✓ normal' : ''; + html += '
'; + html += '
Burn Rate
'; + html += '
'; + html += '
$' + todayCost.toFixed(3) + 'today' + (paceLabel ? '' + paceLabel + '' : '') + '
'; + html += '
$' + last1hCost.toFixed(3) + 'last hour
'; + if (dailyRate > 0) { + html += '
$' + projectedDaily.toFixed(2) + 'projected today
'; + } + html += '
'; + + // Token breakdown + if (data.totalInputTokens !== undefined) { + var totalTok = (data.totalInputTokens || 0) + (data.totalOutputTokens || 0) + (data.totalCacheReadTokens || 0) + (data.totalCacheCreateTokens || 0); + var pctOf = function(n) { return totalTok > 0 ? Math.round(n / totalTok * 100) : 0; }; + html += '
'; + html += '

Token Breakdown

'; + html += '
'; + html += '
' + formatTokens(data.totalInputTokens || 0) + 'Input' + pctOf(data.totalInputTokens || 0) + '%
'; + html += '
' + formatTokens(data.totalOutputTokens || 0) + 'Output' + pctOf(data.totalOutputTokens || 0) + '%
'; + html += '
' + formatTokens(data.totalCacheReadTokens || 0) + 'Cache read' + pctOf(data.totalCacheReadTokens || 0) + '%
'; + html += '
' + formatTokens(data.totalCacheCreateTokens || 0) + 'Cache write' + pctOf(data.totalCacheCreateTokens || 0) + '%
'; + if (data.avgContextPct > 0) { + html += '
' + data.avgContextPct + '%Avg context usedof 200K
'; + } + html += '
'; + } + + // Cost by agent + var agentEntries = Object.entries(data.byAgent || {}).filter(function(e) { return e[1].sessions > 0; }); + if (agentEntries.length >= 1) { + agentEntries.sort(function(a, b) { return b[1].cost - a[1].cost; }); + html += '

Cost by Agent

'; + html += '
'; + var maxAgentCost = agentEntries[0][1].cost || 1; + agentEntries.forEach(function(entry) { + var name = entry[0]; var info = entry[1]; + var p = maxAgentCost > 0 ? (info.cost / maxAgentCost * 100) : 0; + var label = { 'claude': 'Claude Code', 'claude-ext': 'Claude Ext', 'codex': 'Codex', 'opencode': 'OpenCode', 'cursor': 'Cursor', 'kiro': 'Kiro' }[name] || name; + var estMark = info.estimated ? ' ~est.' : ''; + html += '
'; + html += '' + label + estMark + ''; + html += '
'; + html += '$' + info.cost.toFixed(2) + ' (' + info.sessions + ' sess.)'; + html += '
'; + }); + html += '
'; + } + + // Daily cost chart (last 30 days) + var dayKeys = Object.keys(data.byDay || {}).sort(); + var last30 = dayKeys.slice(-30); + if (last30.length > 0) { + var maxCost = Math.max.apply(null, last30.map(function(d) { return data.byDay[d].cost; })); + html += '

Daily Cost (last 30 days)

'; + html += '
'; + last30.forEach(function(d) { + var c = data.byDay[d]; + var p = maxCost > 0 ? (c.cost / maxCost * 100) : 0; + var label = d.slice(5); + html += '
'; + html += '
'; + html += '
' + label + '
'; + html += '
'; + }); + html += '
'; + } + + // Cost by project + var projects = Object.entries(data.byProject || {}).sort(function(a, b) { return b[1].cost - a[1].cost; }); + var topProjects = projects.slice(0, 10); + if (topProjects.length > 0) { + var maxProjCost = topProjects[0][1].cost || 1; + html += '

Cost by Project

'; + html += '
'; + topProjects.forEach(function(entry) { + var name = entry[0]; var info = entry[1]; + var p = maxProjCost > 0 ? (info.cost / maxProjCost * 100) : 0; + html += '
'; + html += '' + escHtml(name) + ''; + html += '
'; + html += '$' + info.cost.toFixed(2) + ''; + html += '
'; + }); + html += '
'; + } + + // Top expensive sessions + if (data.topSessions && data.topSessions.length > 0) { + html += '

Most Expensive Sessions

'; + html += '
'; + data.topSessions.forEach(function(s) { + html += '
'; + html += '$' + s.cost.toFixed(2) + ''; + html += '' + escHtml(s.project || '') + ''; + html += '' + (s.date || '') + ''; + html += '' + (s.id || '').slice(0, 8) + ''; + html += '
'; + }); + html += '
'; + } + + html += '
'; + container.innerHTML = html; +} + function formatTokens(n) { if (n >= 1000000) return (n / 1000000).toFixed(1) + 'M'; if (n >= 1000) return (n / 1000).toFixed(0) + 'K'; diff --git a/src/frontend/app.js b/src/frontend/app.js index 6fd757a..b8ea72f 100644 --- a/src/frontend/app.js +++ b/src/frontend/app.js @@ -691,6 +691,9 @@ function renderCard(s, idx) { } html += '
'; html += '' + s.messages + ' msgs'; + if (s._group_extra > 0) { + html += '+' + s._group_extra + ' more (' + s._group_total_msgs + ' total msgs)'; + } if (s.file_size) { html += '' + formatBytes(s.file_size) + ''; } @@ -823,18 +826,33 @@ async function toggleExpand(sessionId, project, btn) { var deepSearchCache = {}; var deepSearchTimeout = null; +var searchMode = 'hybrid'; // text|semantic|hybrid + +function setSearchMode(mode) { + searchMode = mode; + document.querySelectorAll('.search-mode-btn').forEach(function(b) { + b.classList.toggle('active', b.dataset.mode === mode); + }); + // Re-run current search with new mode + deepSearchCache = {}; + if (searchQuery && searchQuery.length >= 3) { + deepSearch(searchQuery); + } +} async function deepSearch(query) { if (!query || query.length < 3) return; - if (deepSearchCache[query]) { - applyDeepSearchResults(deepSearchCache[query]); + var cacheKey = searchMode + ':' + query; + if (deepSearchCache[cacheKey]) { + applyDeepSearchResults(deepSearchCache[cacheKey]); return; } try { - var resp = await fetch('/api/search?q=' + encodeURIComponent(query)); + var url = '/api/search?q=' + encodeURIComponent(query) + '&mode=' + searchMode; + var resp = await fetch(url); var results = await resp.json(); - deepSearchCache[query] = results; + deepSearchCache[cacheKey] = results; applyDeepSearchResults(results); } catch {} } @@ -968,8 +986,14 @@ function render() { } var renderFn = layout === 'list' ? renderListCard : renderCard; - var visible = sessions.slice(0, renderLimit); - var hasMore = sessions.length > renderLimit; + + // Dedupe conversation groups first (same project+initial prompt collapses + // into a single representative card with "+N more" badge). Shared helper + // used by Timeline / Cloud Sync too. + var conversationGroups = groupSessionsByConversation(sessions); + var dedupedSessions = conversationGroups.map(function(g) { return withGroupBadge(g.representative, g); }); + var visible = dedupedSessions.slice(0, renderLimit); + var hasMore = dedupedSessions.length > renderLimit; if (grouped) { renderGrouped(content, visible, renderFn); @@ -980,7 +1004,7 @@ function render() { } if (hasMore) { - content.innerHTML += '
'; + content.innerHTML += '
'; } } @@ -1024,14 +1048,64 @@ function renderGrouped(container, sessions, renderFn) { container.innerHTML = html; } +// ── Shared session grouping ─────────────────────────────────── +// Collapses near-duplicate sessions (same project + same initial prompt) into +// a single "conversation group". Representative is the newest session; the +// rest are attached as `group_members` so the UI can show "+N more" and let +// the user drill down. Used by Timeline / All Sessions / Cloud Sync views — +// all call this helper, no per-view dedup logic. +function groupSessionsByConversation(sessions) { + var groups = {}; + var order = []; + sessions.forEach(function(s) { + var k = s.group_key || s.id; + if (!groups[k]) { + groups[k] = { + key: k, + representative: s, + members: [s], + total_msgs: s.messages || 0, + total_file_size: s.file_size || 0, + }; + order.push(k); + return; + } + var g = groups[k]; + g.members.push(s); + g.total_msgs += s.messages || 0; + g.total_file_size += s.file_size || 0; + // Newest last_ts wins as representative + if (s.last_ts > g.representative.last_ts) g.representative = s; + }); + return order.map(function(k) { return groups[k]; }); +} + +// Decorate a session with group metadata so renderCard can show "+N more". +function withGroupBadge(s, group) { + if (!group || group.members.length <= 1) return s; + var extra = group.members.length - 1; + // Clone shallow so we don't mutate the original + return Object.assign({}, s, { + _group_count: group.members.length, + _group_extra: extra, + _group_total_msgs: group.total_msgs, + _group_members: group.members, + }); +} + function renderTimeline(container, sessions) { - // Group by date + // Group by date, then within each date dedupe near-duplicate conversations var byDate = {}; sessions.forEach(function(s) { var d = s.date || 'unknown'; if (!byDate[d]) byDate[d] = []; byDate[d].push(s); }); + // Replace each day's flat list with grouped representatives + Object.keys(byDate).forEach(function(d) { + var groups = groupSessionsByConversation(byDate[d]); + byDate[d] = groups.map(function(g) { return withGroupBadge(g.representative, g); }); + }); var dates = Object.keys(byDate).sort().reverse(); if (dates.length === 0) { @@ -1069,6 +1143,9 @@ function renderQACard(s, idx) { html += '' + escHtml((s.first_message || '').slice(0, 160)) + ''; html += ''; html += '' + s.messages + ' msgs'; + if (s._group_extra > 0) { + html += '+' + s._group_extra + ''; + } if (costStr) html += '' + costStr + ''; html += '' + timeAgo(s.last_ts) + ''; html += ''; @@ -1098,17 +1175,27 @@ function renderProjects(container, sessions) { var html = '
'; sorted.forEach(function(entry) { var name = entry[0]; - var list = entry[1].slice().sort(function(a, b) { return b.last_ts - a.last_ts; }); + var rawList = entry[1]; + // Dedupe conversation groups within the project (same first prompt + // collapses into one representative card with "+N more" badge). + var convGroups = groupSessionsByConversation(rawList); + var list = convGroups + .map(function(g) { return withGroupBadge(g.representative, g); }) + .sort(function(a, b) { return b.last_ts - a.last_ts; }); + var color = getProjectColor(name); - var totalMsgs = list.reduce(function(s, e) { return s + (e.messages || 0); }, 0); - var totalCost = list.reduce(function(s, e) { return s + estimateCost(e.file_size); }, 0); + var totalMsgs = rawList.reduce(function(s, e) { return s + (e.messages || 0); }, 0); + var totalCost = rawList.reduce(function(s, e) { return s + estimateCost(e.file_size); }, 0); var costLabel = totalCost > 0 ? ' · ~$' + totalCost.toFixed(2) : ''; + var dedupLabel = rawList.length !== list.length + ? ' (' + rawList.length + ' raw → ' + list.length + ' unique)' + : ''; html += '
'; html += '
'; html += ''; html += '' + escHtml(name) + ''; - html += '' + list.length + ' sessions · ' + totalMsgs + ' msgs' + escHtml(costLabel) + ''; + html += '' + list.length + ' sessions' + dedupLabel + ' · ' + totalMsgs + ' msgs' + escHtml(costLabel) + ''; html += '▼'; html += '
'; html += '
'; @@ -1783,25 +1870,7 @@ async function checkForUpdates() { } localStorage.setItem('codedash-last-version', data.current); - if (data.updateAvailable) { - if (badge) { - badge.textContent = 'v' + data.current + ' → v' + data.latest; - badge.classList.add('update-available'); - badge.title = 'Click to copy update command'; - badge.onclick = function() { - navigator.clipboard.writeText('npm i -g codedash-app@latest').then(function() { - showToast('Copied: npm i -g codedash-app@latest'); - }); - }; - } - var banner = document.getElementById('updateBanner'); - var text = document.getElementById('updateText'); - if (banner && text) { - text.textContent = 'v' + data.latest + ' available — run: npm i -g codedash-app@latest'; - banner.style.display = 'flex'; - banner.dataset.cmd = 'npm i -g codedash-app@latest'; - } - } + // Update banner disabled — dev/fork build, not published to npm } catch {} } diff --git a/src/frontend/cloud.js b/src/frontend/cloud.js index 80ba19a..54cfaa2 100644 --- a/src/frontend/cloud.js +++ b/src/frontend/cloud.js @@ -82,10 +82,17 @@ async function renderCloud(container) { html += '
'; if (cloudLocalSessions) { - for (var i = 0; i < cloudLocalSessions.length; i++) { - var ls = cloudLocalSessions[i]; + // Dedupe near-duplicate conversations via the shared helper — same + // logic as Timeline, All Sessions etc. + var _groups = groupSessionsByConversation(cloudLocalSessions); + for (var i = 0; i < _groups.length; i++) { + var g = _groups[i]; + var ls = g.representative; + var extra = g.members.length - 1; var inCloud = cloudSessionIds.has(ls.id); - var sub = (ls.project_short || '') + ' \u00b7 ' + (ls.messages || 0) + ' msgs'; + var subParts = [(ls.project_short || ''), (ls.messages || 0) + ' msgs']; + if (extra > 0) subParts.push('+' + extra + ' more (' + g.total_msgs + ' total)'); + var sub = subParts.join(' \u00b7 '); var btnHtml = inCloud ? 'in cloud' : ''; diff --git a/src/frontend/detail.js b/src/frontend/detail.js index 1f71257..d5f2e56 100644 --- a/src/frontend/detail.js +++ b/src/frontend/detail.js @@ -1,5 +1,19 @@ // ── Detail panel ─────────────────────────────────────────────── +// Progressive loading page size +var DETAIL_PAGE_SIZE = 50; + +// State for current detail view +var _detailState = { + sessionId: null, + project: null, + allMessages: [], // all loaded messages so far + total: 0, + offset: 0, + hasMore: false, + filter: 'all' // 'all' | 'user' | 'assistant' | 'tool' +}; + async function openDetail(s) { var panel = document.getElementById('detailPanel'); var overlay = document.getElementById('overlay'); @@ -7,6 +21,15 @@ async function openDetail(s) { var body = document.getElementById('detailBody'); if (!panel || !body) return; + // Reset detail state + _detailState.sessionId = s.id; + _detailState.project = s.project || ''; + _detailState.allMessages = []; + _detailState.total = 0; + _detailState.offset = 0; + _detailState.hasMore = false; + _detailState.filter = 'all'; + title.textContent = escHtml(getProjectName(s.project)) + ' / ' + s.id.slice(0, 12); var cost = estimateCost(s.file_size); @@ -82,53 +105,31 @@ async function openDetail(s) { var convertTarget = s.tool === 'codex' ? 'claude' : 'codex'; infoHtml += ''; infoHtml += ''; + infoHtml += ''; } infoHtml += ''; infoHtml += ''; infoHtml += '
'; + // Summary box (hidden by default, shown after summarize) + infoHtml += ''; + + // Message filters + infoHtml += ''; + body.innerHTML = infoHtml + '
Loading messages...
'; panel.classList.add('open'); overlay.classList.add('open'); - // Load messages + // Load messages progressively if (s.has_detail) { - try { - var resp = await fetch('/api/session/' + s.id + '?project=' + encodeURIComponent(s.project || '')); - var data = await resp.json(); - var msgContainer = body.querySelector('.detail-messages'); - if (data.messages && data.messages.length > 0) { - var msgsHtml = '

Conversation

'; - data.messages.forEach(function(m) { - var roleClass = m.role === 'user' ? 'msg-user' : 'msg-assistant'; - var roleLabel = m.role === 'user' ? 'You' : 'Assistant'; - var hasTools = m.tools && m.tools.length > 0; - msgsHtml += '
'; - msgsHtml += '
'; - msgsHtml += '
' + roleLabel + '
'; - msgsHtml += '
' + escHtml(m.content) + '
'; - msgsHtml += '
'; - if (hasTools) { - msgsHtml += '
'; - m.tools.forEach(function(t) { - if (t.type === 'mcp') { - msgsHtml += '' + escHtml(t.tool) + ''; - } else if (t.type === 'skill') { - msgsHtml += '' + escHtml(t.skill) + ''; - } - }); - msgsHtml += '
'; - } - msgsHtml += '
'; - }); - msgContainer.innerHTML = msgsHtml; - } else { - msgContainer.innerHTML = '
No messages found in detail file.
'; - } - } catch (e) { - body.querySelector('.detail-messages').innerHTML = '
Failed to load messages.
'; - } + await _loadDetailPage(s.id, s.project || '', 0, body); } else { body.querySelector('.detail-messages').innerHTML = '
No detail file available for this session.
'; } @@ -197,6 +198,173 @@ async function openDetail(s) { } } +// ── Progressive message loading ─────────────────────────────── + +async function _loadDetailPage(sessionId, project, offset, body) { + try { + var url = '/api/session/' + sessionId + '?project=' + encodeURIComponent(project) + + '&offset=' + offset + '&limit=' + DETAIL_PAGE_SIZE; + var resp = await fetch(url); + var data = await resp.json(); + var msgContainer = body.querySelector('.detail-messages'); + if (!msgContainer) return; + + _detailState.total = data.total || 0; + _detailState.hasMore = !!data.hasMore; + _detailState.offset = (data.offset || 0) + (data.messages ? data.messages.length : 0); + + if (data.messages && data.messages.length > 0) { + // Append to our running list + for (var i = 0; i < data.messages.length; i++) { + _detailState.allMessages.push(data.messages[i]); + } + // Show filter bar + var filterBar = document.getElementById('detailMsgFilters'); + if (filterBar) filterBar.style.display = ''; + // Re-render all visible messages + _renderDetailMessages(msgContainer); + } else if (_detailState.allMessages.length === 0) { + msgContainer.innerHTML = '
No messages found in detail file.
'; + } + } catch (e) { + var mc = body.querySelector('.detail-messages'); + if (mc && _detailState.allMessages.length === 0) { + mc.innerHTML = '
Failed to load messages.
'; + } + } +} + +function _renderSingleMessage(m) { + var roleClass = m.role === 'user' ? 'msg-user' : 'msg-assistant'; + var roleLabel = m.role === 'user' ? 'You' : 'Assistant'; + var hasTools = m.tools && m.tools.length > 0; + var html = '
'; + html += '
'; + html += '
' + roleLabel + '
'; + html += '
' + escHtml(m.content) + '
'; + html += '
'; + if (hasTools) { + html += '
'; + m.tools.forEach(function(t) { + if (t.type === 'mcp') { + html += '' + escHtml(t.tool) + ''; + } else if (t.type === 'skill') { + html += '' + escHtml(t.skill) + ''; + } + }); + html += '
'; + } + html += '
'; + return html; +} + +function _renderDetailMessages(container) { + var msgs = _detailState.allMessages; + var filter = _detailState.filter; + var html = '

Conversation (' + _detailState.allMessages.length; + if (_detailState.total > _detailState.allMessages.length) { + html += ' of ' + _detailState.total; + } + html += ' messages)

'; + html += '
'; + + for (var i = 0; i < msgs.length; i++) { + var m = msgs[i]; + var visible = _matchesFilter(m, filter); + if (visible) { + html += _renderSingleMessage(m); + } + } + html += '
'; + + // "Load more" button + if (_detailState.hasMore) { + var remaining = _detailState.total - _detailState.offset; + html += '
'; + html += ''; + html += '
'; + } + + container.innerHTML = html; +} + +function _matchesFilter(m, filter) { + if (filter === 'all') return true; + if (filter === 'user') return m.role === 'user'; + if (filter === 'assistant') return m.role === 'assistant'; + if (filter === 'tool') return m.tools && m.tools.length > 0; + return true; +} + +async function _onLoadMore() { + var btn = document.getElementById('detailLoadMoreBtn'); + if (btn) { + btn.disabled = true; + btn.textContent = 'Loading...'; + } + var body = document.getElementById('detailBody'); + if (!body) return; + await _loadDetailPage(_detailState.sessionId, _detailState.project, _detailState.offset, body); +} + +// ── Message filters ─────────────────────────────────────────── + +function setDetailFilter(filter) { + _detailState.filter = filter; + // Update active button + var btns = document.querySelectorAll('.detail-msg-filters .filter-btn'); + for (var i = 0; i < btns.length; i++) { + if (btns[i].getAttribute('data-filter') === filter) { + btns[i].classList.add('active'); + } else { + btns[i].classList.remove('active'); + } + } + // Re-render messages (client-side filter, no re-fetch) + var msgContainer = document.querySelector('.detail-messages'); + if (msgContainer) _renderDetailMessages(msgContainer); +} + +// ── Summarize session ───────────────────────────────────────── + +async function summarizeSession(sessionId, project) { + var btn = document.getElementById('detailSummarizeBtn'); + var summaryBox = document.getElementById('detailSummary'); + if (!btn || !summaryBox) return; + + btn.disabled = true; + btn.innerHTML = '⌛ Summarizing...'; + + try { + var resp = await fetch('/api/summarize/' + sessionId + '?project=' + encodeURIComponent(project), { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: '{}' + }); + var data = await resp.json(); + + if (data.ok && data.summary) { + summaryBox.style.display = ''; + summaryBox.innerHTML = '
✨ Summary
' + escHtml(data.summary) + '
'; + btn.innerHTML = '✨ Summarize'; + btn.disabled = false; + } else if (data.error && (data.error.indexOf('not available') >= 0 || data.error.indexOf('not authenticated') >= 0)) { + summaryBox.style.display = ''; + summaryBox.innerHTML = '
✨ Summary
Connect GitHub Copilot in Settings to use Summarize.
'; + btn.innerHTML = '✨ Summarize'; + btn.disabled = false; + } else { + showToast('Summarize failed: ' + (data.error || 'unknown')); + btn.innerHTML = '✨ Summarize'; + btn.disabled = false; + } + } catch (e) { + showToast('Summarize failed: ' + e.message); + btn.innerHTML = '✨ Summarize'; + btn.disabled = false; + } +} + function closeDetail() { var panel = document.getElementById('detailPanel'); var overlay = document.getElementById('overlay'); diff --git a/src/frontend/heatmap.js b/src/frontend/heatmap.js index 17eaef8..7b50d4e 100644 --- a/src/frontend/heatmap.js +++ b/src/frontend/heatmap.js @@ -11,10 +11,16 @@ function renderHeatmap(container) { var now = new Date(); var oneYearAgo = new Date(now.getFullYear() - 1, now.getMonth(), now.getDate()); - // Count sessions per day + by tool + // Dedupe sessions into conversation groups so retry/resume chains count + // as ONE session. Otherwise a `codex exec` loop looks like 1000 sessions + // in a single day. Uses the shared helper from app.js. + var dedupedGroups = groupSessionsByConversation(allSessions); + var dedupedSessions = dedupedGroups.map(function(g) { return g.representative; }); + + // Count sessions per day + by tool (deduped) var counts = {}; var toolCounts = {}; - allSessions.forEach(function(s) { + dedupedSessions.forEach(function(s) { var d = s.date; if (!d) return; counts[d] = (counts[d] || 0) + 1; diff --git a/src/frontend/index.html b/src/frontend/index.html index c8b3fd9..a22c9e1 100644 --- a/src/frontend/index.html +++ b/src/frontend/index.html @@ -111,6 +111,11 @@
+
+ + + +