feat: add initial browser voice input prototype

This commit is contained in:
2026-05-11 22:01:23 -04:00
parent 67bd22b27d
commit 3edc41e86c
10 changed files with 10660 additions and 15 deletions
+33
View File
@@ -0,0 +1,33 @@
# Repository Guidelines
## Project Structure & Module Organization
`cmd/webterm/` contains the CLI entrypoint. Core server, session, Docker, replay, screenshot, and static-serving code lives in `webterm/`. Shared internal helpers live in `internal/`. Frontend terminal code is in `webterm/static/js/terminal.ts`, with the bundled output committed as `webterm/static/js/terminal.js`. Static assets such as fonts, icons, and WASM files live under `webterm/static/`. Documentation and reference media live in `docs/`.
## Build, Test, and Development Commands
- `make install-dev`: install Go and frontend dependencies.
- `make build`: typecheck and bundle frontend assets.
- `make build-fast`: rebuild frontend without TypeScript checking.
- `make build-go`: compile `bin/webterm`.
- `go run ./cmd/webterm`: start the server locally on `http://localhost:8080`.
- `make test`: run all Go tests.
- `make race`: run Go tests with the race detector.
- `make check`: run lint, tests, and coverage.
- `./update.sh`: rebuild, install, and restart the user service.
## Coding Style & Naming Conventions
Use `gofmt` for Go formatting; run `make format` before submitting Go-heavy changes. Follow existing Go naming: exported identifiers use `CamelCase`, internal helpers use `camelCase`, and tests live beside source files. TypeScript in `webterm/static/js/` uses strict mode, 2-space indentation, and direct DOM-oriented code rather than framework abstractions. Keep generated bundles in sync with source changes.
## Testing Guidelines
Go tests use the standard `testing` package. Name tests `TestXxx` and fuzz tests `FuzzXxx`; keep them next to the code they validate. Prefer focused unit tests for `webterm/` and `internal/` changes. Run `make test` for normal work and `make check` before opening a PR. Frontend changes should at minimum pass `bun run typecheck` via `make build`.
## Commit & Pull Request Guidelines
Recent history uses short imperative subjects, sometimes with prefixes such as `feat:`, `fix:`, and `deps:`. Keep commits focused, e.g. `fix: restore websocket reconnect on hidden-tab resume`. PRs should explain user-visible behavior, note test coverage, and include screenshots or recordings for terminal/UI changes. Link related issues when applicable.
## Security & Configuration Tips
By default, the service binds to port `8080`; review `~/.config/systemd/user/webterm.service` before exposing it remotely. Use HTTPS for browser microphone features. Static assets are embedded by default, but `WEBTERM_STATIC_PATH` can override them in development.
+169
View File
@@ -5,6 +5,7 @@
"": { "": {
"name": "webterm-frontend", "name": "webterm-frontend",
"dependencies": { "dependencies": {
"@moonshine-ai/moonshine-js": "^0.1.29",
"ghostty-web": "github:rcarmo/ghostty-web#fcc47d423a7fce1c02c702b6464d0b1ab89175f1", "ghostty-web": "github:rcarmo/ghostty-web#fcc47d423a7fce1c02c702b6464d0b1ab89175f1",
}, },
"devDependencies": { "devDependencies": {
@@ -13,8 +14,176 @@
}, },
}, },
"packages": { "packages": {
"@emnapi/runtime": ["@emnapi/runtime@1.10.0", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA=="],
"@huggingface/jinja": ["@huggingface/jinja@0.5.9", "", {}, "sha512-uWTG+l3VJRsl7EXxYizuL3P+cCPoc3cRqbWWRcQN0FhejRfbdq0RNhCmbY/YDtnTcz9icdLYuLDjsnz4d8JMuw=="],
"@huggingface/transformers": ["@huggingface/transformers@3.8.1", "", { "dependencies": { "@huggingface/jinja": "^0.5.3", "onnxruntime-node": "1.21.0", "onnxruntime-web": "1.22.0-dev.20250409-89f8206ba4", "sharp": "^0.34.1" } }, "sha512-tsTk4zVjImqdqjS8/AOZg2yNLd1z9S5v+7oUPpXaasDRwEDhB+xnglK1k5cad26lL5/ZIaeREgWWy0bs9y9pPA=="],
"@img/colour": ["@img/colour@1.1.0", "", {}, "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ=="],
"@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.2.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w=="],
"@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.2.4" }, "os": "darwin", "cpu": "x64" }, "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw=="],
"@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.2.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g=="],
"@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.2.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg=="],
"@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.2.4", "", { "os": "linux", "cpu": "arm" }, "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A=="],
"@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw=="],
"@img/sharp-libvips-linux-ppc64": ["@img/sharp-libvips-linux-ppc64@1.2.4", "", { "os": "linux", "cpu": "ppc64" }, "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA=="],
"@img/sharp-libvips-linux-riscv64": ["@img/sharp-libvips-linux-riscv64@1.2.4", "", { "os": "linux", "cpu": "none" }, "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA=="],
"@img/sharp-libvips-linux-s390x": ["@img/sharp-libvips-linux-s390x@1.2.4", "", { "os": "linux", "cpu": "s390x" }, "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ=="],
"@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw=="],
"@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw=="],
"@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg=="],
"@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.2.4" }, "os": "linux", "cpu": "arm" }, "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw=="],
"@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg=="],
"@img/sharp-linux-ppc64": ["@img/sharp-linux-ppc64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-ppc64": "1.2.4" }, "os": "linux", "cpu": "ppc64" }, "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA=="],
"@img/sharp-linux-riscv64": ["@img/sharp-linux-riscv64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-riscv64": "1.2.4" }, "os": "linux", "cpu": "none" }, "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw=="],
"@img/sharp-linux-s390x": ["@img/sharp-linux-s390x@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.2.4" }, "os": "linux", "cpu": "s390x" }, "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg=="],
"@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ=="],
"@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg=="],
"@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q=="],
"@img/sharp-wasm32": ["@img/sharp-wasm32@0.34.5", "", { "dependencies": { "@emnapi/runtime": "^1.7.0" }, "cpu": "none" }, "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw=="],
"@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.34.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g=="],
"@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.34.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg=="],
"@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.5", "", { "os": "win32", "cpu": "x64" }, "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw=="],
"@isaacs/fs-minipass": ["@isaacs/fs-minipass@4.0.1", "", { "dependencies": { "minipass": "^7.0.4" } }, "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w=="],
"@moonshine-ai/moonshine-js": ["@moonshine-ai/moonshine-js@0.1.29", "", { "dependencies": { "@huggingface/transformers": "^3.3.3", "@ricky0123/vad-web": "file:../vad-moonshine/packages/web", "llama-tokenizer-js": "^1.2.2", "onnxruntime-web": "^1.22.0" } }, "sha512-Gx1B3mJcbM68ihSy/LyJEuEkGq7sYMqTb04zNdcIeidU0URVlSmXP0GyiQOTTAqDZQXlIC7k+bky+FdXk0UPlg=="],
"@protobufjs/aspromise": ["@protobufjs/aspromise@1.1.2", "", {}, "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="],
"@protobufjs/base64": ["@protobufjs/base64@1.1.2", "", {}, "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="],
"@protobufjs/codegen": ["@protobufjs/codegen@2.0.5", "", {}, "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g=="],
"@protobufjs/eventemitter": ["@protobufjs/eventemitter@1.1.0", "", {}, "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q=="],
"@protobufjs/fetch": ["@protobufjs/fetch@1.1.0", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.1", "@protobufjs/inquire": "^1.1.0" } }, "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ=="],
"@protobufjs/float": ["@protobufjs/float@1.0.2", "", {}, "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="],
"@protobufjs/inquire": ["@protobufjs/inquire@1.1.1", "", {}, "sha512-mnzgDV26ueAvk7rsbt9L7bE0SuAoqyuys/sMMrmVcN5x9VsxpcG3rqAUSgDyLp0UZlmNfIbQ4fHfCtreVBk8Ew=="],
"@protobufjs/path": ["@protobufjs/path@1.1.2", "", {}, "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="],
"@protobufjs/pool": ["@protobufjs/pool@1.1.0", "", {}, "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="],
"@protobufjs/utf8": ["@protobufjs/utf8@1.1.1", "", {}, "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg=="],
"@types/node": ["@types/node@25.7.0", "", { "dependencies": { "undici-types": "~7.21.0" } }, "sha512-z+pdZyxE+RTQE9AcboAZCb4otwcrvgHD+GlBpPgn0emDVt0ohrTMhAwlr2Wd9nZ+nihhYFxO2pThz3C5qSu2Eg=="],
"boolean": ["boolean@3.2.0", "", {}, "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw=="],
"chownr": ["chownr@3.0.0", "", {}, "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g=="],
"define-data-property": ["define-data-property@1.1.4", "", { "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", "gopd": "^1.0.1" } }, "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A=="],
"define-properties": ["define-properties@1.2.1", "", { "dependencies": { "define-data-property": "^1.0.1", "has-property-descriptors": "^1.0.0", "object-keys": "^1.1.1" } }, "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg=="],
"detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="],
"detect-node": ["detect-node@2.1.0", "", {}, "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g=="],
"es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="],
"es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="],
"es6-error": ["es6-error@4.1.1", "", {}, "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg=="],
"escape-string-regexp": ["escape-string-regexp@4.0.0", "", {}, "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="],
"flatbuffers": ["flatbuffers@25.9.23", "", {}, "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ=="],
"ghostty-web": ["ghostty-web@github:rcarmo/ghostty-web#fcc47d4", {}, "rcarmo-ghostty-web-fcc47d4", "sha512-tq0cFciI32VTyOXDoLHQQDndeA6jhFuZ/3TWYx3VlYDzRhYkWAtTBi6t29isYPzdiKNIWggjkn3Ve/+Qub/wBg=="], "ghostty-web": ["ghostty-web@github:rcarmo/ghostty-web#fcc47d4", {}, "rcarmo-ghostty-web-fcc47d4", "sha512-tq0cFciI32VTyOXDoLHQQDndeA6jhFuZ/3TWYx3VlYDzRhYkWAtTBi6t29isYPzdiKNIWggjkn3Ve/+Qub/wBg=="],
"global-agent": ["global-agent@3.0.0", "", { "dependencies": { "boolean": "^3.0.1", "es6-error": "^4.1.1", "matcher": "^3.0.0", "roarr": "^2.15.3", "semver": "^7.3.2", "serialize-error": "^7.0.1" } }, "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q=="],
"globalthis": ["globalthis@1.0.4", "", { "dependencies": { "define-properties": "^1.2.1", "gopd": "^1.0.1" } }, "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ=="],
"gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="],
"guid-typescript": ["guid-typescript@1.0.9", "", {}, "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ=="],
"has-property-descriptors": ["has-property-descriptors@1.0.2", "", { "dependencies": { "es-define-property": "^1.0.0" } }, "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg=="],
"json-stringify-safe": ["json-stringify-safe@5.0.1", "", {}, "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA=="],
"llama-tokenizer-js": ["llama-tokenizer-js@1.2.2", "", {}, "sha512-Wmth393dc3odWU3IzARJ3r2oIfWgw9GdJ5Gm+hGhfECNO18UHLRqEFSf511jn4E9KcQGzuuKw4Wl08pHAemLAw=="],
"long": ["long@5.3.2", "", {}, "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA=="],
"matcher": ["matcher@3.0.0", "", { "dependencies": { "escape-string-regexp": "^4.0.0" } }, "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng=="],
"minipass": ["minipass@7.1.3", "", {}, "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A=="],
"minizlib": ["minizlib@3.1.0", "", { "dependencies": { "minipass": "^7.1.2" } }, "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw=="],
"object-keys": ["object-keys@1.1.1", "", {}, "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="],
"onnxruntime-common": ["onnxruntime-common@1.26.0", "", {}, "sha512-qVyMR4lcWgbkc4getFV+GQijsTnbg/siteoqcDwa3sI/LxbrMSNw4ePyvCq/ymdQaRomCA7YuWmhzsswxvymdw=="],
"onnxruntime-node": ["onnxruntime-node@1.21.0", "", { "dependencies": { "global-agent": "^3.0.0", "onnxruntime-common": "1.21.0", "tar": "^7.0.1" }, "os": [ "linux", "win32", "darwin", ] }, "sha512-NeaCX6WW2L8cRCSqy3bInlo5ojjQqu2fD3D+9W5qb5irwxhEyWKXeH2vZ8W9r6VxaMPUan+4/7NDwZMtouZxEw=="],
"onnxruntime-web": ["onnxruntime-web@1.26.0", "", { "dependencies": { "flatbuffers": "^25.1.24", "guid-typescript": "^1.0.9", "long": "^5.2.3", "onnxruntime-common": "1.26.0", "platform": "^1.3.6", "protobufjs": "^7.2.4" } }, "sha512-LbRr/8zZt2xilI2smrVQGGKINo0U46i8qJp+UXyMBGfqN7KjnH1BiwCwLwyNIVV4i9CKFv7Sf4PwLKWnT8/bEA=="],
"platform": ["platform@1.3.6", "", {}, "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg=="],
"protobufjs": ["protobufjs@7.5.7", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.5", "@protobufjs/eventemitter": "^1.1.0", "@protobufjs/fetch": "^1.1.0", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.1", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.1", "@types/node": ">=13.7.0", "long": "^5.0.0" } }, "sha512-NGnrxS/nLKUo5nkbVQxlC71sB4hdfImdYIbFeSCidxtwATx0AHRPcANSLd0q5Bb2BkoSWo2iisQhGg5/r+ihbA=="],
"roarr": ["roarr@2.15.4", "", { "dependencies": { "boolean": "^3.0.1", "detect-node": "^2.0.4", "globalthis": "^1.0.1", "json-stringify-safe": "^5.0.1", "semver-compare": "^1.0.0", "sprintf-js": "^1.1.2" } }, "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A=="],
"semver": ["semver@7.8.0", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA=="],
"semver-compare": ["semver-compare@1.0.0", "", {}, "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow=="],
"serialize-error": ["serialize-error@7.0.1", "", { "dependencies": { "type-fest": "^0.13.1" } }, "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw=="],
"sharp": ["sharp@0.34.5", "", { "dependencies": { "@img/colour": "^1.0.0", "detect-libc": "^2.1.2", "semver": "^7.7.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.34.5", "@img/sharp-darwin-x64": "0.34.5", "@img/sharp-libvips-darwin-arm64": "1.2.4", "@img/sharp-libvips-darwin-x64": "1.2.4", "@img/sharp-libvips-linux-arm": "1.2.4", "@img/sharp-libvips-linux-arm64": "1.2.4", "@img/sharp-libvips-linux-ppc64": "1.2.4", "@img/sharp-libvips-linux-riscv64": "1.2.4", "@img/sharp-libvips-linux-s390x": "1.2.4", "@img/sharp-libvips-linux-x64": "1.2.4", "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", "@img/sharp-libvips-linuxmusl-x64": "1.2.4", "@img/sharp-linux-arm": "0.34.5", "@img/sharp-linux-arm64": "0.34.5", "@img/sharp-linux-ppc64": "0.34.5", "@img/sharp-linux-riscv64": "0.34.5", "@img/sharp-linux-s390x": "0.34.5", "@img/sharp-linux-x64": "0.34.5", "@img/sharp-linuxmusl-arm64": "0.34.5", "@img/sharp-linuxmusl-x64": "0.34.5", "@img/sharp-wasm32": "0.34.5", "@img/sharp-win32-arm64": "0.34.5", "@img/sharp-win32-ia32": "0.34.5", "@img/sharp-win32-x64": "0.34.5" } }, "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg=="],
"sprintf-js": ["sprintf-js@1.1.3", "", {}, "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA=="],
"tar": ["tar@7.5.15", "", { "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", "minipass": "^7.1.2", "minizlib": "^3.1.0", "yallist": "^5.0.0" } }, "sha512-dzGK0boVlC4W5QFuQN1EFSl3bIDYsk7Tj40U6eIBnK2k/8ml7TZ5agbI5j5+qnoVcAA+rNtBml8SEiLxZpNqRQ=="],
"tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="],
"type-fest": ["type-fest@0.13.1", "", {}, "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg=="],
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
"undici-types": ["undici-types@7.21.0", "", {}, "sha512-w9IMgQrz4O0YN1LtB7K5P63vhlIOvC7opSmouCJ+ZywlPAlO9gIkJ+otk6LvGpAs2wg4econaCz3TvQ9xPoyuQ=="],
"yallist": ["yallist@5.0.0", "", {}, "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw=="],
"@huggingface/transformers/onnxruntime-web": ["onnxruntime-web@1.22.0-dev.20250409-89f8206ba4", "", { "dependencies": { "flatbuffers": "^25.1.24", "guid-typescript": "^1.0.9", "long": "^5.2.3", "onnxruntime-common": "1.22.0-dev.20250409-89f8206ba4", "platform": "^1.3.6", "protobufjs": "^7.2.4" } }, "sha512-0uS76OPgH0hWCPrFKlL8kYVV7ckM7t/36HfbgoFw6Nd0CZVVbQC4PkrR8mBX8LtNUFZO25IQBqV2Hx2ho3FlbQ=="],
"@moonshine-ai/moonshine-js/@ricky0123/vad-web": ["@ricky0123/vad-web@file:../vad-moonshine/packages/web", {}],
"onnxruntime-node/onnxruntime-common": ["onnxruntime-common@1.21.0", "", {}, "sha512-Q632iLLrtCAVOTO65dh2+mNbQir/QNTVBG3h/QdZBpns7mZ0RYbLRBgGABPbpU9351AgYy7SJf1WaeVwMrBFPQ=="],
"@huggingface/transformers/onnxruntime-web/onnxruntime-common": ["onnxruntime-common@1.22.0-dev.20250409-89f8206ba4", "", {}, "sha512-vDJMkfCfb0b1A836rgHj+ORuZf4B4+cc2bASQtpeoJLueuFc5DuYwjIZUBrSvx/fO5IrLjLz+oTrB3pcGlhovQ=="],
} }
} }
+1125 -4
View File
File diff suppressed because it is too large Load Diff
+1
View File
@@ -13,6 +13,7 @@
}, },
"type": "module", "type": "module",
"dependencies": { "dependencies": {
"@moonshine-ai/moonshine-js": "^0.1.29",
"ghostty-web": "github:rcarmo/ghostty-web#fcc47d423a7fce1c02c702b6464d0b1ab89175f1" "ghostty-web": "github:rcarmo/ghostty-web#fcc47d423a7fce1c02c702b6464d0b1ab89175f1"
} }
} }
+281
View File
@@ -0,0 +1,281 @@
# sherpa-onnx Moonshine v2 Migration Plan
## Goal
Replace the current `@moonshine-ai/moonshine-js` browser integration with a `sherpa-onnx` WebAssembly integration that can run Moonshine v2 locally in the browser and feed recognized text into the existing terminal stdin path.
The target behavior is:
- user clicks a voice button in the terminal UI
- browser captures microphone audio
- VAD segments speech locally
- Moonshine v2 recognition runs locally in WASM
- committed transcript is sent into the existing terminal input path
## Why We Are Changing Direction
The current MoonshineJS integration is the wrong runtime for the `medium-streaming-en` bundle we want to use.
Current blockers:
- `moonshine-js` expects old-style ONNX assets such as:
- `quantized/encoder_model.onnx`
- `quantized/decoder_model_merged.onnx`
- the downloaded `medium-streaming-en.zip` contains a newer Moonshine v2 layout:
- `frontend.ort`
- `encoder.ort`
- `decoder_kv.ort`
- `adapter.ort`
- `cross_kv.ort`
- `tokenizer.bin`
- `streaming_config.json`
- self-hosting those files alone does not make the existing `moonshine-js` loader compatible
## Constraints
- keep transcription fully local in the browser
- keep the current terminal WebSocket/stdin path unchanged if possible
- preserve the small voice button UX already added to `webterm/static/js/terminal.ts`
- prefer self-hosted assets served by `webterm/static/`
- do not depend on third-party model/CDN availability at runtime
## Repo Touchpoints
Expected files/modules to change:
- `webterm/static/js/terminal.ts`
- `webterm/static/js/terminal.js`
- `package.json`
- `bun.lock`
- `package-lock.json`
- `webterm/assets_embed.go`
- `webterm/static/...` for new WASM/model assets
- `README.md`
Possible new files:
- `webterm/static/js/sherpa-voice.ts` or similar
- `webterm/static/js/sherpa-onnx.d.ts`
- `webterm/static/models/moonshine-v2-medium/...`
- `scripts/install-sherpa-model.sh` or similar helper
## Proposed Architecture
### 1. Frontend runtime
Use `sherpa-onnx` JavaScript/WebAssembly in the browser instead of `moonshine-js`.
Primary responsibilities:
- initialize sherpa WASM runtime
- load self-hosted Moonshine v2 model files
- initialize VAD + ASR pipeline
- expose simple start/stop API to terminal UI
- emit committed transcript strings
### 2. Audio flow
Use the VAD + offline ASR flow described by sherpa-onnx:
- microphone input
- VAD detects speech boundaries
- finalized speech segment is sent to recognizer
- recognizer returns transcript
- transcript is injected into terminal stdin
### 3. Terminal integration
Keep the existing terminal send path:
- reuse `sendStdin(...)` in `WebTerminal`
- keep current voice button/status UI shell
- replace only the recognition backend
## Implementation Phases
### Phase 1. Remove the current MoonshineJS dependency path
- remove `@moonshine-ai/moonshine-js` import and startup logic
- remove `moonshine-js.d.ts`
- remove MoonshineJS-specific error capture code
- keep the voice UI container, button, and status area
Deliverable:
- terminal still builds
- voice button exists but uses a new backend abstraction
### Phase 2. Stage sherpa-onnx assets locally
- choose exact sherpa-onnx JS/WASM package/version
- download/copy the required runtime assets into `webterm/static/`
- unpack `~/medium-streaming-en.zip` into repo-managed static model directory
- verify final model path layout expected by sherpa-onnx
- ensure static assets are available both in dev mode and embedded mode
Deliverable:
- all WASM/model files are served locally from `/static/...`
### Phase 3. Build a thin browser voice adapter
- create a dedicated module that wraps sherpa-onnx initialization
- define a minimal interface:
- `start()`
- `stop()`
- `isActive()`
- callbacks for:
- status updates
- final transcript
- detailed errors
- keep terminal code from depending directly on low-level sherpa objects
Deliverable:
- one self-contained browser voice adapter module
### Phase 4. Wire VAD + recognition into the terminal UI
- connect voice button click to adapter start/stop
- show clear states:
- ready
- loading runtime
- loading model
- listening
- processing speech
- final transcript sent
- detailed error
- push final transcript into `sendStdin(...)`
- decide whether to append newline automatically
Open question:
- should transcript be inserted as raw text only, or raw text plus `\r`?
Deliverable:
- end-to-end browser speech to terminal input using sherpa-onnx
### Phase 5. Performance and caching
- confirm browser HTTP caching behavior for WASM and model files
- add long-lived cache headers if needed for self-hosted static assets
- optionally add a service worker pre-cache later
- measure first-load vs repeat-load experience
Deliverable:
- repeat visits avoid re-downloading large model assets where possible
### Phase 6. Documentation and deploy
- document required model files and where they live
- document browser requirements
- document secure-origin requirement for microphone access
- document how to update the model bundle in future
Deliverable:
- README instructions for install, deploy, and troubleshooting
## Technical Questions To Resolve Early
1. Which sherpa-onnx JS/WASM distribution should we use?
- npm package
- vendored release bundle
- custom copied example assets
2. Which exact browser API shape should we target?
- direct sherpa recognizer API
- sherpa VAD + non-streaming ASR helper
- example-derived wrapper from sherpa demos
3. What is the expected asset layout for the Moonshine v2 medium zip?
- whether files can be served exactly as unzipped
- whether any extra config or renamed paths are required
4. What transcript commit behavior do we want?
- send text only
- send text plus Enter
- configurable mode
## Risks
### Runtime/API mismatch
Risk:
- sherpa-onnx JS APIs may differ from the examples we choose
Mitigation:
- lock to one verified release
- copy a known working browser example shape before adapting
### Large asset size
Risk:
- medium model load time may be high on first use
Mitigation:
- self-host locally
- keep caching aggressive
- consider fallback option for smaller model later
### Mobile/browser compatibility
Risk:
- WASM + large model + microphone flow may be poor on weaker browsers
Mitigation:
- treat desktop Chromium/Firefox as first target
- gate unsupported browsers with explicit errors
### Current worktree noise
Risk:
- repo already has ongoing voice-related frontend edits
Mitigation:
- isolate the new adapter into its own file/module
- keep migration incremental
## Validation Checklist
- `bun run typecheck` passes
- frontend bundle builds
- `./update.sh` deploys successfully
- remote HTTPS origin still permits microphone access
- first transcription works end-to-end
- repeated transcriptions do not leak memory or duplicate recognizers
- page reload reuses cached assets when possible
- detailed runtime errors are visible in UI and console
## Recommended Execution Order
1. Pick and verify one sherpa-onnx browser example for Moonshine v2
2. Vendor the required WASM/runtime assets
3. Unpack and serve the local model bundle
4. Build a dedicated browser adapter module
5. Rewire the existing voice UI to that adapter
6. Validate microphone -> transcript -> terminal stdin flow
7. Improve caching and docs
## Definition of Done
- no `moonshine-js` dependency remains in the browser path
- voice input uses sherpa-onnx + locally hosted Moonshine v2 assets
- transcripts reach terminal stdin reliably
- remote HTTPS access works
- runtime/model errors are understandable
- setup is documented and reproducible
+1 -1
View File
@@ -12,6 +12,6 @@
"isolatedModules": true, "isolatedModules": true,
"lib": ["ES2022", "DOM", "DOM.Iterable"] "lib": ["ES2022", "DOM", "DOM.Iterable"]
}, },
"include": ["webterm/static/js/**/*.ts"], "include": ["webterm/static/js/**/*"],
"exclude": ["node_modules"] "exclude": ["node_modules"]
} }
+16 -1
View File
@@ -10,10 +10,25 @@ echo "Building Go binary..."
make build-go make build-go
echo "Installing binary..." echo "Installing binary..."
cp bin/webterm ~/go/bin/webterm mkdir -p ~/go/bin
tmp_target=~/go/bin/webterm.new
cp bin/webterm "$tmp_target"
chmod +x "$tmp_target"
mv "$tmp_target" ~/go/bin/webterm
echo "Restarting service..." echo "Restarting service..."
systemctl --user restart webterm.service systemctl --user restart webterm.service
echo "Done. Status:" echo "Done. Status:"
systemctl --user status webterm.service --no-pager systemctl --user status webterm.service --no-pager
echo
echo "Listening sockets:"
ss -ltnp | grep ':8080' || true
echo
echo "Reachable URLs:"
echo " Local: http://127.0.0.1:8080/"
hostname -I 2>/dev/null | tr ' ' '\n' | sed '/^$/d' | while read -r ip; do
echo " LAN: http://$ip:8080/"
done
+35
View File
@@ -0,0 +1,35 @@
declare module "@moonshine-ai/moonshine-js" {
export interface TranscriberCallbacks {
onPermissionsRequested: () => unknown;
onError: (error: unknown) => unknown;
onModelLoadStarted: () => unknown;
onModelLoaded: () => unknown;
onTranscribeStarted: () => unknown;
onTranscribeStopped: () => unknown;
onTranscriptionUpdated: (text: string) => unknown;
onTranscriptionCommitted: (text: string, buffer?: AudioBuffer) => unknown;
onFrame: (probs: unknown, frame: unknown, ema: unknown) => unknown;
onSpeechStart: () => unknown;
onSpeechEnd: () => unknown;
}
export class MicrophoneTranscriber {
constructor(
modelURL: string,
callbacks?: Partial<TranscriberCallbacks>,
useVAD?: boolean,
precision?: string
);
isActive: boolean;
start(): Promise<void>;
stop(): void;
}
export const Settings: {
BASE_ASSET_PATH: {
MOONSHINE: string;
ONNX_RUNTIME: string;
SILERO_VAD: string;
};
};
}
File diff suppressed because one or more lines are too long
+317
View File
@@ -7,6 +7,7 @@
*/ */
import { Terminal, FitAddon, Ghostty, type ITerminalOptions, type ITheme } from "ghostty-web"; import { Terminal, FitAddon, Ghostty, type ITerminalOptions, type ITheme } from "ghostty-web";
import * as Moonshine from "@moonshine-ai/moonshine-js";
/** Maximum queued messages before oldest are dropped */ /** Maximum queued messages before oldest are dropped */
const MAX_MESSAGE_QUEUE_SIZE = 1000; const MAX_MESSAGE_QUEUE_SIZE = 1000;
@@ -21,6 +22,8 @@ const STDIN_BATCH_DELAY_MS = 10;
const STDIN_BATCH_MAX_CHARS = 8192; const STDIN_BATCH_MAX_CHARS = 8192;
const BELL_EMOJI = "🔔"; const BELL_EMOJI = "🔔";
const DEFAULT_MOONSHINE_MODEL_URL = "model/medium-streaming-en";
const VOICE_STATUS_MAX_LENGTH = 48;
/** Shared Ghostty WASM instance (loaded once, reused across all terminals) */ /** Shared Ghostty WASM instance (loaded once, reused across all terminals) */
let sharedGhostty: Ghostty | null = null; let sharedGhostty: Ghostty | null = null;
@@ -697,6 +700,13 @@ class WebTerminal {
private baseTitle: string; private baseTitle: string;
private bellActive = false; private bellActive = false;
private routeKey: string; private routeKey: string;
private voiceControls: HTMLElement | null = null;
private voiceButton: HTMLButtonElement | null = null;
private voiceStatus: HTMLElement | null = null;
private voiceTranscriber: Moonshine.MicrophoneTranscriber | null = null;
private voiceModelURL = DEFAULT_MOONSHINE_MODEL_URL;
private isVoiceStarting = false;
private voiceStartupErrorCleanup: (() => void) | null = null;
private static sharedTextEncoder = new TextEncoder(); private static sharedTextEncoder = new TextEncoder();
private constructor( private constructor(
@@ -873,6 +883,7 @@ class WebTerminal {
// Setup mobile keyboard support // Setup mobile keyboard support
this.setupMobileKeyboard(); this.setupMobileKeyboard();
this.setupTouchSelection(); this.setupTouchSelection();
this.setupVoiceInput();
// Setup mobile extended keybar (only on mobile devices) // Setup mobile extended keybar (only on mobile devices)
if (isMobileDevice()) { if (isMobileDevice()) {
@@ -1338,6 +1349,304 @@ class WebTerminal {
); );
} }
private setupVoiceInput(): void {
this.voiceModelURL =
this.element.dataset.moonshineModelUrl?.trim() || DEFAULT_MOONSHINE_MODEL_URL;
if (window.getComputedStyle(this.element).position === "static") {
this.element.style.position = "relative";
}
const controls = document.createElement("div");
controls.className = "webterm-voice-controls";
controls.innerHTML = `
<button type="button" class="webterm-voice-button" aria-pressed="false" title="Start voice input">
Voice
</button>
<span class="webterm-voice-status">Ready</span>
`;
this.element.appendChild(controls);
this.voiceControls = controls;
this.voiceButton = controls.querySelector(".webterm-voice-button");
this.voiceStatus = controls.querySelector(".webterm-voice-status");
if (!this.voiceButton || !this.voiceStatus) {
return;
}
Object.assign(controls.style, {
position: "absolute",
top: "12px",
right: "12px",
zIndex: "4",
display: "flex",
alignItems: "center",
gap: "8px",
padding: "8px 10px",
borderRadius: "999px",
background: "rgba(9, 14, 19, 0.78)",
backdropFilter: "blur(8px)",
boxShadow: "0 8px 30px rgba(0, 0, 0, 0.28)",
pointerEvents: "auto",
} satisfies Partial<CSSStyleDeclaration>);
Object.assign(this.voiceButton.style, {
border: "1px solid rgba(255, 255, 255, 0.18)",
borderRadius: "999px",
background: "#12202b",
color: "#f3f6fa",
font: '600 12px "Fira Code", "FiraCode Nerd Font", monospace',
padding: "7px 12px",
cursor: "pointer",
} satisfies Partial<CSSStyleDeclaration>);
Object.assign(this.voiceStatus.style, {
color: "#c9d5e0",
font: '500 11px "Fira Code", "FiraCode Nerd Font", monospace',
whiteSpace: "normal",
lineHeight: "1.35",
maxWidth: "44ch",
overflowWrap: "anywhere",
} satisfies Partial<CSSStyleDeclaration>);
this.voiceButton.addEventListener("click", () => {
void this.toggleVoiceInput();
});
if (
typeof navigator === "undefined" ||
!navigator.mediaDevices?.getUserMedia ||
typeof AudioContext === "undefined"
) {
this.setVoiceState("unsupported", "Voice unavailable");
return;
}
this.setVoiceState("idle", "Ready");
}
private async toggleVoiceInput(): Promise<void> {
if (this.isVoiceStarting) {
return;
}
if (this.voiceTranscriber?.isActive) {
this.stopVoiceInput();
return;
}
this.isVoiceStarting = true;
this.armVoiceStartupErrorCapture();
this.setVoiceState("loading", "Starting...");
try {
const transcriber = this.getVoiceTranscriber();
await transcriber.start();
this.setVoiceState("listening", "Listening...");
this.focusTerminalInput();
} catch (error) {
console.error("[webterm] Failed to start Moonshine microphone transcriber:", error);
this.setVoiceState("error", this.describeVoiceError(error));
} finally {
this.isVoiceStarting = false;
}
}
private getVoiceTranscriber(): Moonshine.MicrophoneTranscriber {
if (this.voiceTranscriber) {
return this.voiceTranscriber;
}
this.voiceTranscriber = new Moonshine.MicrophoneTranscriber(
this.voiceModelURL,
{
onPermissionsRequested: () => {
this.setVoiceState("loading", "Allow microphone...");
},
onModelLoadStarted: () => {
this.setVoiceState("loading", "Loading model...");
},
onModelLoaded: () => {
this.setVoiceState("loading", "Model ready");
},
onTranscribeStarted: () => {
this.setVoiceState("listening", "Listening...");
},
onTranscribeStopped: () => {
this.setVoiceState("idle", "Ready");
},
onTranscriptionUpdated: (text: string) => {
const preview = text.trim();
this.setVoiceState("listening", preview ? `Heard: ${preview}` : "Listening...");
},
onTranscriptionCommitted: (text: string) => {
const transcript = text.trim();
if (!transcript) {
return;
}
this.sendStdin(transcript);
this.setVoiceState("listening", `Sent: ${transcript}`);
this.focusTerminalInput();
},
onError: (error: unknown) => {
console.error("[webterm] Moonshine error:", error);
const message = this.describeVoiceError(error);
this.setVoiceState(
"error",
message === "PlatformUnsupported"
? "PlatformUnsupported. Waiting for underlying browser error..."
: message
);
},
},
false
);
return this.voiceTranscriber;
}
private stopVoiceInput(): void {
this.clearVoiceStartupErrorCapture();
if (!this.voiceTranscriber) {
this.setVoiceState("idle", "Ready");
return;
}
this.voiceTranscriber.stop();
this.stopVoiceTracks();
this.setVoiceState("idle", "Ready");
this.focusTerminalInput();
}
private stopVoiceTracks(): void {
const maybeStream = this.voiceTranscriber as unknown as { mediaStream?: MediaStream };
maybeStream.mediaStream?.getTracks().forEach((track) => {
track.stop();
});
}
private describeVoiceError(error: unknown): string {
if (typeof error === "string" && error.trim()) {
return error.trim();
}
if (error instanceof Error && error.message.trim()) {
const cause =
"cause" in error && error.cause
? ` Cause: ${this.describeVoiceError(error.cause)}`
: "";
return `${error.name}: ${error.message.trim()}${cause}`;
}
if (typeof error === "object" && error !== null) {
try {
return JSON.stringify(error);
} catch {
return String(error);
}
}
return "Voice error";
}
private armVoiceStartupErrorCapture(): void {
this.clearVoiceStartupErrorCapture();
const startedAt = Date.now();
const captureWindowMs = 15_000;
const handleFailure = (error: unknown): void => {
if (Date.now() - startedAt > captureWindowMs) {
return;
}
const message = this.describeVoiceError(error);
if (
!message ||
message === "PlatformUnsupported" ||
message === "Voice error"
) {
return;
}
console.error("[webterm] Captured Moonshine startup failure:", error);
this.setVoiceState("error", message);
};
const rejectionHandler = (event: PromiseRejectionEvent) => {
handleFailure(event.reason);
};
const errorHandler = (event: ErrorEvent) => {
handleFailure(event.error ?? event.message);
};
window.addEventListener("unhandledrejection", rejectionHandler);
window.addEventListener("error", errorHandler);
const timeoutId = window.setTimeout(() => {
this.clearVoiceStartupErrorCapture();
}, captureWindowMs);
this.voiceStartupErrorCleanup = () => {
window.removeEventListener("unhandledrejection", rejectionHandler);
window.removeEventListener("error", errorHandler);
clearTimeout(timeoutId);
this.voiceStartupErrorCleanup = null;
};
}
private clearVoiceStartupErrorCapture(): void {
this.voiceStartupErrorCleanup?.();
}
private setVoiceState(
state: "idle" | "loading" | "listening" | "error" | "unsupported",
message: string
): void {
if (!this.voiceButton || !this.voiceStatus) {
return;
}
const clippedMessage =
message.length > VOICE_STATUS_MAX_LENGTH
? `${message.slice(0, VOICE_STATUS_MAX_LENGTH - 1)}`
: message;
const displayMessage = state === "error" ? message : clippedMessage;
this.voiceStatus.textContent = displayMessage;
this.voiceStatus.title = message;
this.voiceButton.disabled = state === "loading" || state === "unsupported";
this.voiceButton.textContent = state === "listening" ? "Stop" : "Voice";
this.voiceButton.setAttribute("aria-pressed", state === "listening" ? "true" : "false");
this.voiceButton.title =
state === "error"
? message
: state === "listening"
? "Stop voice input"
: "Start voice input";
if (state === "listening") {
this.voiceButton.style.background = "#7c1d1d";
this.voiceButton.style.borderColor = "rgba(255, 120, 120, 0.5)";
if (this.voiceControls) {
this.voiceControls.style.background = "rgba(45, 10, 10, 0.78)";
}
} else if (state === "error") {
this.voiceButton.style.background = "#3d2a12";
this.voiceButton.style.borderColor = "rgba(255, 187, 92, 0.55)";
if (this.voiceControls) {
this.voiceControls.style.background = "rgba(46, 29, 8, 0.82)";
}
} else {
this.voiceButton.style.background = "#12202b";
this.voiceButton.style.borderColor = "rgba(255, 255, 255, 0.18)";
if (this.voiceControls) {
this.voiceControls.style.background = "rgba(9, 14, 19, 0.78)";
}
}
}
private focusTerminalInput(): void {
if (isMobileDevice()) {
this.focusMobileInput();
} else {
this.terminal.focus();
}
}
private keybarButtonHeight = 44; private keybarButtonHeight = 44;
/** Setup bottom-docked mobile extended keyboard bar */ /** Setup bottom-docked mobile extended keyboard bar */
@@ -1971,6 +2280,7 @@ class WebTerminal {
/** Clean up resources */ /** Clean up resources */
dispose(): void { dispose(): void {
this.stopVoiceInput();
this.stopResourceCleanup(); this.stopResourceCleanup();
this.stopHeartbeatWatchdog(); this.stopHeartbeatWatchdog();
if (this.pendingStdinTimer) { if (this.pendingStdinTimer) {
@@ -2009,6 +2319,13 @@ class WebTerminal {
this.mobileKeybarStyle.remove(); this.mobileKeybarStyle.remove();
this.mobileKeybarStyle = null; this.mobileKeybarStyle = null;
} }
if (this.voiceControls) {
this.voiceControls.remove();
this.voiceControls = null;
}
this.voiceButton = null;
this.voiceStatus = null;
this.voiceTranscriber = null;
this.fitAddon.dispose(); this.fitAddon.dispose();
this.terminal.dispose(); this.terminal.dispose();
} }