diff --git a/.github/workflows/speech.yml b/.github/workflows/speech.yml new file mode 100644 index 0000000..fed4311 --- /dev/null +++ b/.github/workflows/speech.yml @@ -0,0 +1,51 @@ +name: Native speech + +on: + push: + paths: + - crates/speech/** + - Cargo.toml + - Cargo.lock + - .github/workflows/speech.yml + pull_request: + paths: + - crates/speech/** + - Cargo.toml + - Cargo.lock + - .github/workflows/speech.yml + +jobs: + native: + name: Speech (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [macos-14, macos-15-intel, windows-2022, ubuntu-latest] + env: + MACOSX_DEPLOYMENT_TARGET: '11.0' + steps: + - uses: actions/checkout@v5 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + - run: cargo clippy -p robius-speech --all-targets -- -D warnings + - run: cargo test -p robius-speech + - name: Apple native lifecycle regressions + if: runner.os == 'macOS' + run: bash crates/speech/swift/tests/run.sh + - name: Compile iOS backends + if: matrix.os == 'macos-14' + run: | + rustup target add aarch64-apple-ios aarch64-apple-ios-sim + cargo check -p robius-speech --target aarch64-apple-ios + cargo check -p robius-speech --target aarch64-apple-ios-sim + - uses: actions/setup-java@v4 + if: matrix.os == 'macos-14' + with: + distribution: temurin + java-version: '17' + - name: Android speech lifecycle regressions + if: matrix.os == 'macos-14' + run: python3 crates/speech/tests/android_retry_test.py diff --git a/Cargo.lock b/Cargo.lock index 44f682f..da4d5e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -992,8 +992,8 @@ dependencies = [ "objc2-local-authentication", "retry", "robius-android-env", - "windows", - "windows-core", + "windows 0.56.0", + "windows-core 0.56.0", "zbus", "zbus_polkit", ] @@ -1045,7 +1045,7 @@ dependencies = [ "objc2-foundation", "robius-android-env", "tokio", - "windows", + "windows 0.56.0", "zbus", ] @@ -1063,7 +1063,7 @@ dependencies = [ "objc2-foundation", "objc2-ui-kit", "robius-android-env", - "windows", + "windows 0.56.0", ] [[package]] @@ -1081,7 +1081,16 @@ dependencies = [ "objc2-ui-kit", "robius-android-env", "robius-common", - "windows", + "windows 0.56.0", +] + +[[package]] +name = "robius-speech" +version = "0.3.1" +dependencies = [ + "jni", + "robius-android-env", + "windows 0.61.3", ] [[package]] @@ -1561,22 +1570,68 @@ version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1de69df01bdf1ead2f4ac895dc77c9351aefff65b2f3db429a343f9cbf05e132" dependencies = [ - "windows-core", + "windows-core 0.56.0", "windows-targets 0.52.5", ] +[[package]] +name = "windows" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +dependencies = [ + "windows-collections", + "windows-core 0.61.2", + "windows-future", + "windows-link 0.1.3", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" +dependencies = [ + "windows-core 0.61.2", +] + [[package]] name = "windows-core" version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4698e52ed2d08f8658ab0c39512a7c00ee5fe2688c65f8c0a4f06750d729f2a6" dependencies = [ - "windows-implement", - "windows-interface", - "windows-result", + "windows-implement 0.56.0", + "windows-interface 0.56.0", + "windows-result 0.1.1", "windows-targets 0.52.5", ] +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement 0.60.2", + "windows-interface 0.59.3", + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", + "windows-threading", +] + [[package]] name = "windows-implement" version = "0.56.0" @@ -1588,6 +1643,17 @@ dependencies = [ "syn", ] +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-interface" version = "0.56.0" @@ -1599,12 +1665,39 @@ dependencies = [ "syn", ] +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", +] + [[package]] name = "windows-result" version = "0.1.1" @@ -1614,6 +1707,24 @@ dependencies = [ "windows-targets 0.52.5", ] +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-sys" version = "0.45.0" @@ -1638,7 +1749,7 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -1672,6 +1783,15 @@ dependencies = [ "windows_x86_64_msvc 0.52.5", ] +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" diff --git a/README.md b/README.md index 4a4a18a..9cad7b1 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,9 @@ There's also a [status table](#crate--platform-status-table) that shows what eac * [`robius-share`](crates/share/): opens the native system share sheet so you can share a file or other content with a different app on your system. * This is implemented using `Intent.createChooser` on Android, `UIActivityViewController` on iOS, `NSSharingServicePicker` on macOS, and the WinRT Share UI on Windows. * Linux doesn't really have a system share sheet, so we wrote a custom XDG portal connection that still supports every possible type of share payload. +* [`robius-speech`](crates/speech/): streams native speech-to-text results and microphone levels on macOS, iOS, Android, and Windows. + * Provides recording sessions with partial/final transcripts, graceful stop, and cancellation, independently of the UI toolkit. + * Linux doesn't offer any OS-provided speech-to-text functionality, so this crate can't do anything on Linux. * [`robius-web-auth-session`](crates/web_auth_session/): runs an OAuth/SSO login process in the OS's own in-app browser session, and then sends the result back to your app. * Currently this is for iOS only, based on `ASWebAuthenticationSession`. There's no other safe/supported way to do web login on iOS, because otherwise iOS will suspend your app while showing the browser. @@ -49,6 +52,7 @@ Symbol legend: ✅ fully supported · ⚠️ partial, or has issues · 🚧 unde | [`robius-location`](crates/location/) | ✅ `CLLocationManager` (CoreLocation) | ✅ `CLLocationManager` (CoreLocation) | ✅ `LocationManager` | ✅ `Geolocator` (`Windows.Devices.Geolocation`, WinRT) | ✅ XDG Location portal, with a `GeoClue` fallback | | [`robius-open`](crates/open/) | ✅ `NSWorkspace.openURL` | ✅ `UIApplication.openURL` | ✅ `Intent` (`ACTION_VIEW`) | ✅ `Launcher.LaunchUriAsync` (WinRT) | ✅ `xdg-open` | | [`robius-share`](crates/share/) | ✅ `NSSharingServicePicker` | ✅ `UIActivityViewController` | ✅ `ACTION_SEND` / `ACTION_SEND_MULTIPLE` via `Intent.createChooser` | ✅ WinRT Share UI (`DataTransferManager`) | ✅ XDG portal "Open With" chooser (`OpenURI` / `OpenFile`), or its `SaveFiles` dialog for multi-item payloads; `xdg-open` fallback | +| [`robius-speech`](crates/speech/) | ✅ `SFSpeechRecognizer` + `AVAudioEngine` | ✅ `SFSpeechRecognizer` + `AVAudioEngine` | ✅ `SpeechRecognizer` (needs an installed recognition service) | ✅ SAPI dictation (needs an installed speech language) | ❌ no OS-native speech-to-text service exists | | [`robius-web-auth-session`](crates/web_auth_session/) | ❌ not supported | ✅ `ASWebAuthenticationSession` | 🚧 planned (custom chrome tabs) | ❌ not supported | ❌ not supported | diff --git a/crates/speech/.gitignore b/crates/speech/.gitignore new file mode 100644 index 0000000..a254237 --- /dev/null +++ b/crates/speech/.gitignore @@ -0,0 +1,3 @@ +/target +/Cargo.lock +__pycache__/ diff --git a/crates/speech/Cargo.toml b/crates/speech/Cargo.toml new file mode 100644 index 0000000..177ab25 --- /dev/null +++ b/crates/speech/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "robius-speech" +version.workspace = true +edition.workspace = true +authors.workspace = true +description = "Native streaming speech-to-text input and microphone capture for Rust apps" +documentation = "https://docs.rs/robius-speech" +homepage.workspace = true +keywords = ["robius", "speech", "dictation", "microphone", "STT"] +categories.workspace = true +license.workspace = true +repository.workspace = true +readme = "README.md" + +[lints.rust] +keyword_idents_2024 = "forbid" +non_ascii_idents = "forbid" +non_local_definitions = "forbid" +unsafe_op_in_unsafe_fn = "forbid" +unnameable_types = "warn" +unused_import_braces = "warn" + +[lints.clippy] +collapsible_if = "allow" +collapsible_else_if = "allow" +uninlined_format_args = "allow" + +[dependencies] + +[target.'cfg(target_os = "windows")'.dependencies] +windows = { version = "0.61.3", default-features = false, features = ["Win32_Media_Speech", "Win32_System_Com", "Win32_Globalization"] } + +[target.'cfg(target_os = "android")'.dependencies] +jni.workspace = true +robius-android-env.workspace = true diff --git a/crates/speech/LICENSE b/crates/speech/LICENSE new file mode 100644 index 0000000..ea5839f --- /dev/null +++ b/crates/speech/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Project Robius Developers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/speech/README.md b/crates/speech/README.md new file mode 100644 index 0000000..9117725 --- /dev/null +++ b/crates/speech/README.md @@ -0,0 +1,152 @@ +# Robius speech + +`robius-speech` provides native streaming speech-to-text input and microphone capture for Rust apps. + +* This crate currently only allows access to speech recognition (speech-to-text) services, not text-to-speech. +* This crate doesn't offer integration with third-party transcription services or model downloads. + +| Platform | Native service | Requirements | +| --- | --- | --- | +| macOS | ✅ SFSpeechRecognizer and AVAudioEngine | macOS 11+, microphone and speech permission | +| iOS | ✅ SFSpeechRecognizer and AVAudioEngine | iOS 13+, microphone and speech permission | +| Android | ✅ SpeechRecognizer | API 26+, installed recognition service, microphone permission | +| Windows | ✅ SAPI dictation and native audio input | Installed system speech language and microphone access | +| Linux, Web | ❌ Unavailable | `is_supported()` returns false | + +* Apple and Android prefer on-device recognition when supported, but each platform may use +the system provider's network service when a local recognizer is unavailable. +* Windows SAPI uses entirely on-device recognizers that are already installed. +* Linux simply doesn't offer a built-in platform-native speech to text service, so there's nothing we can do. + + +## Usage + +```rust,no_run +use robius_speech::{NativeSpeechEvent, NativeSpeechOptions, NativeSpeechSession, SpeechError}; + +fn start_dictation() -> Result { + NativeSpeechSession::start(NativeSpeechOptions::default(), |event| { + match event { + NativeSpeechEvent::Transcript { text, is_final } => { + // Forward to your UI thread: replace the current utterance on + // partial results and commit it when is_final is true. + println!("{text} (final: {is_final})"); + } + NativeSpeechEvent::Error(error) => eprintln!("{} ({:?})", error, error.kind()), + _ => {} + } + }) +} +``` + +The session (`NativeSpeechSession` object) must stay alive for as long as you are dictating; +dropping it releases the microphone and ends recognition. +The `start()` function returns immediately rather than blocking on the permission prompt, +and the `Started` event tells you when permission has been granted and the microphone is actually recording. + +Each `AudioLevel` event includes an amplitude level that's normalized between 0 and 1, +which can be used to display a level meter or similar sound wave animation. +Callbacks arrive on platform-dependent native OS threads, so you'll want to forward them +to your main UI thread or event loop. + +Within a session, partial transcripts replace the current "utterance", +but final transcripts commit the full utterance, with recognition continuing across +as many utterances as the user speaks. An "utterance" is just a chunk of words spoken by the user. + +Calling `stop()` closes the microphone but still allows the final result to arrive, +whereas `cancel()` (or just dropping the session) discards anything that's still pending. +Either way, the session ends with exactly one terminal event: `Stopped` or `Error`. +Note that it's possible for an in-progress callback to continue executing after you cancel it. +This crate also provides `cancel_all()` for things like suspending/quitting the app. + +Importantly, only one session can be active at a time. + +Errors have a `SpeechErrorKind` so that you can handle the exact cause. +For ex, `PermissionDenied` usually means that you should inform the user that they need +to enable audio and/or speech permissions in system settings. +Similarly, `Unavailable` means that the audio input or speech recognition service +just doesn't exist and retrying it will never work. + + +## Putting the words into a text field + +Transcripts are only half the job; a text field wants "replace these bytes with this text". +`Dictation` does that bookkeeping for you, and it knows nothing about any UI toolkit: + +```rust +use robius_speech::Dictation; + +// Speech replaces the current selection and carries on from there. +let mut dictation = Dictation::new(&field.text(), field.selection()); + +// For each `Transcript` event, forwarded to your UI thread: +if let Some(edit) = dictation.transcript(&text, is_final) { + field.replace_range(edit.range, &edit.text); + dictation.applied(); +} +``` + +Partials revise the current utterance in place and finals commit it, with a space kept +between the dictated words and whatever the user typed (but not before punctuation, +or after an opening bracket, or between CJK characters). + +Users also keep typing and clicking while they dictate, and that shouldn't lose or repeat +a word. When you see the user about to change the field (a keystroke, a click that moves +the caret), call `interrupt()` before their edit lands, and `settle()` with the field's new +text and selection once it has. Dictation then picks up again from wherever the caret is, +adding only the words that aren't on screen yet. If the platform IME is mid-composition, +treat that as an interruption too, and don't `settle()` until it has committed. + +Each `Replacement` also says whether it `continues` an earlier one, so you can group a +whole run of revisions into a single undo step. + + +## Platform integration + +**Apple.** Your application must declare `NSMicrophoneUsageDescription` and +`NSSpeechRecognitionUsageDescription` in its Info.plist. +This crate will request both permissions by itself, if needed. +Sandboxed or hardened macOS applications (which is typical for any distributed app bundle) +will also need the `com.apple.security.device.audio-input` entitlement. + +Note that running a raw executable (e.g., `cargo run`) won't work with audio/speech, +you need an app bundle that has an embedded plist. +The apple backend in this crate will detect that case and report an error. + +**Android.** Declare `android.permission.RECORD_AUDIO` and a `queries` intent for +`android.speech.RecognitionService` in your manifest. The runtime permission +request is handled for you, through a headless fragment whose result routes back +to the crate rather than to your activity's `onRequestPermissionsResult`, so no +permission plumbing of your own is needed. This works just like other robius crates. + +## Building and validation + +Apple builds always compile the Swift bridge, and a missing toolchain will fail the build. +You need to install the Xcode command line tools (`xcode-select --install`), which are +already required by Rust itself... so you probably have that already taken care of. +On iOS, you need a full Xcode installation just like every other crate/app. + +For macOS builds, set the `MACOSX_DEPLOYMENT_TARGET` env var to 11.0 or newer. +This mostly just matters on Intel x86 macs, not Apple silicon (ARM aarch64), +but it doesn't hurt to always set it. + +Android builds need an SDK platform jar, `d8`, and a JDK, which the build script +locates through `ANDROID_HOME` (or `ANDROID_SDK_ROOT`), `ANDROID_PLATFORM`, +`ANDROID_BUILD_TOOLS_VERSION`, and `JAVA_HOME`. When those tools are absent the +build still succeeds, emitting a warning and compiling a backend that reports +itself as unavailable, which keeps plain `cargo check` and rust-analyzer working. +If you install them afterwards, run `cargo clean -p robius-speech` so that the +build script looks for them again. + +For this we recommend using the [`android-build`](https://crates.io/crates/android-build) crate +like all Makepad + Robius apps do. + +```sh +MACOSX_DEPLOYMENT_TARGET=11.0 cargo test -p robius-speech +MACOSX_DEPLOYMENT_TARGET=11.0 cargo clippy -p robius-speech --all-targets -- -D warnings +bash crates/speech/swift/tests/run.sh +python3 crates/speech/tests/android_retry_test.py +``` + +The Swift test harness requires macOS, and the Android harness requires a JDK and a C compiler +on either macOS or Linux. diff --git a/crates/speech/android_build.rs b/crates/speech/android_build.rs new file mode 100644 index 0000000..a6ab76c --- /dev/null +++ b/crates/speech/android_build.rs @@ -0,0 +1,67 @@ +use std::{env, fs, path::{Path, PathBuf}, process::Command}; + +fn run(command: &mut Command) { + let output = command.output().expect("failed to launch Android speech bridge compiler"); + assert!(output.status.success(), "Android speech bridge compilation failed:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); +} + +fn newest_child(root: &Path, required: &str) -> Option { + let mut paths: Vec<_> = fs::read_dir(root).ok()? + .filter_map(Result::ok).map(|entry| entry.path()) + .filter(|path| path.join(required).is_file()).collect(); + paths.sort_by_key(|path| { + path.file_name().unwrap().to_string_lossy().split(|c: char| !c.is_ascii_digit()) + .filter_map(|part| part.parse::().ok()).collect::>() + }); + paths.pop() +} + +pub fn build() { + println!("cargo:rerun-if-changed=java"); + for name in ["ANDROID_HOME", "ANDROID_SDK_ROOT", "ANDROID_PLATFORM", "ANDROID_BUILD_TOOLS_VERSION", "JAVA_HOME", "PATH"] { + println!("cargo:rerun-if-env-changed={name}"); + } + let unavailable = || println!("cargo:warning=Android native dictation disabled: an Android SDK, platform jar, d8, and JDK are required. Configure ANDROID_HOME and JAVA_HOME to enable it."); + let Some(sdk) = env::var_os("ANDROID_HOME").or_else(|| env::var_os("ANDROID_SDK_ROOT")).map(PathBuf::from) else { + unavailable(); + return; + }; + let platform = env::var_os("ANDROID_PLATFORM").map(|name| sdk.join("platforms").join(name)) + .or_else(|| newest_child(&sdk.join("platforms"), "android.jar")); + let build_tools = env::var_os("ANDROID_BUILD_TOOLS_VERSION").map(|name| sdk.join("build-tools").join(name)) + .or_else(|| newest_child(&sdk.join("build-tools"), "lib/d8.jar")); + let java_bin = |name: &str| { + let name = if cfg!(windows) { format!("{name}.exe") } else { name.to_owned() }; + env::var_os("JAVA_HOME").map(|root| PathBuf::from(root).join("bin").join(&name)) + .unwrap_or_else(|| PathBuf::from(name)) + }; + let (Some(platform), Some(build_tools)) = (platform, build_tools) else { + unavailable(); + return; + }; + if !platform.join("android.jar").is_file() || !build_tools.join("lib/d8.jar").is_file() + || ["javac", "java"].iter().any(|name| { + !Command::new(java_bin(name)).arg("-version").output().is_ok_and(|output| output.status.success()) + }) + { + unavailable(); + return; + } + let output = PathBuf::from(env::var_os("OUT_DIR").unwrap()); + let classes = output.join("speech-java"); + if classes.exists() { fs::remove_dir_all(&classes).unwrap(); } + fs::create_dir_all(&classes).unwrap(); + run(Command::new(java_bin("javac")).args(["-source", "8", "-target", "8", "-classpath"]) + .arg(platform.join("android.jar")).arg("-d").arg(&classes) + .arg("java/dev/robius/speech/NativeSpeech.java") + .arg("java/dev/robius/speech/SpeechPermissionFragment.java")); + let mut class_files: Vec<_> = fs::read_dir(classes.join("dev/robius/speech")).unwrap() + .filter_map(Result::ok).map(|entry| entry.path()) + .filter(|path| path.extension().is_some_and(|ext| ext == "class")).collect(); + class_files.sort(); + run(Command::new(java_bin("java")).arg("-cp").arg(build_tools.join("lib/d8.jar")) + .args(["com.android.tools.r8.D8", "--min-api", "26", "--lib"]) + .arg(platform.join("android.jar")).arg("--output").arg(&output).args(class_files)); + println!("cargo:rustc-cfg=native_speech_android"); +} diff --git a/crates/speech/build.rs b/crates/speech/build.rs new file mode 100644 index 0000000..d68a894 --- /dev/null +++ b/crates/speech/build.rs @@ -0,0 +1,104 @@ +use std::{env, path::PathBuf, process::Command}; + +mod android_build; + +fn main() { + println!("cargo:rustc-check-cfg=cfg(native_speech_android)"); + let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap(); + if target_os == "android" { + android_build::build(); + return; + } + if target_os != "macos" && target_os != "ios" { return; } + apple_build(&target_os); +} + +/// Compiles and links the Swift bridge. Dictation ships in every Apple build, so +/// a missing Swift toolchain fails the build rather than quietly dropping the +/// feature on you. +fn apple_build(target_os: &str) { + // The iOS SDKs only come with full Xcode, whereas macOS needs nothing more + // than the command line tools. Suggest whichever one fits the target. + let install_hint = if target_os == "macos" { + "Install the Xcode command line tools with `xcode-select --install`." + } else { + "The iOS SDK ships only with full Xcode. Install it, then select it with \ + `sudo xcode-select -s /Applications/Xcode.app`." + }; + // The toolchain is missing or misconfigured, so installing it will fix things. + let missing_toolchain = |reason: &str| -> ! { + panic!("robius-speech could not build its Swift bridge: {reason}.\n\ + Native dictation is part of every Apple build. {install_hint}\n\ + `xcode-select -p` prints which developer directory is currently selected."); + }; + // The toolchain ran fine and rejected our own source, so installing + // something won't help. Don't send anyone off to the App Store for this. + let bridge_failed = |reason: &str| -> ! { + panic!("robius-speech could not build its Swift bridge: {reason}"); + }; + + println!("cargo:rerun-if-changed=swift/NativeSpeech.swift"); + for key in ["MACOSX_DEPLOYMENT_TARGET", "IPHONEOS_DEPLOYMENT_TARGET", "IPHONESIMULATOR_DEPLOYMENT_TARGET", "DEVELOPER_DIR", "SDKROOT"] { + println!("cargo:rerun-if-env-changed={key}"); + } + let out = PathBuf::from(env::var_os("OUT_DIR").unwrap()); + let arch = match env::var("CARGO_CFG_TARGET_ARCH").unwrap().as_str() { + "aarch64" => "arm64", + "x86_64" => "x86_64", + other => bridge_failed(&format!("unsupported Apple architecture {other}")), + }; + let simulator = target_os == "ios" && (env::var("CARGO_CFG_TARGET_ABI").as_deref() == Ok("sim") || arch == "x86_64"); + let (sdk, deployment_key, default_deployment, platform) = match (target_os, simulator) { + ("macos", _) => ("macosx", "MACOSX_DEPLOYMENT_TARGET", "11.0", "macosx"), + (_, true) => ("iphonesimulator", "IPHONESIMULATOR_DEPLOYMENT_TARGET", "13.0", "ios"), + _ => ("iphoneos", "IPHONEOS_DEPLOYMENT_TARGET", "13.0", "ios"), + }; + let configured_deployment = env::var(deployment_key).ok(); + if target_os == "macos" && arch == "x86_64" && configured_deployment.is_none() { + panic!("robius-speech requires macOS 11.0 or newer. Set MACOSX_DEPLOYMENT_TARGET=11.0 (or newer) for the entire Cargo invocation, including the final executable. Rust's default Intel deployment target is too old for this Swift bridge."); + } + let deployment = configured_deployment.unwrap_or_else(|| default_deployment.into()); + if target_os == "macos" { + let major = deployment.split('.').next().and_then(|part| part.parse::().ok()).unwrap_or(0); + assert!(major >= 11, "robius-speech requires MACOSX_DEPLOYMENT_TARGET=11.0 or newer for the entire Cargo invocation"); + } + let target = format!("{arch}-apple-{platform}{deployment}{}", if simulator { "-simulator" } else { "" }); + let Ok(sdk_result) = Command::new("xcrun").args(["--sdk", sdk, "--show-sdk-path"]).output() else { + missing_toolchain("xcrun is not installed"); + }; + if !sdk_result.status.success() { + missing_toolchain(&format!("cannot locate the {sdk} SDK: {}", String::from_utf8_lossy(&sdk_result.stderr).trim())); + } + let sdk_path = String::from_utf8(sdk_result.stdout).unwrap(); + let sdk_path = sdk_path.trim(); + let library = out.join("librobius_speech.a"); + let result = Command::new("xcrun").args([ + "--sdk", sdk, "swiftc", "-swift-version", "5", "-O", "-emit-library", "-static", "-parse-as-library", + "-module-name", "RobiusSpeech", "-target", &target, "-sdk", sdk_path, + "-module-cache-path", + ]).arg(out.join("swift-module-cache")).arg("swift/NativeSpeech.swift").arg("-o").arg(&library).output(); + let Ok(result) = result else { missing_toolchain("the Swift compiler is not installed") }; + if !result.status.success() { + bridge_failed(&format!("the Swift bridge did not compile:\n{}", String::from_utf8_lossy(&result.stderr).trim())); + } + println!("cargo:rustc-link-search=native={}", out.display()); + println!("cargo:rustc-link-search=native={sdk_path}/usr/lib/swift"); + println!("cargo:rustc-link-search=native=/usr/lib/swift"); + println!("cargo:rustc-link-lib=static=robius_speech"); + for framework in ["Speech", "AVFoundation", "Foundation"] { + println!("cargo:rustc-link-lib=framework={framework}"); + } + // Swift's compiler supplies compatibility libraries from its toolchain. + let Ok(info) = Command::new("xcrun").args(["--sdk", sdk, "swiftc", "-target", &target, "-sdk", sdk_path, "-print-target-info"]).output() else { + missing_toolchain("the Swift compiler is not installed"); + }; + if !info.status.success() { + missing_toolchain("cannot query the Swift runtime search paths"); + } + for line in String::from_utf8_lossy(&info.stdout).lines() { + let path = line.trim().trim_end_matches(',').trim_matches('"'); + if path.starts_with('/') && path.contains("/lib/swift") { + println!("cargo:rustc-link-search=native={path}"); + } + } +} diff --git a/crates/speech/java/dev/robius/speech/NativeSpeech.java b/crates/speech/java/dev/robius/speech/NativeSpeech.java new file mode 100644 index 0000000..906fd35 --- /dev/null +++ b/crates/speech/java/dev/robius/speech/NativeSpeech.java @@ -0,0 +1,380 @@ +package dev.robius.speech; + +import android.Manifest; +import android.app.Activity; +import android.app.Application; +import android.content.Context; +import android.content.Intent; +import android.content.pm.PackageManager; +import android.os.Build; +import android.os.Bundle; +import android.os.Handler; +import android.os.Looper; +import android.speech.RecognitionListener; +import android.speech.RecognizerIntent; +import android.speech.SpeechRecognizer; +import java.util.ArrayList; +import java.util.HashMap; + +/** + * One dictation session, driving the system SpeechRecognizer. + * + * This is loaded from the crate's embedded DEX, so an app doesn't need a custom + * Activity or any Java of its own. Everything here runs on the main looper. + */ +public final class NativeSpeech implements RecognitionListener, Application.ActivityLifecycleCallbacks { + private static final Handler MAIN = new Handler(Looper.getMainLooper()); + private static final HashMap SESSIONS = new HashMap<>(); + private static native void event(long id, int kind, String text, float level); + // Error kinds, matching `codes` in lib.rs. + private static final int ERR_OTHER = 5; + private static final int ERR_PERMISSION = 6; + private static final int ERR_UNAVAILABLE = 7; + private static final int ERR_LANGUAGE = 8; + private static final int ERR_AUDIO = 9; + + private final Activity activity; + private final long id; + private final String locale; + private final boolean preferOnDevice; + private SpeechRecognizer recognizer; + private SpeechPermissionFragment permissionRequest; + private boolean listening = true; + private boolean awaitingResult; + private boolean started; + private boolean disposed; + private boolean triedSystemFallback; + private boolean askedForPermission; + private int recognizerGeneration; + private int transientFailures; + private static final int MAX_TRANSIENT_RETRIES = 3; + private String pending = ""; + private final Runnable restart = () -> listen(); + private final Runnable finalTimeout = () -> finish(); + + private NativeSpeech(Activity activity, long id, String locale, boolean preferOnDevice) { + this.activity = activity; + this.id = id; + this.locale = locale; + this.preferOnDevice = preferOnDevice; + } + + public static boolean supported(Activity activity) { + // Querying the service does not start it or request any permission. + return SpeechRecognizer.isRecognitionAvailable(activity) || onDeviceAvailable(activity); + } + + private static boolean onDeviceAvailable(Context context) { + if (Build.VERSION.SDK_INT < 31) return false; + try { + return (Boolean) SpeechRecognizer.class.getMethod("isOnDeviceRecognitionAvailable", Context.class) + .invoke(null, context); + } catch (ReflectiveOperationException | RuntimeException error) { + return false; + } + } + + public static void start(Activity activity, long id, String locale, boolean preferOnDevice) { + MAIN.post(() -> { + NativeSpeech session = new NativeSpeech(activity, id, locale, preferOnDevice); + SESSIONS.put(id, session); + session.begin(); + }); + } + + public static void stop(long id, boolean cancel) { + MAIN.post(() -> { + NativeSpeech session = SESSIONS.get(id); + if (session == null) return; + session.listening = false; + MAIN.removeCallbacks(session.restart); + if (cancel) { + session.dispose(); + } else if (!session.awaitingResult) { + session.finish(); + } else { + try { + session.recognizer.stopListening(); + MAIN.postDelayed(session.finalTimeout, 3000); + } catch (RuntimeException error) { + session.finish(); + } + } + }); + } + + private void begin() { + if (activity.isFinishing() || activity.isDestroyed()) { + // Bound to this discarded Activity, not to the device. + fail("The application is no longer active."); + return; + } + if (activity.checkSelfPermission(Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) { + // Ask once, then resume from the top. The guard also stops a recognizer + // that reports the permission as missing even after a grant from looping. + if (askedForPermission) { + fail("Allow microphone access in Android settings to use speech input.", ERR_PERMISSION); + return; + } + askedForPermission = true; + permissionRequest = SpeechPermissionFragment.request(activity, outcome -> { + if (disposed) return; + permissionRequest = null; + if (outcome == SpeechPermissionFragment.Outcome.GRANTED) { + begin(); + } else if (outcome == SpeechPermissionFragment.Outcome.CANCELLED) { + finish(); + } else { + fail("Microphone access is needed for speech input.", ERR_PERMISSION); + } + }); + return; + } + try { + activity.getApplication().registerActivityLifecycleCallbacks(this); + listen(); + } catch (RuntimeException error) { + fail("Unable to start the system speech recognition service.", ERR_UNAVAILABLE); + } + } + + private SpeechRecognizer createRecognizer() { + if (preferOnDevice && !triedSystemFallback && onDeviceAvailable(activity)) { + try { + SpeechRecognizer deviceRecognizer = (SpeechRecognizer) SpeechRecognizer.class + .getMethod("createOnDeviceSpeechRecognizer", Context.class).invoke(null, activity); + if (deviceRecognizer != null) return deviceRecognizer; + } catch (ReflectiveOperationException | RuntimeException error) { + // Some vendor services advertise on-device support without providing it. + } + } + return SpeechRecognizer.isRecognitionAvailable(activity) + ? SpeechRecognizer.createSpeechRecognizer(activity) : null; + } + + private void listen() { + if (disposed || !listening) return; + pending = ""; + Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH); + intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM); + intent.putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true); + intent.putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 1); + if (locale != null && !locale.isEmpty()) intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE, locale); + // On older devices this is a preference: the installed system service may ignore it. + intent.putExtra(RecognizerIntent.EXTRA_PREFER_OFFLINE, preferOnDevice && !triedSystemFallback); + try { + if (recognizer == null) recognizer = createRecognizer(); + if (recognizer == null) { + fail("No speech recognition service is installed on this device.", ERR_UNAVAILABLE); + return; + } + // Each attempt gets a new listener generation. Late callbacks from + // an utterance or cancelled client must not affect its successor. + installListener(); + awaitingResult = true; + recognizer.startListening(intent); + } catch (RuntimeException error) { + fail("Unable to start microphone recording for speech input.", ERR_AUDIO); + } + } + + private void installListener() { + // Destroyed services can still have Binder callbacks queued. Keep those + // from committing an old hypothesis after switching to the fallback. + final int generation = ++recognizerGeneration; + recognizer.setRecognitionListener(new RecognitionListener() { + private boolean current() { return !disposed && generation == recognizerGeneration; } + @Override public void onReadyForSpeech(Bundle params) { + if (current()) NativeSpeech.this.onReadyForSpeech(params); + } + @Override public void onBeginningOfSpeech() {} + @Override public void onRmsChanged(float rms) { + if (current()) NativeSpeech.this.onRmsChanged(rms); + } + @Override public void onBufferReceived(byte[] buffer) {} + @Override public void onEndOfSpeech() { + if (current()) NativeSpeech.this.onEndOfSpeech(); + } + @Override public void onError(int error) { + if (current()) NativeSpeech.this.onError(error); + } + @Override public void onResults(Bundle results) { + if (current()) NativeSpeech.this.onResults(results); + } + @Override public void onPartialResults(Bundle results) { + if (current()) NativeSpeech.this.onPartialResults(results); + } + @Override public void onEvent(int type, Bundle params) {} + }); + } + + private String transcript(Bundle results) { + ArrayList values = results.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION); + return values == null || values.isEmpty() || values.get(0) == null ? "" : values.get(0); + } + + private void commitPending() { + if (!pending.isEmpty()) { + event(id, 2, pending, 0); + pending = ""; + } + } + + private void nextUtterance() { + awaitingResult = false; + event(id, 3, null, 0); + if (listening) MAIN.postDelayed(restart, 150); + else finish(); + } + + private void retryTransientError(int error) { + commitPending(); + if (transientFailures >= MAX_TRANSIENT_RETRIES) { + fail("The system speech service is still unavailable. Try again in a moment."); + return; + } + ++recognizerGeneration; + // A client error can mean its Binder connection has died. Recreate + // that client after releasing the old one, preserving on-device choice. + if (error == SpeechRecognizer.ERROR_CLIENT) { + try { recognizer.cancel(); } catch (RuntimeException ignored) {} + try { recognizer.destroy(); } catch (RuntimeException ignored) {} + recognizer = null; + } + event(id, 3, null, 0); + MAIN.removeCallbacks(restart); + // Do not reset this on onReadyForSpeech: a broken service can report + // ready and fail repeatedly without ever completing an utterance. + MAIN.postDelayed(restart, 500L << transientFailures++); + } + + private void finish() { + if (disposed) return; + commitPending(); + dispose(); + event(id, 4, null, 0); + } + + private void fail(String message) { + fail(message, ERR_OTHER); + } + + private void fail(String message, int kind) { + if (disposed) return; + dispose(); + event(id, kind, message, 0); + } + + private void dispose() { + if (disposed) return; + disposed = true; + listening = false; + MAIN.removeCallbacks(restart); + MAIN.removeCallbacks(finalTimeout); + SESSIONS.remove(id); + if (permissionRequest != null) { + permissionRequest.cancel(); + permissionRequest = null; + } + activity.getApplication().unregisterActivityLifecycleCallbacks(this); + if (recognizer != null) { + try { recognizer.cancel(); } catch (RuntimeException ignored) {} + try { recognizer.destroy(); } catch (RuntimeException ignored) {} + recognizer = null; + } + } + + @Override public void onReadyForSpeech(Bundle params) { + if (disposed || !listening || !awaitingResult) return; + if (!started) { + started = true; + event(id, 0, null, 0); + } + } + @Override public void onBeginningOfSpeech() {} + @Override public void onRmsChanged(float rms) { + if (!disposed && listening) event(id, 3, null, Float.isNaN(rms) ? 0 : Math.max(0, Math.min(1, rms / 10.0f))); + } + @Override public void onBufferReceived(byte[] buffer) {} + @Override public void onEndOfSpeech() { + if (!disposed) event(id, 3, null, 0); + } + @Override public void onPartialResults(Bundle results) { + if (disposed || !awaitingResult || results == null) return; + String text = transcript(results); + if (!text.isEmpty() && !text.equals(pending)) { + pending = text; + event(id, 1, text, 0); + } + } + @Override public void onResults(Bundle results) { + if (disposed || !awaitingResult) return; + awaitingResult = false; + transientFailures = 0; + String text = results == null ? "" : transcript(results); + if (!text.isEmpty()) pending = text; + commitPending(); + nextUtterance(); + } + @Override public void onError(int error) { + if (disposed || !awaitingResult) return; + awaitingResult = false; + if (error == SpeechRecognizer.ERROR_NO_MATCH || error == SpeechRecognizer.ERROR_SPEECH_TIMEOUT) { + transientFailures = 0; + commitPending(); + nextUtterance(); + return; + } + if (!listening) { + finish(); + return; + } + if (error == SpeechRecognizer.ERROR_RECOGNIZER_BUSY || error == SpeechRecognizer.ERROR_CLIENT) { + retryTransientError(error); + return; + } + // On-device recognition may exist without a model for the requested + // language. Let the regular native service handle it when available. + if ((error == 12 || error == 13) && preferOnDevice && !triedSystemFallback + && SpeechRecognizer.isRecognitionAvailable(activity)) { + triedSystemFallback = true; + try { + ++recognizerGeneration; + recognizer.destroy(); + recognizer = SpeechRecognizer.createSpeechRecognizer(activity); + installListener(); + commitPending(); + nextUtterance(); + return; + } catch (RuntimeException ignored) { + // Continue with the actionable language error below. + } + } + switch (error) { + case SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS: + fail("Allow microphone access in Android settings to use speech input.", ERR_PERMISSION); break; + case SpeechRecognizer.ERROR_AUDIO: + fail("The microphone is unavailable. Check whether another app is using it.", ERR_AUDIO); break; + case SpeechRecognizer.ERROR_NETWORK: + case SpeechRecognizer.ERROR_NETWORK_TIMEOUT: + fail("The system speech service needs a network connection. Check your connection and try again."); break; + case 12: // ERROR_LANGUAGE_NOT_SUPPORTED (API 31) + case 13: // ERROR_LANGUAGE_UNAVAILABLE (API 31) + fail("The system speech service does not have recognition support for this language.", ERR_LANGUAGE); break; + default: + fail("The system speech service stopped (error " + error + "). Try again."); + } + } + @Override public void onEvent(int type, Bundle params) {} + @Override public void onActivityPaused(Activity other) { + if (other == activity) finish(); + } + @Override public void onActivityDestroyed(Activity other) { + if (other == activity) finish(); + } + @Override public void onActivityCreated(Activity activity, Bundle state) {} + @Override public void onActivityStarted(Activity activity) {} + @Override public void onActivityResumed(Activity activity) {} + @Override public void onActivityStopped(Activity activity) {} + @Override public void onActivitySaveInstanceState(Activity activity, Bundle state) {} +} diff --git a/crates/speech/java/dev/robius/speech/SpeechPermissionFragment.java b/crates/speech/java/dev/robius/speech/SpeechPermissionFragment.java new file mode 100644 index 0000000..741d514 --- /dev/null +++ b/crates/speech/java/dev/robius/speech/SpeechPermissionFragment.java @@ -0,0 +1,168 @@ +package dev.robius.speech; + +import android.app.Activity; +import android.app.Fragment; +import android.app.FragmentManager; +import android.content.pm.PackageManager; +import android.os.Bundle; + +/** + * Headless fragment that asks for RECORD_AUDIO. `Fragment.requestPermissions` routes the + * result back here rather than to the activity's own `onRequestPermissionsResult`, so an + * application needs no permission plumbing of its own. The request belongs to the current + * Activity and is cancelled if that Activity is recreated or destroyed. + */ +public final class SpeechPermissionFragment extends Fragment { + public enum Outcome { GRANTED, DENIED, CANCELLED } + /** Delivered once on the UI thread, unless the caller cancels first. */ + public interface Result { void onResult(Outcome outcome); } + + // Must be <= 0xffff: android.app.Fragment encodes its index in the upper 16 bits. + private static final int REQUEST_CODE = 0x5350; + private static final String HOST_TAG = "dev.robius.speech.PermissionHost"; + private static final String TAG = "dev.robius.speech.SpeechPermissionFragment"; + private static final String[] PERMISSIONS = { android.Manifest.permission.RECORD_AUDIO }; + + // Null after delivery or cancellation; the OS dialog can outlive its caller. + private Result callback; + private boolean launched; + private boolean completed; + private Fragment host; + private FragmentManager manager; + private FragmentManager.FragmentLifecycleCallbacks lifecycle; + + /** Required for Fragment transactions; this helper is never saved for restoration. */ + public SpeechPermissionFragment() {} + + private SpeechPermissionFragment(Result callback) { + this.callback = callback; + } + + /** Asks without blocking. Called on the UI thread by NativeSpeech. */ + public static SpeechPermissionFragment request(Activity activity, Result callback) { + SpeechPermissionFragment fragment = null; + try { + if (activity.isFinishing() || activity.isDestroyed()) { + callback.onResult(Outcome.CANCELLED); + return null; + } + FragmentManager manager = activity.getFragmentManager(); + Fragment existing = manager.findFragmentByTag(HOST_TAG); + if (existing != null) { + Fragment child = existing.getChildFragmentManager().findFragmentByTag(TAG); + if (child instanceof SpeechPermissionFragment) { + SpeechPermissionFragment pending = (SpeechPermissionFragment) child; + if (!pending.completed) { + if (pending.callback != null) { + callback.onResult(Outcome.CANCELLED); + return null; + } + // Cancelling a session cannot dismiss the OS dialog. Transfer + // its pending result instead of launching a duplicate request. + pending.callback = callback; + return pending; + } + } + // Includes an empty framework host restored after process death. + manager.beginTransaction().remove(existing).commitNowAllowingStateLoss(); + } + fragment = new SpeechPermissionFragment(callback); + fragment.manager = manager; + fragment.host = new Fragment(); + fragment.lifecycle = fragment.hostLifecycle(); + manager.registerFragmentLifecycleCallbacks(fragment.lifecycle, false); + manager.beginTransaction().add(fragment.host, HOST_TAG).commitNowAllowingStateLoss(); + fragment.host.getChildFragmentManager().beginTransaction() + .add(fragment, TAG).commitNowAllowingStateLoss(); + return fragment.completed ? null : fragment; + } catch (RuntimeException error) { + if (fragment == null) { + callback.onResult(Outcome.CANCELLED); + } else { + fragment.deliver(Outcome.CANCELLED); + if (fragment.host.isAdded()) fragment.removeSelf(); + else fragment.unregisterLifecycle(); + } + return null; + } + } + + private FragmentManager.FragmentLifecycleCallbacks hostLifecycle() { + return new FragmentManager.FragmentLifecycleCallbacks() { + @Override public void onFragmentSaveInstanceState(FragmentManager fm, Fragment f, Bundle state) { + // The helper comes from an embedded child DEX, which Android's Activity + // class loader cannot restore. Persist only our plain framework host. + // This callback runs after child state is saved. The host is exclusively + // ours and has no other state; never modify another fragment's bundle. + if (f == host) state.clear(); + } + @Override public void onFragmentDetached(FragmentManager fm, Fragment f) { + if (f == host) unregisterLifecycle(); + } + }; + } + + private void unregisterLifecycle() { + if (lifecycle != null) { + manager.unregisterFragmentLifecycleCallbacks(lifecycle); + lifecycle = null; + } + } + + /** Release only this session's callback; a launched dialog may still return. */ + public void cancel() { + callback = null; + if (!launched) { + completed = true; + removeSelf(); + } + } + + @Override public void onResume() { + super.onResume(); + if (completed || (!launched && callback == null)) { + removeSelf(); + return; + } + if (!launched) { + launched = true; + try { + requestPermissions(PERMISSIONS, REQUEST_CODE); + } catch (RuntimeException error) { + deliver(Outcome.CANCELLED); + removeSelf(); + } + } + } + + @Override public void onRequestPermissionsResult(int requestCode, String[] permissions, int[] results) { + if (requestCode != REQUEST_CODE) return; + // An empty array means the request was interrupted, which counts as denied. + boolean granted = results.length > 0 && results[0] == PackageManager.PERMISSION_GRANTED; + deliver(granted ? Outcome.GRANTED : Outcome.DENIED); + removeSelf(); + } + + @Override public void onDestroy() { + super.onDestroy(); + // Neither fragment is retained. End the old Activity's session now; a + // later grant must never resume recognition against its destroyed Activity. + deliver(Outcome.CANCELLED); + } + + private void deliver(Outcome outcome) { + if (completed) return; + completed = true; + Result pending = callback; + callback = null; + if (pending != null) pending.onResult(outcome); + } + + private void removeSelf() { + // Removal can be requested from within a Fragment lifecycle callback. + // Keep the save-state guard until the host actually detaches. + if (host != null && host.isAdded()) { + manager.beginTransaction().remove(host).commitAllowingStateLoss(); + } + } +} diff --git a/crates/speech/src/android.rs b/crates/speech/src/android.rs new file mode 100644 index 0000000..5483b3c --- /dev/null +++ b/crates/speech/src/android.rs @@ -0,0 +1,120 @@ +//! Android dictation, bridged to the system `SpeechRecognizer`. +//! +//! The Java side in `java/dev/robius/speech/` does the real work, including the +//! runtime permission request; this module just loads it and calls into it. That +//! Java is compiled to DEX and embedded in the library, so an app doesn't need to +//! add anything to its own Activity. + +use std::sync::{Mutex, OnceLock}; +use jni::{JNIEnv, NativeMethod, objects::{GlobalRef, JClass, JObject, JString, JValue}, sys::{jfloat, jint, jlong}}; +use crate::{codes, error_kind_from_code, NativeSpeechEvent, NativeSpeechOptions, SpeechError, SpeechErrorKind}; + +static BRIDGE: OnceLock>> = OnceLock::new(); + +fn with_env(run: impl FnOnce(&mut JNIEnv, &JObject) -> jni::errors::Result) -> Result { + // The environment provider selects the host toolkit's current Activity. + // ndk-context can panic before initialization; report unavailable instead + // of unwinding through a caller merely checking for speech support. + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + robius_android_env::with_activity(|env, activity| { + let result = env.with_local_frame(32, |env| run(env, activity)); + if env.exception_check().unwrap_or(false) { + // Never leave an exception pending on the application's thread. + let _ = env.exception_clear(); + } + result + }) + })).map_err(|_| SpeechError::new( + SpeechErrorKind::Unavailable, + "Unable to access the Android application context. Initialize it before using speech recognition.", + ))?; + result.and_then(|inner| inner).map_err(|error| SpeechError::new( + SpeechErrorKind::Unavailable, + format!("Unable to access Android speech recognition: {error}"), + )) +} + +pub(super) const ENGINE: &str = "android-speechrecognizer"; + +fn bridge(env: &mut JNIEnv, activity: &JObject) -> jni::errors::Result { + let mut cached = BRIDGE.get_or_init(|| Mutex::new(None)).lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if let Some(class) = cached.as_ref() { + return Ok(class.clone()); + } + if activity.is_null() { + return Err(jni::errors::Error::NullPtr("Android Activity")); + } + // Borrow the current Activity without caching it: Android can replace it + // after rotation or recreation. Only the loaded bridge class is retained. + let parent = env.call_method(activity, "getClassLoader", "()Ljava/lang/ClassLoader;", &[])?.l()?; + let dex = env.byte_array_from_slice(include_bytes!(concat!(env!("OUT_DIR"), "/classes.dex")))?; + let bytes = env.call_static_method("java/nio/ByteBuffer", "wrap", "([B)Ljava/nio/ByteBuffer;", &[JValue::Object(&dex)])?.l()?; + let loader = env.new_object("dalvik/system/InMemoryDexClassLoader", "(Ljava/nio/ByteBuffer;Ljava/lang/ClassLoader;)V", + &[JValue::Object(&bytes), JValue::Object(&parent)])?; + let name = env.new_string("dev.robius.speech.NativeSpeech")?; + let class = JClass::from(env.call_method(loader, "loadClass", "(Ljava/lang/String;)Ljava/lang/Class;", &[JValue::Object(&name)])?.l()?); + env.register_native_methods(&class, &[NativeMethod { + name: "event".into(), + sig: "(JILjava/lang/String;F)V".into(), + fn_ptr: native_event as *mut std::ffi::c_void, + }])?; + let global = env.new_global_ref(class)?; + *cached = Some(global.clone()); + Ok(global) +} + +extern "system" fn native_event(mut env: JNIEnv, _: JClass, id: jlong, kind: jint, text: JString, level: jfloat) { + // Java invokes this on the main thread. Decoding/callback failures must not + // unwind through JNI, and app callbacks are isolated by crate::emit as well. + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let text = if text.is_null() { String::new() } + else { env.get_string(&text).map(String::from).unwrap_or_default() }; + let event = match kind { + codes::STARTED => NativeSpeechEvent::Started, + codes::PARTIAL | codes::FINAL => NativeSpeechEvent::Transcript { text, is_final: kind == codes::FINAL }, + codes::LEVEL => NativeSpeechEvent::AudioLevel(if level.is_finite() { level.clamp(0.0, 1.0) } else { 0.0 }), + codes::STOPPED => NativeSpeechEvent::Stopped, + codes::ERROR..=codes::ERROR_AUDIO => { + NativeSpeechEvent::Error(SpeechError::new(error_kind_from_code(kind), text)) + } + _ => return, + }; + crate::emit(id as u64, event); + })); + if env.exception_check().unwrap_or(false) { let _ = env.exception_clear(); } +} + +pub(super) fn is_supported() -> bool { + with_env(|env, activity| { + if activity.is_null() { return Ok(false); } + let class = bridge(env, activity)?; + let class: &JClass = class.as_obj().into(); + env.call_static_method(class, "supported", "(Landroid/app/Activity;)Z", &[JValue::Object(activity)])?.z() + }).unwrap_or(false) +} + +pub(super) fn start(id: u64, options: &NativeSpeechOptions) -> Result<(), SpeechError> { + with_env(|env, activity| { + if activity.is_null() { return Err(jni::errors::Error::NullPtr("Android Activity")); } + let class = bridge(env, activity)?; + let class: &JClass = class.as_obj().into(); + let locale = env.new_string(options.locale.as_deref().unwrap_or(""))?; + env.call_static_method(class, "start", "(Landroid/app/Activity;JLjava/lang/String;Z)V", &[ + JValue::Object(activity), JValue::Long(id as jlong), JValue::Object(&locale), + JValue::Bool(options.prefer_on_device.into()), + ])?; + Ok(()) + }) +} + +pub(super) fn stop(id: u64, cancel: bool) { + let result = with_env(|env, activity| { + let class = bridge(env, activity)?; + let class: &JClass = class.as_obj().into(); + env.call_static_method(class, "stop", "(JZ)V", &[JValue::Long(id as jlong), JValue::Bool(cancel.into())])?; + Ok(()) + }); + if let Err(error) = result { + crate::emit(id, NativeSpeechEvent::Error(error)); + } +} diff --git a/crates/speech/src/apple.rs b/crates/speech/src/apple.rs new file mode 100644 index 0000000..7ca4811 --- /dev/null +++ b/crates/speech/src/apple.rs @@ -0,0 +1,43 @@ +//! macOS and iOS dictation, bridged to `SFSpeechRecognizer` and `AVAudioEngine`. +//! +//! The Swift side in `swift/NativeSpeech.swift` does the real work, including +//! asking for both permissions; this module is just the C ABI between the two. + +use std::ffi::{c_char, CStr, CString}; +use crate::{codes, emit, error_kind_from_code, NativeSpeechEvent, NativeSpeechOptions, SpeechError}; + +extern "C" { + fn robius_speech_start(id: u64, locale: *const c_char, prefer_on_device: bool); + fn robius_speech_stop(id: u64, cancel: bool); +} + +pub(super) const ENGINE: &str = "apple-sfspeech"; + +pub(super) fn is_supported() -> bool { true } + +pub(super) fn start(id: u64, options: &NativeSpeechOptions) -> Result<(), SpeechError> { + let locale = CString::new(options.locale.as_deref().unwrap_or_default()).unwrap(); + unsafe { robius_speech_start(id, locale.as_ptr(), options.prefer_on_device); } + Ok(()) +} + +pub(super) fn stop(id: u64, cancel: bool) { + unsafe { robius_speech_stop(id, cancel); } +} + +#[no_mangle] +extern "C" fn robius_speech_event(id: u64, kind: i32, text: *const c_char, level: f32) { + let text = || { + if text.is_null() { String::new() } + else { unsafe { CStr::from_ptr(text) }.to_string_lossy().into_owned() } + }; + let event = match kind { + codes::STARTED => NativeSpeechEvent::Started, + codes::PARTIAL => NativeSpeechEvent::Transcript { text: text(), is_final: false }, + codes::FINAL => NativeSpeechEvent::Transcript { text: text(), is_final: true }, + codes::LEVEL => NativeSpeechEvent::AudioLevel(if level.is_finite() { level.clamp(0.0, 1.0) } else { 0.0 }), + codes::STOPPED => NativeSpeechEvent::Stopped, + _ => NativeSpeechEvent::Error(SpeechError::new(error_kind_from_code(kind), text())), + }; + emit(id, event); +} diff --git a/crates/speech/src/dictation.rs b/crates/speech/src/dictation.rs new file mode 100644 index 0000000..0918fdb --- /dev/null +++ b/crates/speech/src/dictation.rs @@ -0,0 +1,558 @@ +//! Puts dictated words into a text field. +//! +//! A recognizer hands you transcripts, but a text field wants "replace these +//! bytes with this text". [`Dictation`] does the bookkeeping in between: it +//! remembers where the words go, revises the current utterance in place as +//! partial transcripts arrive, keeps a space between speech and whatever the +//! user typed, and copes with the user editing the field mid-sentence without +//! losing or repeating a word. It knows nothing about any UI toolkit; you apply +//! the [`Replacement`]s it returns to your own text field. + +use std::ops::Range; + +/// One edit to make to the field: replace `range` (byte offsets) with `text`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Replacement { + pub range: Range, + pub text: String, + /// Whether this revises what an earlier replacement wrote, rather than + /// starting to write somewhere new. Handy for grouping undo: a whole run of + /// revisions of one phrase reads best as a single undo step. + pub continues: bool, +} + +/// Where dictated words go in a text field, and what is there now. +/// +/// The cycle is: feed each transcript to [`transcript`](Self::transcript), +/// apply the replacement it returns, then call [`applied`](Self::applied). +/// When the user is about to change the field themselves (a keystroke, a click +/// that moves the caret), call [`interrupt`](Self::interrupt) first so nothing +/// is written underneath their edit, and once it has landed call +/// [`settle`](Self::settle) with the field's new text and selection. Dictation +/// then carries on from wherever the caret is, adding only the words that +/// aren't on screen yet. Calling `settle` after every event is fine; it also +/// notices changes you didn't see coming. +#[derive(Clone, Debug)] +pub struct Dictation { + /// The field's text when we last anchored, and the range of it we write over. + draft: String, + start: usize, + end: usize, + /// Utterances the recognizer has finished, and the one it is still revising. + committed: String, + pending: String, + /// What has actually been written over `start..end`; `None` until the first + /// replacement lands. + shown: Option, + /// A replacement handed out but not yet confirmed by `applied`. + offered: Option, + /// Set while the user is editing. `true` means an utterance was mid-flight + /// when they started, so we wait for it to finish before re-anchoring. + interrupted: Option, +} + +impl Dictation { + /// Speech replaces `selection` (byte offsets into `text`) and carries on from there. + pub fn new(text: &str, selection: Range) -> Self { + let Range { start, end } = ordered(selection); + let start = floor_char_boundary(text, start); + let end = floor_char_boundary(text, end); + Self { + draft: text.to_owned(), + start, + end, + committed: String::new(), + pending: String::new(), + shown: None, + offered: None, + interrupted: None, + } + } + + /// A transcript from the recognizer. A partial revises the current utterance, + /// a final commits it. Returns the edit that brings the field up to date, if + /// there is one; apply it and call [`applied`](Self::applied). + pub fn transcript(&mut self, text: &str, is_final: bool) -> Option { + self.pending = text.trim().to_owned(); + if is_final { + append_words(&mut self.committed, &self.pending); + self.pending.clear(); + } + if let Some(awaiting_final) = self.interrupted.as_mut() { + if is_final { + *awaiting_final = false; + } + return None; + } + self.offer() + } + + /// The replacement handed out last has been applied to the field. + pub fn applied(&mut self) { + if let Some(offered) = self.offered.take() { + self.shown = Some(offered); + } + } + + /// The user is changing the field or moving the caret. Nothing more is + /// written until [`settle`](Self::settle) sees their change has landed. + pub fn interrupt(&mut self) { + if self.interrupted.is_none() { + self.interrupted = Some(!self.pending.is_empty()); + self.offered = None; + } + } + + pub fn is_interrupted(&self) -> bool { + self.interrupted.is_some() + } + + /// The field's text and selection now that the current event has been + /// handled. Anything we didn't write ourselves counts as an interruption, + /// and an interruption that turns out to have changed nothing (End with the + /// caret already at the end, a click on the caret) is called off. Once an + /// interrupted utterance has finished (or none was in flight), dictation + /// re-anchors at the caret and returns whatever was said since that isn't + /// on screen yet, if anything. + pub fn settle(&mut self, text: &str, selection: Range) -> Option { + let selection = ordered(selection); + if self.matches(text, &selection) { + self.interrupted = None; + return self.offer(); + } + self.interrupt(); + if self.interrupted != Some(false) { + return None; + } + let unseen = words_beyond(self.shown.as_deref().unwrap_or(""), &self.committed); + *self = Self { + committed: unseen, + pending: std::mem::take(&mut self.pending), + ..Self::new(text, selection) + }; + self.offer() + } + + fn offer(&mut self) -> Option { + let insertion = self.insertion(); + let same = match &self.shown { + Some(shown) => *shown == insertion, + None => insertion.is_empty(), + }; + if same { + return None; + } + let range = match &self.shown { + Some(shown) => self.start..self.start + shown.len(), + None => self.start..self.end, + }; + self.offered = Some(insertion.clone()); + Some(Replacement { range, text: insertion, continues: self.shown.is_some() }) + } + + /// Whether the field holds exactly what we last left in it. + fn matches(&self, text: &str, selection: &Range) -> bool { + let Some(shown) = &self.shown else { + return text == self.draft && *selection == (self.start..self.end); + }; + let caret = self.start + shown.len(); + *selection == (caret..caret) + && text.len() == self.draft.len() - (self.end - self.start) + shown.len() + && text[..self.start] == self.draft[..self.start] + && text[self.start..caret] == **shown + && text[caret..] == self.draft[self.end..] + } + + /// Everything said since the anchor, spaced off the draft around it. + fn insertion(&self) -> String { + let mut spoken = self.committed.clone(); + append_words(&mut spoken, &self.pending); + if spoken.is_empty() { + return spoken; + } + if needs_space(&self.draft[..self.start], &spoken) { + spoken.insert(0, ' '); + } + if needs_space(&spoken, &self.draft[self.end..]) { + spoken.push(' '); + } + spoken + } +} + +fn ordered(range: Range) -> Range { + range.start.min(range.end)..range.start.max(range.end) +} + +fn floor_char_boundary(text: &str, index: usize) -> usize { + let mut index = index.min(text.len()); + while !text.is_char_boundary(index) { + index -= 1; + } + index +} + +/// The part of `spoken` that goes beyond what the user can already see in +/// `shown`. Words are compared loosely, because a recognizer routinely re-cases +/// or re-punctuates what it already sent. Returns nothing when the two share no +/// leading words at all, so nothing is ever inserted twice. +fn words_beyond(shown: &str, spoken: &str) -> String { + if shown.trim().is_empty() { + return spoken.trim().to_owned(); + } + let loose = |word: &str| word.trim_matches(|c: char| !c.is_alphanumeric()).to_lowercase(); + let shown_words: Vec<_> = shown.split_whitespace().map(loose).collect(); + let mut matched = 0; + let mut rest = spoken.trim(); + for word in spoken.split_whitespace() { + if matched >= shown_words.len() || loose(word) != shown_words[matched] { + break; + } + matched += 1; + rest = rest[word.len()..].trim_start(); + } + // Every word the user can see must be accounted for. If the revision inserted + // a word inside them, the tail is not safely separable, so insert nothing. + if matched < shown_words.len() { String::new() } else { rest.to_owned() } +} + +fn append_words(destination: &mut String, text: &str) { + if needs_space(destination, text) { + destination.push(' '); + } + destination.push_str(text); +} + +/// Whether dictated text needs a space to keep it off whatever precedes it. +/// +/// Speech joins onto a draft the user may have typed, so the default is to +/// separate them. The exceptions are all cases where jamming them together is +/// what was meant: an opening bracket the user just typed, punctuation that the +/// recognizer itself supplies, and scripts that don't space their words. +/// An apostrophe is only a joiner on the right, for endings like `'s`; a draft +/// that happens to end in one still gets its space. +fn needs_space(left: &str, right: &str) -> bool { + let (Some(left), Some(right)) = (left.chars().last(), right.chars().next()) else { return false }; + let cjk = |c| matches!(c, '\u{3000}'..='\u{30ff}' | '\u{3400}'..='\u{9fff}' | '\u{ac00}'..='\u{d7af}' | '\u{f900}'..='\u{faff}'); + !left.is_whitespace() && !right.is_whitespace() + && !matches!(left, '(' | '[' | '{') + && !matches!(right, '.' | ',' | '!' | '?' | ':' | ';' | ')' | ']' | '}' | '\'' | '\u{2019}') + && !cjk(left) && !cjk(right) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A stand-in text field: applies each replacement and reports back. + struct Field { + text: String, + caret: usize, + } + + impl Field { + fn new(text: &str, caret: usize) -> Self { + Self { text: text.into(), caret } + } + + fn apply(&mut self, dictation: &mut Dictation, replacement: Option) { + if let Some(replacement) = replacement { + self.text.replace_range(replacement.range.clone(), &replacement.text); + self.caret = replacement.range.start + replacement.text.len(); + dictation.applied(); + } + } + + fn say(&mut self, dictation: &mut Dictation, text: &str, is_final: bool) { + let replacement = dictation.transcript(text, is_final); + self.apply(dictation, replacement); + } + + /// The user types at the caret, as a toolkit event would deliver it. + fn type_text(&mut self, dictation: &mut Dictation, text: &str) { + dictation.interrupt(); + self.text.insert_str(self.caret, text); + self.caret += text.len(); + self.settle(dictation); + } + + fn settle(&mut self, dictation: &mut Dictation) { + let replacement = dictation.settle(&self.text, self.caret..self.caret); + self.apply(dictation, replacement); + } + } + + #[test] + fn partials_replace_the_utterance_and_finals_append() { + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "I scream", false); + assert_eq!(field.text, "I scream"); + field.say(&mut dictation, "Ice cream.", true); + assert_eq!(field.text, "Ice cream."); + field.say(&mut dictation, "Please", false); + assert_eq!(field.text, "Ice cream. Please"); + field.say(&mut dictation, "Please!", true); + assert_eq!(field.text, "Ice cream. Please!"); + assert_eq!(field.caret, field.text.len()); + } + + #[test] + fn the_first_replacement_starts_a_run_and_later_ones_continue_it() { + let mut dictation = Dictation::new("Hi old friend", 3..6); + let first = dictation.transcript("new", false).unwrap(); + assert_eq!(first, Replacement { range: 3..6, text: "new".into(), continues: false }); + dictation.applied(); + let second = dictation.transcript("dear", false).unwrap(); + assert_eq!(second, Replacement { range: 3..6, text: "dear".into(), continues: true }); + // A revision that changes nothing is not an edit, nor is the final that + // merely confirms it. + dictation.applied(); + assert_eq!(dictation.transcript("dear", false), None); + assert_eq!(dictation.transcript("dear", true), None); + } + + #[test] + fn a_replacement_that_never_landed_is_simply_offered_again() { + let mut dictation = Dictation::new("", 0..0); + assert_eq!(dictation.transcript("one", false).unwrap().range, 0..0); + // Not applied, so the field still holds nothing; the next one starts over. + let again = dictation.transcript("one two", false).unwrap(); + assert_eq!(again, Replacement { range: 0..0, text: "one two".into(), continues: false }); + dictation.applied(); + assert_eq!(dictation.transcript("one two three", true).unwrap().range, 0..7); + } + + #[test] + fn speech_is_spaced_off_the_draft_but_not_off_punctuation_or_brackets() { + let after = |draft: &str, spoken: &str| { + let mut dictation = Dictation::new(draft, draft.len()..draft.len()); + let mut field = Field::new(draft, draft.len()); + field.say(&mut dictation, spoken, true); + field.text + }; + assert_eq!(after("hello", "world"), "hello world"); + assert_eq!(after("hello,", "world"), "hello, world"); + assert_eq!(after("hello'", "world"), "hello' world"); + assert_eq!(after("hello ", "world"), "hello world"); + assert_eq!(after("hello\n", "world"), "hello\nworld"); + assert_eq!(after("hello(", "world"), "hello(world"); + assert_eq!(after("你好", "世界"), "你好世界"); + assert_eq!(after("", "say enter"), "say enter"); + + // Recognizer punctuation joins onto the words before it. + let mut dictation = Dictation::new("(", 1..1); + let mut field = Field::new("(", 1); + field.say(&mut dictation, "say enter", true); + field.say(&mut dictation, ", then stop)", true); + assert_eq!(field.text, "(say enter, then stop)"); + + // And a caret parked mid-draft gets a space on both sides. + let mut dictation = Dictation::new("abcdef", 3..3); + let mut field = Field::new("abcdef", 3); + field.say(&mut dictation, "MID", true); + assert_eq!(field.text, "abc MID def"); + + // Silence replaces nothing and adds no whitespace. + let mut dictation = Dictation::new("keep this", 0..9); + assert_eq!(dictation.transcript(" ", false), None); + assert_eq!(dictation.transcript("", true), None); + } + + #[test] + fn selection_offsets_are_ordered_and_kept_on_char_boundaries() { + let draft = "Hi 🦀, old text today"; + let start = draft.find("old").unwrap(); + let end = draft.find(" today").unwrap(); + let mut dictation = Dictation::new(draft, end..start); + assert_eq!(dictation.transcript("new text", false).unwrap().range, start..end); + // Inside the crab: floored to its start. Past the end: clamped. + let inside = draft.find('🦀').unwrap() + 1; + assert_eq!(Dictation::new(draft, inside..inside).start, inside - 1); + assert_eq!(Dictation::new(draft, 999..999).start, draft.len()); + } + + #[test] + fn typing_mid_utterance_keeps_the_words_spoken_afterwards() { + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "hello world", false); + field.type_text(&mut dictation, "X"); + assert!(dictation.is_interrupted(), "still waiting for the utterance to finish"); + // Revisions in the meantime are not written under the user's edit. + field.say(&mut dictation, "hello world and", false); + assert_eq!(field.text, "hello worldX"); + field.say(&mut dictation, "hello world and more", true); + field.settle(&mut dictation); + assert_eq!(field.text, "hello worldX and more"); + assert!(!dictation.is_interrupted()); + } + + #[test] + fn typing_between_utterances_re_anchors_at_once() { + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "first", true); + field.type_text(&mut dictation, "X"); + assert!(!dictation.is_interrupted()); + field.say(&mut dictation, "second", true); + assert_eq!(field.text, "firstX second"); + + // Typing before anything was recognized at all. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.type_text(&mut dictation, "typed first"); + field.say(&mut dictation, "spoken after", true); + assert_eq!(field.text, "typed first spoken after"); + } + + #[test] + fn a_revision_after_an_edit_never_undoes_it_or_repeats_itself() { + // Backspace, then the same utterance again: nothing to add. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "typo here", false); + dictation.interrupt(); + field.text.pop(); + field.caret -= 1; + field.settle(&mut dictation); + field.say(&mut dictation, "typo here", false); + field.say(&mut dictation, "typo here.", true); + field.settle(&mut dictation); + assert_eq!(field.text, "typo her"); + + // Wiping the dictated text and typing over it. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "wipe me", false); + dictation.interrupt(); + field.text = "fresh".into(); + field.caret = 5; + field.settle(&mut dictation); + field.say(&mut dictation, "wipe me", true); + field.settle(&mut dictation); + field.say(&mut dictation, "then more", true); + assert_eq!(field.text, "fresh then more"); + + // A wholesale revision that shares no words with what was shown is + // dropped rather than repeated; speech resumes with the next utterance. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "keep these words", false); + dictation.interrupt(); + field.caret = 4; + field.settle(&mut dictation); + field.say(&mut dictation, "stale revision", true); + field.settle(&mut dictation); + assert_eq!(field.text, "keep these words"); + field.say(&mut dictation, "and more", true); + assert_eq!(field.text, "keep and more these words"); + } + + #[test] + fn a_final_that_arrives_during_the_edit_waits_for_it() { + // The final is fed before the edit lands, exactly as when both happen in + // one event; the words it adds still go after the edit. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "keep these words", false); + dictation.interrupt(); + assert_eq!(dictation.transcript("keep these words and more", true), None); + field.text = "keep words".into(); + field.caret = 5; + field.settle(&mut dictation); + assert_eq!(field.text, "keep and more words"); + + // Two finals in the same gap both make it in. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "hello", false); + dictation.interrupt(); + assert_eq!(dictation.transcript("hello world", true), None); + assert_eq!(dictation.transcript("again", true), None); + field.text = "hello!".into(); + field.caret = 6; + field.settle(&mut dictation); + assert_eq!(field.text, "hello! world again"); + } + + #[test] + fn settle_notices_changes_that_arrived_without_warning() { + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "hello", false); + // Nothing changed: not an interruption. + field.settle(&mut dictation); + assert!(!dictation.is_interrupted()); + // The caret moved: an interruption, resolved at the utterance boundary. + field.caret = 0; + field.settle(&mut dictation); + assert!(dictation.is_interrupted()); + field.say(&mut dictation, "hello there", true); + field.settle(&mut dictation); + // The caret follows the words, including the space that keeps them apart. + assert_eq!(field.text, "there hello"); + assert_eq!(field.caret, 6); + } + + #[test] + fn an_interruption_that_changed_nothing_is_called_off() { + // End with the caret already at the end, or a click on the caret: the + // field is untouched, so the utterance carries on, revisions included. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "I scream", false); + dictation.interrupt(); + assert_eq!(dictation.transcript("I scream for", false), None, "held while the edit is pending"); + field.settle(&mut dictation); + assert!(!dictation.is_interrupted()); + assert_eq!(field.text, "I scream for", "the held revision lands as soon as nothing changed"); + field.say(&mut dictation, "Ice cream please", true); + assert_eq!(field.text, "Ice cream please"); + + // The same when the final itself arrived while the interruption was pending. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "hello world", false); + dictation.interrupt(); + assert_eq!(dictation.transcript("Hello, world.", true), None); + field.settle(&mut dictation); + assert_eq!(field.text, "Hello, world."); + } + + #[test] + fn a_partial_that_arrives_during_an_edit_is_written_after_it() { + // Between utterances the user types, and the next utterance's first + // partial is fed before their edit lands. Nothing was in flight, so the + // edit re-anchors at once and the partial goes after it. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "first", true); + dictation.interrupt(); + assert_eq!(dictation.transcript("second", false), None); + field.text.push('X'); + field.caret += 1; + field.settle(&mut dictation); + assert_eq!(field.text, "firstX second"); + assert!(!dictation.is_interrupted()); + field.say(&mut dictation, "second one", true); + assert_eq!(field.text, "firstX second one"); + } + + #[test] + fn words_beyond_only_returns_what_was_not_already_shown() { + assert_eq!(words_beyond("hello world", "hello world and more"), "and more"); + // Re-casing and re-punctuating is not new speech. + assert_eq!(words_beyond("hello world", "Hello, world! And more"), "And more"); + assert_eq!(words_beyond("hello world", "Hello world."), ""); + assert_eq!(words_beyond("", "all of it"), "all of it"); + assert_eq!(words_beyond(" ", "all of it"), "all of it"); + // No shared leading words: lose the tail rather than repeat the draft. + assert_eq!(words_beyond("hello world", "goodbye everyone"), ""); + // A word inserted inside what is shown is not separable either. + assert_eq!(words_beyond("hello world", "hello big world and more"), ""); + assert_eq!(words_beyond("the cat sat", "the cat quickly sat down"), ""); + // A shorter final is a revision, not new speech. + assert_eq!(words_beyond("hello world and more", "hello world"), ""); + } +} diff --git a/crates/speech/src/lib.rs b/crates/speech/src/lib.rs new file mode 100644 index 0000000..c2a3323 --- /dev/null +++ b/crates/speech/src/lib.rs @@ -0,0 +1,322 @@ +//! Native streaming speech-to-text input and microphone capture for Rust apps. +//! +//! Recognition keeps going across as many utterances as the user speaks, until you +//! stop it. An "utterance" is just a chunk of words spoken by the user: partial +//! transcripts replace the current one, whereas a final transcript commits it. +//! +//! Note that speech always stays as plain text here. This crate never interprets +//! spoken works as commands or synthesizes key events from them, +//! so saying something like "enter" will just type the word "enter". +//! +//! [`Dictation`] turns those transcripts into edits of a text field, so an app +//! only has to apply the replacements it hands back. +//! +//! Apple builds need the Xcode command line tools (`xcode-select --install`), and +//! support macOS 11+ and iOS 13+. For macOS builds, set the `MACOSX_DEPLOYMENT_TARGET` +//! env var to 11.0 or newer for the whole Cargo invocation, including the final app. +//! This mostly just matters on Intel x86 macs, but it doesn't hurt to always set it. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; +use std::sync::atomic::{AtomicU64, Ordering}; + +pub mod dictation; +pub use dictation::{Dictation, Replacement}; + +#[cfg(any(target_os = "macos", target_os = "ios"))] +mod apple; +#[cfg(all(target_os = "android", native_speech_android))] +mod android; +#[cfg(target_os = "windows")] +mod windows; + +#[cfg(any(target_os = "macos", target_os = "ios"))] +use apple as backend; +#[cfg(all(target_os = "android", native_speech_android))] +use android as backend; +#[cfg(target_os = "windows")] +use windows as backend; + +#[derive(Clone, Debug)] +pub struct NativeSpeechOptions { + /// A BCP-47 locale (e.g., `"en-US"`), or the OS's default speech language. + pub locale: Option, + /// Prefer on-device recognition when the native service supports it. + /// Otherwise the system service may need a network connection. + pub prefer_on_device: bool, +} + +impl Default for NativeSpeechOptions { + fn default() -> Self { + Self { locale: None, prefer_on_device: true } + } +} + +/// The cause of a speech failure, so you can handle each one properly +/// instead of matching on message text. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +#[non_exhaustive] +pub enum SpeechErrorKind { + /// The user denied permission, or your app is missing the privacy + /// declaration it needs in order to even ask. + /// Usually you'll want to tell the user to enable audio and/or speech + /// permissions in their system settings. + PermissionDenied, + /// The audio input or speech recognition service just doesn't exist here, + /// so retrying will never work. + Unavailable, + /// The recognizer doesn't support the language you asked for. + Language, + /// The microphone is missing, busy, or changed mid-session. + /// Starting things again will likely work. + Audio, + /// Another recording is already active; only one session can be active at a time. + Busy, + /// Anything else that the OS reported. + Other, +} + +/// A [`SpeechErrorKind`] that you can match on, plus the message the OS gave us. +/// `Display` writes just that message, which is already worded for end users. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SpeechError { + kind: SpeechErrorKind, + message: String, +} + +impl SpeechError { + pub fn new(kind: SpeechErrorKind, message: impl Into) -> Self { + Self { kind, message: message.into() } + } + + pub fn kind(&self) -> SpeechErrorKind { + self.kind + } + + pub fn message(&self) -> &str { + &self.message + } +} + +impl std::fmt::Display for SpeechError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.message) + } +} + +impl std::error::Error for SpeechError {} + +#[derive(Clone, Debug, PartialEq)] +pub enum NativeSpeechEvent { + /// Permissions were granted and the microphone is actually recording. + Started, + /// Recognized text is available. + /// + /// If `is_final` is `false`, the text is a partial transcript that may be updated/changed later. + /// If `is_final` is `true`, the text is a final transcript that is fully "committed". + Transcript { + text: String, + is_final: bool, + }, + /// Microphone amplitude, normalized between 0 and 1, which you can use to + /// display a level meter or similar sound wave animation. + AudioLevel(f32), + /// Capture and final recognition are done. This is a final event. + Stopped, + /// A final error, e.g., denied permissions or unavailable hardware. + Error(SpeechError), +} + +/// Event codes shared with the Swift and Java bridges. The error kinds are just +/// extra values of the same `kind` argument, so classifying them needs no ABI change. +#[cfg(any(target_os = "macos", target_os = "ios", all(target_os = "android", native_speech_android)))] +#[allow(dead_code)] // Not every backend uses every code. +pub(crate) mod codes { + pub(crate) const STARTED: i32 = 0; + pub(crate) const PARTIAL: i32 = 1; + pub(crate) const FINAL: i32 = 2; + pub(crate) const LEVEL: i32 = 3; + pub(crate) const STOPPED: i32 = 4; + pub(crate) const ERROR: i32 = 5; + pub(crate) const ERROR_PERMISSION: i32 = 6; + pub(crate) const ERROR_UNAVAILABLE: i32 = 7; + pub(crate) const ERROR_LANGUAGE: i32 = 8; + pub(crate) const ERROR_AUDIO: i32 = 9; +} + +/// Unknown codes fall back to `Other`, so a newer bridge still reports its message. +#[cfg(any(target_os = "macos", target_os = "ios", all(target_os = "android", native_speech_android)))] +pub(crate) fn error_kind_from_code(code: i32) -> SpeechErrorKind { + match code { + codes::ERROR_PERMISSION => SpeechErrorKind::PermissionDenied, + codes::ERROR_UNAVAILABLE => SpeechErrorKind::Unavailable, + codes::ERROR_LANGUAGE => SpeechErrorKind::Language, + codes::ERROR_AUDIO => SpeechErrorKind::Audio, + _ => SpeechErrorKind::Other, + } +} + +type Callback = Arc; +static SESSIONS: OnceLock>> = OnceLock::new(); +static NEXT_SESSION: AtomicU64 = AtomicU64::new(1); + +fn sessions() -> &'static Mutex> { + SESSIONS.get_or_init(|| Mutex::new(HashMap::new())) +} + +/// Owns the system microphone and recognition task; dropping it cancels both. +/// +/// Importantly, only one session can be active at a time. +/// +/// Callbacks arrive on platform-dependent native OS threads, so you'll want to +/// forward them to your main UI thread or event loop. Note that it's possible for +/// an in-progress callback to continue executing after you cancel the session. +pub struct NativeSpeechSession { + id: u64, +} + +/// The short, stable name of the recognizer this build uses, e.g., `"apple-sfspeech"`. +/// Handy for logs and bug reports. +pub fn engine_name() -> &'static str { + backend::ENGINE +} + +impl NativeSpeechSession { + /// Whether this platform offers a native speech recognizer at all. + /// + /// This is false on Linux and the web, so you'll usually want to hide any + /// dictation button entirely rather than let it fail when pressed. + pub fn is_supported() -> bool { + backend::is_supported() + } + + /// Starts dictating, requesting microphone (and on Apple, speech recognition) + /// permission if needed, so you don't need any platform-specific permission code. + /// + /// This returns immediately rather than blocking on the permission prompt. + /// The `Started` event tells you when the microphone is actually recording, + /// and a refusal arrives as [`SpeechErrorKind::PermissionDenied`]. + pub fn start( + options: NativeSpeechOptions, + callback: impl Fn(NativeSpeechEvent) + Send + Sync + 'static, + ) -> Result { + if !Self::is_supported() { + return Err(SpeechError::new( + SpeechErrorKind::Unavailable, + "Native speech recognition is not available on this platform.", + )); + } + if options.locale.as_ref().is_some_and(|locale| locale.contains('\0')) { + return Err(SpeechError::new( + SpeechErrorKind::Language, + "The speech recognition language is invalid.", + )); + } + let id = NEXT_SESSION.fetch_add(1, Ordering::Relaxed); + { + let mut sessions = sessions().lock().unwrap(); + if !sessions.is_empty() { + return Err(SpeechError::new( + SpeechErrorKind::Busy, + "Another speech recording is already active.", + )); + } + sessions.insert(id, Arc::new(callback)); + } + if let Err(error) = backend::start(id, &options) { + sessions().lock().unwrap().remove(&id); + return Err(error); + } + Ok(Self { id }) + } + + /// Closes the microphone, but still lets the final recognition result arrive. + pub fn stop(&self) { + backend::stop(self.id, false); + } + + /// Stops immediately and discards anything that's still pending. + pub fn cancel(&self) { + sessions().lock().unwrap().remove(&self.id); + backend::stop(self.id, true); + } +} + +impl Drop for NativeSpeechSession { + fn drop(&mut self) { + self.cancel(); + } +} + +/// Cancels every active session, for things like suspending or quitting the app. +pub fn cancel_all() { + let ids: Vec<_> = sessions().lock().unwrap().drain().map(|(id, _)| id).collect(); + for id in ids { + backend::stop(id, true); + } +} + +#[cfg(any(test, target_os = "macos", target_os = "ios", all(target_os = "android", native_speech_android), target_os = "windows"))] +pub(crate) fn emit(id: u64, event: NativeSpeechEvent) { + let callback = { + let mut sessions = sessions().lock().unwrap(); + if matches!(event, NativeSpeechEvent::Stopped | NativeSpeechEvent::Error(_)) { + sessions.remove(&id) + } else { + sessions.get(&id).cloned() + } + }; + if let Some(callback) = callback { + // Never unwind through a native callback boundary. + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| callback(event))); + } +} + +#[cfg(not(any(target_os = "macos", target_os = "ios", all(target_os = "android", native_speech_android), target_os = "windows")))] +mod backend { + use super::{NativeSpeechOptions, SpeechError, SpeechErrorKind}; + pub(super) const ENGINE: &str = "none"; + pub(super) fn is_supported() -> bool { false } + pub(super) fn start(_: u64, _: &NativeSpeechOptions) -> Result<(), SpeechError> { + Err(SpeechError::new( + SpeechErrorKind::Unavailable, + "Native speech recognition is not available on this platform.", + )) + } + pub(super) fn stop(_: u64, _: bool) {} +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn terminal_and_cancelled_sessions_discard_late_native_results() { + let received = Arc::new(Mutex::new(Vec::new())); + let output = received.clone(); + sessions().lock().unwrap().insert(1001, Arc::new(move |event| { + output.lock().unwrap().push(event); + })); + emit(1001, NativeSpeechEvent::Transcript { text: "draft".into(), is_final: false }); + emit(1001, NativeSpeechEvent::Stopped); + emit(1001, NativeSpeechEvent::Transcript { text: "stale".into(), is_final: true }); + assert_eq!(*received.lock().unwrap(), vec![ + NativeSpeechEvent::Transcript { text: "draft".into(), is_final: false }, + NativeSpeechEvent::Stopped, + ]); + + let output = received.clone(); + sessions().lock().unwrap().insert(1002, Arc::new(move |event| { + output.lock().unwrap().push(event); + })); + // A previous session's delayed callback must not be delivered to its + // successor, even when recognition is restarted immediately. + emit(1001, NativeSpeechEvent::Error(SpeechError::new(SpeechErrorKind::Other, "late failure"))); + NativeSpeechSession { id: 1002 }.cancel(); + emit(1002, NativeSpeechEvent::Started); + assert_eq!(received.lock().unwrap().len(), 2); + let sessions = sessions().lock().unwrap(); + assert!(!sessions.contains_key(&1001)); + assert!(!sessions.contains_key(&1002)); + } +} diff --git a/crates/speech/src/windows.rs b/crates/speech/src/windows.rs new file mode 100644 index 0000000..ee0a8f5 --- /dev/null +++ b/crates/speech/src/windows.rs @@ -0,0 +1,440 @@ +//! Windows dictation through SAPI and its native microphone input. +//! +//! We use SAPI rather than the WinRT `SpeechRecognizer` because SAPI works in a +//! plain unpackaged desktop app, whereas WinRT dictation needs package identity. +//! SAPI also uses entirely on-device recognizers that are already installed. +//! +//! Note that every COM interface here stays on one dedicated MTA thread, +//! including when it gets released. +//! See + +use crate::{emit, NativeSpeechEvent, NativeSpeechOptions, SpeechError, SpeechErrorKind}; +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock, mpsc}; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::time::{Duration, Instant}; +use windows::core::{Interface, IUnknown, PCWSTR, PWSTR, HRESULT}; +use windows::Win32::Globalization::LocaleNameToLCID; +use windows::Win32::Media::Speech::*; +use windows::Win32::System::Com::{ + CoCreateInstance, CoInitializeEx, CoTaskMemFree, CoUninitialize, + CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED, +}; + +const RECORDING: u8 = 0; +const STOPPING: u8 = 1; +const CANCELLED: u8 = 2; + +struct Control { + state: AtomicU8, + wake: mpsc::Sender<()>, +} + +static CONTROLS: OnceLock>>> = OnceLock::new(); +// Cancellation can release the public session before its native worker has +// closed the microphone. Serialize native ownership across that short interval. +static MICROPHONE: Mutex<()> = Mutex::new(()); + +fn controls() -> &'static Mutex>> { + CONTROLS.get_or_init(|| Mutex::new(HashMap::new())) +} + +pub(super) fn is_supported() -> bool { + static SUPPORTED: OnceLock = OnceLock::new(); + *SUPPORTED.get_or_init(|| { + // Only query installed engine tokens: do not open the microphone or + // load a recognition model while checking whether to show the control. + // A separate apartment also avoids changing the UI thread's COM model. + std::thread::Builder::new().name("speech-availability".into()).spawn(|| { + let Ok(_apartment) = Apartment::new() else { return false }; + unsafe { + let Ok(category): Result = + CoCreateInstance(&SpObjectTokenCategory, None, CLSCTX_INPROC_SERVER) + else { return false }; + if category.SetId(SPCAT_RECOGNIZERS, false).is_err() { return false; } + let Ok(tokens) = category.EnumTokens(PCWSTR::null(), PCWSTR::null()) else { return false }; + let mut count = 0; + tokens.GetCount(&mut count).is_ok() && count > 0 + } + }).ok().and_then(|thread| thread.join().ok()).unwrap_or(false) + }) +} + +pub(super) fn start(id: u64, options: &NativeSpeechOptions) -> Result<(), SpeechError> { + let (wake, receiver) = mpsc::channel(); + let control = Arc::new(Control { state: AtomicU8::new(RECORDING), wake }); + controls().lock().unwrap_or_else(|poisoned| poisoned.into_inner()).insert(id, control.clone()); + let options = options.clone(); + let spawned = std::thread::Builder::new().name("native-speech".into()).spawn(move || { + worker(id, || { + if control.state.load(Ordering::Acquire) != RECORDING { + Ok(()) + } else { + run(id, &options, &control, &receiver) + } + }); + }); + if let Err(error) = spawned { + controls().lock().unwrap_or_else(|poisoned| poisoned.into_inner()).remove(&id); + return Err(SpeechError::new(SpeechErrorKind::Other, format!("Could not start Windows speech recognition: {error}"))); + } + Ok(()) +} + +fn worker(id: u64, run: impl FnOnce() -> Result<(), SpeechError>) { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _microphone = MICROPHONE.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + run() + })).unwrap_or_else(|_| Err(SpeechError::new(SpeechErrorKind::Other, "Windows speech recognition stopped unexpectedly. Please try again."))); + controls().lock().unwrap_or_else(|poisoned| poisoned.into_inner()).remove(&id); + // COM/audio resources unwind before this terminal callback releases the + // public session, including when an unexpected worker panic occurs. + emit(id, match result { + Ok(()) => NativeSpeechEvent::Stopped, + Err(error) => NativeSpeechEvent::Error(error), + }); +} + +pub(super) fn stop(id: u64, cancel: bool) { + if let Some(control) = controls().lock().unwrap_or_else(|poisoned| poisoned.into_inner()).get(&id) { + // A repeated graceful stop must never undo a cancellation. + control.state.fetch_max(if cancel { CANCELLED } else { STOPPING }, Ordering::AcqRel); + let _ = control.wake.send(()); + } +} + +struct Apartment; + +impl Apartment { + fn new() -> Result { + unsafe { CoInitializeEx(None, COINIT_MULTITHREADED).ok() } + .map_err(|error| failure("Could not initialize Windows speech recognition", error))?; + Ok(Self) + } +} + +impl Drop for Apartment { + fn drop(&mut self) { + unsafe { CoUninitialize() }; + } +} + +struct Recognition { + recognizer: ISpRecognizer, + context: ISpRecoContext, + _grammar: ISpRecoGrammar, +} + +impl Drop for Recognition { + fn drop(&mut self) { + // This is our in-process recognizer, so stopping it cannot affect + // another application's dictation or Windows voice controls. + let _ = unsafe { self.recognizer.SetRecoState(SPRST_INACTIVE_WITH_PURGE) }; + } +} + +pub(super) const ENGINE: &str = "windows-sapi"; + +fn failure(context: &str, error: windows::core::Error) -> SpeechError { + failure_kind(SpeechErrorKind::Other, context, error) +} + +fn failure_kind(kind: SpeechErrorKind, context: &str, error: windows::core::Error) -> SpeechError { + SpeechError::new(kind, format!("{context}. Check Windows microphone access and that a speech recognition language is installed. {error}")) +} + +fn run( + id: u64, + options: &NativeSpeechOptions, + control: &Control, + receiver: &mpsc::Receiver<()>, +) -> Result<(), SpeechError> { + // Declared first so COM is uninitialized after every interface is dropped. + let _apartment = Apartment::new()?; + let recognition = create_recognition(options)?; + if control.state.load(Ordering::Acquire) != RECORDING { + return Ok(()); + } + unsafe { recognition._grammar.SetDictationState(SPRS_ACTIVE) } + .map_err(|error| failure("Could not open the microphone for dictation", error))?; + if control.state.load(Ordering::Acquire) == CANCELLED { + return Ok(()); + } + emit(id, NativeSpeechEvent::Started); + let mut stop_deadline = None; + let mut partial = String::new(); + loop { + let state = control.state.load(Ordering::Acquire); + if state == CANCELLED { + return Ok(()); + } + if state == STOPPING && stop_deadline.is_none() { + // INACTIVE closes native capture and lets buffered audio finish. + // PURGE is reserved for cancellation and the cleanup guard. + unsafe { recognition.recognizer.SetRecoState(SPRST_INACTIVE) } + .map_err(|error| failure("Could not finish speech recognition", error))?; + stop_deadline = Some(Instant::now() + Duration::from_secs(5)); + } + while let Some(event) = next_event(&recognition.context)? { + if control.state.load(Ordering::Acquire) == CANCELLED { + return Ok(()); + } + if handle_event(id, &event, &mut partial, control, stop_deadline.is_some())? { + return Ok(()); + } + } + if stop_deadline.is_some_and(|deadline| Instant::now() >= deadline) { + // Some engines never send END_SR_STREAM on a silent recording. + // Preserve the visible hypothesis and always release the device. + finish_partial(id, &mut partial, control); + return Ok(()); + } + // SAPI queues events on its own worker threads. Poll only while this + // session exists; stop/cancel wakes this wait immediately. + let _ = receiver.recv_timeout(Duration::from_millis(20)); + } +} + +/// Returns true when SAPI has ended this stream. +fn handle_event(id: u64, event: &Event, partial: &mut String, control: &Control, stopping: bool) -> Result { + match SPEVENTENUM(event.0._bitfield & 0xffff) { + SPEI_HYPOTHESIS | SPEI_RECOGNITION => { + let text = event.text()?; + let is_final = event.0._bitfield & 0xffff == SPEI_RECOGNITION.0; + if is_final { + partial.clear(); + } else { + partial.clone_from(&text); + } + emit(id, NativeSpeechEvent::Transcript { text, is_final }); + } + SPEI_FALSE_RECOGNITION => { + if control.state.load(Ordering::Acquire) == STOPPING { + // Stopping mid-phrase can make an engine reject its buffered + // tail. Keep the words already shown instead of erasing them. + finish_partial(id, partial, control); + } else { + // Remove a rejected utterance while recording, without + // committing speech the engine did not recognize. + partial.clear(); + emit(id, NativeSpeechEvent::Transcript { text: String::new(), is_final: true }); + } + } + SPEI_SR_AUDIO_LEVEL => { + emit(id, NativeSpeechEvent::AudioLevel((event.0.wParam.0 as f32 / 100.0).clamp(0.0, 1.0))); + } + SPEI_END_SR_STREAM => { + // Finalize the displayed hypothesis even when the stream ended + // with an error, before the terminal callback releases the session. + finish_partial(id, partial, control); + let status = HRESULT(event.0.lParam.0 as i32); + if let Err(error) = status.ok() { + return Err(failure_kind(SpeechErrorKind::Audio, "Windows speech recognition lost its audio input", error)); + } + if !stopping { + return Err(SpeechError::new(SpeechErrorKind::Audio, "Windows speech recognition stopped receiving microphone audio. Check the microphone and try again.")); + } + return Ok(true); + } + _ => {} + } + Ok(false) +} + +fn finish_partial(id: u64, partial: &mut String, control: &Control) { + if !partial.is_empty() && control.state.load(Ordering::Acquire) != CANCELLED { + emit(id, NativeSpeechEvent::Transcript { text: std::mem::take(partial), is_final: true }); + } +} + +fn create_recognition(options: &NativeSpeechOptions) -> Result { + unsafe { + let recognizer: ISpRecognizer = CoCreateInstance(&SpInprocRecognizer, None, CLSCTX_INPROC_SERVER) + .map_err(|error| failure_kind(SpeechErrorKind::Unavailable, "Windows speech recognition is unavailable", error))?; + // SAPI's recognizers run on-device, also when prefer_on_device is false. + if let Some(locale) = options.locale.as_deref() { + let name: Vec = locale.encode_utf16().chain(Some(0)).collect(); + let language = LocaleNameToLCID(PCWSTR(name.as_ptr()), 0) & 0xffff; + if language == 0 { + return Err(SpeechError::new(SpeechErrorKind::Language, format!("Windows does not recognize the speech language {locale}."))); + } + let category: ISpObjectTokenCategory = CoCreateInstance(&SpObjectTokenCategory, None, CLSCTX_INPROC_SERVER) + .map_err(|error| failure("Could not find Windows speech recognition languages", error))?; + category.SetId(SPCAT_RECOGNIZERS, false) + .map_err(|error| failure("Could not find Windows speech recognition languages", error))?; + let attributes: Vec = format!("Language={language:x}").encode_utf16().chain(Some(0)).collect(); + let tokens = category.EnumTokens(PCWSTR(attributes.as_ptr()), PCWSTR::null()) + .map_err(|error| failure_kind(SpeechErrorKind::Language, "Could not find the requested speech recognition language", error))?; + let token = tokens.Item(0) + .map_err(|_| SpeechError::new(SpeechErrorKind::Language, format!("Install the Windows speech recognition language for {locale} before dictating in that language.")))?; + recognizer.SetRecognizer(&token) + .map_err(|error| failure_kind(SpeechErrorKind::Language, "Could not use the requested speech recognition language", error))?; + } + let input: ISpObjectToken = default_token(SPCAT_AUDIOIN) + .map_err(|error| failure_kind(SpeechErrorKind::Audio, "No Windows microphone is available", error))?; + recognizer.SetInput(&input, true) + .map_err(|error| failure_kind(SpeechErrorKind::Audio, "Could not select the Windows microphone", error))?; + let context = recognizer.CreateRecoContext() + .map_err(|error| failure("Could not create Windows dictation", error))?; + // SPFEI also includes these two reserved flags, as required by SAPI. + let interest = [SPEI_HYPOTHESIS, SPEI_RECOGNITION, SPEI_FALSE_RECOGNITION, + SPEI_SR_AUDIO_LEVEL, SPEI_END_SR_STREAM, SPEI_RESERVED1, SPEI_RESERVED2] + .into_iter().fold(0, |mask, event| mask | (1u64 << event.0)); + context.SetInterest(interest, interest) + .map_err(|error| failure("Could not subscribe to Windows dictation", error))?; + let grammar = context.CreateGrammar(1) + .map_err(|error| failure("Could not create the dictation grammar", error))?; + grammar.LoadDictation(PCWSTR::null(), SPLO_STATIC) + .map_err(|error| failure_kind(SpeechErrorKind::Language, "The installed Windows speech language does not support dictation", error))?; + Ok(Recognition { recognizer, context, _grammar: grammar }) + } +} + +unsafe fn default_token(category_id: PCWSTR) -> windows::core::Result { + unsafe { + let category: ISpObjectTokenCategory = CoCreateInstance(&SpObjectTokenCategory, None, CLSCTX_INPROC_SERVER)?; + category.SetId(category_id, false)?; + let id = ComText(category.GetDefaultTokenId()?); + let token: ISpObjectToken = CoCreateInstance(&SpObjectToken, None, CLSCTX_INPROC_SERVER)?; + token.SetId(PCWSTR::null(), PCWSTR(id.0.0), false)?; + Ok(token) + } +} + +struct ComText(PWSTR); + +impl Drop for ComText { + fn drop(&mut self) { + unsafe { CoTaskMemFree(Some(self.0.0.cast())) }; + } +} + +struct Event(SPEVENT); + +impl Event { + fn text(&self) -> Result { + let raw = self.0.lParam.0 as *mut std::ffi::c_void; + unsafe { + let result = ISpRecoResult::from_raw_borrowed(&raw) + .ok_or_else(|| SpeechError::new(SpeechErrorKind::Other, "Windows speech recognition returned an empty result."))?; + let mut text = ComText(PWSTR::null()); + result.GetText(0, u32::MAX, true, &mut text.0, None) + .map_err(|error| failure("Could not read Windows dictation text", error))?; + if text.0.is_null() { + return Ok(String::new()); + } + text.0.to_string().map_err(|_| SpeechError::new(SpeechErrorKind::Other, "Windows speech recognition returned invalid text.")) + } + } +} + +impl Drop for Event { + fn drop(&mut self) { + let pointer = self.0.lParam.0 as *mut std::ffi::c_void; + if pointer.is_null() { + return; + } + // SpClearEvent's ownership rules, including failed/ignored events. + match SPEVENTLPARAMTYPE((self.0._bitfield >> 16) & 0xffff) { + SPET_LPARAM_IS_TOKEN | SPET_LPARAM_IS_OBJECT => unsafe { + drop(IUnknown::from_raw(pointer)); + }, + SPET_LPARAM_IS_POINTER | SPET_LPARAM_IS_STRING => unsafe { + CoTaskMemFree(Some(pointer)); + }, + _ => {} + } + } +} + +fn next_event(context: &ISpRecoContext) -> Result, SpeechError> { + let mut event = Event(SPEVENT::default()); + let mut fetched = 0; + unsafe { context.GetEvents(1, &mut event.0, &mut fetched) } + .map_err(|error| failure("Could not receive Windows dictation", error))?; + Ok((fetched != 0).then_some(event)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn recording() -> (u64, Arc>>) { + let id = crate::NEXT_SESSION.fetch_add(1, Ordering::Relaxed) + 10_000; + let events = Arc::new(Mutex::new(Vec::new())); + let output = events.clone(); + crate::sessions().lock().unwrap().insert(id, Arc::new(move |event| { + output.lock().unwrap().push(event); + })); + (id, events) + } + + #[test] + fn a_panicking_worker_releases_the_session_and_does_not_block_its_successor() { + let (id, events) = recording(); + let (wake, _receiver) = mpsc::channel(); + controls().lock().unwrap().insert(id, Arc::new(Control { state: AtomicU8::new(RECORDING), wake })); + worker(id, || panic!("injected worker failure; no COM or microphone is used")); + assert!(matches!(events.lock().unwrap().as_slice(), [NativeSpeechEvent::Error(_)])); + assert!(!crate::sessions().lock().unwrap().contains_key(&id)); + assert!(!controls().lock().unwrap().contains_key(&id)); + + let (next, next_events) = recording(); + worker(next, || Ok(())); + assert_eq!(*next_events.lock().unwrap(), vec![NativeSpeechEvent::Stopped]); + } + + #[test] + fn rejecting_the_last_utterance_during_stop_preserves_it_exactly_once() { + let (id, events) = recording(); + let (wake, _receiver) = mpsc::channel(); + let control = Control { state: AtomicU8::new(STOPPING), wake }; + let mut pending = "last spoken words".to_owned(); + let mut event = Event(SPEVENT::default()); + worker(id, || { + event.0._bitfield = SPEI_FALSE_RECOGNITION.0; + assert!(!handle_event(id, &event, &mut pending, &control, true)?); + event.0._bitfield = SPEI_END_SR_STREAM.0; + assert!(handle_event(id, &event, &mut pending, &control, true)?); + Ok(()) + }); + assert_eq!(*events.lock().unwrap(), vec![ + NativeSpeechEvent::Transcript { text: "last spoken words".into(), is_final: true }, + NativeSpeechEvent::Stopped, + ]); + } + + #[test] + fn a_failed_stream_finalizes_visible_words_before_reporting_the_error() { + let (id, events) = recording(); + let (wake, _receiver) = mpsc::channel(); + let control = Control { state: AtomicU8::new(STOPPING), wake }; + let mut pending = "keep this interrupted utterance".to_owned(); + let mut event = Event(SPEVENT::default()); + event.0._bitfield = SPEI_END_SR_STREAM.0; + event.0.lParam.0 = 0x80004005u32 as i32 as isize; // E_FAIL, not an owned pointer. + worker(id, || handle_event(id, &event, &mut pending, &control, true).map(|_| ())); + let events = events.lock().unwrap(); + assert!(matches!(events.as_slice(), [ + NativeSpeechEvent::Transcript { text, is_final: true }, NativeSpeechEvent::Error(_) + ] if text == "keep this interrupted utterance")); + } + + #[test] + fn normal_rejection_clears_the_hypothesis_and_cancellation_does_not_commit_it() { + let (id, events) = recording(); + let (wake, _receiver) = mpsc::channel(); + let control = Control { state: AtomicU8::new(RECORDING), wake }; + let mut pending = "unrecognized words".to_owned(); + let mut event = Event(SPEVENT::default()); + event.0._bitfield = SPEI_FALSE_RECOGNITION.0; + assert!(!handle_event(id, &event, &mut pending, &control, false).unwrap()); + assert_eq!(*events.lock().unwrap(), vec![NativeSpeechEvent::Transcript { text: String::new(), is_final: true }]); + assert!(pending.is_empty()); + control.state.store(CANCELLED, Ordering::Release); + pending = "discard this queued result".to_owned(); + finish_partial(id, &mut pending, &control); + assert_eq!(events.lock().unwrap().len(), 1); + crate::sessions().lock().unwrap().remove(&id); + } +} diff --git a/crates/speech/swift/NativeSpeech.swift b/crates/speech/swift/NativeSpeech.swift new file mode 100644 index 0000000..377d210 --- /dev/null +++ b/crates/speech/swift/NativeSpeech.swift @@ -0,0 +1,568 @@ +// macOS and iOS dictation, using SFSpeechRecognizer plus AVAudioEngine. +// +// This side asks for both permissions, owns the microphone, and restarts the +// recognizer between utterances. Rust only sees the events sent to the callback +// below. Note that session state lives on the main queue; only the audio tap runs +// elsewhere, and it hands its buffers over through a synchronized capture sink. + +import Foundation +import AVFoundation +import Speech + +@_silgen_name("robius_speech_event") +private func nativeEvent(_ id: UInt64, _ kind: Int32, _ text: UnsafePointer?, _ level: Float) + +// Session/recognizer state lives on the main queue. Audio taps use a synchronized +// capture sink and marshal meter updates and errors back to that queue. +private var sessions: [UInt64: NativeSpeech] = [:] + +// Error kinds, matching `codes` in lib.rs. Not private: the regression test +// asserts against these rather than repeating the numbers. +let errOther: Int32 = 5 +let errPermission: Int32 = 6 +let errUnavailable: Int32 = 7 +let errLanguage: Int32 = 8 +let errAudio: Int32 = 9 + +#if os(iOS) +// The HFP spelling was introduced by the iOS 26 SDK. This older spelling has +// the identical value and also compiles with older supported Xcode releases. +private let speechAudioOptions: AVAudioSession.CategoryOptions = [.defaultToSpeaker, .allowBluetooth, .mixWithOthers] +#endif + +// NotificationCenter waits for an observer even when it targets OperationQueue.main. +// Audio-engine notifications originate on an internal queue that engine teardown +// also waits for. Return to that queue before touching the engine on the main queue. +func observeAudioNotification(_ name: Notification.Name, object: Any?, handler: @escaping () -> Void) -> NSObjectProtocol { + NotificationCenter.default.addObserver(forName: name, object: object, queue: nil) { _ in + DispatchQueue.main.async { handler() } + } +} + +struct SpeechRetryPolicy { + private(set) var consecutiveNoSpeech = 0 + + mutating func madeProgress() { consecutiveNoSpeech = 0 } + + mutating func retryDelay(after error: NSError) -> TimeInterval? { + // Apple's documented no-speech error. Code 203 is a generic failure, + // not a documented timeout; authorization/network failures stay visible. + // https://developer.apple.com/documentation/speech/sfspeechrecognitiontask/error + guard error.domain == "kAFAssistantErrorDomain", error.code == 1110, consecutiveNoSpeech < 3 else { return nil } + consecutiveNoSpeech += 1 + return 0.25 * Double(1 << (consecutiveNoSpeech - 1)) + } +} + +// The engine keeps recording while a recognizer finalizes an utterance. Only +// that short interval needs copies; otherwise buffers go straight to the request. +// No engine operations or UI callbacks run while this lock is held. +final class SpeechAudioCapture { + private let lock = NSLock() + private var appendToRequest: ((AVAudioPCMBuffer) -> Void)? + private var requestGeneration: UInt64? + private var pending: [AVAudioPCMBuffer] = [] + private var pendingDuration: TimeInterval = 0 + private var active = false + private var overflowed = false + + func prepareRequest(_ generation: UInt64) { + lock.lock() + active = true + requestGeneration = generation + appendToRequest = nil + lock.unlock() + } + + func beginRequest(generation: UInt64? = nil, _ append: @escaping (AVAudioPCMBuffer) -> Void) { + lock.lock() + defer { lock.unlock() } + // A task may finish immediately, before recognitionTask() even returns. + // Do not bind its request after that callback has already detached it. + if let generation = generation, requestGeneration != generation { return } + active = true + appendToRequest = append + for buffer in pending { append(buffer) } + pending.removeAll(keepingCapacity: true) + pendingDuration = 0 + } + + func endRequest(ifCurrent generation: UInt64? = nil) { + lock.lock() + defer { lock.unlock() } + if let generation = generation, requestGeneration != generation { return } + requestGeneration = nil + appendToRequest = nil + } + + // False reports one overflow/allocation error. Further buffers are discarded + // until the main queue handles that terminal error and stops capture. + func append(_ buffer: AVAudioPCMBuffer) -> Bool { + lock.lock() + defer { lock.unlock() } + guard active, !overflowed else { return true } + if let append = appendToRequest { append(buffer); return true } + let duration = Double(buffer.frameLength) / buffer.format.sampleRate + guard duration.isFinite, pendingDuration + duration <= 4, + let copy = AVAudioPCMBuffer(pcmFormat: buffer.format, frameCapacity: buffer.frameLength) + else { + overflowed = true + pending.removeAll() + pendingDuration = 0 + return false + } + copy.frameLength = buffer.frameLength + let source = UnsafeMutableAudioBufferListPointer(UnsafeMutablePointer(mutating: buffer.audioBufferList)) + let destination = UnsafeMutableAudioBufferListPointer(copy.mutableAudioBufferList) + guard source.count == destination.count, + source.indices.allSatisfy({ source[$0].mDataByteSize <= destination[$0].mDataByteSize }) + else { + overflowed = true + pending.removeAll() + pendingDuration = 0 + return false + } + for index in source.indices { + if let from = source[index].mData, let to = destination[index].mData { + memcpy(to, from, Int(source[index].mDataByteSize)) + } + } + pending.append(copy) + pendingDuration += duration + return true + } + + var hasPendingAudio: Bool { + lock.lock() + defer { lock.unlock() } + return !pending.isEmpty + } + + func stop(discardPending: Bool = true) { + lock.lock() + active = false + requestGeneration = nil + appendToRequest = nil + if discardPending { + pending.removeAll() + pendingDuration = 0 + } + lock.unlock() + } +} + +final class NativeSpeech { + let id: UInt64 + let locale: Locale? + let preferOnDevice: Bool + // Do not initialize audio hardware before the privacy checks and permissions. + var engine: AVAudioEngine? + let capture = SpeechAudioCapture() + var recognizer: SFSpeechRecognizer? + var request: SFSpeechAudioBufferRecognitionRequest? + var task: SFSpeechRecognitionTask? + var stopping = false + var finished = false + var tapInstalled = false + var started = false + var lastPartial = "" + var generation: UInt64 = 0 + var retryPolicy = SpeechRetryPolicy() + var endingUtterance = false + var rollover: DispatchWorkItem? + var pendingRestart: DispatchWorkItem? + var observers: [NSObjectProtocol] = [] + #if os(iOS) + var previousAudioConfiguration: (AVAudioSession.Category, AVAudioSession.Mode, AVAudioSession.CategoryOptions)? + var activatedAudioSession = false + #endif + + init(id: UInt64, locale: String, preferOnDevice: Bool) { + self.id = id + self.locale = locale.isEmpty ? nil : Locale(identifier: locale) + self.preferOnDevice = preferOnDevice + } + + func send(_ kind: Int32, _ text: String = "", _ level: Float = 0) { + guard !finished else { return } + text.withCString { nativeEvent(id, kind, $0, level) } + } + + func authorize() { + let usageKeys = ["NSMicrophoneUsageDescription", "NSSpeechRecognitionUsageDescription"] + #if os(macOS) + // An embedded __info_plist satisfies Bundle.main, but TCC can still + // attribute an unbundled process to its launching terminal or editor. + // Those apps lack our speech description, and TCC terminates the process. + let bundle = Bundle.main.bundleURL + guard bundle.pathExtension == "app", + let data = try? Data(contentsOf: bundle.appendingPathComponent("Contents/Info.plist")), + let plist = (try? PropertyListSerialization.propertyList(from: data, format: nil)) as? [String: Any], + usageKeys.allSatisfy({ (plist[$0] as? String)?.isEmpty == false }) + else { + fail("Launch the application from its .app bundle to use speech input.", errPermission) + return + } + #endif + // Missing privacy keys cause a process termination in Apple's APIs. + // Report an actionable error before invoking either permission prompt. + for key in usageKeys { + guard let reason = Bundle.main.object(forInfoDictionaryKey: key) as? String, !reason.isEmpty else { + fail("This app is missing its \(key) privacy description.", errPermission) + return + } + } + #if ROBIUS_SPEECH_TESTS + // Native lifecycle regressions must never request real microphone or + // speech permission, including when testing an invalid launch context. + preconditionFailure("Unexpected permission request in native speech tests") + #else + SFSpeechRecognizer.requestAuthorization { [weak self] status in + DispatchQueue.main.async { + guard let self = self, !self.finished, !self.stopping else { return } + guard status == .authorized else { + self.fail("Speech recognition permission was denied. Allow speech recognition for this app in system privacy settings.", errPermission) + return + } + AVCaptureDevice.requestAccess(for: .audio) { [weak self] granted in + DispatchQueue.main.async { + guard let self = self, !self.finished, !self.stopping else { return } + guard granted else { + self.fail("Microphone permission was denied. Allow microphone access for this app in system privacy settings.", errPermission) + return + } + self.start() + } + } + } + } + #endif + } + + func start() { + // Use the native default-language selection rather than a region-based + // Locale.current such as en_NL. Explicit locales retain Apple's built-in + // fallback to the keyboard's dictation language if unsupported. + let selectedRecognizer = locale.map { SFSpeechRecognizer(locale: $0) } ?? SFSpeechRecognizer() + guard let recognizer = selectedRecognizer, recognizer.isAvailable else { + fail("Speech recognition is unavailable for the current language. Check the system speech settings and network connection.", errLanguage) + return + } + self.recognizer = recognizer + recognizer.defaultTaskHint = .dictation + // A busy UI must not delay detaching a completed task's audio sink. + // Only the synchronized sink is touched here; session state stays on main. + let callbacks = OperationQueue() + callbacks.name = "org.robius.speech.recognition" + callbacks.maxConcurrentOperationCount = 1 + callbacks.qualityOfService = .userInitiated + recognizer.queue = callbacks + #if os(iOS) + do { + let audio = AVAudioSession.sharedInstance() + previousAudioConfiguration = (audio.category, audio.mode, audio.categoryOptions) + try audio.setCategory(.playAndRecord, mode: .measurement, options: speechAudioOptions) + try audio.setActive(true) + activatedAudioSession = true + } catch { + fail("Could not activate the microphone: \(error.localizedDescription)", errAudio) + return + } + observers.append(observeAudioNotification(AVAudioSession.interruptionNotification, object: nil) { [weak self] in + guard let self = self, !self.stopping, !self.finished else { return } + self.fail("Speech recording was interrupted by another audio session.", errAudio) + }) + #endif + let engine = AVAudioEngine() + self.engine = engine + observers.append(observeAudioNotification(.AVAudioEngineConfigurationChange, object: engine) { [weak self] in + guard let self = self, !self.stopping, !self.finished else { return } + self.fail("The microphone changed or disconnected. Start dictation again to use the new microphone.", errAudio) + }) + beginUtterance() + } + + func beginUtterance(finishing: Bool = false) { + guard !finished, (!stopping || finishing), let recognizer = recognizer, let engine = engine else { return } + guard recognizer.isAvailable else { + // isAvailable also drops on a brief network loss for server-backed + // locales, so this is retryable rather than a missing recognizer. + fail("The system speech recognition service became unavailable.") + return + } + generation &+= 1 + let utterance = generation + endingUtterance = false + let request = SFSpeechAudioBufferRecognitionRequest() + request.shouldReportPartialResults = true + request.taskHint = .dictation + if preferOnDevice && recognizer.supportsOnDeviceRecognition { + request.requiresOnDeviceRecognition = true + } + if #available(macOS 13.0, iOS 16.0, *) { request.addsPunctuation = true } + self.request = request + lastPartial = "" + let capture = self.capture + capture.prepareRequest(utterance) + task = recognizer.recognitionTask(with: request) { [weak self] result, error in + if result?.isFinal == true || error != nil { + // Start buffering immediately, before the main queue handles the + // transcript. A late previous callback cannot detach its successor. + capture.endRequest(ifCurrent: utterance) + } + DispatchQueue.main.async { + guard let self = self, !self.finished, self.generation == utterance else { return } + if let result = result { + self.lastPartial = result.bestTranscription.formattedString + if !self.lastPartial.isEmpty { self.retryPolicy.madeProgress() } + self.send(result.isFinal ? 2 : 1, self.lastPartial) + if result.isFinal { + self.lastPartial = "" + if self.stopping { self.finishStoppedUtterance() } + else { self.restartUtterance(after: 0.15) } + return + } + } + if let error = error { + if self.stopping { + // Preserve the most recent partial if a system recognizer + // ends without delivering a final result after endAudio. + self.finishStoppedUtterance() + } else if self.endingUtterance { + self.commitPartial() + self.restartUtterance(after: 0.15) + } else if let delay = self.retryPolicy.retryDelay(after: error as NSError) { + self.commitPartial() + self.restartUtterance(after: delay) + } else { + self.fail("Speech recognition failed: \(error.localizedDescription)") + } + } + } + } + capture.beginRequest(generation: utterance) { request.append($0) } + if finishing { endRequest(); scheduleStopTimeout(); return } + // The existing tap now targets the new request, including PCM saved + // during finalization/backoff. Keeping the engine running avoids gaps. + if tapInstalled { scheduleRollover(utterance); return } + let input = engine.inputNode + let format = input.outputFormat(forBus: 0) + guard format.sampleRate > 0 && format.channelCount > 0 else { + fail("No working microphone is available.", errAudio) + return + } + var meterFrames: AVAudioFrameCount = 0 + input.installTap(onBus: 0, bufferSize: 1024, format: format) { [weak self] buffer, _ in + if !capture.append(buffer) { + DispatchQueue.main.async { [weak self] in + self?.fail("Speech recognition took too long to resume. Your draft has been kept; please start dictation again.") + } + } + meterFrames += buffer.frameLength + guard meterFrames >= AVAudioFrameCount(format.sampleRate / 30) else { return } + meterFrames = 0 + guard let samples = buffer.floatChannelData, buffer.frameLength > 0 else { return } + var energy: Float = 0 + let channels = Int(buffer.format.channelCount) + let count = Int(buffer.frameLength) + for channel in 0.., _ preferOnDevice: Bool) { + let locale = String(cString: locale) + DispatchQueue.main.async { + let session = NativeSpeech(id: id, locale: locale, preferOnDevice: preferOnDevice) + sessions[id] = session + session.authorize() + } +} + +@_cdecl("robius_speech_stop") +public func stopNativeSpeech(_ id: UInt64, _ cancel: Bool) { + DispatchQueue.main.async { sessions[id]?.stop(cancel: cancel) } +} diff --git a/crates/speech/swift/tests/SpeechTests.swift b/crates/speech/swift/tests/SpeechTests.swift new file mode 100644 index 0000000..8afbeca --- /dev/null +++ b/crates/speech/swift/tests/SpeechTests.swift @@ -0,0 +1,269 @@ +// Host-side regressions for the Swift bridge, driving NativeSpeech.swift's own +// types directly. Note that nothing here opens a microphone or touches a +// permission API, so these are safe to run anywhere macOS builds. + +import Foundation +import AVFoundation + +private var nativeEvents: [(id: UInt64, kind: Int32, text: String)] = [] + +// The production bridge links this callback from Rust. The test build traps +// before any permission API, even if the unbundled-launch guard regresses. +@_cdecl("robius_speech_event") +func testNativeSpeechEvent(_ id: UInt64, _ kind: Int32, _ text: UnsafePointer?, _ level: Float) { + precondition(Thread.isMainThread) + nativeEvents.append((id, kind, text.map(String.init(cString:)) ?? "")) +} + +@main +enum SpeechTests { + static let cases: [(String, () -> Void)] = [ + ("audio notifications defer to the main queue", audioNotificationsDeferToMainQueue), + ("an unbundled launch is rejected", unbundledLaunchIsRejected), + ("silence retries are bounded", silenceRetriesAreBounded), + ("capture buffers audio across a request handoff", captureBuffersAcrossHandoff), + ("capture stops allocating when a recognizer stalls", captureStopsAllocatingWhenStalled), + ("capture keeps every buffer under concurrent handoff", captureKeepsEveryBufferConcurrently), + ("capture binds audio to the current request generation", captureBindsAudioToItsGeneration), + ("utterance rollover commits the draft", rolloverCommitsTheDraft), + ("cancelling during rollover emits nothing", cancellingDuringRolloverIsSilent), + ("a graceful stop replays buffered speech", gracefulStopReplaysBufferedSpeech), + ] + + static func main() { + for (name, run) in cases { + // Flush before running: a precondition failure aborts the process, + // and piped stdout would otherwise lose the name of the failing case. + print(" \(name)") + fflush(stdout) + run() + } + print("Passed \(cases.count) native speech regressions; no microphone used.") + } + + // MARK: - Notification delivery + + static func audioNotificationsDeferToMainQueue() { + let name = Notification.Name("RobiusSpeech.NotificationDeliveryTest") + let source = NSObject() + let posted = DispatchSemaphore(value: 0) + var received = 0 + let observer = observeAudioNotification(name, object: source) { + precondition(Thread.isMainThread, "Audio state must remain on the main thread") + received += 1 + } + defer { NotificationCenter.default.removeObserver(observer) } + + DispatchQueue.global().async { + NotificationCenter.default.post(name: name, object: source) + posted.signal() + } + // Model the main thread waiting for an engine operation. The engine's + // notification-publishing worker must be able to return independently. + precondition(posted.wait(timeout: .now() + 2) == .success, "Audio notification blocked its posting thread waiting for the main queue") + precondition(received == 0, "Observer ran before the main queue could process it") + drainMainQueue(until: { received == 1 }) + + // Even a notification posted on main must defer engine teardown until + // the posting operation has returned, rather than reentering the engine. + NotificationCenter.default.post(name: name, object: source) + precondition(received == 1, "Audio notification ran its handler reentrantly") + drainMainQueue(until: { received == 2 }) + } + + // MARK: - Privacy and permissions + + static func unbundledLaunchIsRejected() { + // Reproduce cargo-run's misleading embedded privacy descriptions. TCC + // can ignore them and attribute access to the parent editor instead. + precondition(Bundle.main.bundleURL.pathExtension != "app") + for key in ["NSMicrophoneUsageDescription", "NSSpeechRecognitionUsageDescription"] { + precondition(Bundle.main.object(forInfoDictionaryKey: key) as? String != nil, "The test executable must embed its privacy descriptions") + } + nativeEvents.removeAll() + "".withCString { startNativeSpeech(42, $0, true) } + drainMainQueue(until: { !nativeEvents.isEmpty }) + precondition(nativeEvents.count == 1) + precondition(nativeEvents[0].id == 42 && nativeEvents[0].kind == errPermission) + precondition(nativeEvents[0].text == "Launch the application from its .app bundle to use speech input.") + } + + // MARK: - Retry policy + + static func silenceRetriesAreBounded() { + var retries = SpeechRetryPolicy() + let noSpeech = NSError(domain: "kAFAssistantErrorDomain", code: 1110) + precondition(retries.retryDelay(after: noSpeech) == 0.25) + precondition(retries.retryDelay(after: noSpeech) == 0.5) + precondition(retries.retryDelay(after: noSpeech) == 1.0) + precondition(retries.retryDelay(after: noSpeech) == nil, "Silence retries must be bounded") + retries.madeProgress() + precondition(retries.retryDelay(after: noSpeech) == 0.25, "Recognized speech resets the retry budget") + for code in [203, 1100, 1101, 1107, 1700] { + precondition(retries.retryDelay(after: NSError(domain: "kAFAssistantErrorDomain", code: code)) == nil, "A generic failure is not a documented timeout") + } + precondition(retries.retryDelay(after: NSError(domain: "UnrelatedDomain", code: 1110)) == nil) + } + + // MARK: - Audio capture + + static func captureBuffersAcrossHandoff() { + let capture = SpeechAudioCapture() + var samples: [Float] = [] + capture.beginRequest { samples.append($0.floatChannelData![0][0]) } + precondition(capture.append(audioBuffer(sample: 1))) + capture.endRequest() + let reusedBuffer = audioBuffer(sample: 2) + precondition(capture.append(reusedBuffer)) + reusedBuffer.floatChannelData![0][0] = 99 + precondition(capture.append(audioBuffer(sample: 3))) + precondition(samples == [1], "Finalizing a task must buffer new audio") + capture.beginRequest { samples.append($0.floatChannelData![0][0]) } + precondition(capture.append(audioBuffer(sample: 4))) + precondition(samples == [1, 2, 3, 4], "Copied rollover audio must precede live audio without reuse corruption") + capture.endRequest() + precondition(capture.append(audioBuffer(sample: 5))) + capture.stop(discardPending: false) + precondition(capture.hasPendingAudio, "Graceful stop must keep buffered speech for final recognition") + precondition(capture.append(audioBuffer(sample: 99))) + capture.beginRequest { samples.append($0.floatChannelData![0][0]) } + precondition(samples == [1, 2, 3, 4, 5], "Final recognition must replay saved speech but no post-stop audio") + capture.stop() + } + + static func captureStopsAllocatingWhenStalled() { + let bounded = SpeechAudioCapture() + bounded.beginRequest { _ in } + bounded.endRequest() + for _ in 0..<4 { precondition(bounded.append(audioBuffer(sample: 0, frames: 8_000))) } + precondition(!bounded.append(audioBuffer(sample: 0)), "A stalled recognizer must not allocate unbounded PCM storage") + precondition(bounded.append(audioBuffer(sample: 0)), "Buffer overflow must report one terminal failure") + precondition(!bounded.hasPendingAudio) + bounded.stop() + } + + static func captureKeepsEveryBufferConcurrently() { + let concurrent = SpeechAudioCapture() + var ordered: [Int] = [] + let receive: (AVAudioPCMBuffer) -> Void = { ordered.append(Int($0.floatChannelData![0][0])) } + concurrent.beginRequest(receive) + let producerFinished = DispatchSemaphore(value: 0) + DispatchQueue.global().async { + for value in 0..<2_000 { precondition(concurrent.append(audioBuffer(sample: Float(value)))) } + producerFinished.signal() + } + for _ in 0..<200 { + concurrent.endRequest() + concurrent.beginRequest(receive) + } + precondition(producerFinished.wait(timeout: .now() + 2) == .success) + concurrent.beginRequest(receive) + precondition(ordered == Array(0..<2_000), "Concurrent request handoff must preserve every PCM buffer in order") + concurrent.stop() + } + + static func captureBindsAudioToItsGeneration() { + let generations = SpeechAudioCapture() + var oldRequest: [Float] = [] + var newRequest: [Float] = [] + generations.prepareRequest(10) + generations.beginRequest(generation: 10) { oldRequest.append($0.floatChannelData![0][0]) } + precondition(generations.append(audioBuffer(sample: 10))) + let detached = DispatchSemaphore(value: 0) + DispatchQueue.global().async { + generations.endRequest(ifCurrent: 10) + precondition(generations.append(audioBuffer(sample: 11))) + detached.signal() + } + // Simulate the UI being blocked while a terminal task callback arrives. + precondition(detached.wait(timeout: .now() + 2) == .success) + precondition(oldRequest == [10], "PCM after a terminal callback must buffer without waiting for UI dispatch") + generations.prepareRequest(20) + generations.beginRequest(generation: 20) { newRequest.append($0.floatChannelData![0][0]) } + generations.endRequest(ifCurrent: 10) + precondition(generations.append(audioBuffer(sample: 12))) + precondition(newRequest == [11, 12], "A delayed old callback must not detach the active successor") + generations.endRequest(ifCurrent: 20) + generations.prepareRequest(30) + generations.endRequest(ifCurrent: 30) + generations.beginRequest(generation: 30) { _ in preconditionFailure("An already-completed request must not be rebound") } + precondition(generations.append(audioBuffer(sample: 13))) + generations.prepareRequest(40) + generations.beginRequest(generation: 40) { newRequest.append($0.floatChannelData![0][0]) } + precondition(newRequest == [11, 12, 13], "Immediate task completion must preserve subsequent PCM for its successor") + generations.stop() + } + + // MARK: - Session lifecycle + + static func rolloverCommitsTheDraft() { + nativeEvents.removeAll() + let rollover = NativeSpeech(id: 43, locale: "", preferOnDevice: true) + precondition(rollover.locale == nil, "An unspecified locale must use native default-language selection") + precondition(rollover.engine == nil, "Creating a session must not initialize microphone hardware") + rollover.started = true + rollover.generation = 7 + rollover.lastPartial = "Keep this draft" + rollover.endUtteranceBeforeLimit(6, finalizationTimeout: 0) + precondition(!rollover.endingUtterance, "An old utterance timer must not affect its successor") + rollover.endUtteranceBeforeLimit(7, finalizationTimeout: 0) + drainMainQueue(until: { !nativeEvents.isEmpty }) + precondition(nativeEvents.count == 1 && nativeEvents[0].kind == 2 && nativeEvents[0].text == "Keep this draft") + precondition(rollover.generation == 8 && rollover.pendingRestart != nil) + rollover.stop(cancel: false) + precondition(rollover.finished && rollover.pendingRestart == nil, "Stopping during backoff must cancel the pending restart") + precondition(nativeEvents.count == 2 && nativeEvents[1].kind == 4) + } + + static func cancellingDuringRolloverIsSilent() { + let before = nativeEvents.count + let cancelled = NativeSpeech(id: 44, locale: "en-US", preferOnDevice: true) + cancelled.generation = 1 + cancelled.lastPartial = "Cancelled partial" + cancelled.endUtteranceBeforeLimit(1, finalizationTimeout: 0) + cancelled.stop(cancel: true) + var drained = false + DispatchQueue.main.async { drained = true } + drainMainQueue(until: { drained }) + precondition(nativeEvents.count == before, "Cancelled rollover must not emit a transcript or restart") + } + + static func gracefulStopReplaysBufferedSpeech() { + nativeEvents.removeAll() + let stopping = NativeSpeech(id: 45, locale: "", preferOnDevice: true) + stopping.stopping = true + stopping.lastPartial = "Previous utterance" + stopping.capture.beginRequest { _ in } + stopping.capture.endRequest() + precondition(stopping.capture.append(audioBuffer(sample: 6))) + stopping.endCapture(discardPending: false) + stopping.finishStoppedUtterance() + precondition(nativeEvents.count == 1 && nativeEvents[0].text == "Previous utterance") + precondition(!stopping.finished && stopping.pendingRestart != nil, "Queued speech must finish before the stopped event") + var replayed: [Float] = [] + stopping.capture.beginRequest { replayed.append($0.floatChannelData![0][0]) } + precondition(replayed == [6]) + stopping.lastPartial = "Buffered final words" + stopping.finishStoppedUtterance() + precondition(nativeEvents.map(\.kind) == [2, 2, 4]) + precondition(nativeEvents[1].text == "Buffered final words" && stopping.finished) + } + + // MARK: - Helpers + + private static func audioBuffer(sample: Float, frames: AVAudioFrameCount = 1) -> AVAudioPCMBuffer { + let format = AVAudioFormat(standardFormatWithSampleRate: 8_000, channels: 1)! + let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frames)! + buffer.frameLength = frames + for frame in 0.. Bool) { + let deadline = Date(timeIntervalSinceNow: 2) + while !complete() && Date() < deadline { + _ = RunLoop.main.run(mode: .default, before: Date(timeIntervalSinceNow: 0.01)) + } + precondition(complete(), "Deferred audio notification was never delivered") + } +} diff --git a/crates/speech/swift/tests/run.sh b/crates/speech/swift/tests/run.sh new file mode 100755 index 0000000..f3d7e47 --- /dev/null +++ b/crates/speech/swift/tests/run.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Builds NativeSpeech.swift together with its regressions and runs them. +# You need macOS and the Xcode command line tools. No microphone is used. +set -euo pipefail + +tests_dir=$(cd "$(dirname "$0")" && pwd) +tmp=$(mktemp -d "${TMPDIR:-/tmp}/robius-speech-tests.XXXXXX") +trap 'rm -rf "$tmp"' EXIT + +# The privacy keys must be embedded for the unbundled-launch regression, which +# checks that the bridge still refuses to touch a permission API without a bundle. +cat > "$tmp/Info.plist" <<'PLIST' + + + + + CFBundleIdentifier + org.robius.speech.tests + NSMicrophoneUsageDescription + Privacy validation regression only; these tests do not access the microphone. + NSSpeechRecognitionUsageDescription + Privacy validation regression only; these tests do not request speech recognition. + + +PLIST + +xcrun --sdk macosx swiftc -swift-version 5 -O -parse-as-library -D ROBIUS_SPEECH_TESTS \ + "$tests_dir/../NativeSpeech.swift" "$tests_dir/SpeechTests.swift" \ + -Xlinker -sectcreate -Xlinker __TEXT -Xlinker __info_plist -Xlinker "$tmp/Info.plist" \ + -o "$tmp/speech-tests" +"$tmp/speech-tests" diff --git a/crates/speech/tests/android_retry_test.py b/crates/speech/tests/android_retry_test.py new file mode 100644 index 0000000..71d965a --- /dev/null +++ b/crates/speech/tests/android_retry_test.py @@ -0,0 +1,734 @@ +#!/usr/bin/env python3 +"""Runs the real Android session logic against deterministic host-side fakes. + +You need a JDK and a C compiler. Note that this never talks to a device or opens +a microphone: the fakes stand in for the Android APIs, and the JNI shim just +records the events that the production NativeSpeech.java emits. +""" + +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + + +SOURCES = { + "android/Manifest.java": """package android; +public class Manifest { public static class permission { public static final String RECORD_AUDIO = "audio"; } } +""", + "android/content/Context.java": "package android.content; public class Context {}", + "android/content/Intent.java": """package android.content; +public class Intent { + public Intent(String action) {} + public Intent putExtra(String key, String value) { return this; } + public Intent putExtra(String key, boolean value) { return this; } + public Intent putExtra(String key, int value) { return this; } +} +""", + "android/content/pm/PackageManager.java": """package android.content.pm; +public class PackageManager { public static final int PERMISSION_GRANTED = 0; } +""", + "android/app/Activity.java": """package android.app; +public class Activity extends android.content.Context { + private final Application application; + private final FragmentManager fragments = new FragmentManager(this); + // Set by tests: 0 grants, -1 denies until the user answers the prompt. + public static int permission = 0; + public boolean resumed = true; + private boolean destroyed, changingConfigurations; + public Activity() { this(new Application()); } + public Activity(Application application) { this.application = application; } + public boolean isFinishing() { return false; } + public boolean isDestroyed() { return destroyed; } + public boolean isChangingConfigurations() { return changingConfigurations; } + public int checkSelfPermission(String name) { return permission; } + public Application getApplication() { return application; } + public FragmentManager getFragmentManager() { return fragments; } + public ClassLoader getClassLoader() { return Activity.class.getClassLoader(); } + public void runOnUiThread(Runnable runnable) { runnable.run(); } + public void pause() { + if (!resumed) return; + resumed = false; + fragments.pauseAll(); + application.paused(this); + } + public void resume() { + if (destroyed || resumed) return; + resumed = true; + application.resumed(this); + fragments.resumeAll(); + } + public void destroy(boolean configurationChange) { + changingConfigurations = configurationChange; + pause(); + destroyed = true; + fragments.destroy(); + application.destroyed(this); + } +} +""", + "android/app/Fragment.java": """package android.app; +public class Fragment { + private Activity activity; + private FragmentManager manager; + private final FragmentManager children = new FragmentManager(this); + public static Fragment pending; + public static int requests; + void attach(Activity activity, FragmentManager manager) { + this.activity = activity; this.manager = manager; + } + void detach() { this.activity = null; this.manager = null; } + public Activity getActivity() { return activity; } + public FragmentManager getFragmentManager() { return manager; } + public FragmentManager getChildFragmentManager() { return children; } + public boolean isAdded() { return manager != null && manager.contains(this); } + public void setRetainInstance(boolean retain) { + if (retain) throw new AssertionError("permission helpers must not retain discarded Activities"); + } + public void onCreate(android.os.Bundle state) {} + public void onResume() {} + public void onPause() {} + public void onDestroy() {} + public void onDetach() {} + public void onSaveInstanceState(android.os.Bundle state) {} + public void onRequestPermissionsResult(int code, String[] names, int[] results) {} + // Records the in-flight request so a test can answer it, as the OS would. + public void requestPermissions(String[] names, int code) { + if (pending != null) throw new AssertionError("a second system prompt was requested while one was pending"); + requests++; + pending = this; pendingCode = code; + activity.pause(); + } + public static int pendingCode; + /** Answer the prompt the way the user would. */ + public static void answer(boolean granted) { + Fragment fragment = pending; pending = null; + if (fragment == null) throw new AssertionError("no permission request is pending"); + Activity.permission = granted ? 0 : -1; + Activity owner = fragment.getActivity(); + // Activity.findFragmentByWho drops results for a detached request fragment. + if (owner != null) { + fragment.onRequestPermissionsResult(pendingCode, new String[] {"audio"}, + new int[] { granted ? 0 : -1 }); + owner.resume(); + } + } +} +""", + "android/app/FragmentManager.java": """package android.app; +public class FragmentManager { + private final Activity activity; + private final Fragment parent; + private boolean destroyed, executing; + private final java.util.LinkedHashMap byTag = new java.util.LinkedHashMap<>(); + private final java.util.ArrayList callbacks = new java.util.ArrayList<>(); + private final java.util.ArrayList transactions = new java.util.ArrayList<>(); + public FragmentManager(Activity activity) { this.activity = activity; this.parent = null; } + public FragmentManager(Fragment parent) { this.activity = null; this.parent = parent; } + private Activity activity() { return parent == null ? activity : parent.getActivity(); } + public boolean isDestroyed() { return destroyed; } + public boolean isStateSaved() { return false; } + public Fragment findFragmentByTag(String tag) { return byTag.get(tag); } + public FragmentTransaction beginTransaction() { return new FragmentTransaction(this); } + public boolean contains(Fragment fragment) { return byTag.containsValue(fragment); } + public void registerFragmentLifecycleCallbacks(FragmentLifecycleCallbacks callback, boolean recursive) { + callbacks.add(callback); + } + public void unregisterFragmentLifecycleCallbacks(FragmentLifecycleCallbacks callback) { callbacks.remove(callback); } + public int callbackCount() { return callbacks.size(); } + public static abstract class FragmentLifecycleCallbacks { + public void onFragmentSaveInstanceState(FragmentManager manager, Fragment fragment, android.os.Bundle state) {} + public void onFragmentDetached(FragmentManager manager, Fragment fragment) {} + public void onFragmentDestroyed(FragmentManager manager, Fragment fragment) {} + } + void commit(Runnable action, boolean immediate) { + if (destroyed) return; // commitAllowingStateLoss drops work on a destroyed host. + if (immediate) { execute(action); return; } + transactions.add(action); + new android.os.Handler(android.os.Looper.getMainLooper()).post(() -> { + if (transactions.remove(action) && !destroyed) execute(action); + }); + } + private void execute(Runnable action) { + if (executing) throw new IllegalStateException("FragmentManager is already executing transactions"); + executing = true; + try { action.run(); } finally { executing = false; } + } + public boolean executePendingTransactions() { + boolean any = !transactions.isEmpty(); + for (Runnable action : new java.util.ArrayList<>(transactions)) { + transactions.remove(action); + if (!destroyed) execute(action); + } + return any; + } + void add(String tag, Fragment fragment) { + byTag.put(tag, fragment); + fragment.attach(activity(), this); + fragment.onCreate(null); + if (activity().resumed) fragment.onResume(); + } + void remove(Fragment fragment) { + if (!contains(fragment)) return; + byTag.values().remove(fragment); + fragment.getChildFragmentManager().destroy(); + fragment.onDestroy(); + for (FragmentLifecycleCallbacks callback : new java.util.ArrayList<>(callbacks)) { + callback.onFragmentDestroyed(this, fragment); + } + fragment.onDetach(); + fragment.detach(); + for (FragmentLifecycleCallbacks callback : new java.util.ArrayList<>(callbacks)) { + callback.onFragmentDetached(this, fragment); + } + } + public void pauseAll() { + for (Fragment fragment : new java.util.ArrayList<>(byTag.values())) { + fragment.getChildFragmentManager().pauseAll(); fragment.onPause(); + } + } + public void resumeAll() { + for (Fragment fragment : new java.util.ArrayList<>(byTag.values())) { + fragment.onResume(); fragment.getChildFragmentManager().resumeAll(); + } + } + public void destroy() { + destroyed = true; + transactions.clear(); + for (Fragment fragment : new java.util.ArrayList<>(byTag.values())) remove(fragment); + } + public static final class SavedFragment { + public final String className, tag; + public final android.os.Bundle state; + SavedFragment(Fragment fragment, String tag, android.os.Bundle state) { + this.className = fragment.getClass().getName(); this.tag = tag; this.state = state; + } + } + public java.util.List saveAllState() { + executePendingTransactions(); + java.util.List result = new java.util.ArrayList<>(); + for (java.util.Map.Entry entry : byTag.entrySet()) { + Fragment fragment = entry.getValue(); + android.os.Bundle state = new android.os.Bundle(); + // The framework snapshots the class name before requesting this Bundle. + SavedFragment saved = new SavedFragment(fragment, entry.getKey(), state); + fragment.onSaveInstanceState(state); + java.util.List childState = fragment.getChildFragmentManager().saveAllState(); + if (!childState.isEmpty()) state.put("children", childState); + // Android's callback runs after performSaveInstanceState saved children. + for (FragmentLifecycleCallbacks callback : new java.util.ArrayList<>(callbacks)) { + callback.onFragmentSaveInstanceState(this, fragment, state); + } + result.add(saved); + } + return result; + } + @SuppressWarnings("unchecked") + public void restoreAllState(java.util.List saved, ClassLoader loader) throws Exception { + for (SavedFragment entry : saved) { + Fragment fragment = (Fragment) loader.loadClass(entry.className).getConstructor().newInstance(); + byTag.put(entry.tag, fragment); + fragment.attach(activity(), this); + fragment.onCreate(entry.state); + Object children = entry.state.get("children"); + if (children != null) fragment.getChildFragmentManager().restoreAllState((java.util.List) children, loader); + } + } + public int count() { return byTag.size(); } +} +""", + "android/app/FragmentTransaction.java": """package android.app; +public class FragmentTransaction { + private final FragmentManager manager; + private String tag; private Fragment added, removed; + public FragmentTransaction(FragmentManager manager) { this.manager = manager; } + public FragmentTransaction add(Fragment fragment, String tag) { + this.added = fragment; this.tag = tag; return this; + } + public FragmentTransaction remove(Fragment fragment) { this.removed = fragment; return this; } + public void commitAllowingStateLoss() { manager.commit(this::apply, false); } + public void commitNowAllowingStateLoss() { manager.commit(this::apply, true); } + private void apply() { + if (removed != null) manager.remove(removed); + if (added != null) manager.add(tag, added); + } +} +""", + "android/app/Application.java": """package android.app; +public class Application { + private final java.util.ArrayList callbacks = new java.util.ArrayList<>(); + public void registerActivityLifecycleCallbacks(ActivityLifecycleCallbacks callback) { callbacks.add(callback); } + public void unregisterActivityLifecycleCallbacks(ActivityLifecycleCallbacks callback) { callbacks.remove(callback); } + public void paused(Activity activity) { + for (ActivityLifecycleCallbacks callback : new java.util.ArrayList<>(callbacks)) callback.onActivityPaused(activity); + } + public void resumed(Activity activity) { + for (ActivityLifecycleCallbacks callback : new java.util.ArrayList<>(callbacks)) callback.onActivityResumed(activity); + } + public void destroyed(Activity activity) { + for (ActivityLifecycleCallbacks callback : new java.util.ArrayList<>(callbacks)) callback.onActivityDestroyed(activity); + } + public interface ActivityLifecycleCallbacks { + void onActivityCreated(Activity activity, android.os.Bundle state); + void onActivityStarted(Activity activity); + void onActivityResumed(Activity activity); + void onActivityPaused(Activity activity); + void onActivityStopped(Activity activity); + void onActivitySaveInstanceState(Activity activity, android.os.Bundle state); + void onActivityDestroyed(Activity activity); + } +} +""", + "android/os/Build.java": """package android.os; +public class Build { public static class VERSION { public static int SDK_INT = 33; } } +""", + "android/os/Bundle.java": """package android.os; +public class Bundle { + private java.util.ArrayList values; + private final java.util.HashMap data = new java.util.HashMap<>(); + public void clear() { data.clear(); values = null; } + public boolean isEmpty() { return data.isEmpty() && values == null; } + public void put(String key, Object value) { data.put(key, value); } + public Object get(String key) { return data.get(key); } + public void putString(String key, String value) { data.put(key, value); } + public String getString(String key) { return (String) data.get(key); } + public void putStringArrayList(String key, java.util.ArrayList value) { values = value; } + public java.util.ArrayList getStringArrayList(String key) { return values; } +} +""", + "android/os/Looper.java": """package android.os; +public class Looper { public static Looper getMainLooper() { return new Looper(); } } +""", + "android/os/Handler.java": """package android.os; +public class Handler { + private static final class Task { + Runnable runnable; long time; + Task(Runnable runnable, long time) { this.runnable = runnable; this.time = time; } + } + private static final java.util.ArrayList tasks = new java.util.ArrayList<>(); + private static long now; + public Handler(Looper looper) {} + public boolean post(Runnable runnable) { return postDelayed(runnable, 0); } + public boolean postDelayed(Runnable runnable, long delay) { + tasks.add(new Task(runnable, now + delay)); + tasks.sort(java.util.Comparator.comparingLong(task -> task.time)); + return true; + } + public void removeCallbacks(Runnable runnable) { tasks.removeIf(task -> task.runnable == runnable); } + public static int count() { return tasks.size(); } + public static long nextDelay() { return tasks.get(0).time - now; } + public static void runNext() { + Task task = tasks.remove(0); + now = task.time; + task.runnable.run(); + } + public static void reset() { tasks.clear(); now = 0; } + public static void drain() { + int limit = 100; + while (!tasks.isEmpty() && tasks.get(0).time == now) { + if (--limit == 0) throw new AssertionError("unbounded immediate work"); + runNext(); + } + } +} +""", + "harness/HostStateFragment.java": """package harness; +public class HostStateFragment extends android.app.Fragment { + public String value = "host draft must survive"; + @Override public void onCreate(android.os.Bundle state) { + if (state != null) value = state.getString("draft"); + } + @Override public void onSaveInstanceState(android.os.Bundle state) { state.putString("draft", value); } +} +""", + "harness/Launcher.java": """package harness; +public class Launcher { + public static void main(String[] args) throws Exception { + java.net.URL url = new java.io.File(args[0]).toURI().toURL(); + try (java.net.URLClassLoader child = new java.net.URLClassLoader(new java.net.URL[] {url}, Launcher.class.getClassLoader())) { + Class test = child.loadClass("dev.robius.speech.NativeSpeechRetryTest"); + test.getMethod("main", String[].class).invoke(null, (Object) new String[] {args[1]}); + } + } +} +""", + "android/speech/RecognitionListener.java": """package android.speech; +public interface RecognitionListener { + void onReadyForSpeech(android.os.Bundle params); + void onBeginningOfSpeech(); + void onRmsChanged(float rms); + void onBufferReceived(byte[] buffer); + void onEndOfSpeech(); + void onError(int error); + void onResults(android.os.Bundle results); + void onPartialResults(android.os.Bundle results); + void onEvent(int type, android.os.Bundle params); +} +""", + "android/speech/RecognizerIntent.java": """package android.speech; +public class RecognizerIntent { + public static final String ACTION_RECOGNIZE_SPEECH = "speech", EXTRA_LANGUAGE_MODEL = "model", + LANGUAGE_MODEL_FREE_FORM = "free", EXTRA_PARTIAL_RESULTS = "partial", EXTRA_MAX_RESULTS = "max", + EXTRA_LANGUAGE = "language", EXTRA_PREFER_OFFLINE = "offline"; +} +""", + "android/speech/SpeechRecognizer.java": """package android.speech; +public class SpeechRecognizer { + public static final String RESULTS_RECOGNITION = "results"; + public static final int ERROR_NETWORK_TIMEOUT = 1, ERROR_NETWORK = 2, ERROR_AUDIO = 3, + ERROR_CLIENT = 5, ERROR_SPEECH_TIMEOUT = 6, ERROR_NO_MATCH = 7, + ERROR_RECOGNIZER_BUSY = 8, ERROR_INSUFFICIENT_PERMISSIONS = 9; + public static SpeechRecognizer last; + public static int starts; + public RecognitionListener listener; + public boolean destroyed; + public static boolean isRecognitionAvailable(android.content.Context context) { return true; } + public static boolean isOnDeviceRecognitionAvailable(android.content.Context context) { return false; } + public static SpeechRecognizer createSpeechRecognizer(android.content.Context context) { + return last = new SpeechRecognizer(); + } + public static SpeechRecognizer createOnDeviceSpeechRecognizer(android.content.Context context) { + return createSpeechRecognizer(context); + } + public void setRecognitionListener(RecognitionListener listener) { this.listener = listener; } + public void startListening(android.content.Intent intent) { starts++; } + public void stopListening() {} + public void cancel() {} + public void destroy() { destroyed = true; } +} +""", + "dev/robius/speech/NativeSpeechRetryTest.java": """package dev.robius.speech; +import android.app.Activity; +import android.os.Bundle; +import android.os.Handler; +import android.speech.RecognitionListener; +import android.speech.SpeechRecognizer; + +public class NativeSpeechRetryTest { + private static final java.util.ArrayList events = new java.util.ArrayList<>(); + private static final java.util.ArrayList eventIds = new java.util.ArrayList<>(); + public static void record(long id, int kind, String text, float level) { + events.add(kind + ":" + (text == null ? "" : text)); + eventIds.add(id); + } + private static void check(boolean condition, String message) { + if (!condition) throw new AssertionError(message + " events=" + events); + } + private static Bundle words(String text) { + Bundle bundle = new Bundle(); + bundle.putStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION, + new java.util.ArrayList<>(java.util.Arrays.asList(text))); + return bundle; + } + private static void start(long id) { + Handler.reset(); events.clear(); eventIds.clear(); SpeechRecognizer.starts = 0; + NativeSpeech.start(new Activity(), id, "", false); + Handler.runNext(); + SpeechRecognizer.last.listener.onReadyForSpeech(new Bundle()); + } + private static void busyIsBounded() { + start(1); + for (int retry = 0; retry < 3; retry++) { + RecognitionListener old = SpeechRecognizer.last.listener; + old.onError(SpeechRecognizer.ERROR_RECOGNIZER_BUSY); + check(Handler.count() == 1 && Handler.nextDelay() == (500L << retry), "bounded exponential backoff"); + int count = events.size(); + old.onPartialResults(words("stale")); + old.onError(SpeechRecognizer.ERROR_RECOGNIZER_BUSY); + check(events.size() == count && Handler.count() == 1, "late callbacks cannot schedule retries or insert words"); + Handler.runNext(); + SpeechRecognizer.last.listener.onReadyForSpeech(new Bundle()); + } + SpeechRecognizer.last.listener.onError(SpeechRecognizer.ERROR_RECOGNIZER_BUSY); + check(Handler.count() == 0 && SpeechRecognizer.last.destroyed, "exhausted retry budget releases recognizer"); + check(events.stream().filter(event -> event.startsWith("5:")).count() == 1, "one terminal error after retries"); + check(SpeechRecognizer.starts == 4, "initial attempt and only three retries"); + } + private static void clientRecreatesAndSuccessResetsBudget() { + start(2); + SpeechRecognizer old = SpeechRecognizer.last; + old.listener.onPartialResults(words("keep this")); + old.listener.onError(SpeechRecognizer.ERROR_CLIENT); + check(old.destroyed && events.contains("2:keep this"), "client reconnect commits visible text and destroys old client"); + Handler.runNext(); + check(SpeechRecognizer.last != old, "client error creates a fresh recognizer"); + int count = events.size(); + old.listener.onResults(words("obsolete")); + check(events.size() == count, "old client cannot append late final results"); + SpeechRecognizer.last.listener.onResults(words("next utterance")); + check(Handler.nextDelay() == 150, "successful utterance resumes normal cadence"); + Handler.runNext(); + SpeechRecognizer.last.listener.onError(SpeechRecognizer.ERROR_RECOGNIZER_BUSY); + check(Handler.nextDelay() == 500, "completed utterance resets transient failure budget"); + NativeSpeech.stop(2, true); + Handler.runNext(); + check(Handler.count() == 0, "cancel removes delayed reconnect"); + } + private static void stopDuringBackoffNeverRestarts() { + start(3); + SpeechRecognizer.last.listener.onError(SpeechRecognizer.ERROR_RECOGNIZER_BUSY); + NativeSpeech.stop(3, false); + Handler.runNext(); + check(Handler.count() == 0 && SpeechRecognizer.starts == 1, "stop during backoff does not restart capture"); + check(events.stream().filter(event -> event.equals("4:")).count() == 1, "graceful stop emits one terminal event"); + } + private static void permissionIsRequestedThenSessionProceeds() { + Handler.reset(); events.clear(); SpeechRecognizer.starts = 0; + android.app.Activity.permission = -1; // not granted yet + android.app.Activity activity = new android.app.Activity(); + NativeSpeech.start(activity, 10, "", false); + Handler.runNext(); + check(android.app.Fragment.pending != null, "a missing permission must raise the system prompt"); + check(SpeechRecognizer.starts == 0, "recognition must not begin before the prompt is answered"); + android.app.Fragment.answer(true); + check(activity.getFragmentManager().callbackCount() == 1, + "the save-state guard remains until queued removal actually detaches its host"); + Handler.drain(); + check(activity.getFragmentManager().count() == 0, "the fragment removes itself once answered"); + SpeechRecognizer.last.listener.onReadyForSpeech(new Bundle()); + check(SpeechRecognizer.starts == 1 && events.contains("0:"), "granting resumes the session"); + NativeSpeech.stop(10, true); + Handler.drain(); + android.app.Activity.permission = 0; + } + private static void permissionDenialIsReportedOnce() { + Handler.reset(); events.clear(); SpeechRecognizer.starts = 0; + android.app.Activity.permission = -1; + android.app.Activity activity = new android.app.Activity(); + NativeSpeech.start(activity, 11, "", false); + Handler.runNext(); + android.app.Fragment.answer(false); + Handler.drain(); + check(SpeechRecognizer.starts == 0, "a denied prompt must not open the microphone"); + check(events.size() == 1 && events.get(0).startsWith("6:"), "denial reports PermissionDenied once, events=" + events); + check(activity.getFragmentManager().count() == 0, "the fragment removes itself after a denial"); + android.app.Activity.permission = 0; + } + private static void permissionsAreNotRetried() { + start(4); + SpeechRecognizer.last.listener.onError(SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS); + check(Handler.count() == 0 && SpeechRecognizer.last.destroyed, "permission errors stay terminal"); + } + private static Activity missingPermission() { + Handler.reset(); events.clear(); eventIds.clear(); SpeechRecognizer.starts = 0; + Activity.permission = -1; + check(android.app.Fragment.pending == null, "the preceding case must release its system prompt"); + android.app.Fragment.requests = 0; + return new Activity(); + } + private static void cancelDiscardsLateGrant() { + Activity activity = missingPermission(); + NativeSpeech.start(activity, 20, "", false); + Handler.runNext(); + NativeSpeech.stop(20, true); + Handler.drain(); + check(events.isEmpty() && SpeechRecognizer.starts == 0, "canceling a pending session is silent"); + android.app.Fragment.answer(true); + Handler.drain(); + check(events.isEmpty() && SpeechRecognizer.starts == 0, "a late grant cannot restart a canceled session"); + check(activity.getFragmentManager().count() == 0 && activity.getFragmentManager().callbackCount() == 0, + "answering an abandoned request releases its parent and save listener"); + } + private static void nextSessionAdoptsPendingPrompt() { + Activity activity = missingPermission(); + NativeSpeech.start(activity, 21, "", false); + Handler.runNext(); + android.app.Fragment pending = android.app.Fragment.pending; + // Both operations are queued before the old request receives its answer. + NativeSpeech.stop(21, true); + NativeSpeech.start(activity, 22, "", false); + Handler.drain(); + check(events.isEmpty(), "a successor must not receive a false permission denial"); + check(android.app.Fragment.pending == pending && android.app.Fragment.requests == 1, + "the successor adopts the existing system prompt without requesting a second one"); + android.app.Fragment.answer(true); + Handler.drain(); + check(SpeechRecognizer.starts == 1, "grant starts only the successor"); + SpeechRecognizer.last.listener.onReadyForSpeech(new Bundle()); + check(events.size() == 1 && events.get(0).equals("0:") && eventIds.get(0) == 22L, + "the canceled session cannot receive the successor's result"); + check(activity.getFragmentManager().count() == 0 && activity.getFragmentManager().callbackCount() == 0, + "adopted prompt cleanup releases all helper resources"); + NativeSpeech.stop(22, true); + Handler.drain(); + } + private static void cancelBeforeResumeDoesNotAsk() { + Activity activity = missingPermission(); + activity.resumed = false; + NativeSpeech.start(activity, 23, "", false); + Handler.runNext(); + check(android.app.Fragment.pending == null, "a paused owner does not launch the prompt yet"); + NativeSpeech.stop(23, true); + Handler.runNext(); + // Resume before the queued fragment removal executes. + activity.resume(); + Handler.drain(); + check(android.app.Fragment.requests == 0 && SpeechRecognizer.starts == 0 && events.isEmpty(), + "cancel must suppress a deferred prompt even before its transaction is removed"); + check(activity.getFragmentManager().count() == 0 && activity.getFragmentManager().callbackCount() == 0, + "cancel before prompt launch releases the parent and listener"); + } + private static void restartWhileOldParentRemovalIsQueued() { + Activity activity = missingPermission(); + activity.resumed = false; + NativeSpeech.start(activity, 24, "", false); + Handler.runNext(); + NativeSpeech.stop(24, true); + NativeSpeech.start(activity, 25, "", false); + Handler.runNext(); // cancel queues a removal behind the already queued start + Handler.runNext(); + activity.resume(); + Handler.drain(); + check(events.isEmpty() && android.app.Fragment.requests == 1, + "an old removal cannot cancel the replacement request or cause a false denial"); + android.app.Fragment.answer(true); + Handler.drain(); + check(SpeechRecognizer.starts == 1, "the replacement still starts after old removal drains"); + SpeechRecognizer.last.listener.onReadyForSpeech(new Bundle()); + check(eventIds.size() == 1 && eventIds.get(0) == 25L, "only the replacement receives Started"); + NativeSpeech.stop(25, true); + Handler.drain(); + } + private static void configurationTeardownStopsPendingSession() { + Activity activity = missingPermission(); + NativeSpeech.start(activity, 26, "", false); + Handler.runNext(); + java.util.List saved = activity.getFragmentManager().saveAllState(); + activity.destroy(true); + Handler.drain(); + check(events.size() == 1 && events.get(0).equals("4:") && eventIds.get(0) == 26L, + "configuration teardown ends the pending session once with Stopped, not permission denial"); + check(activity.getFragmentManager().callbackCount() == 0, "discarded Activity releases its save listener"); + android.app.Fragment.answer(true); + Handler.drain(); + check(events.size() == 1 && SpeechRecognizer.starts == 0, "a grant cannot restart the discarded Activity"); + Activity replacement = new Activity(activity.getApplication()); + try { replacement.getFragmentManager().restoreAllState(saved, replacement.getClassLoader()); } + catch (Exception error) { throw new AssertionError("configuration state must be restorable", error); } + NativeSpeech.start(replacement, 27, "", false); + Handler.drain(); + check(SpeechRecognizer.starts == 1, "the replacement Activity can start a new session after grant"); + SpeechRecognizer.last.listener.onReadyForSpeech(new Bundle()); + check(eventIds.get(eventIds.size() - 1) == 27L, "the new Activity owns the restarted session"); + NativeSpeech.stop(27, true); + Handler.drain(); + } + private static void processRestoreContainsOnlyHostVisibleClasses() { + Activity activity = missingPermission(); + check(NativeSpeech.class.getClassLoader() != activity.getClassLoader(), + "production speech must be loaded in a separate child classloader"); + try { + activity.getClassLoader().loadClass("dev.robius.speech.SpeechPermissionFragment"); + throw new AssertionError("host loader must be unable to load the embedded helper"); + } catch (ClassNotFoundException expected) {} + activity.getFragmentManager().beginTransaction().add(new harness.HostStateFragment(), "unrelated") + .commitNowAllowingStateLoss(); + NativeSpeech.start(activity, 28, "", false); + Handler.runNext(); + java.util.List saved = activity.getFragmentManager().saveAllState(); + check(saved.size() == 2, "saving preserves the unrelated app fragment and framework permission parent"); + // Repeated saves must remain safe while the same OS request is pending. + saved = activity.getFragmentManager().saveAllState(); + Activity restored = new Activity(); + try { restored.getFragmentManager().restoreAllState(saved, restored.getClassLoader()); } + catch (Exception error) { throw new AssertionError("process restoration cannot load child DEX classes", error); } + harness.HostStateFragment unrelated = (harness.HostStateFragment) restored.getFragmentManager().findFragmentByTag("unrelated"); + check(unrelated != null && "host draft must survive".equals(unrelated.value), + "clearing permission state cannot alter an unrelated app fragment's state"); + android.app.Fragment host = restored.getFragmentManager().findFragmentByTag("dev.robius.speech.PermissionHost"); + check(host != null && host.getClass() == android.app.Fragment.class && host.getChildFragmentManager().count() == 0, + "the restored helper host contains no custom child fragment"); + // A fresh process would lose these callbacks; explicitly release the old process's fake state. + NativeSpeech.stop(28, true); + Handler.drain(); + android.app.Fragment.answer(false); + Handler.drain(); + check(events.isEmpty(), "an abandoned old-process request has no recipient"); + NativeSpeech.start(restored, 29, "", false); + Handler.drain(); + check(android.app.Fragment.pending != null && restored.getFragmentManager().count() == 2, + "a new request replaces the empty restored host without disturbing app fragments"); + android.app.Fragment.answer(true); + Handler.drain(); + check(SpeechRecognizer.starts == 1, "speech works after restoring a process with a pending permission request"); + NativeSpeech.stop(29, true); + Handler.drain(); + } + private interface Case { void run(); } + public static void main(String[] args) { + System.load(args[0]); + Object[][] cases = { + {"a busy recognizer retries with bounded backoff", (Case) NativeSpeechRetryTest::busyIsBounded}, + {"a client error recreates the recognizer", (Case) NativeSpeechRetryTest::clientRecreatesAndSuccessResetsBudget}, + {"stopping during backoff never restarts", (Case) NativeSpeechRetryTest::stopDuringBackoffNeverRestarts}, + {"permission errors are not retried", (Case) NativeSpeechRetryTest::permissionsAreNotRetried}, + {"a missing permission is requested, then the session proceeds", (Case) NativeSpeechRetryTest::permissionIsRequestedThenSessionProceeds}, + {"a denied permission is reported once", (Case) NativeSpeechRetryTest::permissionDenialIsReportedOnce}, + {"canceling a pending prompt discards a late grant", (Case) NativeSpeechRetryTest::cancelDiscardsLateGrant}, + {"a new session adopts an abandoned system prompt", (Case) NativeSpeechRetryTest::nextSessionAdoptsPendingPrompt}, + {"cancel before resume never launches the deferred prompt", (Case) NativeSpeechRetryTest::cancelBeforeResumeDoesNotAsk}, + {"queued old removal cannot remove a new permission request", (Case) NativeSpeechRetryTest::restartWhileOldParentRemovalIsQueued}, + {"configuration teardown stops the old pending session", (Case) NativeSpeechRetryTest::configurationTeardownStopsPendingSession}, + {"saved permission state restores through the host classloader", (Case) NativeSpeechRetryTest::processRestoreContainsOnlyHostVisibleClasses}, + }; + for (Object[] entry : cases) { + System.out.println(" " + entry[0]); + ((Case) entry[1]).run(); + } + System.out.println("Passed " + cases.length + " Android speech regressions; no microphone used."); + } +} +""", +} + +JNI = """#include +JNIEXPORT void JNICALL Java_dev_robius_speech_NativeSpeech_event( + JNIEnv *env, jclass owner, jlong id, jint kind, jstring text, jfloat level) { + (void)owner; + jclass test = (*env)->FindClass(env, "dev/robius/speech/NativeSpeechRetryTest"); + jmethodID record = (*env)->GetStaticMethodID(env, test, "record", "(JILjava/lang/String;F)V"); + (*env)->CallStaticVoidMethod(env, test, record, id, kind, text, level); +} +""" + + +def main(): + java_home = os.environ.get("JAVA_HOME") + if not java_home and sys.platform == "darwin": + java_home = subprocess.check_output(["/usr/libexec/java_home"], text=True).strip() + if not java_home: + java_home = str(Path(shutil.which("javac") or "javac").resolve().parent.parent) + java_home = Path(java_home) + crate = Path(__file__).resolve().parents[1] + with tempfile.TemporaryDirectory(prefix="native-speech-retry-") as temp: + root = Path(temp) + for name, source in SOURCES.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source) + for name in ("NativeSpeech.java", "SpeechPermissionFragment.java"): + shutil.copyfile(crate / "java/dev/robius/speech" / name, root / "dev/robius/speech" / name) + # Keep platform/host classes on the parent loader and embedded speech + # classes on a child loader, matching the Android InMemoryDexClassLoader. + host_classes = root / "host-classes" + speech_classes = root / "speech-classes" + subprocess.run([str(java_home / "bin/javac"), "-d", str(host_classes), + *map(str, (root / "android").rglob("*.java")), + *map(str, (root / "harness").rglob("*.java"))], check=True) + subprocess.run([str(java_home / "bin/javac"), "-classpath", str(host_classes), "-d", str(speech_classes), + *map(str, (root / "dev").rglob("*.java"))], check=True) + c_file = root / "events.c" + c_file.write_text(JNI) + platform = "darwin" if sys.platform == "darwin" else "linux" + library = root / ("events.dylib" if platform == "darwin" else "events.so") + subprocess.run([os.environ.get("CC", "cc"), "-shared", "-fPIC", + "-I" + str(java_home / "include"), "-I" + str(java_home / "include" / platform), + str(c_file), "-o", str(library)], check=True) + subprocess.run([str(java_home / "bin/java"), "-cp", str(host_classes), + "harness.Launcher", str(speech_classes), str(library)], check=True) + + +if __name__ == "__main__": + main()