Add SpeechTextInput for dictation support in any text input

Cargo.toml

@@ -17,16 +17,16 @@ metadata.makepad-auto-version = "zqpv-Yj-K7WNVK2I8h5Okhho46Q="
 # makepad-widgets = { git = "https://github.com/makepad/makepad", branch = "dev", features = ["serde"] }
 # makepad-code-editor = { git = "https://github.com/makepad/makepad", branch = "dev" }
 
-makepad-widgets = { git = "https://github.com/kevinaboos/makepad", branch = "button-grab-key-focus", features = ["serde"] }
-makepad-code-editor = { git = "https://github.com/kevinaboos/makepad", branch = "button-grab-key-focus" }
+makepad-widgets = { git = "https://github.com/kevinaboos/makepad", branch = "text-input-scroll-bar", features = ["serde"] }
+makepad-code-editor = { git = "https://github.com/kevinaboos/makepad", branch = "text-input-scroll-bar" }
 
 
 robius-directories = { git = "https://github.com/project-robius/robius" }
 robius-file-picker = { git = "https://github.com/project-robius/robius" }
 robius-location    = { git = "https://github.com/project-robius/robius" }
 robius-open        = { git = "https://github.com/project-robius/robius" }
 robius-share       = { git = "https://github.com/project-robius/robius" }
-robius-speech      = { git = "https://github.com/project-robius/robius", branch = "speech_to_text" }
+robius-speech      = { git = "https://github.com/project-robius/robius" }
 robius-use-makepad = "0.1.1" ## auto-configures all `robius-*` crates to work with Makepad.
 
 

src/app.rs

@@ -17,7 +17,7 @@ use crate::{
         event_source_modal::{EventSourceModalAction, EventSourceModalWidgetRefExt}, invite_modal::{InviteModalAction, InviteModalWidgetRefExt}, main_desktop_ui::MainDesktopUiAction, navigation_tab_bar::{NavigationBarAction, SelectedTab}, new_message_context_menu::NewMessageContextMenuWidgetRefExt, room_context_menu::RoomContextMenuWidgetRefExt, room_screen::{InviteAction, MessageAction, clear_timeline_states, invalidate_single_timeline_state}, rooms_list::{RoomsListAction, RoomsListRef, RoomsListUpdate, clear_all_invited_rooms, enqueue_rooms_list_update}
     }, join_leave_room_modal::{
         JoinLeaveModalKind, JoinLeaveRoomModalAction, JoinLeaveRoomModalWidgetRefExt
-    }, login::login_screen::LoginAction, logout::logout_confirm_modal::{LogoutAction, LogoutConfirmModalAction, LogoutConfirmModalWidgetRefExt}, persistence, profile::user_profile_cache::clear_user_profile_cache, room::BasicRoomDetails, settings::app_preferences::{AppPreferences, UiZoom}, shared::{confirmation_modal::{ConfirmationModalContent, ConfirmationModalWidgetRefExt}, context_menu::{ContextMenuClosed, menu_position_margin}, image_viewer::{ImageViewerAction, LoadState}, popup_list::{PopupKind, enqueue_popup_notification}}, sliding_sync::{DirectMessageRoomAction, MatrixRequest, TimelineKind, current_user_id, submit_async_request}, utils::RoomNameId, verification::VerificationAction, verification_modal::{
+    }, login::login_screen::LoginAction, logout::logout_confirm_modal::{LogoutAction, LogoutConfirmModalAction, LogoutConfirmModalWidgetRefExt}, persistence, profile::user_profile_cache::clear_user_profile_cache, room::BasicRoomDetails, settings::app_preferences::{AppPreferences, UiZoom}, shared::{confirmation_modal::{ConfirmationModalContent, ConfirmationModalWidgetRefExt}, context_menu::{ContextMenuClosed, menu_position_margin}, image_viewer::{ImageViewerAction, LoadState}, popup_list::{PopupKind, enqueue_popup_notification}, speech_text_input::cancel_all_dictation}, sliding_sync::{DirectMessageRoomAction, MatrixRequest, TimelineKind, current_user_id, submit_async_request}, utils::RoomNameId, verification::VerificationAction, verification_modal::{
         VerificationModalAction,
         VerificationModalWidgetRefExt,
     }
@@ -273,7 +273,7 @@ impl MatchEvent for App {
 
             match action.downcast_ref() {
                 Some(LogoutAction::LogoutSuccess) => {
-                    robius_speech::cancel_all();
+                    cancel_all_dictation();
                     self.app_state.logged_in = false;
                     self.ui.modal(cx, ids!(logout_confirm_modal)).close(cx);
                     self.update_login_visibility(cx);
@@ -308,6 +308,7 @@ impl MatchEvent for App {
             if let Some(LoginAction::LoginFailure(_)) = action.downcast_ref() {
                 if self.app_state.logged_in {
                     log!("Received LoginAction::LoginFailure while logged in; showing login screen.");
+                    cancel_all_dictation();
                     self.app_state.logged_in = false;
                     self.update_login_visibility(cx);
                     self.ui.redraw(cx);
@@ -859,7 +860,7 @@ impl App {
                 crate::sliding_sync::set_sync_service_desired_running(true, "app resume");
             }
             Event::Background => {
-                robius_speech::cancel_all();
+                cancel_all_dictation();
                 if self.lifecycle.is_foreground {
                     log!("App entered background; persisting state and stopping Matrix sync.");
                     self.lifecycle.is_foreground = false;
@@ -880,7 +881,7 @@ impl App {
                 crate::sliding_sync::set_sync_service_desired_running(true, "app foreground");
             }
             Event::Shutdown => {
-                robius_speech::cancel_all();
+                cancel_all_dictation();
                 self.handle_shutdown(cx);
             }
             _ => {}

src/home/editing_pane.rs

@@ -237,6 +237,7 @@ impl Widget for EditingPane {
             if self.button(cx, ids!(cancel_button)).clicked(actions)
                 || edit_text_input.escaped(actions)
             {
+                mentionable_input.cancel_dictation(cx);
                 self.animator_play(cx, ids!(panel.hide));
                 self.redraw(cx);
                 return;
@@ -247,6 +248,8 @@ impl Widget for EditingPane {
             if self.button(cx, ids!(accept_button)).clicked(actions)
                 || edit_text_input.returned(actions).is_some()
             {
+                // The edit is the text as shown now, so stop adding dictated words to it.
+                mentionable_input.cancel_dictation(cx);
                 let edited_text = edit_text_input.text().trim().to_string();
                 let edited_content = match info.event_tl_item.content() {
                     TimelineItemContent::MsgLike(msg_like_content) => {
@@ -325,6 +328,7 @@ impl Widget for EditingPane {
                                             PopupKind::Error,
                                             None,
                                         );
+                                        mentionable_input.cancel_dictation(cx);
                                         self.animator_play(cx, ids!(panel.hide));
                                         self.redraw(cx);
                                         return;
@@ -489,6 +493,8 @@ impl EditingPane {
         }
         match edit_result {
             Ok(()) => {
+                self.mentionable_text_input(cx, ids!(editing_content.edit_text_input))
+                    .cancel_dictation(cx);
                 self.animator_play(cx, ids!(panel.hide));
             },
             Err(e) => {
@@ -562,6 +568,7 @@ impl EditingPane {
     pub fn save_state(&self) -> Option<EditingPaneState> {
         let info = self.info.as_ref()?;
         let mentionable_input = self.child_by_path(ids!(editing_content.edit_text_input)).as_mentionable_text_input();
+        mentionable_input.release_microphone();
         Some(EditingPaneState {
             event_tl_item: info.event_tl_item.clone(),
             mentionable_input_state: mentionable_input.save_state(),
@@ -664,6 +671,8 @@ impl EditingPaneRef {
     /// This function *DOES NOT* emit an [`EditingPaneAction::Hidden`] action.
     pub fn force_reset_hide(&self, cx: &mut Cx) {
         let Some(mut inner) = self.borrow_mut() else { return };
+        inner.mentionable_text_input(cx, ids!(editing_content.edit_text_input))
+            .cancel_dictation(cx);
         if inner.visible {
             cx.revert_key_focus();
         }

src/home/home_screen.rs

@@ -14,8 +14,8 @@ use crate::{
         app_preferences::{AppPreferencesGlobal, AppPreferencesAction, ViewModeOverride},
         settings_screen::SettingsScreenWidgetRefExt,
     },
-    shared::room_filter_input_bar::{MainFilterAction, RoomFilterInputBarWidgetExt},
     shared::mention_popup::MentionablePopupRef,
+    shared::speech_text_input::cancel_all_dictation,
     utils::RoomNameId,
 };
 
@@ -263,7 +263,7 @@ script_mod! {
                             align: Align{y: 0.5}
 
                             CachedWidget {
-                                room_filter_input_bar := RoomFilterInputBar {}
+                                room_filter_input_bar := RoomFilterInputBar { is_main_filter: true }
                             }
 
                             // Hide this until it's implemented.
@@ -482,6 +482,16 @@ pub fn effective_is_desktop(cx: &mut Cx) -> bool {
         .unwrap_or(true) // Before the first selection, default to desktop mode
 }
 
+/// Returns the id of the page that shows the given navigation tab.
+fn page_for_tab(tab: &SelectedTab) -> LiveId {
+    match tab {
+        SelectedTab::Space { .. }
+        | SelectedTab::Home => id!(home_page),
+        SelectedTab::Settings => id!(settings_page),
+        SelectedTab::AddRoom => id!(add_room_page),
+    }
+}
+
 
 #[derive(Script, Widget)]
 pub struct HomeScreen {
@@ -522,14 +532,6 @@ impl ScriptHook for HomeScreen {
 impl Widget for HomeScreen {
     fn handle_event(&mut self, cx: &mut Cx, event: &Event, scope: &mut Scope) {
         if let Event::Actions(actions) = event {
-            // On desktop, the RoomFilterInputBar is inside this HomeScreen.
-            // Check if it changed and re-emit as a MainFilterAction so that
-            // RoomsList and SpacesBar can respond without cross-talk from
-            // other RoomFilterInputBar instances (e.g., SpaceLobbyScreen's).
-            if let Some(keywords) = self.view.room_filter_input_bar(cx, ids!(room_filter_input_bar)).changed(actions) {
-                cx.action(MainFilterAction::Changed(keywords));
-            }
-
             let app_state = scope.data.get_mut::<AppState>().unwrap();
             for action in actions {
                 match action.downcast_ref() {
@@ -545,6 +547,7 @@ impl Widget for HomeScreen {
                     // Only open the settings screen if it is not currently open.
                     Some(NavigationBarAction::OpenSettings) => {
                         if !matches!(app_state.selected_tab, SelectedTab::Settings) {
+                            self.cancel_dictation_if_page_changes(cx, &app_state.selected_tab, &SelectedTab::Settings);
                             self.previous_selection = std::mem::replace(&mut app_state.selected_tab, SelectedTab::Settings);
                             cx.action(NavigationBarAction::TabSelected(app_state.selected_tab.clone()));
                             if let Some(settings_page) = self.update_active_page_from_selection(cx, app_state) {
@@ -748,15 +751,16 @@ impl HomeScreen {
     ) -> Option<WidgetRef> {
         self.view
             .page_flip(cx, ids!(home_screen_page_flip))
-            .set_active_page(
-                cx,
-                match app_state.selected_tab {
-                    SelectedTab::Space { .. }
-                    | SelectedTab::Home => id!(home_page),
-                    SelectedTab::Settings => id!(settings_page),
-                    SelectedTab::AddRoom => id!(add_room_page),
-                },
-            )
+            .set_active_page(cx, page_for_tab(&app_state.selected_tab))
+    }
+
+    /// Cancels dictation if showing `new_tab` instead of `old_tab` hides the page the user is looking at.
+    fn cancel_dictation_if_page_changes(&self, cx: &mut Cx, old_tab: &SelectedTab, new_tab: &SelectedTab) {
+        // On mobile, a pushed screen stays in front of whichever page is shown.
+        let is_page_in_front = self.view.stack_navigation(cx, ids!(view_stack)).current_view().is_none();
+        if is_page_in_front && page_for_tab(old_tab) != page_for_tab(new_tab) {
+            cancel_all_dictation();
+        }
     }
 
     /// Populates a `StackNavigationView` with the given room/screen's info.
@@ -897,13 +901,16 @@ impl HomeScreen {
             }
         }
         app_state.selected_room = Some(sr);
+        // The pushed screen covers whatever the user was dictating into.
+        cancel_all_dictation();
         stack_navigation.push(cx, view_id);
         self.view.redraw(cx);
     }
 
     /// Switches to (selects) the given navigation tab, if it isn't already the selected one.
     fn switch_to_tab(&mut self, cx: &mut Cx, app_state: &mut AppState, new_tab: SelectedTab) {
         if app_state.selected_tab == new_tab { return }
+        self.cancel_dictation_if_page_changes(cx, &app_state.selected_tab, &new_tab);
         self.previous_selection = std::mem::replace(&mut app_state.selected_tab, new_tab);
         cx.action(NavigationBarAction::TabSelected(app_state.selected_tab.clone()));
         self.update_active_page_from_selection(cx, app_state);
@@ -952,6 +959,8 @@ impl HomeScreen {
         if stack_nav.is_transitioning() {
             return;
         }
+        // The popped screen is what the user was dictating into.
+        cancel_all_dictation();
         let Some(current_screen) = app_state.selected_room.take() else {
             // If we didn't have a current screen, something's buggy,
             // so the safest option is to clear the mobile stack and start over. nbd.

src/home/loading_pane.rs

@@ -1,6 +1,7 @@
 use makepad_widgets::*;
 use matrix_sdk::ruma::{EventId, OwnedEventId};
 
+use crate::shared::speech_text_input::escape_stopped_dictation;
 use crate::sliding_sync::TimelineRequestSender;
 
 
@@ -172,7 +173,7 @@ impl Widget for LoadingPane {
             )
             || event.back_pressed()
             || match event.hits_with_capture_overload(cx, area, true) {
-                Hit::KeyUp(key) => key.key_code == KeyCode::Escape,
+                Hit::KeyUp(key) => key.key_code == KeyCode::Escape && !escape_stopped_dictation(),
                 Hit::FingerDown(_fde) => {
                     cx.set_key_focus(area);
                     false

src/home/main_desktop_ui.rs

@@ -3,7 +3,7 @@ use ruma::OwnedRoomId;
 use tokio::sync::Notify;
 use std::{collections::{HashMap, HashSet}, sync::Arc};
 
-use crate::{app::{AppState, AppStateAction, SavedDockState, SelectedRoom}, home::{navigation_tab_bar::{NavigationBarAction, SelectedTab}, rooms_list::RoomsListRef, space_lobby::SpaceLobbyScreenWidgetRefExt}, utils::RoomNameId};
+use crate::{app::{AppState, AppStateAction, SavedDockState, SelectedRoom}, home::{navigation_tab_bar::{NavigationBarAction, SelectedTab}, rooms_list::RoomsListRef, space_lobby::SpaceLobbyScreenWidgetRefExt}, shared::speech_text_input::cancel_all_dictation, utils::RoomNameId};
 use super::{invite_screen::InviteScreenWidgetRefExt, room_screen::RoomScreenWidgetRefExt, rooms_list::{AcceptedInviteKind, RoomsListAction}, spaces_bar::SpacesBarAction};
 
 script_mod! {
@@ -166,6 +166,13 @@ impl MainDesktopUI {
         }
     }
 
+    /// Cancels dictation unless `room` is already shown, since showing it hides the current tab.
+    fn cancel_dictation_unless_shown(&self, room: &SelectedRoom) {
+        if self.most_recently_selected_room.as_ref() != Some(room) {
+            cancel_all_dictation();
+        }
+    }
+
     /// Focuses on a room if it is already open, otherwise creates a new tab for the room.
     fn focus_or_create_tab(&mut self, cx: &mut Cx, room: SelectedRoom) {
         // Do nothing if the room to select is already created and focused.
@@ -546,7 +553,9 @@ impl WidgetMatchEvent for MainDesktopUI {
                     self.switch_dock_to_space(cx, app_state, None);
                 }
                 cx.action(NavigationBarAction::GoToHome);
-                self.focus_or_create_tab(cx, SelectedRoom::InvitedRoom { room_name_id: space_name_id });
+                let invite = SelectedRoom::InvitedRoom { room_name_id: space_name_id };
+                self.cancel_dictation_unless_shown(&invite);
+                self.focus_or_create_tab(cx, invite);
                 continue;
             }
 
@@ -572,6 +581,12 @@ impl WidgetMatchEvent for MainDesktopUI {
             match widget_action.cast() {
                 // Whenever a tab (except for the home_tab) is pressed, notify the app state.
                 DockAction::TabWasPressed(tab_id) => {
+                    // Switching tabs hides whatever the user was dictating into.
+                    let current_tab_id = self.most_recently_selected_room.as_ref()
+                        .map_or(id!(home_tab), SelectedRoom::tab_id);
+                    if tab_id != current_tab_id {
+                        cancel_all_dictation();
+                    }
                     if tab_id == id!(home_tab) {
                         self.select_room(cx, None);
                     }
@@ -622,6 +637,7 @@ impl WidgetMatchEvent for MainDesktopUI {
             // Handle RoomsList actions, which are updates from the rooms list.
             match widget_action.cast_ref() {
                 RoomsListAction::Selected(selected_room) => {
+                    self.cancel_dictation_unless_shown(selected_room);
                     // Note that this cannot be performed within draw_walk() as the draw flow prevents from
                     // performing actions that would trigger a redraw, and the Dock internally performs (and expects)
                     // a redraw to be happening in order to draw the tab content.

src/home/new_message_context_menu.rs

@@ -6,7 +6,7 @@ use makepad_widgets::*;
 use matrix_sdk::ruma::{OwnedEventId, events::room::message::MessageType};
 use matrix_sdk_ui::timeline::{EventSendState, EventTimelineItem, MsgLikeContent, MsgLikeKind, TimelineEventItemId};
 
-use crate::{home::send_status_indicator::is_send_error_retryable, shared::context_menu::{BUTTON_HEIGHT, ContextMenuClosed, expected_menu_size}, sliding_sync::UserPowerLevels};
+use crate::{home::send_status_indicator::is_send_error_retryable, shared::{context_menu::{BUTTON_HEIGHT, ContextMenuClosed, expected_menu_size}, speech_text_input::escape_stopped_dictation}, sliding_sync::UserPowerLevels};
 
 use super::room_screen::MessageAction;
 
@@ -311,7 +311,8 @@ impl Widget for NewMessageContextMenu {
         let close_menu = {
             event.back_pressed()
             || match event.hits_with_capture_overload(cx, area, true) {
-                Hit::KeyUp(key) => key.key_code == KeyCode::Escape,
+                // An `Escape` that stopped dictation shouldn't also close this menu.
+                Hit::KeyUp(key) => key.key_code == KeyCode::Escape && !escape_stopped_dictation(),
                 Hit::FingerDown(fde) => {
                     let reaction_text_input = self.view.text_input(cx, ids!(reaction_input_view.reaction_text_input));
                     if reaction_text_input.area().rect(cx).contains(fde.abs) {
@@ -355,7 +356,7 @@ impl WidgetMatchEvent for NewMessageContextMenu {
             );
             close_menu = true;
         }
-        else if reaction_text_input.escaped(actions) {
+        else if reaction_text_input.escaped(actions) && !escape_stopped_dictation() {
             close_menu = true;
         }
         else if self.button(cx, ids!(react_button)).clicked(actions) {