mimick/library/lightbox/
ocr.rs

1//! Recognized-text (OCR) overlay in the lightbox: loading boxes, mouse and touch selection, and copying.
2
3use std::cell::Cell;
4use std::rc::Rc;
5
6use gtk::prelude::*;
7
8use super::Lightbox;
9
10mod geometry;
11mod layer;
12mod state;
13
14pub(super) use layer::OcrLayer;
15
16/// Floating button over the photo that copies the selection, or all text when nothing is selected.
17pub(super) fn copy_button() -> gtk::Button {
18    gtk::Button::builder()
19        .label("Copy All Text")
20        .halign(gtk::Align::Center)
21        .valign(gtk::Align::End)
22        .margin_bottom(12)
23        .css_classes(["osd", "pill"])
24        .visible(false)
25        .build()
26}
27
28impl Lightbox {
29    /// Fetches the photo's OCR boxes; results for a photo the user has left are dropped.
30    pub(super) fn load_ocr(self: &Rc<Self>, asset_id: String, image: (f64, f64)) {
31        let generation = self.load_gen.get();
32        let lb = self.clone();
33        glib::MainContext::default().spawn_local(async move {
34            let result = lb.ui.ctx.api_client.fetch_asset_ocr(&asset_id).await;
35            if lb.load_gen.get() != generation {
36                return;
37            }
38            match result {
39                Ok(boxes) if !boxes.is_empty() => {
40                    lb.picture.ocr.set_boxes(boxes, image);
41                    lb.sync_ocr_controls();
42                }
43                Ok(_) => {}
44                Err(err) => log::debug!("OCR fetch failed for {asset_id}: {err}"),
45            }
46        });
47    }
48
49    /// All OCR input goes through the picture scroller, since the drawing layer can't take input.
50    pub(super) fn connect_ocr(self: &Rc<Self>) {
51        self.connect_ocr_toggle();
52        self.connect_ocr_mouse_selection();
53        self.connect_ocr_taps();
54        self.connect_ocr_hover();
55        let lb = self.clone();
56        self.picture
57            .ocr_copy
58            .connect_clicked(move |_| lb.copy_ocr_text());
59    }
60
61    fn connect_ocr_toggle(self: &Rc<Self>) {
62        let lb = self.clone();
63        self.actions.ocr_toggle.connect_toggled(move |btn| {
64            btn.set_tooltip_text(Some(if btn.is_active() {
65                "Hide recognized text (T)"
66            } else {
67                "Show recognized text (T)"
68            }));
69            if !btn.is_active() {
70                lb.picture.ocr.clear_selection();
71            }
72            lb.sync_ocr_controls();
73        });
74    }
75
76    /// T, Ctrl+A and Ctrl+C; returns whether the key was handled.
77    pub(super) fn handle_ocr_key(self: &Rc<Self>, key: gtk::gdk::Key, ctrl: bool) -> bool {
78        use gtk::gdk::Key;
79        let shown = self.picture.ocr.area.is_visible();
80        match (ctrl, key) {
81            (false, Key::t | Key::T) if self.actions.ocr_toggle.is_visible() => self
82                .actions
83                .ocr_toggle
84                .set_active(!self.actions.ocr_toggle.is_active()),
85            (true, Key::a | Key::A) if shown => {
86                self.picture.ocr.select_all();
87                self.sync_ocr_controls();
88            }
89            (true, Key::c | Key::C) if shown => self.copy_ocr_text(),
90            _ => return false,
91        }
92        true
93    }
94
95    /// Mouse presses on a box select text instead of panning, swiping or dragging the photo out.
96    fn connect_ocr_mouse_selection(self: &Rc<Self>) {
97        let drag = gtk::GestureDrag::new();
98        drag.set_button(gtk::gdk::BUTTON_PRIMARY);
99        drag.set_propagation_phase(gtk::PropagationPhase::Capture);
100        let start = Rc::new(Cell::new(None::<(f64, f64)>));
101        let begin = start.clone();
102        let lb = self.clone();
103        drag.connect_drag_begin(move |gesture, x, y| {
104            begin.set(None);
105            let Some(idx) = (!is_touch(gesture))
106                .then(|| lb.ocr_box_under(x, y))
107                .flatten()
108            else {
109                gesture.set_state(gtk::EventSequenceState::Denied);
110                return;
111            };
112            gesture.set_state(gtk::EventSequenceState::Claimed);
113            let mods = gesture.current_event_state();
114            let shift = mods.contains(gtk::gdk::ModifierType::SHIFT_MASK);
115            let ctrl = mods.contains(gtk::gdk::ModifierType::CONTROL_MASK);
116            if lb.picture.ocr.press(idx, shift, ctrl) {
117                begin.set(Some((x, y)));
118            }
119            lb.sync_ocr_controls();
120        });
121        let lb = self.clone();
122        drag.connect_drag_update(move |_, dx, dy| {
123            let Some((x, y)) = start.get() else {
124                return;
125            };
126            if let Some(idx) = lb.ocr_box_under(x + dx, y + dy) {
127                lb.picture.ocr.extend_to(idx);
128                lb.sync_ocr_controls();
129            }
130        });
131        self.picture.scrolled.add_controller(drag);
132    }
133
134    /// Touch taps toggle boxes; drags are left to panning, pinching and swiping.
135    fn connect_ocr_taps(self: &Rc<Self>) {
136        let click = gtk::GestureClick::new();
137        click.set_button(gtk::gdk::BUTTON_PRIMARY);
138        click.set_propagation_phase(gtk::PropagationPhase::Capture);
139        let dragged = Rc::new(Cell::new(false));
140        let pressed = dragged.clone();
141        click.connect_pressed(move |_, _, _, _| pressed.set(false));
142        let stopped = dragged.clone();
143        click.connect_stopped(move |_| stopped.set(true));
144        let lb = self.clone();
145        click.connect_released(move |gesture, _, x, y| {
146            if dragged.get() || !lb.picture.ocr.area.is_visible() {
147                return;
148            }
149            let modified = gesture.current_event_state().intersects(
150                gtk::gdk::ModifierType::SHIFT_MASK | gtk::gdk::ModifierType::CONTROL_MASK,
151            );
152            match lb.ocr_box_under(x, y) {
153                Some(idx) if is_touch(gesture) => {
154                    gesture.set_state(gtk::EventSequenceState::Claimed);
155                    lb.picture.ocr.click(idx, true);
156                }
157                None if !modified => lb.picture.ocr.clear_selection(),
158                _ => return,
159            }
160            lb.sync_ocr_controls();
161        });
162        self.picture.scrolled.add_controller(click);
163    }
164
165    /// Text cursor and highlight over a box, with its text as a tooltip.
166    fn connect_ocr_hover(self: &Rc<Self>) {
167        let scrolled = &self.picture.scrolled;
168        let motion = gtk::EventControllerMotion::new();
169        let lb = self.clone();
170        motion.connect_motion(move |_, x, y| lb.set_ocr_hover(lb.ocr_box_under(x, y)));
171        let lb = self.clone();
172        motion.connect_leave(move |_| lb.set_ocr_hover(None));
173        scrolled.add_controller(motion);
174
175        scrolled.set_has_tooltip(true);
176        let lb = self.clone();
177        scrolled.connect_query_tooltip(move |_, x, y, _, tooltip| {
178            let text = lb
179                .ocr_box_under(f64::from(x), f64::from(y))
180                .and_then(|idx| lb.picture.ocr.text_at(idx));
181            match text {
182                Some(text) => {
183                    tooltip.set_text(Some(&text));
184                    true
185                }
186                None => false,
187            }
188        });
189    }
190
191    fn set_ocr_hover(&self, idx: Option<usize>) {
192        if self.picture.ocr.set_hovered(idx) {
193            self.picture
194                .scrolled
195                .set_cursor_from_name(idx.map(|_| "text"));
196        }
197    }
198
199    /// Copies the selection, or all text when nothing is selected.
200    pub(super) fn copy_ocr_text(self: &Rc<Self>) {
201        let ocr = &self.picture.ocr;
202        let Some(text) = ocr.selected_text().or_else(|| ocr.all_text()) else {
203            return;
204        };
205        self.picture.scrolled.clipboard().set_text(&text);
206        self.picture.ocr_copy.set_label("Copied");
207        let lb = self.clone();
208        glib::timeout_add_local_once(std::time::Duration::from_millis(1500), move || {
209            lb.sync_ocr_controls()
210        });
211    }
212
213    /// Box under a point in picture-scroller coordinates, while the text is shown.
214    pub(super) fn ocr_box_under(&self, x: f64, y: f64) -> Option<usize> {
215        let layer = &self.picture.ocr.area;
216        if !layer.is_visible() {
217            return None;
218        }
219        let point = self
220            .picture
221            .scrolled
222            .compute_point(layer, &gtk::graphene::Point::new(x as f32, y as f32))?;
223        self.picture
224            .ocr
225            .box_at((f64::from(point.x()), f64::from(point.y())))
226    }
227
228    /// Shows the toggle, layer and copy button to match the current photo's boxes.
229    pub(super) fn sync_ocr_controls(&self) {
230        let ocr = &self.picture.ocr;
231        let has_boxes = ocr.has_boxes();
232        let shown = self.actions.ocr_toggle.is_active() && has_boxes;
233        self.actions.ocr_toggle.set_visible(has_boxes);
234        ocr.area.set_visible(shown);
235        self.picture.ocr_copy.set_visible(shown);
236        self.picture.ocr_copy.set_label(if ocr.has_selection() {
237            "Copy Selected Text"
238        } else {
239            "Copy All Text"
240        });
241        if !shown {
242            ocr.set_hovered(None);
243            self.picture.scrolled.set_cursor(None::<&gtk::gdk::Cursor>);
244        }
245    }
246}
247
248/// Touch presses must not claim the sequence, or panning and pinch-zoom stop working.
249fn is_touch(controller: &impl IsA<gtk::EventController>) -> bool {
250    controller
251        .current_event_device()
252        .is_some_and(|device| device.source() == gtk::gdk::InputSource::Touchscreen)
253}
mimick/library/lightbox/ocr.rs

mimick/library/lightbox/
ocr.rs
