From d77b62bb6d1cbbb466f7764f248e6666c4b1dcf6 Mon Sep 17 00:00:00 2001 From: Abhijeet Arch Date: Sat, 24 Oct 2026 19:16:05 +0530 Subject: [PATCH 1/3] feat(linux): add Hyprland cursor tracking and position zoom --- crates/compositor/src/frame_geometry.rs | 36 +- electron/ipc/handlers.ts | 19 +- .../capture/linuxNativeCaptureSession.ts | 124 ++++++- .../recording/hyprlandCursorTracker.test.ts | 327 +++++++++++++++++ .../cursor/recording/hyprlandCursorTracker.ts | 329 ++++++++++++++++++ .../recording/pipeWireCursorAccumulator.ts | 8 +- .../native/pipewire-capture/csrc/pw_shim.c | 6 +- .../native/pipewire-capture/src/events.rs | 16 +- electron/native/pipewire-capture/src/main.rs | 81 ++--- .../native/pipewire-capture/src/portal.rs | 100 +++--- scripts/build-whisper-stt.sh | 4 + src/i18n/locales/en/timeline.json | 4 +- .../timeline/zoom-suggestions.test.ts | 6 +- .../ai-edition/timeline/zoom-suggestions.ts | 76 +++- src/lib/nativeLinuxRecording.test.ts | 13 +- src/lib/nativeLinuxRecording.ts | 5 +- 16 files changed, 986 insertions(+), 168 deletions(-) create mode 100644 electron/native-bridge/cursor/recording/hyprlandCursorTracker.test.ts create mode 100644 electron/native-bridge/cursor/recording/hyprlandCursorTracker.ts diff --git a/crates/compositor/src/frame_geometry.rs b/crates/compositor/src/frame_geometry.rs index f189f6f379..4db1f162c2 100644 --- a/crates/compositor/src/frame_geometry.rs +++ b/crates/compositor/src/frame_geometry.rs @@ -3834,11 +3834,11 @@ impl<'a> CursorPlacer<'a> { } } - /// La pointe à `t`, posée sur le point cliqué brut à chaque clic (`pinned_at`), pas sur la piste - /// lissée qui traîne derrière la souris. Sprite plat comme modèle 3D : le rebond, l'anneau - /// d'impact et `ctx.clicks` se centrent sur ce point brut, la pointe doit s'y trouver. + /// La pointe à `t`. Le curseur modélisé touche le plan à chaque clic : il se pose sur le point + /// cliqué brut (`pinned_at`), pas sur la piste lissée qui traîne derrière la souris. Le sprite + /// plat garde la piste telle quelle, son rendu est celui d'avant. fn at(&self, t: f32) -> Option<(f32, f32)> { - self.track.pinned_at(t) + if self.model3d { self.track.pinned_at(t) } else { self.track.at(t) } } /// `None` hors de la coupe courante (zoom serré, hors écran). @@ -9279,28 +9279,20 @@ mod tests { assert_eq!(fx(2.5).cursor, Some(drawn.placement.upright_center()), "là où le curseur se dessine"); } - /// Le sprite plat aussi se pose sur le clic brut : à fort lissage, le ressort traîne loin derrière - /// une souris qui clique dès l'arrivée, mais la pointe dessinée, `ctx.cursor` et `ctx.clicks` - /// tombent sur le même pixel pendant le contact. + /// Hyprland position-only sidecars have no bitmap assets or cursor type. The + /// renderer must still select OpenScreen's bundled arrow sprite, with its + /// configured hotspot, for both preview and export. #[test] - fn the_flat_cursor_lands_on_the_click_under_smoothing() { - const TC: f32 = 1.5; + fn position_only_track_uses_the_bundled_arrow_sprite() { let scene = model_scene(); let cfg = crate::config::all().pop().expect("cfg"); let fg = full_frame_geometry(); - let raw = approach_track([0.08, 0.05], [0.7, 0.6], [0.2, 0.9], 2.0, 0.0, TC); - for smoothing in [0.0, 0.3, 0.67, 1.0] { - let track = raw.smoothed(smoothing); - for ms in [0, 40, 100] { - let input = cursor_input(&scene, &cfg, &track, TC + ms as f32 / 1000.0); - let fx = effect_pointer(&fg, &input); - let (cursor, click) = (fx.cursor.expect("pointe"), fx.clicks[0].0); - let drawn = plan_cursor(&fg, &input).expect("visible").placement.upright_center(); - let off = |a: [f32; 2], b: [f32; 2]| ((a[0] - b[0]) * 1920.0).hypot((a[1] - b[1]) * 1080.0); - assert!(off(cursor, click) < 0.5, "lissage {smoothing}, {ms} ms : {} px", off(cursor, click)); - assert!(off(drawn, click) < 0.5, "lissage {smoothing}, {ms} ms : {} px", off(drawn, click)); - } - } + let track = still_track(vec![]); + let plan = plan_cursor(&fg, &cursor_input(&scene, &cfg, &track, 1.0)).expect("visible"); + + assert_eq!(plan.cursor_type, None, "position-only samples have no native shape"); + assert!(plan.placement.upright_center()[0] > 0.0); + assert!(plan.placement.upright_center()[1] > 0.0); } /// Masqué par l'utilisateur, le curseur n'est plus dessiné, mais les effets gardent la pointe diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 8ac4658c79..9af6e04b72 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -102,6 +102,7 @@ import { LinuxNativeCaptureSession, } from "../native-bridge/capture/linuxNativeCaptureSession"; import { compactCursorTelemetryPauseRanges } from "../native-bridge/cursor/recording/cursorPauseRanges"; +import { HyprlandCursorTracker } from "../native-bridge/cursor/recording/hyprlandCursorTracker"; import { createCursorRecordingSession } from "../native-bridge/cursor/recording/factory"; import { isMacCursorHelperUnavailable, @@ -2630,7 +2631,11 @@ export function registerIpcHandlers( const session = new LinuxNativeCaptureSession({ outputPath, - cursorMode: portalCursorMode(cursorCaptureMode), + cursorMode: portalCursorMode( + cursorCaptureMode, + HyprlandCursorTracker.isAvailable(), + ), + sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS, fps: request.video.fps, ...(request.video.bitrate ? { bitrate: request.video.bitrate } : {}), audio: { @@ -2716,7 +2721,11 @@ export function registerIpcHandlers( prepared ?? new LinuxNativeCaptureSession({ outputPath, - cursorMode: portalCursorMode(cursorCaptureMode), + cursorMode: portalCursorMode( + cursorCaptureMode, + HyprlandCursorTracker.isAvailable(), + ), + sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS, fps: request.video.fps, ...(request.video.bitrate ? { bitrate: request.video.bitrate } : {}), audio: { @@ -2822,7 +2831,11 @@ export function registerIpcHandlers( // The helper collects cursor samples itself, from the same portal // session that produced the pixels, so there is no separate sampler // to stop and no clock offset to correct — the two are one recording. - if (cursorCaptureMode === "editable-overlay" && result.cursor.samples.length > 0) { + // Keep the sidecar even when a run produced zero samples. An empty + // sidecar is an explicit native-capture diagnostic; omitting it made a + // failed Hyprland IPC session indistinguishable from the Chromium + // fallback and hid the real cause of an invisible cursor. + if (cursorCaptureMode === "editable-overlay") { await fs.writeFile( `${result.path}.cursor.json`, JSON.stringify(result.cursor, null, 2), diff --git a/electron/native-bridge/capture/linuxNativeCaptureSession.ts b/electron/native-bridge/capture/linuxNativeCaptureSession.ts index 19a83b5c60..a32f343c62 100644 --- a/electron/native-bridge/capture/linuxNativeCaptureSession.ts +++ b/electron/native-bridge/capture/linuxNativeCaptureSession.ts @@ -5,6 +5,7 @@ import { type CursorPauseRange, compactCursorTelemetryPauseRanges, } from "../cursor/recording/cursorPauseRanges"; +import { HyprlandCursorTracker } from "../cursor/recording/hyprlandCursorTracker"; import { NdjsonLineReader, PipeWireCursorAccumulator, @@ -56,7 +57,8 @@ const HELPER_OUTPUT_MAX_BYTES = 64 * 1024; export interface LinuxCaptureConfig { outputPath: string; - cursorMode: "metadata" | "embedded"; + cursorMode: "auto" | "metadata" | "embedded" | "hidden"; + sampleIntervalMs?: number; fps: number; bitrate?: number; audio: { @@ -119,6 +121,9 @@ export class LinuxNativeCaptureSession { private lastError: string | null = null; private paused = false; private helperOutputText = ""; + private effectiveCursorMode: "metadata" | "embedded" | "hidden" = "metadata"; + private cursorMetadataSupported: boolean | null = null; + private hyprlandCursor: HyprlandCursorTracker | null = null; private cursorRecordingStartMs = 0; private cursorPauseStartedAtMs: number | null = null; private cursorPauseRanges: CursorPauseRange[] = []; @@ -317,6 +322,7 @@ export class LinuxNativeCaptureSession { if (this.paused) { return; } + this.hyprlandCursor?.pause(); this.write("pause"); this.paused = true; this.cursorPauseStartedAtMs = Date.now(); @@ -326,6 +332,7 @@ export class LinuxNativeCaptureSession { if (!this.paused) { return; } + this.hyprlandCursor?.resume(); this.write("resume"); this.completeCursorPauseRange(); this.paused = false; @@ -344,6 +351,11 @@ export class LinuxNativeCaptureSession { return this.paused; } + /** The cursor mode actually negotiated by the portal. */ + get cursorMode(): LinuxCaptureConfig["cursorMode"] { + return this.negotiatedCursorMode ?? this.config.cursorMode; + } + /** * What the portal granted. Known from `stream-started`, so it is populated by * the time [`waitUntilCapturing`] resolves. @@ -356,15 +368,6 @@ export class LinuxNativeCaptureSession { return this.sourceKind; } - /** - * The cursor mode the helper really negotiates, known once `start()` resolves. - * A portal without EMBEDDED gets METADATA instead (#1357), so the cursor has - * to be drawn by the editor even though the system cursor was asked for. - */ - get cursorMode(): LinuxCaptureConfig["cursorMode"] { - return this.negotiatedCursorMode ?? this.config.cursorMode; - } - /** * What the helper said in this session, for Save Diagnostics' `helperOutput.linux`: * its stderr as written and every stdout line except `cursor-sample`, which comes @@ -449,6 +452,8 @@ export class LinuxNativeCaptureSession { } private kill() { + this.hyprlandCursor?.stop(); + this.hyprlandCursor = null; const child = this.process; if (!child || child.killed) { return; @@ -480,12 +485,42 @@ export class LinuxNativeCaptureSession { private handleEvent(payload: PipeWireHelperEvent) { switch (payload.event) { - case "ready": - if (payload.cursorMode === "metadata" || payload.cursorMode === "embedded") { + case "ready": { + if ( + payload.cursorMode === "metadata" || + payload.cursorMode === "embedded" || + payload.cursorMode === "hidden" + ) { this.negotiatedCursorMode = payload.cursorMode; } + this.cursorMetadataSupported = + typeof payload.cursorMetadataSupported === "boolean" + ? payload.cursorMetadataSupported + : null; + this.effectiveCursorMode = + payload.cursorMode === "hidden" + ? "hidden" + : payload.cursorMode === "embedded" + ? "embedded" + : "metadata"; + const useHyprlandPositionTelemetry = + this.config.cursorMode === "auto" && + (this.effectiveCursorMode !== "metadata" || this.cursorMetadataSupported === false); + this.ensureHyprlandCursorTracker( + useHyprlandPositionTelemetry || this.effectiveCursorMode === "hidden", + ); + console.info( + "[capture-linux] cursor negotiation", + JSON.stringify({ + requested: this.config.cursorMode, + effective: this.effectiveCursorMode, + metadataSupported: payload.cursorMetadataSupported ?? null, + hyprlandTracker: Boolean(this.hyprlandCursor), + }), + ); this.resolveReady(); return; + } case "source-selected": if (payload.sourceKind) { @@ -519,6 +554,24 @@ export class LinuxNativeCaptureSession { sourceKind: payload.sourceKind ?? null, }), ); + // Some helper/portal combinations report the negotiated mode before + // the stream details are available. Re-evaluate here so hidden-cursor + // capture cannot silently lose its Hyprland position telemetry. + this.ensureHyprlandCursorTracker( + (this.config.cursorMode === "auto" && + (this.effectiveCursorMode !== "metadata" || this.cursorMetadataSupported === false)) || + this.effectiveCursorMode === "hidden", + ); + if (this.hyprlandCursor && !this.hyprlandCursor.hasCaptureGeometry()) { + this.hyprlandCursor.setCaptureGeometry({ + position: + payload.positionX != null && payload.positionY != null + ? { x: payload.positionX, y: payload.positionY } + : null, + width: payload.logicalWidth ?? payload.width, + height: payload.logicalHeight ?? payload.height, + }); + } return; case "audio-source": @@ -555,6 +608,22 @@ export class LinuxNativeCaptureSession { // picker is dropped rather than left pinned to the start. this.cursor.rebase(payload.timestampMs); this.cursorRecordingStartMs = payload.timestampMs; + this.ensureHyprlandCursorTracker( + (this.config.cursorMode === "auto" && + (this.effectiveCursorMode !== "metadata" || this.cursorMetadataSupported === false)) || + this.effectiveCursorMode === "hidden", + ); + this.hyprlandCursor?.rebase(); + // Older helper builds and some portal backends may omit stream-started + // geometry. The encoded frame dimensions are still sufficient for the + // monitor-size/origin fallback in HyprlandCursorTracker. + if (this.hyprlandCursor && !this.hyprlandCursor.hasCaptureGeometry()) { + this.hyprlandCursor.setCaptureGeometry({ + position: null, + width: payload.width, + height: payload.height, + }); + } this.capturing = true; console.info( "[capture-linux] capture started", @@ -566,19 +635,20 @@ export class LinuxNativeCaptureSession { return; case "cursor-sample": - this.cursor.addSample(payload); + if (this.effectiveCursorMode === "metadata") this.cursor.addSample(payload); return; - case "capture-stopped": + case "capture-stopped": { + const cursor = + this.hyprlandCursor?.stop() ?? + compactCursorTelemetryPauseRanges(this.cursor.toRecordingData(), this.cursorPauseRanges); + this.hyprlandCursor = null; this.stopped = { path: payload.path, durationMs: payload.durationMs, frames: payload.frames, droppedFrames: payload.dropped, - cursor: compactCursorTelemetryPauseRanges( - this.cursor.toRecordingData(), - this.cursorPauseRanges, - ), + cursor, ...(this.sourceKind ? { sourceKind: this.sourceKind } : {}), ...(this.videoEncoder ? { videoEncoder: this.videoEncoder } : {}), }; @@ -591,9 +661,12 @@ export class LinuxNativeCaptureSession { convertMs: payload.convertMs, uploadMs: payload.uploadMs, encodeMs: payload.encodeMs, + cursorProvider: cursor.provider, + cursorSamples: cursor.samples.length, }), ); return; + } case "warning": console.warn(`[capture-linux] ${payload.code}: ${payload.message}`); @@ -616,6 +689,21 @@ export class LinuxNativeCaptureSession { } } + private ensureHyprlandCursorTracker(needed: boolean) { + if (!needed || this.hyprlandCursor) return; + if (!HyprlandCursorTracker.isAvailable()) { + console.warn( + "[capture-linux] position-only cursor capture requested, but the Hyprland IPC socket is unavailable", + ); + return; + } + this.hyprlandCursor = new HyprlandCursorTracker({ + sampleIntervalMs: this.config.sampleIntervalMs ?? 33, + maxSamples: this.config.maxCursorSamples, + }); + this.hyprlandCursor.start(); + } + private resolveReady() { const resolve = this.readyResolve; this.clearReadyState(); diff --git a/electron/native-bridge/cursor/recording/hyprlandCursorTracker.test.ts b/electron/native-bridge/cursor/recording/hyprlandCursorTracker.test.ts new file mode 100644 index 0000000000..c96cb0a24a --- /dev/null +++ b/electron/native-bridge/cursor/recording/hyprlandCursorTracker.test.ts @@ -0,0 +1,327 @@ +import net from "node:net"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { HyprlandCursorTracker, mapHyprlandCursorToCapture } from "./hyprlandCursorTracker"; + +const originalRuntime = process.env.XDG_RUNTIME_DIR; +const originalSignature = process.env.HYPRLAND_INSTANCE_SIGNATURE; + +afterEach(() => { + if (originalRuntime === undefined) delete process.env.XDG_RUNTIME_DIR; + else process.env.XDG_RUNTIME_DIR = originalRuntime; + if (originalSignature === undefined) delete process.env.HYPRLAND_INSTANCE_SIGNATURE; + else process.env.HYPRLAND_INSTANCE_SIGNATURE = originalSignature; + vi.restoreAllMocks(); +}); + +describe("mapHyprlandCursorToCapture", () => { + it("maps a monitor with a non-zero origin and Y offset", () => { + const result = mapHyprlandCursorToCapture( + { x: 5058, y: 673 }, + [ + { x: 0, y: 494, width: 1366, height: 768, scale: 1 }, + { x: 1366, y: 236, width: 2560, height: 1600, scale: 1 }, + { x: 3926, y: 0, width: 3840, height: 2160, scale: 1 }, + ], + { x: 3926, y: 0, width: 3840, height: 2160 }, + ); + + expect(result).toEqual({ cx: 1132 / 3840, cy: 673 / 2160 }); + }); + + it("supports negative origins and fractional scaling", () => { + const result = mapHyprlandCursorToCapture( + { x: -640, y: 100 }, + [{ x: -1280, y: 0, width: 1280, height: 720, scale: 2 }], + { x: -1280, y: 0, width: 1280, height: 720 }, + ); + + expect(result.cx).toBeCloseTo(0.5); + expect(result.cy).toBeCloseTo(100 / 720); + }); + + it("rotates coordinates for transformed monitors", () => { + const result = mapHyprlandCursorToCapture( + { x: 25, y: 50 }, + [{ x: 0, y: 0, width: 100, height: 200, scale: 1, transform: 1 }], + { x: 0, y: 0, width: 100, height: 200 }, + ); + + expect(result).toEqual({ cx: 0.75, cy: 0.25 }); + }); + + it("clamps cursor positions outside a selected recording region", () => { + expect( + mapHyprlandCursorToCapture({ x: 2000, y: -100 }, [], { + x: 0, + y: 0, + width: 1000, + height: 1000, + }), + ).toEqual({ cx: 1, cy: 0 }); + }); + + it("recovers from an IPC disconnect without spawning a process per sample", async () => { + process.env.XDG_RUNTIME_DIR = "/tmp/openscreen-hypr-runtime"; + process.env.HYPRLAND_INSTANCE_SIGNATURE = "test-instance"; + let connections = 0; + vi.spyOn(net, "createConnection").mockImplementation(() => { + connections++; + const listeners = new Map void>(); + const socket = { + writable: true, + destroyed: false, + setTimeout: () => socket, + on: (event: string, listener: (...args: unknown[]) => void) => { + listeners.set(event, listener); + if (event === "connect") queueMicrotask(listener); + return socket; + }, + write: (command: string) => { + if (connections === 1) { + listeners.get("error")?.(new Error("compositor restarted")); + return true; + } + if (command === "j/cursorpos") listeners.get("data")?.('{"x":50,"y":25}'); + if (command === "j/monitors") { + listeners.get("data")?.( + '[{"x":0,"y":0,"width":100,"height":50,"scale":1,"transform":0}]', + ); + } + return true; + }, + destroy: () => { + socket.destroyed = true; + }, + }; + return socket as unknown as net.Socket; + }); + + const tracker = new HyprlandCursorTracker({ sampleIntervalMs: 20, maxSamples: 10 }); + tracker.setCaptureRect({ x: 0, y: 0, width: 100, height: 50 }); + tracker.start(); + await new Promise((resolve) => setTimeout(resolve, 120)); + const result = tracker.stop(); + + expect(connections).toBeGreaterThanOrEqual(2); + expect(result.samples.length).toBeGreaterThan(0); + }); + + it("resolves an omitted portal origin from a matching Hyprland monitor", async () => { + process.env.XDG_RUNTIME_DIR = "/tmp/openscreen-hypr-runtime"; + process.env.HYPRLAND_INSTANCE_SIGNATURE = "geometry-fallback"; + vi.spyOn(net, "createConnection").mockImplementation(() => { + const listeners = new Map void>(); + const socket = { + writable: true, + destroyed: false, + setTimeout: () => socket, + on: (event: string, listener: (...args: unknown[]) => void) => { + listeners.set(event, listener); + if (event === "connect") queueMicrotask(listener); + return socket; + }, + write: (command: string) => { + if (command === "j/cursorpos") listeners.get("data")?.('{"x":1500,"y":500}'); + if (command === "j/monitors") { + listeners.get("data")?.( + '[{"x":0,"y":494,"width":1366,"height":768,"scale":1,"transform":0},{"x":1366,"y":236,"width":2560,"height":1600,"scale":1,"transform":0}]', + ); + } + return true; + }, + destroy: () => { + socket.destroyed = true; + }, + }; + return socket as unknown as net.Socket; + }); + + const tracker = new HyprlandCursorTracker({ sampleIntervalMs: 10, maxSamples: 10 }); + tracker.setCaptureGeometry({ width: 2560, height: 1600 }); + tracker.start(); + await new Promise((resolve) => setTimeout(resolve, 120)); + + const sample = tracker.stop().samples[0]; + expect(sample).toMatchObject({ cx: 134 / 2560, cy: 264 / 1600, visible: true }); + }); + + it("matches a scaled monitor using its logical capture size", async () => { + process.env.XDG_RUNTIME_DIR = "/tmp/openscreen-hypr-runtime"; + process.env.HYPRLAND_INSTANCE_SIGNATURE = "scaled-geometry-fallback"; + vi.spyOn(net, "createConnection").mockImplementation(() => { + const listeners = new Map void>(); + const socket = { + writable: true, + destroyed: false, + setTimeout: () => socket, + on: (event: string, listener: (...args: unknown[]) => void) => { + listeners.set(event, listener); + if (event === "connect") queueMicrotask(listener); + return socket; + }, + write: (command: string) => { + if (command === "j/cursorpos") listeners.get("data")?.('{"x":500,"y":300}'); + if (command === "j/monitors") { + listeners.get("data")?.( + '[{"x":0,"y":0,"width":2560,"height":1600,"scale":2,"transform":0}]', + ); + } + return true; + }, + destroy: () => { + socket.destroyed = true; + }, + }; + return socket as unknown as net.Socket; + }); + + const tracker = new HyprlandCursorTracker({ sampleIntervalMs: 10, maxSamples: 10 }); + tracker.setCaptureGeometry({ width: 1280, height: 800 }); + tracker.start(); + await new Promise((resolve) => setTimeout(resolve, 60)); + + const sample = tracker.stop().samples[0]; + expect(sample).toMatchObject({ cx: 500 / 1280, cy: 300 / 800, visible: true }); + }); + + it("uses the containing monitor for a smaller window stream", async () => { + process.env.XDG_RUNTIME_DIR = "/tmp/openscreen-hypr-runtime"; + process.env.HYPRLAND_INSTANCE_SIGNATURE = "window-geometry-fallback"; + vi.spyOn(net, "createConnection").mockImplementation(() => { + const listeners = new Map void>(); + const socket = { + writable: true, + destroyed: false, + setTimeout: () => socket, + on: (event: string, listener: (...args: unknown[]) => void) => { + listeners.set(event, listener); + if (event === "connect") queueMicrotask(listener); + return socket; + }, + write: (command: string) => { + if (command === "j/cursorpos") listeners.get("data")?.('{"x":1500,"y":500}'); + if (command === "j/monitors") { + listeners.get("data")?.( + '[{"x":0,"y":494,"width":1366,"height":768,"scale":1,"transform":0},{"x":1366,"y":236,"width":2560,"height":1600,"scale":1,"transform":0}]', + ); + } + return true; + }, + destroy: () => { + socket.destroyed = true; + }, + }; + return socket as unknown as net.Socket; + }); + + const tracker = new HyprlandCursorTracker({ sampleIntervalMs: 10, maxSamples: 10 }); + tracker.setCaptureGeometry({ width: 1234, height: 720 }); + tracker.start(); + await new Promise((resolve) => setTimeout(resolve, 60)); + + const sample = tracker.stop().samples[0]; + expect(sample).toMatchObject({ cx: 134 / 1234, cy: 264 / 720, visible: true }); + }); + + it("corrects a zero portal origin for a uniquely sized monitor", async () => { + process.env.XDG_RUNTIME_DIR = "/tmp/openscreen-hypr-runtime"; + process.env.HYPRLAND_INSTANCE_SIGNATURE = "exact-monitor-origin"; + vi.spyOn(net, "createConnection").mockImplementation(() => { + const listeners = new Map void>(); + const socket = { + writable: true, + destroyed: false, + setTimeout: () => socket, + on: (event: string, listener: (...args: unknown[]) => void) => { + listeners.set(event, listener); + if (event === "connect") queueMicrotask(listener); + return socket; + }, + write: (command: string) => { + if (command === "j/cursorpos") listeners.get("data")?.('{"x":4479,"y":1080}'); + if (command === "j/monitors") { + listeners.get("data")?.( + '[{"x":1366,"y":236,"width":2560,"height":1600},{"x":3926,"y":0,"width":3840,"height":2160}]', + ); + } + return true; + }, + destroy: () => { + socket.destroyed = true; + }, + }; + return socket as unknown as net.Socket; + }); + + const tracker = new HyprlandCursorTracker({ sampleIntervalMs: 10, maxSamples: 5 }); + tracker.setCaptureGeometry({ position: { x: 0, y: 0 }, width: 3840, height: 2160 }); + tracker.start(); + await new Promise((resolve) => setTimeout(resolve, 30)); + const sample = tracker.stop().samples[0]; + + expect(sample?.cx).toBeCloseTo((4479 - 3926) / 3840); + expect(sample?.cy).toBeCloseTo(1080 / 2160); + }); + + it("does not produce samples when the Hyprland socket is unavailable", async () => { + process.env.XDG_RUNTIME_DIR = "/tmp/openscreen-no-such-runtime"; + process.env.HYPRLAND_INSTANCE_SIGNATURE = "missing"; + const tracker = new HyprlandCursorTracker({ sampleIntervalMs: 10, maxSamples: 10 }); + tracker.setCaptureRect({ x: 0, y: 0, width: 100, height: 50 }); + tracker.start(); + await new Promise((resolve) => setTimeout(resolve, 40)); + expect(tracker.stop().samples).toEqual([]); + }); + + it("rebases timestamps and excludes paused time", async () => { + process.env.XDG_RUNTIME_DIR = "/tmp/openscreen-hypr-runtime"; + process.env.HYPRLAND_INSTANCE_SIGNATURE = "timestamp-test"; + vi.spyOn(net, "createConnection").mockImplementation(() => { + const listeners = new Map void>(); + const socket = { + writable: true, + destroyed: false, + setTimeout: () => socket, + on: (event: string, listener: (...args: unknown[]) => void) => { + listeners.set(event, listener); + if (event === "connect") queueMicrotask(listener); + return socket; + }, + write: (command: string) => { + if (command === "j/cursorpos") listeners.get("data")?.('{"x":50,"y":25}'); + if (command === "j/monitors") { + listeners.get("data")?.( + '[{"x":0,"y":0,"width":100,"height":50,"scale":1,"transform":0}]', + ); + } + return true; + }, + destroy: () => { + socket.destroyed = true; + }, + }; + return socket as unknown as net.Socket; + }); + + const tracker = new HyprlandCursorTracker({ sampleIntervalMs: 15, maxSamples: 20 }); + tracker.setCaptureRect({ x: 0, y: 0, width: 100, height: 50 }); + tracker.start(); + await new Promise((resolve) => setTimeout(resolve, 45)); + tracker.rebase(); + await new Promise((resolve) => setTimeout(resolve, 30)); + tracker.pause(); + await new Promise((resolve) => setTimeout(resolve, 80)); + tracker.resume(); + await new Promise((resolve) => setTimeout(resolve, 45)); + + const recording = tracker.stop(); + const samples = recording.samples; + expect(recording.provider).toBe("native"); + expect(samples.length).toBeGreaterThan(1); + expect(samples[0].timeMs).toBeLessThan(30); + expect(samples.at(-1)?.timeMs ?? 0).toBeLessThan(110); + expect( + samples.every((sample, index) => index === 0 || sample.timeMs >= samples[index - 1].timeMs), + ).toBe(true); + }); +}); diff --git a/electron/native-bridge/cursor/recording/hyprlandCursorTracker.ts b/electron/native-bridge/cursor/recording/hyprlandCursorTracker.ts new file mode 100644 index 0000000000..4153aa3353 --- /dev/null +++ b/electron/native-bridge/cursor/recording/hyprlandCursorTracker.ts @@ -0,0 +1,329 @@ +import net from "node:net"; +import { existsSync, readdirSync } from "node:fs"; +import { hrtime } from "node:process"; +import type { CursorRecordingData, CursorRecordingSample } from "../../../../src/native/contracts"; + +export interface HyprlandMonitorGeometry { + x: number; + y: number; + width: number; + height: number; + scale?: number; + transform?: number; +} + +export interface HyprlandCaptureRect { + x: number; + y: number; + width: number; + height: number; +} + +export interface HyprlandCaptureGeometry { + position?: { x: number; y: number } | null; + width: number; + height: number; +} + +export function mapHyprlandCursorToCapture( + point: { x: number; y: number }, + monitors: HyprlandMonitorGeometry[], + capture: HyprlandCaptureRect, +) { + const monitor = monitors.find( + (candidate) => + point.x >= candidate.x && + point.y >= candidate.y && + point.x < candidate.x + candidate.width && + point.y < candidate.y + candidate.height, + ); + const transform = monitor?.transform ?? 0; + const logicalX = point.x - capture.x; + const logicalY = point.y - capture.y; + // Hyprland cursor coordinates and portal stream positions/sizes are in the + // compositor's logical coordinate space. The PipeWire frame dimensions are + // physical pixels and must not be used for this ratio, especially on scaled + // monitors. `capture` is populated from the portal's logical `size`. + const logicalWidth = Math.max(1, capture.width); + const logicalHeight = Math.max(1, capture.height); + let x = logicalX / logicalWidth; + let y = logicalY / logicalHeight; + + if (transform === 1) { + [x, y] = [1 - y, x]; + } else if (transform === 2) { + [x, y] = [1 - x, 1 - y]; + } else if (transform === 3) { + [x, y] = [y, 1 - x]; + } + + return { + cx: Math.min(1, Math.max(0, x)), + cy: Math.min(1, Math.max(0, y)), + }; +} + +function hyprlandSocketPath() { + const uid = typeof process.getuid === "function" ? process.getuid() : null; + const runtime = process.env.XDG_RUNTIME_DIR?.trim() || (uid === null ? null : `/run/user/${uid}`); + if (!runtime) return null; + + const configuredSignature = process.env.HYPRLAND_INSTANCE_SIGNATURE?.trim(); + if (configuredSignature) { + return `${runtime}/hypr/${configuredSignature}/.socket.sock`; + } + + // Electron launched from a desktop entry may not inherit Hyprland's + // instance variables even though it runs in the same user session. Discover + // the one live socket rather than disabling position telemetry in that case. + try { + const hyprlandRoot = `${runtime}/hypr`; + const signature = readdirSync(hyprlandRoot, { withFileTypes: true }) + .find( + (entry) => + entry.isDirectory() && existsSync(`${hyprlandRoot}/${entry.name}/.socket.sock`), + ) + ?.name; + return signature ? `${hyprlandRoot}/${signature}/.socket.sock` : null; + } catch { + return null; + } +} + +function monotonicMs() { + return Number(hrtime.bigint() / 1_000_000n); +} + +export class HyprlandCursorTracker { + private sockets = new Set(); + private interval: NodeJS.Timeout | null = null; + private sampleInFlight = false; + private monitorCache: HyprlandMonitorGeometry[] = []; + private monitorCacheAt = 0; + private samples: CursorRecordingSample[] = []; + private startMs = 0; + private pausedAt: number | null = null; + private pausedDurationMs = 0; + private captureRect: HyprlandCaptureRect | null = null; + private captureSize: { width: number; height: number } | null = null; + private lastIpcError: string | null = null; + + constructor(private readonly options: { sampleIntervalMs: number; maxSamples: number }) {} + + static isAvailable() { + return Boolean(hyprlandSocketPath()); + } + + start() { + this.stopSocket(); + this.samples = []; + this.startMs = monotonicMs(); + this.pausedAt = null; + this.pausedDurationMs = 0; + this.interval = setInterval(() => void this.sample(), this.options.sampleIntervalMs); + this.interval.unref(); + void this.sample(); + } + + stop(): CursorRecordingData { + if (this.interval) clearInterval(this.interval); + this.interval = null; + this.stopSocket(); + return { version: 2, provider: "native", samples: this.samples, assets: [] }; + } + + pause() { + if (this.pausedAt === null) this.pausedAt = monotonicMs(); + } + + resume() { + if (this.pausedAt !== null) { + this.pausedDurationMs += Math.max(0, monotonicMs() - this.pausedAt); + this.pausedAt = null; + } + } + + rebase() { + this.startMs = monotonicMs(); + this.pausedDurationMs = 0; + this.pausedAt = null; + this.samples = []; + } + + setCaptureRect(rect: HyprlandCaptureRect | null) { + this.captureRect = rect; + this.captureSize = rect ? { width: rect.width, height: rect.height } : null; + } + + /** + * Some portal backends omit Stream.position even for a monitor stream. Keep + * sampling in that case and resolve the origin from Hyprland's monitor list + * using the logical stream size. An omitted portal position must not turn a + * valid recording into an empty cursor sidecar. + */ + setCaptureGeometry(geometry: HyprlandCaptureGeometry | null) { + if (!geometry) { + this.captureRect = null; + this.captureSize = null; + return; + } + this.captureSize = { + width: Math.max(1, geometry.width), + height: Math.max(1, geometry.height), + }; + this.captureRect = geometry.position + ? { + x: geometry.position.x, + y: geometry.position.y, + width: this.captureSize.width, + height: this.captureSize.height, + } + : null; + } + + hasCaptureGeometry() { + return this.captureRect !== null || this.captureSize !== null; + } + + private async sample() { + if (this.sampleInFlight || this.pausedAt !== null || (!this.captureRect && !this.captureSize)) return; + this.sampleInFlight = true; + try { + const point = await this.request<{ x: number; y: number }>("j/cursorpos"); + const now = monotonicMs(); + const monitors = + now - this.monitorCacheAt > 1000 + ? await this.request("j/monitors") + : this.monitorCache; + this.monitorCache = Array.isArray(monitors) ? monitors : this.monitorCache; + this.monitorCacheAt = now; + this.correctPortalOriginForExactMonitor(point); + if (!this.captureRect && this.captureSize) { + const matches = this.monitorCache.filter( + (monitor) => { + const scale = monitor.scale && monitor.scale > 0 ? monitor.scale : 1; + const widths = [monitor.width, monitor.width / scale].map(Math.round); + const heights = [monitor.height, monitor.height / scale].map(Math.round); + return widths.includes(this.captureSize?.width ?? -1) && + heights.includes(this.captureSize?.height ?? -1); + }, + ); + const containing = this.monitorCache.find( + (monitor) => + point.x >= monitor.x && + point.y >= monitor.y && + point.x < monitor.x + monitor.width && + point.y < monitor.y + monitor.height, + ); + // Exact size wins for monitor captures. For a window capture whose + // portal size is smaller than the monitor, the containing monitor is + // the best available origin when the portal omitted Stream.position. + const monitor = matches.length === 1 ? matches[0] : containing; + if (monitor) { + this.captureRect = { + x: monitor.x, + y: monitor.y, + width: this.captureSize.width, + height: this.captureSize.height, + }; + } + } + if (!this.captureRect) return; + const mapped = mapHyprlandCursorToCapture(point, this.monitorCache, this.captureRect); + this.samples.push({ + timeMs: Math.max(0, now - this.startMs - this.pausedDurationMs), + ...mapped, + visible: true, + interactionType: "move", + }); + if (this.samples.length > this.options.maxSamples) this.samples.shift(); + } catch (error) { + // IPC is best-effort. A compositor restart is recovered on the next sample. + const message = error instanceof Error ? error.message : String(error); + if (message !== this.lastIpcError) { + this.lastIpcError = message; + console.warn("[hyprland-cursor] sample failed:", message); + } + } finally { + this.sampleInFlight = false; + } + } + + /** + * Hyprland's portal backend can report a full-monitor stream's origin as + * (0,0), even when the selected monitor has a negative or non-zero virtual + * desktop origin. If the logical stream size uniquely identifies a monitor + * and the pointer is inside it, trust Hyprland's monitor geometry for the + * origin. Window streams remain untouched because their size normally does + * not uniquely identify a monitor. + */ + private correctPortalOriginForExactMonitor(point: { x: number; y: number }) { + if (!this.captureRect) return; + const matches = this.monitorCache.filter((monitor) => { + const scale = monitor.scale && monitor.scale > 0 ? monitor.scale : 1; + const widths = [monitor.width, monitor.width / scale].map(Math.round); + const heights = [monitor.height, monitor.height / scale].map(Math.round); + return ( + widths.includes(Math.round(this.captureRect!.width)) && + heights.includes(Math.round(this.captureRect!.height)) + ); + }); + const monitor = matches.find( + (candidate) => + point.x >= candidate.x && + point.y >= candidate.y && + point.x < candidate.x + candidate.width && + point.y < candidate.y + candidate.height, + ); + if (matches.length === 1 && monitor) { + this.captureRect.x = monitor.x; + this.captureRect.y = monitor.y; + } + } + + private request(command: string): Promise { + return new Promise((resolve, reject) => { + const socketPath = hyprlandSocketPath(); + if (!socketPath) { + reject(new Error("Hyprland IPC socket is unavailable")); + return; + } + + const socket = net.createConnection(socketPath); + this.sockets.add(socket); + let response = ""; + let settled = false; + const settle = (error?: Error, value?: T) => { + if (settled) return; + settled = true; + this.sockets.delete(socket); + if (error) reject(error); + else resolve(value as T); + }; + + socket.setTimeout(Math.max(250, this.options.sampleIntervalMs * 2)); + socket.on("connect", () => socket.write(command)); + socket.on("data", (chunk) => { + response += chunk.toString(); + try { + settle(undefined, JSON.parse(response) as T); + socket.destroy(); + } catch { + // Wait for the rest of a JSON response. + } + }); + socket.on("error", (error) => settle(error)); + socket.on("timeout", () => settle(new Error("Hyprland IPC timeout"))); + socket.on("close", () => { + if (!settled) settle(new Error("Hyprland IPC connection closed")); + }); + }); + } + + private stopSocket() { + if (this.interval) clearInterval(this.interval); + this.interval = null; + for (const socket of this.sockets) socket.destroy(); + this.sockets.clear(); + } +} diff --git a/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts b/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts index 9a233c7d26..26c5e98b77 100644 --- a/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts +++ b/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts @@ -31,8 +31,8 @@ export type PipeWireHelperEvent = event: "ready"; timestampMs: number; pipewireVersion?: string | null; - /** The mode really negotiated; absent from helpers older than #1357. */ - cursorMode?: "metadata" | "embedded" | "hidden"; + cursorMetadataSupported?: boolean; + cursorMode?: "auto" | "metadata" | "embedded" | "hidden"; } | { /** The picker has been answered. Fires before any pixel moves. */ @@ -49,6 +49,10 @@ export type PipeWireHelperEvent = nodeId: number; width: number; height: number; + positionX?: number | null; + positionY?: number | null; + logicalWidth?: number | null; + logicalHeight?: number | null; /** What the compositor actually handed over. Absent on backends that * omit it — treat that as unknown, never as a screen. */ sourceKind?: "monitor" | "window" | "virtual" | null; diff --git a/electron/native/pipewire-capture/csrc/pw_shim.c b/electron/native/pipewire-capture/csrc/pw_shim.c index d28e6c9909..f4c9bc629e 100644 --- a/electron/native/pipewire-capture/csrc/pw_shim.c +++ b/electron/native/pipewire-capture/csrc/pw_shim.c @@ -612,9 +612,11 @@ int osc_pw_enum_format_accepts_dmabuf_producer(int with_modifier, int64_t produc const struct spa_pod *consumer; const struct spa_pod *producer; - /* The unit test exercises the full tiled offer, so enumerate unconditionally. */ + /* Keep this synthetic intersection test independent of the host EGL stack. + * Runtime negotiation still calls the same builder with prefer_dmabuf=1; + * the test only needs the guaranteed CPU-readable fallback modifiers. */ consumer = - with_modifier ? osc_build_enum_format_dmabuf(&ours, 1) : osc_build_enum_format(&ours); + with_modifier ? osc_build_enum_format_dmabuf(&ours, 0) : osc_build_enum_format(&ours); if (consumer == NULL) { return -1; } diff --git a/electron/native/pipewire-capture/src/events.rs b/electron/native/pipewire-capture/src/events.rs index 9bc4d4c7a4..b8ac85d1b6 100644 --- a/electron/native/pipewire-capture/src/events.rs +++ b/electron/native/pipewire-capture/src/events.rs @@ -48,9 +48,8 @@ pub enum Event { pipewire_version: Option, /// Whether the portal advertises METADATA cursor mode at all. cursor_metadata_supported: bool, - /// The cursor mode actually negotiated, which can differ from the one - /// asked for (see `CursorMode::resolve`). - cursor_mode: &'static str, + /// The mode selected for this session after capability negotiation. + cursor_mode: String, }, /// The user has answered the compositor's picker and a source was granted. /// @@ -78,6 +77,10 @@ pub enum Event { /// space, when the portal reports one (monitor streams only). position_x: Option, position_y: Option, + /// Logical compositor-space size. This can differ from the PipeWire + /// frame size on scaled monitors. + logical_width: Option, + logical_height: Option, /// `"monitor"`, `"window"` or `"virtual"` — what the compositor /// actually handed over, straight from the portal's reply. Absent when /// the backend omits it. This is the only honest answer to "what am I @@ -256,7 +259,7 @@ mod tests { timestamp_ms: 42, pipewire_version: Some("1.0.5".to_owned()), cursor_metadata_supported: true, - cursor_mode: "metadata", + cursor_mode: "metadata".to_owned(), }, false, ); @@ -279,13 +282,12 @@ mod tests { timestamp_ms: 7, pipewire_version: None, cursor_metadata_supported: false, - cursor_mode: "embedded", + cursor_mode: "hidden".to_owned(), }); assert_eq!(value["event"], "ready"); assert_eq!(value["timestampMs"], 7); assert!(value["pipewireVersion"].is_null()); assert_eq!(value["cursorMetadataSupported"], false); - assert_eq!(value["cursorMode"], "embedded"); } #[test] @@ -297,6 +299,8 @@ mod tests { height: 1440, position_x: Some(0), position_y: Some(-1080), + logical_width: Some(1920), + logical_height: Some(1080), source_kind: Some("window".to_owned()), }); assert_eq!(value["event"], "stream-started"); diff --git a/electron/native/pipewire-capture/src/main.rs b/electron/native/pipewire-capture/src/main.rs index aac59ec4dc..8ed4b3337d 100644 --- a/electron/native/pipewire-capture/src/main.rs +++ b/electron/native/pipewire-capture/src/main.rs @@ -145,10 +145,11 @@ impl Request { fn cursor_mode(&self) -> Result { match self.cursor_mode.as_deref() { None | Some("") | Some("metadata") => Ok(portal::CursorMode::Metadata), + Some("auto") => Ok(portal::CursorMode::Auto), Some("embedded") => Ok(portal::CursorMode::Embedded), Some("hidden") => Ok(portal::CursorMode::Hidden), Some(other) => Err(format!( - "cursorMode must be one of metadata, embedded, hidden — got {other:?}" + "cursorMode must be one of auto, metadata, embedded, hidden — got {other:?}" )), } } @@ -249,7 +250,7 @@ fn main() { fail(&mut emitter, "invalid-arguments", &message); } }; - let cursor_mode = match request.cursor_mode() { + let requested_cursor_mode = match request.cursor_mode() { Ok(mode) => mode, Err(message) => fail(&mut emitter, "invalid-arguments", &message), }; @@ -280,18 +281,18 @@ fn main() { } // Probed before `ready` so the event can report it, and so an unsupported - // compositor fails immediately instead of after a pointless picker. Only - // fatal when METADATA is the mode actually requested — a recording that - // wants the system cursor painted in has no use for it. - let offered_cursor_modes = match pollster::block_on(portal::offered_cursor_modes()) { - Ok(modes) => modes, + // compositor fails immediately instead of after a pointless picker. Auto + // chooses metadata first, then hidden, then embedded based on the actual + // bit flags advertised by the portal. + let cursor_capabilities = match pollster::block_on(portal::cursor_capabilities()) { + Ok(capabilities) => capabilities, Err(error) => fail(&mut emitter, error.code(), &error.message()), }; - let cursor_metadata_supported = offered_cursor_modes.contains(&portal::CursorMode::Metadata); - // Resolved before anything reads it, so the portal, the click readers and - // `ready` all see the mode that is really negotiated. - let cursor_mode = cursor_mode.resolve(&offered_cursor_modes); - if !cursor_metadata_supported && cursor_mode.reports_cursor() { + let cursor_mode = requested_cursor_mode.resolve( + cursor_capabilities.metadata, + cursor_capabilities.hidden, + ); + if !cursor_capabilities.metadata && cursor_mode.reports_cursor() { let error = portal::PortalError::CursorMetadataUnsupported; fail(&mut emitter, error.code(), &error.message()); } @@ -299,8 +300,8 @@ fn main() { let _ = emitter.emit(&Event::Ready { timestamp_ms: timestamp_ms(), pipewire_version: shim::library_version(), - cursor_metadata_supported, - cursor_mode: cursor_mode.as_str(), + cursor_metadata_supported: cursor_capabilities.metadata, + cursor_mode: cursor_mode.as_str().to_owned(), }); if request.probe_only { @@ -578,7 +579,12 @@ fn begin_stream( Ok(started) => { *session = Some(started); *granted_kind = source_kind; - *portal_stream = Some(StreamInfo { node_id, position, source_kind }); + *portal_stream = Some(StreamInfo { + node_id, + position, + size, + source_kind, + }); Ok(()) } Err(message) => { @@ -596,6 +602,7 @@ fn begin_stream( struct StreamInfo { node_id: u32, position: Option<(i32, i32)>, + size: Option<(i32, i32)>, source_kind: Option, } @@ -608,20 +615,6 @@ struct SharedMemoryFallback { source_kind: Option, } -/// Whether the negotiated buffers carry a SPA_META_VideoCrop block, read from -/// the shim's "Header:12,VideoCrop:16" list. -/// -/// Only then is waiting for a window's crop worth anything. mutter negotiates -/// the block and fills it a few frames late; KWin never negotiates it — it sizes -/// a window stream to the window itself — and sends one frame when the stream -/// starts, then only when the window repaints. Waiting on KWin therefore threw -/// away the one frame a still window ever sends, and the start timed out. -fn negotiates_video_crop(metas: &str) -> bool { - metas - .split(',') - .any(|meta| meta.split(':').next() == Some("VideoCrop")) -} - fn should_retry_shared_memory( state: &str, error: Option<&str>, @@ -663,9 +656,6 @@ fn run( let mut granted_kind: Option = None; // Frames skipped while waiting for a window's first usable crop rectangle. let mut frames_awaiting_crop: u32 = 0; - // Set from the first `BufferInfo`, which the shim sends before that same - // buffer's frame. See `negotiates_video_crop`. - let mut crop_meta_negotiated = false; // A crop change is reported once, not once per frame. let mut crop_change_reported = false; // A `deferStart` grant held while the caller runs its countdown. @@ -835,11 +825,8 @@ fn run( // the encoder to that frame would pin a window // recording at monitor size for its whole duration, // which is the very bug being fixed, only intermittent. - // A compositor that never negotiated the crop meta - // (KWin) is not waited on at all. - let expecting_crop = granted_kind == Some(portal::SourceKind::Window) - && crop_meta_negotiated - && !frame.has_crop; + let expecting_crop = + granted_kind == Some(portal::SourceKind::Window) && !frame.has_crop; if expecting_crop && frames_awaiting_crop < MAX_FRAMES_AWAITING_CROP { frames_awaiting_crop += 1; mailbox.recycle(frame.pixels); @@ -1055,9 +1042,9 @@ fn run( Ok(stream) => { // The portal's size is in the compositor's coordinate space // and can differ from the negotiated pixel size on a scaled - // display. Logged rather than used: cursor positions arrive - // in stream pixels, and `content_rect` places them from - // there. + // display. The logical size is forwarded for Hyprland cursor + // normalization; the PipeWire frame size remains the video + // encoder's physical-pixel geometry. let _ = emitter.emit(&Event::Debug { code: "portal-stream".to_owned(), data: json_map([ @@ -1138,6 +1125,8 @@ fn run( height: format.height, position_x: stream.position.map(|(x, _)| x), position_y: stream.position.map(|(_, y)| y), + logical_width: stream.size.map(|(width, _)| width), + logical_height: stream.size.map(|(_, height)| height), source_kind: stream .source_kind .map(|kind| kind.as_str().to_owned()), @@ -1180,7 +1169,6 @@ fn run( // bearing field: a missing Cursor next to a present Header means // our ParamMeta was sent and lost the size intersection, which is // otherwise indistinguishable from never having sent it. - crop_meta_negotiated = negotiates_video_crop(&metas); let _ = emitter.emit(&Event::Debug { code: "buffer-info".to_owned(), data: json_map([ @@ -1862,16 +1850,7 @@ mod microphone_resolution_tests { #[cfg(test)] mod capture_fallback_tests { - use super::{negotiates_video_crop, should_retry_shared_memory, stream_error_is_fatal}; - - #[test] - fn only_a_negotiated_video_crop_block_is_worth_waiting_for() { - // mutter: the block is there, its rect may arrive a few frames late. - assert!(negotiates_video_crop("Header:12,VideoCrop:16,Cursor:589872")); - // KWin: no block, so waiting would discard a still window's only frame. - assert!(!negotiates_video_crop("Header:12,VideoDamage:256,Cursor:589872")); - assert!(!negotiates_video_crop("")); - } + use super::{should_retry_shared_memory, stream_error_is_fatal}; #[test] fn a_stream_error_is_fatal_only_before_capture_starts() { diff --git a/electron/native/pipewire-capture/src/portal.rs b/electron/native/pipewire-capture/src/portal.rs index 68ef7156de..046369a473 100644 --- a/electron/native/pipewire-capture/src/portal.rs +++ b/electron/native/pipewire-capture/src/portal.rs @@ -26,7 +26,7 @@ use ashpd::desktop::PersistMode; use ashpd::enumflags2::BitFlags; use ashpd::WindowIdentifier; -/// Which of the portal's three cursor modes to ask for. +/// Which cursor mode to ask the portal for. /// /// This is the HUD's cursor-mode toggle, one layer down. `Metadata` is what /// makes the editable cursor possible at all — the compositor keeps the pointer @@ -35,6 +35,7 @@ use ashpd::WindowIdentifier; /// leaves it alone. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum CursorMode { + Auto, Metadata, Embedded, Hidden, @@ -43,43 +44,37 @@ pub enum CursorMode { impl CursorMode { fn to_portal(self) -> PortalCursorMode { match self { + Self::Auto => PortalCursorMode::Metadata, Self::Metadata => PortalCursorMode::Metadata, Self::Embedded => PortalCursorMode::Embedded, Self::Hidden => PortalCursorMode::Hidden, } } - /// Only METADATA yields cursor samples; in the other modes the pointer is - /// either painted into the frames or absent, and there is no metadata to - /// read either way. + /// Only METADATA yields portal cursor samples; in the other modes the + /// pointer is either painted into the frames or absent, and there is no + /// portal metadata to read either way. pub fn reports_cursor(self) -> bool { matches!(self, Self::Metadata) } + pub fn resolve(self, metadata_supported: bool, hidden_supported: bool) -> Self { + match self { + Self::Auto if metadata_supported => Self::Metadata, + Self::Auto if hidden_supported => Self::Hidden, + Self::Auto => Self::Embedded, + other => other, + } + } + pub fn as_str(self) -> &'static str { match self { + Self::Auto => "auto", Self::Metadata => "metadata", Self::Embedded => "embedded", Self::Hidden => "hidden", } } - - /// The mode to actually negotiate, given what the portal offers. - /// - /// Some portals offer no EMBEDDED at all (niri's, #1357). Refusing to record - /// there leaves the user nothing, so EMBEDDED falls back to METADATA: the - /// pointer stays out of the pixels and the editor draws it from the samples - /// instead. Any other miss is returned unchanged, for the negotiation to - /// refuse with its own error. - pub fn resolve(self, offered: &[CursorMode]) -> CursorMode { - if self == Self::Embedded - && !offered.contains(&Self::Embedded) - && offered.contains(&Self::Metadata) - { - return Self::Metadata; - } - self - } } /// Which kind of source the compositor actually handed over. @@ -192,20 +187,40 @@ fn portal_unavailable(error: impl std::fmt::Display) -> PortalError { )) } -/// Cheap, non-interactive probe: which cursor modes does this portal offer? +/// Cheap, non-interactive probe for the cursor modes needed by auto selection. /// /// Split out from [`negotiate`] so the helper can answer that question — and /// emit `ready` — before anything raises a dialog. -pub async fn offered_cursor_modes() -> Result, PortalError> { +pub struct CursorCapabilities { + pub metadata: bool, + pub hidden: bool, +} + +pub async fn cursor_capabilities() -> Result { let proxy = Screencast::new().await.map_err(portal_unavailable)?; let cursor_modes = proxy .available_cursor_modes() .await .map_err(|error| failed("AvailableCursorModes", error))?; - Ok([CursorMode::Metadata, CursorMode::Embedded, CursorMode::Hidden] - .into_iter() - .filter(|mode| cursor_modes.contains(mode.to_portal())) - .collect()) + Ok(CursorCapabilities { + metadata: cursor_modes.contains(PortalCursorMode::Metadata), + hidden: cursor_modes.contains(PortalCursorMode::Hidden), + }) +} + +#[cfg(test)] +mod tests { + use super::CursorMode; + + #[test] + fn auto_prefers_metadata_then_hidden_then_embedded() { + assert_eq!( + CursorMode::Auto.resolve(true, true), + CursorMode::Metadata + ); + assert_eq!(CursorMode::Auto.resolve(false, true), CursorMode::Hidden); + assert_eq!(CursorMode::Auto.resolve(false, false), CursorMode::Embedded); + } } /// Runs the negotiation to completion. Blocks on the user for as long as the @@ -223,9 +238,12 @@ pub async fn negotiate(cursor_mode: CursorMode) -> Result PortalError::CursorMetadataUnsupported, @@ -304,25 +322,3 @@ pub async fn negotiate(cursor_mode: CursorMode) -> Result { expect(plan(telemetry)).toEqual([]); }); - it("places no zoom where the pointer only sits still", () => { - expect(plan(park(2000, 0.5, 0.5))).toEqual([]); + it("uses a stable position when the provider has no click telemetry", () => { + const suggestions = plan(park(2000, 0.5, 0.5)); + expect(suggestions.length).toBeGreaterThan(0); + expect(suggestions[0].focus).toEqual({ cx: 0.5, cy: 0.5 }); }); it("returns nothing without telemetry, or for a lone move or mouseup", () => { diff --git a/src/lib/ai-edition/timeline/zoom-suggestions.ts b/src/lib/ai-edition/timeline/zoom-suggestions.ts index 8df686f48d..1b2bafa2df 100644 --- a/src/lib/ai-edition/timeline/zoom-suggestions.ts +++ b/src/lib/ai-edition/timeline/zoom-suggestions.ts @@ -7,9 +7,9 @@ // zoom, which holds while the camera follows the cursor. A zoom never comes straight back after // leaving. The video ends on the whole screen. // -// A still pointer is no reason to zoom: at rest it sits far from where the viewer looks, and -// Screen Studio, Cap and Tella all zoom on clicks alone. A take recorded without clicks gets no -// automatic zoom. +// Clicks remain the strongest signal. Position-only providers such as Hyprland cannot observe +// button presses without privileged input monitoring, so a no-click track falls back to stable +// cursor-position moments instead of disabling auto-zoom altogether. import type { CursorTelemetryPoint, ZoomFocus } from "@/components/video-editor/types"; import { zoomTransitionMs } from "@/lib/zoomMath/constants"; @@ -86,6 +86,9 @@ interface RecordedClick { focus: ZoomFocus; } +const POSITION_CLUSTER_RADIUS = 0.045; +const POSITION_MOMENT_MIN_MS = 350; + /** The recorded interactions that count as a click; `move` and `mouseup` do not. */ const CLICK_INTERACTION_TYPES: ReadonlySet> = new Set(["click", "double-click", "right-click", "middle-click"]); @@ -144,6 +147,65 @@ function visibleClicks( ); } +/** + * Position-only fallback for Wayland providers. A moment is the midpoint of a + * contiguous cursor dwell cluster; movement to a new area starts another one. + * It intentionally runs only when no clicks exist, so metadata-capable systems + * keep their existing click-driven timing and never receive duplicate zooms. + */ +function visiblePositionMoments( + samples: CursorTelemetryPoint[], + totalMs: number, + ignoreFromMs: number, +): RecordedClick[] { + const points = normalizeCursorTelemetry(samples, totalMs).filter( + (sample) => sample.visible !== false && sample.timeMs < ignoreFromMs, + ); + if (points.length === 0) return []; + + const moments: RecordedClick[] = []; + let cluster: CursorTelemetryPoint[] = []; + let sumX = 0; + let sumY = 0; + let center = { cx: 0, cy: 0 }; + const flush = () => { + if (cluster.length < 2) return; + const start = cluster[0].timeMs; + const end = cluster[cluster.length - 1].timeMs; + if (end - start < POSITION_MOMENT_MIN_MS) return; + const middle = cluster[Math.floor(cluster.length / 2)]; + moments.push({ timeMs: middle.timeMs, focus: { cx: middle.cx, cy: middle.cy } }); + }; + + for (const point of points) { + const distance = Math.hypot(point.cx - center.cx, point.cy - center.cy); + if (cluster.length > 0 && distance > POSITION_CLUSTER_RADIUS) { + flush(); + cluster = []; + sumX = 0; + sumY = 0; + } + cluster.push(point); + sumX += point.cx; + sumY += point.cy; + center = { + cx: sumX / cluster.length, + cy: sumY / cluster.length, + }; + } + flush(); + return moments; +} + +function recordedZoomMoments( + samples: CursorTelemetryPoint[], + totalMs: number, + ignoreFromMs: number, +): RecordedClick[] { + const clicks = visibleClicks(samples, totalMs, ignoreFromMs); + return clicks.length > 0 ? clicks : visiblePositionMoments(samples, totalMs, ignoreFromMs); +} + /** When the recording stopped: its telemetry runs until the take ends. */ function lastSampleMs(samples: CursorTelemetryPoint[]): number { let last = 0; @@ -342,7 +404,11 @@ export function buildAutoZoomSuggestions(options: { }): AutoZoomSuggestion[] { const { cursorTelemetry, totalMs, existingRegions } = options; if (totalMs <= 0 || cursorTelemetry.length === 0) return []; - const clicks = visibleClicks(cursorTelemetry, totalMs, options.ignoreClicksFromMs ?? Infinity); + const clicks = recordedZoomMoments( + cursorTelemetry, + totalMs, + options.ignoreClicksFromMs ?? Infinity, + ); const shots = planShots( clicks, totalMs, @@ -398,7 +464,7 @@ export function buildAutoZoomSuggestionsForClips(options: { (sample) => sample.timeMs >= sourceOffsetMs && sample.timeMs <= sourceOffsetMs + windowMs, ) .map((sample) => ({ ...sample, timeMs: sample.timeMs - sourceOffsetMs })); - const clicks = visibleClicks( + const clicks = recordedZoomMoments( clipTelemetry, windowMs, lastSampleMs(telemetry) - STOP_CLICK_GUARD_MS - sourceOffsetMs, diff --git a/src/lib/nativeLinuxRecording.test.ts b/src/lib/nativeLinuxRecording.test.ts index ef9342657a..c23333fd58 100644 --- a/src/lib/nativeLinuxRecording.test.ts +++ b/src/lib/nativeLinuxRecording.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { portalOwnsSourceSelection } from "./nativeLinuxRecording"; +import { portalCursorMode, portalOwnsSourceSelection } from "./nativeLinuxRecording"; function probe(platform: string, available: boolean | Error) { return { @@ -61,3 +61,14 @@ describe("portalOwnsSourceSelection", () => { ).resolves.toBe(false); }); }); + +describe("portalCursorMode", () => { + it("keeps metadata negotiation off Hyprland", () => { + expect(portalCursorMode("editable-overlay")).toBe("metadata"); + }); + + it("uses capability-aware auto negotiation on Hyprland", () => { + expect(portalCursorMode("editable-overlay", true)).toBe("auto"); + expect(portalCursorMode("system", true)).toBe("embedded"); + }); +}); diff --git a/src/lib/nativeLinuxRecording.ts b/src/lib/nativeLinuxRecording.ts index 3beee7c521..849719ac11 100644 --- a/src/lib/nativeLinuxRecording.ts +++ b/src/lib/nativeLinuxRecording.ts @@ -100,6 +100,7 @@ export type NativeLinuxRecordingStartResult = { */ export function portalCursorMode( mode: import("./recordingSession").CursorCaptureMode, -): "metadata" | "embedded" { - return mode === "system" ? "embedded" : "metadata"; + hyprland = false, +): "auto" | "metadata" | "embedded" { + return mode === "system" ? "embedded" : hyprland ? "auto" : "metadata"; } From ef89c465b656d133d10d995ec88b0a6d30a38d7a Mon Sep 17 00:00:00 2001 From: Abhijeet Arch Date: Sat, 24 Oct 2026 20:31:29 +0530 Subject: [PATCH 2/3] fix: keep unrelated changes out of Hyprland cursor PR --- crates/compositor/src/frame_geometry.rs | 32 ++++++++++++++++--- .../native/pipewire-capture/csrc/pw_shim.c | 6 ++-- scripts/build-whisper-stt.sh | 4 --- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/crates/compositor/src/frame_geometry.rs b/crates/compositor/src/frame_geometry.rs index 4db1f162c2..b5a7db7ea8 100644 --- a/crates/compositor/src/frame_geometry.rs +++ b/crates/compositor/src/frame_geometry.rs @@ -3834,11 +3834,11 @@ impl<'a> CursorPlacer<'a> { } } - /// La pointe à `t`. Le curseur modélisé touche le plan à chaque clic : il se pose sur le point - /// cliqué brut (`pinned_at`), pas sur la piste lissée qui traîne derrière la souris. Le sprite - /// plat garde la piste telle quelle, son rendu est celui d'avant. + /// La pointe à `t`, posée sur le point cliqué brut à chaque clic (`pinned_at`), pas sur la piste + /// lissée qui traîne derrière la souris. Sprite plat comme modèle 3D : le rebond, l'anneau + /// d'impact et `ctx.clicks` se centrent sur ce point brut, la pointe doit s'y trouver. fn at(&self, t: f32) -> Option<(f32, f32)> { - if self.model3d { self.track.pinned_at(t) } else { self.track.at(t) } + self.track.pinned_at(t) } /// `None` hors de la coupe courante (zoom serré, hors écran). @@ -9279,6 +9279,30 @@ mod tests { assert_eq!(fx(2.5).cursor, Some(drawn.placement.upright_center()), "là où le curseur se dessine"); } + /// Le sprite plat aussi se pose sur le clic brut : à fort lissage, le ressort traîne loin derrière + /// une souris qui clique dès l'arrivée, mais la pointe dessinée, `ctx.cursor` et `ctx.clicks` + /// tombent sur le même pixel pendant le contact. + #[test] + fn the_flat_cursor_lands_on_the_click_under_smoothing() { + const TC: f32 = 1.5; + let scene = model_scene(); + let cfg = crate::config::all().pop().expect("cfg"); + let fg = full_frame_geometry(); + let raw = approach_track([0.08, 0.05], [0.7, 0.6], [0.2, 0.9], 2.0, 0.0, TC); + for smoothing in [0.0, 0.3, 0.67, 1.0] { + let track = raw.smoothed(smoothing); + for ms in [0, 40, 100] { + let input = cursor_input(&scene, &cfg, &track, TC + ms as f32 / 1000.0); + let fx = effect_pointer(&fg, &input); + let (cursor, click) = (fx.cursor.expect("pointe"), fx.clicks[0].0); + let drawn = plan_cursor(&fg, &input).expect("visible").placement.upright_center(); + let off = |a: [f32; 2], b: [f32; 2]| ((a[0] - b[0]) * 1920.0).hypot((a[1] - b[1]) * 1080.0); + assert!(off(cursor, click) < 0.5, "lissage {smoothing}, {ms} ms : {} px", off(cursor, click)); + assert!(off(drawn, click) < 0.5, "lissage {smoothing}, {ms} ms : {} px", off(drawn, click)); + } + } + } + /// Hyprland position-only sidecars have no bitmap assets or cursor type. The /// renderer must still select OpenScreen's bundled arrow sprite, with its /// configured hotspot, for both preview and export. diff --git a/electron/native/pipewire-capture/csrc/pw_shim.c b/electron/native/pipewire-capture/csrc/pw_shim.c index f4c9bc629e..d28e6c9909 100644 --- a/electron/native/pipewire-capture/csrc/pw_shim.c +++ b/electron/native/pipewire-capture/csrc/pw_shim.c @@ -612,11 +612,9 @@ int osc_pw_enum_format_accepts_dmabuf_producer(int with_modifier, int64_t produc const struct spa_pod *consumer; const struct spa_pod *producer; - /* Keep this synthetic intersection test independent of the host EGL stack. - * Runtime negotiation still calls the same builder with prefer_dmabuf=1; - * the test only needs the guaranteed CPU-readable fallback modifiers. */ + /* The unit test exercises the full tiled offer, so enumerate unconditionally. */ consumer = - with_modifier ? osc_build_enum_format_dmabuf(&ours, 0) : osc_build_enum_format(&ours); + with_modifier ? osc_build_enum_format_dmabuf(&ours, 1) : osc_build_enum_format(&ours); if (consumer == NULL) { return -1; } diff --git a/scripts/build-whisper-stt.sh b/scripts/build-whisper-stt.sh index 5714543c79..23b33ce3e3 100644 --- a/scripts/build-whisper-stt.sh +++ b/scripts/build-whisper-stt.sh @@ -86,10 +86,6 @@ readonly MACOS_DEPLOYMENT_TARGET="13.0" # Determine the default backend flag for this host. backend_flag_for_host() { - if [[ "${OPENSCREEN_WHISPER_BACKEND:-}" == "cpu" ]]; then - echo "-DOSC_ENABLE_VULKAN=OFF" - return - fi case "${OS_ARCH}" in darwin-arm64) echo "-DOSC_ENABLE_METAL=ON" ;; darwin-x64) echo "" ;; From 9bfa00be2a4cd0adcdd905b5669308f282ec07cd Mon Sep 17 00:00:00 2001 From: Abhijeet Arch Date: Sat, 24 Oct 2026 20:39:49 +0530 Subject: [PATCH 3/3] fix(linux): address cursor review findings --- electron/ipc/handlers.ts | 12 +- .../capture/linuxNativeCaptureSession.ts | 13 +- .../recording/hyprlandCursorTracker.test.ts | 7 +- .../cursor/recording/hyprlandCursorTracker.ts | 119 +++++++++++------- electron/native/pipewire-capture/src/main.rs | 1 + .../native/pipewire-capture/src/portal.rs | 25 +++- .../timeline/zoom-suggestions.test.ts | 29 +++++ .../ai-edition/timeline/zoom-suggestions.ts | 31 +++-- 8 files changed, 162 insertions(+), 75 deletions(-) diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 9af6e04b72..07cf566b17 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -102,8 +102,8 @@ import { LinuxNativeCaptureSession, } from "../native-bridge/capture/linuxNativeCaptureSession"; import { compactCursorTelemetryPauseRanges } from "../native-bridge/cursor/recording/cursorPauseRanges"; -import { HyprlandCursorTracker } from "../native-bridge/cursor/recording/hyprlandCursorTracker"; import { createCursorRecordingSession } from "../native-bridge/cursor/recording/factory"; +import { HyprlandCursorTracker } from "../native-bridge/cursor/recording/hyprlandCursorTracker"; import { isMacCursorHelperUnavailable, requestMacCursorAccessibilityAccess, @@ -2631,10 +2631,7 @@ export function registerIpcHandlers( const session = new LinuxNativeCaptureSession({ outputPath, - cursorMode: portalCursorMode( - cursorCaptureMode, - HyprlandCursorTracker.isAvailable(), - ), + cursorMode: portalCursorMode(cursorCaptureMode, HyprlandCursorTracker.isAvailable()), sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS, fps: request.video.fps, ...(request.video.bitrate ? { bitrate: request.video.bitrate } : {}), @@ -2721,10 +2718,7 @@ export function registerIpcHandlers( prepared ?? new LinuxNativeCaptureSession({ outputPath, - cursorMode: portalCursorMode( - cursorCaptureMode, - HyprlandCursorTracker.isAvailable(), - ), + cursorMode: portalCursorMode(cursorCaptureMode, HyprlandCursorTracker.isAvailable()), sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS, fps: request.video.fps, ...(request.video.bitrate ? { bitrate: request.video.bitrate } : {}), diff --git a/electron/native-bridge/capture/linuxNativeCaptureSession.ts b/electron/native-bridge/capture/linuxNativeCaptureSession.ts index a32f343c62..6d1bc8d552 100644 --- a/electron/native-bridge/capture/linuxNativeCaptureSession.ts +++ b/electron/native-bridge/capture/linuxNativeCaptureSession.ts @@ -180,6 +180,7 @@ export class LinuxNativeCaptureSession { }); child.once("exit", (code, signal) => { this.process = null; + if (!this.stopped) this.stopHyprlandCursorTracker(); const reason = this.lastError ?? `the Linux capture helper exited (code=${code ?? "null"}, signal=${signal ?? "null"})`; @@ -206,6 +207,7 @@ export class LinuxNativeCaptureSession { }); child.once("error", (error) => { this.process = null; + this.stopHyprlandCursorTracker(); this.rejectReady(error); }); @@ -560,7 +562,7 @@ export class LinuxNativeCaptureSession { this.ensureHyprlandCursorTracker( (this.config.cursorMode === "auto" && (this.effectiveCursorMode !== "metadata" || this.cursorMetadataSupported === false)) || - this.effectiveCursorMode === "hidden", + this.effectiveCursorMode === "hidden", ); if (this.hyprlandCursor && !this.hyprlandCursor.hasCaptureGeometry()) { this.hyprlandCursor.setCaptureGeometry({ @@ -570,6 +572,7 @@ export class LinuxNativeCaptureSession { : null, width: payload.logicalWidth ?? payload.width, height: payload.logicalHeight ?? payload.height, + allowMonitorOriginFallback: payload.sourceKind !== "window", }); } return; @@ -611,7 +614,7 @@ export class LinuxNativeCaptureSession { this.ensureHyprlandCursorTracker( (this.config.cursorMode === "auto" && (this.effectiveCursorMode !== "metadata" || this.cursorMetadataSupported === false)) || - this.effectiveCursorMode === "hidden", + this.effectiveCursorMode === "hidden", ); this.hyprlandCursor?.rebase(); // Older helper builds and some portal backends may omit stream-started @@ -622,6 +625,7 @@ export class LinuxNativeCaptureSession { position: null, width: payload.width, height: payload.height, + allowMonitorOriginFallback: this.sourceKind !== "window", }); } this.capturing = true; @@ -704,6 +708,11 @@ export class LinuxNativeCaptureSession { this.hyprlandCursor.start(); } + private stopHyprlandCursorTracker() { + this.hyprlandCursor?.stop(); + this.hyprlandCursor = null; + } + private resolveReady() { const resolve = this.readyResolve; this.clearReadyState(); diff --git a/electron/native-bridge/cursor/recording/hyprlandCursorTracker.test.ts b/electron/native-bridge/cursor/recording/hyprlandCursorTracker.test.ts index c96cb0a24a..7e986108d5 100644 --- a/electron/native-bridge/cursor/recording/hyprlandCursorTracker.test.ts +++ b/electron/native-bridge/cursor/recording/hyprlandCursorTracker.test.ts @@ -184,7 +184,7 @@ describe("mapHyprlandCursorToCapture", () => { expect(sample).toMatchObject({ cx: 500 / 1280, cy: 300 / 800, visible: true }); }); - it("uses the containing monitor for a smaller window stream", async () => { + it("does not invent a window origin when the portal omits it", async () => { process.env.XDG_RUNTIME_DIR = "/tmp/openscreen-hypr-runtime"; process.env.HYPRLAND_INSTANCE_SIGNATURE = "window-geometry-fallback"; vi.spyOn(net, "createConnection").mockImplementation(() => { @@ -215,12 +215,11 @@ describe("mapHyprlandCursorToCapture", () => { }); const tracker = new HyprlandCursorTracker({ sampleIntervalMs: 10, maxSamples: 10 }); - tracker.setCaptureGeometry({ width: 1234, height: 720 }); + tracker.setCaptureGeometry({ width: 1234, height: 720, allowMonitorOriginFallback: false }); tracker.start(); await new Promise((resolve) => setTimeout(resolve, 60)); - const sample = tracker.stop().samples[0]; - expect(sample).toMatchObject({ cx: 134 / 1234, cy: 264 / 720, visible: true }); + expect(tracker.stop().samples).toEqual([]); }); it("corrects a zero portal origin for a uniquely sized monitor", async () => { diff --git a/electron/native-bridge/cursor/recording/hyprlandCursorTracker.ts b/electron/native-bridge/cursor/recording/hyprlandCursorTracker.ts index 4153aa3353..473c6c47bc 100644 --- a/electron/native-bridge/cursor/recording/hyprlandCursorTracker.ts +++ b/electron/native-bridge/cursor/recording/hyprlandCursorTracker.ts @@ -1,5 +1,5 @@ -import net from "node:net"; import { existsSync, readdirSync } from "node:fs"; +import net from "node:net"; import { hrtime } from "node:process"; import type { CursorRecordingData, CursorRecordingSample } from "../../../../src/native/contracts"; @@ -23,6 +23,17 @@ export interface HyprlandCaptureGeometry { position?: { x: number; y: number } | null; width: number; height: number; + /** False for window streams when the portal omitted the window origin. */ + allowMonitorOriginFallback?: boolean; +} + +function monitorLogicalSize(monitor: HyprlandMonitorGeometry) { + const scale = monitor.scale && monitor.scale > 0 ? monitor.scale : 1; + const swap = (monitor.transform ?? 0) % 2 === 1; + return { + width: (swap ? monitor.height : monitor.width) / scale, + height: (swap ? monitor.width : monitor.height) / scale, + }; } export function mapHyprlandCursorToCapture( @@ -30,13 +41,15 @@ export function mapHyprlandCursorToCapture( monitors: HyprlandMonitorGeometry[], capture: HyprlandCaptureRect, ) { - const monitor = monitors.find( - (candidate) => + const monitor = monitors.find((candidate) => { + const { width, height } = monitorLogicalSize(candidate); + return ( point.x >= candidate.x && point.y >= candidate.y && - point.x < candidate.x + candidate.width && - point.y < candidate.y + candidate.height, - ); + point.x < candidate.x + width && + point.y < candidate.y + height + ); + }); const transform = monitor?.transform ?? 0; const logicalX = point.x - capture.x; const logicalY = point.y - capture.y; @@ -78,12 +91,9 @@ function hyprlandSocketPath() { // the one live socket rather than disabling position telemetry in that case. try { const hyprlandRoot = `${runtime}/hypr`; - const signature = readdirSync(hyprlandRoot, { withFileTypes: true }) - .find( - (entry) => - entry.isDirectory() && existsSync(`${hyprlandRoot}/${entry.name}/.socket.sock`), - ) - ?.name; + const signature = readdirSync(hyprlandRoot, { withFileTypes: true }).find( + (entry) => entry.isDirectory() && existsSync(`${hyprlandRoot}/${entry.name}/.socket.sock`), + )?.name; return signature ? `${hyprlandRoot}/${signature}/.socket.sock` : null; } catch { return null; @@ -106,7 +116,9 @@ export class HyprlandCursorTracker { private pausedDurationMs = 0; private captureRect: HyprlandCaptureRect | null = null; private captureSize: { width: number; height: number } | null = null; + private allowMonitorOriginFallback = true; private lastIpcError: string | null = null; + private generation = 0; constructor(private readonly options: { sampleIntervalMs: number; maxSamples: number }) {} @@ -115,6 +127,7 @@ export class HyprlandCursorTracker { } start() { + this.generation += 1; this.stopSocket(); this.samples = []; this.startMs = monotonicMs(); @@ -126,6 +139,7 @@ export class HyprlandCursorTracker { } stop(): CursorRecordingData { + this.generation += 1; if (this.interval) clearInterval(this.interval); this.interval = null; this.stopSocket(); @@ -144,6 +158,7 @@ export class HyprlandCursorTracker { } rebase() { + this.generation += 1; this.startMs = monotonicMs(); this.pausedDurationMs = 0; this.pausedAt = null; @@ -165,19 +180,21 @@ export class HyprlandCursorTracker { if (!geometry) { this.captureRect = null; this.captureSize = null; + this.allowMonitorOriginFallback = true; return; } + this.allowMonitorOriginFallback = geometry.allowMonitorOriginFallback ?? true; this.captureSize = { width: Math.max(1, geometry.width), height: Math.max(1, geometry.height), }; this.captureRect = geometry.position ? { - x: geometry.position.x, - y: geometry.position.y, - width: this.captureSize.width, - height: this.captureSize.height, - } + x: geometry.position.x, + y: geometry.position.y, + width: this.captureSize.width, + height: this.captureSize.height, + } : null; } @@ -186,35 +203,40 @@ export class HyprlandCursorTracker { } private async sample() { - if (this.sampleInFlight || this.pausedAt !== null || (!this.captureRect && !this.captureSize)) return; + if (this.sampleInFlight || this.pausedAt !== null || (!this.captureRect && !this.captureSize)) + return; + const generation = this.generation; this.sampleInFlight = true; try { const point = await this.request<{ x: number; y: number }>("j/cursorpos"); const now = monotonicMs(); - const monitors = - now - this.monitorCacheAt > 1000 - ? await this.request("j/monitors") - : this.monitorCache; - this.monitorCache = Array.isArray(monitors) ? monitors : this.monitorCache; - this.monitorCacheAt = now; + if (now - this.monitorCacheAt > 1000 || this.monitorCacheAt === 0) { + const refreshed = await this.request("j/monitors"); + if (Array.isArray(refreshed)) { + this.monitorCache = refreshed; + this.monitorCacheAt = now; + } + } this.correctPortalOriginForExactMonitor(point); - if (!this.captureRect && this.captureSize) { - const matches = this.monitorCache.filter( - (monitor) => { - const scale = monitor.scale && monitor.scale > 0 ? monitor.scale : 1; - const widths = [monitor.width, monitor.width / scale].map(Math.round); - const heights = [monitor.height, monitor.height / scale].map(Math.round); - return widths.includes(this.captureSize?.width ?? -1) && - heights.includes(this.captureSize?.height ?? -1); - }, - ); - const containing = this.monitorCache.find( - (monitor) => + if (!this.captureRect && this.captureSize && this.allowMonitorOriginFallback) { + const matches = this.monitorCache.filter((monitor) => { + const scale = monitor.scale && monitor.scale > 0 ? monitor.scale : 1; + const widths = [monitor.width, monitor.width / scale].map(Math.round); + const heights = [monitor.height, monitor.height / scale].map(Math.round); + return ( + widths.includes(this.captureSize?.width ?? -1) && + heights.includes(this.captureSize?.height ?? -1) + ); + }); + const containing = this.monitorCache.find((monitor) => { + const { width, height } = monitorLogicalSize(monitor); + return ( point.x >= monitor.x && point.y >= monitor.y && - point.x < monitor.x + monitor.width && - point.y < monitor.y + monitor.height, - ); + point.x < monitor.x + width && + point.y < monitor.y + height + ); + }); // Exact size wins for monitor captures. For a window capture whose // portal size is smaller than the monitor, the containing monitor is // the best available origin when the portal omitted Stream.position. @@ -228,7 +250,7 @@ export class HyprlandCursorTracker { }; } } - if (!this.captureRect) return; + if (!this.captureRect || generation !== this.generation) return; const mapped = mapHyprlandCursorToCapture(point, this.monitorCache, this.captureRect); this.samples.push({ timeMs: Math.max(0, now - this.startMs - this.pausedDurationMs), @@ -268,13 +290,15 @@ export class HyprlandCursorTracker { heights.includes(Math.round(this.captureRect!.height)) ); }); - const monitor = matches.find( - (candidate) => + const monitor = matches.find((candidate) => { + const { width, height } = monitorLogicalSize(candidate); + return ( point.x >= candidate.x && point.y >= candidate.y && - point.x < candidate.x + candidate.width && - point.y < candidate.y + candidate.height, - ); + point.x < candidate.x + width && + point.y < candidate.y + height + ); + }); if (matches.length === 1 && monitor) { this.captureRect.x = monitor.x; this.captureRect.y = monitor.y; @@ -313,7 +337,10 @@ export class HyprlandCursorTracker { } }); socket.on("error", (error) => settle(error)); - socket.on("timeout", () => settle(new Error("Hyprland IPC timeout"))); + socket.on("timeout", () => { + socket.destroy(); + settle(new Error("Hyprland IPC timeout")); + }); socket.on("close", () => { if (!settled) settle(new Error("Hyprland IPC connection closed")); }); diff --git a/electron/native/pipewire-capture/src/main.rs b/electron/native/pipewire-capture/src/main.rs index 8ed4b3337d..66f238ae9a 100644 --- a/electron/native/pipewire-capture/src/main.rs +++ b/electron/native/pipewire-capture/src/main.rs @@ -291,6 +291,7 @@ fn main() { let cursor_mode = requested_cursor_mode.resolve( cursor_capabilities.metadata, cursor_capabilities.hidden, + cursor_capabilities.embedded, ); if !cursor_capabilities.metadata && cursor_mode.reports_cursor() { let error = portal::PortalError::CursorMetadataUnsupported; diff --git a/electron/native/pipewire-capture/src/portal.rs b/electron/native/pipewire-capture/src/portal.rs index 046369a473..a71f2b7ed2 100644 --- a/electron/native/pipewire-capture/src/portal.rs +++ b/electron/native/pipewire-capture/src/portal.rs @@ -58,10 +58,16 @@ impl CursorMode { matches!(self, Self::Metadata) } - pub fn resolve(self, metadata_supported: bool, hidden_supported: bool) -> Self { + pub fn resolve( + self, + metadata_supported: bool, + hidden_supported: bool, + embedded_supported: bool, + ) -> Self { match self { Self::Auto if metadata_supported => Self::Metadata, Self::Auto if hidden_supported => Self::Hidden, + Self::Embedded if !embedded_supported && metadata_supported => Self::Metadata, Self::Auto => Self::Embedded, other => other, } @@ -194,6 +200,7 @@ fn portal_unavailable(error: impl std::fmt::Display) -> PortalError { pub struct CursorCapabilities { pub metadata: bool, pub hidden: bool, + pub embedded: bool, } pub async fn cursor_capabilities() -> Result { @@ -205,6 +212,7 @@ pub async fn cursor_capabilities() -> Result { Ok(CursorCapabilities { metadata: cursor_modes.contains(PortalCursorMode::Metadata), hidden: cursor_modes.contains(PortalCursorMode::Hidden), + embedded: cursor_modes.contains(PortalCursorMode::Embedded), }) } @@ -215,11 +223,19 @@ mod tests { #[test] fn auto_prefers_metadata_then_hidden_then_embedded() { assert_eq!( - CursorMode::Auto.resolve(true, true), + CursorMode::Auto.resolve(true, true, true), CursorMode::Metadata ); - assert_eq!(CursorMode::Auto.resolve(false, true), CursorMode::Hidden); - assert_eq!(CursorMode::Auto.resolve(false, false), CursorMode::Embedded); + assert_eq!(CursorMode::Auto.resolve(false, true, true), CursorMode::Hidden); + assert_eq!(CursorMode::Auto.resolve(false, false, true), CursorMode::Embedded); + assert_eq!( + CursorMode::Embedded.resolve(true, true, false), + CursorMode::Metadata + ); + assert_eq!( + CursorMode::Embedded.resolve(true, true, true), + CursorMode::Embedded + ); } } @@ -243,6 +259,7 @@ pub async fn negotiate(cursor_mode: CursorMode) -> Result { expect(suggestions[0].focus).toEqual({ cx: 0.5, cy: 0.5 }); }); + it("does not join position dwells across a hidden interval", () => { + const suggestions = plan( + [ + { timeMs: 1000, cx: 0.5, cy: 0.5, visible: true }, + { timeMs: 1200, cx: 0.5, cy: 0.5, visible: true }, + { timeMs: 1400, cx: 0.5, cy: 0.5, visible: true }, + { timeMs: 1500, cx: 0.5, cy: 0.5, visible: false }, + { timeMs: 8000, cx: 0.5, cy: 0.5, visible: true }, + { timeMs: 8200, cx: 0.5, cy: 0.5, visible: true }, + { timeMs: 8400, cx: 0.5, cy: 0.5, visible: true }, + ], + 10000, + ); + expect(suggestions).toHaveLength(2); + }); + + it("places an irregular position dwell at its temporal midpoint", () => { + const suggestions = plan( + [ + { timeMs: 0, cx: 0.4, cy: 0.6 }, + { timeMs: 100, cx: 0.4, cy: 0.6 }, + { timeMs: 200, cx: 0.4, cy: 0.6 }, + { timeMs: 5000, cx: 0.4, cy: 0.6 }, + ], + 5000, + ); + expect(suggestions[0]?.span).toEqual({ start: 2000, end: 4000 }); + }); + it("returns nothing without telemetry, or for a lone move or mouseup", () => { expect(plan([])).toEqual([]); for (const kind of ["move", "mouseup"] as const) { diff --git a/src/lib/ai-edition/timeline/zoom-suggestions.ts b/src/lib/ai-edition/timeline/zoom-suggestions.ts index 1b2bafa2df..7257687e8d 100644 --- a/src/lib/ai-edition/timeline/zoom-suggestions.ts +++ b/src/lib/ai-edition/timeline/zoom-suggestions.ts @@ -158,32 +158,43 @@ function visiblePositionMoments( totalMs: number, ignoreFromMs: number, ): RecordedClick[] { - const points = normalizeCursorTelemetry(samples, totalMs).filter( - (sample) => sample.visible !== false && sample.timeMs < ignoreFromMs, - ); - if (points.length === 0) return []; + const normalized = normalizeCursorTelemetry(samples, totalMs); + if (normalized.length === 0) return []; const moments: RecordedClick[] = []; let cluster: CursorTelemetryPoint[] = []; let sumX = 0; let sumY = 0; let center = { cx: 0, cy: 0 }; + const reset = () => { + cluster = []; + sumX = 0; + sumY = 0; + center = { cx: 0, cy: 0 }; + }; const flush = () => { if (cluster.length < 2) return; const start = cluster[0].timeMs; const end = cluster[cluster.length - 1].timeMs; if (end - start < POSITION_MOMENT_MIN_MS) return; - const middle = cluster[Math.floor(cluster.length / 2)]; - moments.push({ timeMs: middle.timeMs, focus: { cx: middle.cx, cy: middle.cy } }); + const timeMs = (start + end) / 2; + const middle = cluster.reduce((nearest, sample) => + Math.abs(sample.timeMs - timeMs) < Math.abs(nearest.timeMs - timeMs) ? sample : nearest, + ); + moments.push({ timeMs, focus: { cx: middle.cx, cy: middle.cy } }); }; - for (const point of points) { + for (const point of normalized) { + if (point.timeMs >= ignoreFromMs) break; + if (point.visible === false) { + flush(); + reset(); + continue; + } const distance = Math.hypot(point.cx - center.cx, point.cy - center.cy); if (cluster.length > 0 && distance > POSITION_CLUSTER_RADIUS) { flush(); - cluster = []; - sumX = 0; - sumY = 0; + reset(); } cluster.push(point); sumX += point.cx;