diff --git a/crates/compositor/src/frame_geometry.rs b/crates/compositor/src/frame_geometry.rs index f189f6f37..b5a7db7ea 100644 --- a/crates/compositor/src/frame_geometry.rs +++ b/crates/compositor/src/frame_geometry.rs @@ -9303,6 +9303,22 @@ mod tests { } } + /// Hyprland position-only sidecars have no bitmap assets or cursor type. The + /// renderer must still select OpenScreen's bundled arrow sprite, with its + /// configured hotspot, for both preview and export. + #[test] + fn position_only_track_uses_the_bundled_arrow_sprite() { + let scene = model_scene(); + let cfg = crate::config::all().pop().expect("cfg"); + let fg = full_frame_geometry(); + let track = still_track(vec![]); + let plan = plan_cursor(&fg, &cursor_input(&scene, &cfg, &track, 1.0)).expect("visible"); + + assert_eq!(plan.cursor_type, None, "position-only samples have no native shape"); + assert!(plan.placement.upright_center()[0] > 0.0); + assert!(plan.placement.upright_center()[1] > 0.0); + } + /// Masqué par l'utilisateur, le curseur n'est plus dessiné, mais les effets gardent la pointe /// et les clics : un halo suit toujours la souris, une onde part toujours du clic. Quand c'est /// l'enregistrement qui cache la pointe, les effets la perdent, et gardent les clics d'avant. diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index ec56e955f..58ae64597 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -103,6 +103,7 @@ import { } from "../native-bridge/capture/linuxNativeCaptureSession"; import { compactCursorTelemetryPauseRanges } from "../native-bridge/cursor/recording/cursorPauseRanges"; import { createCursorRecordingSession } from "../native-bridge/cursor/recording/factory"; +import { HyprlandCursorTracker } from "../native-bridge/cursor/recording/hyprlandCursorTracker"; import { isMacCursorHelperUnavailable, requestMacCursorAccessibilityAccess, @@ -2630,7 +2631,8 @@ export function registerIpcHandlers( const session = new LinuxNativeCaptureSession({ outputPath, - cursorMode: portalCursorMode(cursorCaptureMode), + cursorMode: portalCursorMode(cursorCaptureMode, HyprlandCursorTracker.isAvailable()), + sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS, fps: request.video.fps, ...(request.video.bitrate ? { bitrate: request.video.bitrate } : {}), audio: { @@ -2716,7 +2718,8 @@ export function registerIpcHandlers( prepared ?? new LinuxNativeCaptureSession({ outputPath, - cursorMode: portalCursorMode(cursorCaptureMode), + cursorMode: portalCursorMode(cursorCaptureMode, HyprlandCursorTracker.isAvailable()), + sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS, fps: request.video.fps, ...(request.video.bitrate ? { bitrate: request.video.bitrate } : {}), audio: { @@ -2822,7 +2825,11 @@ export function registerIpcHandlers( // The helper collects cursor samples itself, from the same portal // session that produced the pixels, so there is no separate sampler // to stop and no clock offset to correct — the two are one recording. - if (cursorCaptureMode === "editable-overlay" && result.cursor.samples.length > 0) { + // Keep the sidecar even when a run produced zero samples. An empty + // sidecar is an explicit native-capture diagnostic; omitting it made a + // failed Hyprland IPC session indistinguishable from the Chromium + // fallback and hid the real cause of an invisible cursor. + if (cursorCaptureMode === "editable-overlay") { await fs.writeFile( `${result.path}.cursor.json`, JSON.stringify(result.cursor, null, 2), diff --git a/electron/native-bridge/capture/linuxNativeCaptureSession.ts b/electron/native-bridge/capture/linuxNativeCaptureSession.ts index 19a83b5c6..6d1bc8d55 100644 --- a/electron/native-bridge/capture/linuxNativeCaptureSession.ts +++ b/electron/native-bridge/capture/linuxNativeCaptureSession.ts @@ -5,6 +5,7 @@ import { type CursorPauseRange, compactCursorTelemetryPauseRanges, } from "../cursor/recording/cursorPauseRanges"; +import { HyprlandCursorTracker } from "../cursor/recording/hyprlandCursorTracker"; import { NdjsonLineReader, PipeWireCursorAccumulator, @@ -56,7 +57,8 @@ const HELPER_OUTPUT_MAX_BYTES = 64 * 1024; export interface LinuxCaptureConfig { outputPath: string; - cursorMode: "metadata" | "embedded"; + cursorMode: "auto" | "metadata" | "embedded" | "hidden"; + sampleIntervalMs?: number; fps: number; bitrate?: number; audio: { @@ -119,6 +121,9 @@ export class LinuxNativeCaptureSession { private lastError: string | null = null; private paused = false; private helperOutputText = ""; + private effectiveCursorMode: "metadata" | "embedded" | "hidden" = "metadata"; + private cursorMetadataSupported: boolean | null = null; + private hyprlandCursor: HyprlandCursorTracker | null = null; private cursorRecordingStartMs = 0; private cursorPauseStartedAtMs: number | null = null; private cursorPauseRanges: CursorPauseRange[] = []; @@ -175,6 +180,7 @@ export class LinuxNativeCaptureSession { }); child.once("exit", (code, signal) => { this.process = null; + if (!this.stopped) this.stopHyprlandCursorTracker(); const reason = this.lastError ?? `the Linux capture helper exited (code=${code ?? "null"}, signal=${signal ?? "null"})`; @@ -201,6 +207,7 @@ export class LinuxNativeCaptureSession { }); child.once("error", (error) => { this.process = null; + this.stopHyprlandCursorTracker(); this.rejectReady(error); }); @@ -317,6 +324,7 @@ export class LinuxNativeCaptureSession { if (this.paused) { return; } + this.hyprlandCursor?.pause(); this.write("pause"); this.paused = true; this.cursorPauseStartedAtMs = Date.now(); @@ -326,6 +334,7 @@ export class LinuxNativeCaptureSession { if (!this.paused) { return; } + this.hyprlandCursor?.resume(); this.write("resume"); this.completeCursorPauseRange(); this.paused = false; @@ -344,6 +353,11 @@ export class LinuxNativeCaptureSession { return this.paused; } + /** The cursor mode actually negotiated by the portal. */ + get cursorMode(): LinuxCaptureConfig["cursorMode"] { + return this.negotiatedCursorMode ?? this.config.cursorMode; + } + /** * What the portal granted. Known from `stream-started`, so it is populated by * the time [`waitUntilCapturing`] resolves. @@ -356,15 +370,6 @@ export class LinuxNativeCaptureSession { return this.sourceKind; } - /** - * The cursor mode the helper really negotiates, known once `start()` resolves. - * A portal without EMBEDDED gets METADATA instead (#1357), so the cursor has - * to be drawn by the editor even though the system cursor was asked for. - */ - get cursorMode(): LinuxCaptureConfig["cursorMode"] { - return this.negotiatedCursorMode ?? this.config.cursorMode; - } - /** * What the helper said in this session, for Save Diagnostics' `helperOutput.linux`: * its stderr as written and every stdout line except `cursor-sample`, which comes @@ -449,6 +454,8 @@ export class LinuxNativeCaptureSession { } private kill() { + this.hyprlandCursor?.stop(); + this.hyprlandCursor = null; const child = this.process; if (!child || child.killed) { return; @@ -480,12 +487,42 @@ export class LinuxNativeCaptureSession { private handleEvent(payload: PipeWireHelperEvent) { switch (payload.event) { - case "ready": - if (payload.cursorMode === "metadata" || payload.cursorMode === "embedded") { + case "ready": { + if ( + payload.cursorMode === "metadata" || + payload.cursorMode === "embedded" || + payload.cursorMode === "hidden" + ) { this.negotiatedCursorMode = payload.cursorMode; } + this.cursorMetadataSupported = + typeof payload.cursorMetadataSupported === "boolean" + ? payload.cursorMetadataSupported + : null; + this.effectiveCursorMode = + payload.cursorMode === "hidden" + ? "hidden" + : payload.cursorMode === "embedded" + ? "embedded" + : "metadata"; + const useHyprlandPositionTelemetry = + this.config.cursorMode === "auto" && + (this.effectiveCursorMode !== "metadata" || this.cursorMetadataSupported === false); + this.ensureHyprlandCursorTracker( + useHyprlandPositionTelemetry || this.effectiveCursorMode === "hidden", + ); + console.info( + "[capture-linux] cursor negotiation", + JSON.stringify({ + requested: this.config.cursorMode, + effective: this.effectiveCursorMode, + metadataSupported: payload.cursorMetadataSupported ?? null, + hyprlandTracker: Boolean(this.hyprlandCursor), + }), + ); this.resolveReady(); return; + } case "source-selected": if (payload.sourceKind) { @@ -519,6 +556,25 @@ export class LinuxNativeCaptureSession { sourceKind: payload.sourceKind ?? null, }), ); + // Some helper/portal combinations report the negotiated mode before + // the stream details are available. Re-evaluate here so hidden-cursor + // capture cannot silently lose its Hyprland position telemetry. + this.ensureHyprlandCursorTracker( + (this.config.cursorMode === "auto" && + (this.effectiveCursorMode !== "metadata" || this.cursorMetadataSupported === false)) || + this.effectiveCursorMode === "hidden", + ); + if (this.hyprlandCursor && !this.hyprlandCursor.hasCaptureGeometry()) { + this.hyprlandCursor.setCaptureGeometry({ + position: + payload.positionX != null && payload.positionY != null + ? { x: payload.positionX, y: payload.positionY } + : null, + width: payload.logicalWidth ?? payload.width, + height: payload.logicalHeight ?? payload.height, + allowMonitorOriginFallback: payload.sourceKind !== "window", + }); + } return; case "audio-source": @@ -555,6 +611,23 @@ export class LinuxNativeCaptureSession { // picker is dropped rather than left pinned to the start. this.cursor.rebase(payload.timestampMs); this.cursorRecordingStartMs = payload.timestampMs; + this.ensureHyprlandCursorTracker( + (this.config.cursorMode === "auto" && + (this.effectiveCursorMode !== "metadata" || this.cursorMetadataSupported === false)) || + this.effectiveCursorMode === "hidden", + ); + this.hyprlandCursor?.rebase(); + // Older helper builds and some portal backends may omit stream-started + // geometry. The encoded frame dimensions are still sufficient for the + // monitor-size/origin fallback in HyprlandCursorTracker. + if (this.hyprlandCursor && !this.hyprlandCursor.hasCaptureGeometry()) { + this.hyprlandCursor.setCaptureGeometry({ + position: null, + width: payload.width, + height: payload.height, + allowMonitorOriginFallback: this.sourceKind !== "window", + }); + } this.capturing = true; console.info( "[capture-linux] capture started", @@ -566,19 +639,20 @@ export class LinuxNativeCaptureSession { return; case "cursor-sample": - this.cursor.addSample(payload); + if (this.effectiveCursorMode === "metadata") this.cursor.addSample(payload); return; - case "capture-stopped": + case "capture-stopped": { + const cursor = + this.hyprlandCursor?.stop() ?? + compactCursorTelemetryPauseRanges(this.cursor.toRecordingData(), this.cursorPauseRanges); + this.hyprlandCursor = null; this.stopped = { path: payload.path, durationMs: payload.durationMs, frames: payload.frames, droppedFrames: payload.dropped, - cursor: compactCursorTelemetryPauseRanges( - this.cursor.toRecordingData(), - this.cursorPauseRanges, - ), + cursor, ...(this.sourceKind ? { sourceKind: this.sourceKind } : {}), ...(this.videoEncoder ? { videoEncoder: this.videoEncoder } : {}), }; @@ -591,9 +665,12 @@ export class LinuxNativeCaptureSession { convertMs: payload.convertMs, uploadMs: payload.uploadMs, encodeMs: payload.encodeMs, + cursorProvider: cursor.provider, + cursorSamples: cursor.samples.length, }), ); return; + } case "warning": console.warn(`[capture-linux] ${payload.code}: ${payload.message}`); @@ -616,6 +693,26 @@ export class LinuxNativeCaptureSession { } } + private ensureHyprlandCursorTracker(needed: boolean) { + if (!needed || this.hyprlandCursor) return; + if (!HyprlandCursorTracker.isAvailable()) { + console.warn( + "[capture-linux] position-only cursor capture requested, but the Hyprland IPC socket is unavailable", + ); + return; + } + this.hyprlandCursor = new HyprlandCursorTracker({ + sampleIntervalMs: this.config.sampleIntervalMs ?? 33, + maxSamples: this.config.maxCursorSamples, + }); + this.hyprlandCursor.start(); + } + + private stopHyprlandCursorTracker() { + this.hyprlandCursor?.stop(); + this.hyprlandCursor = null; + } + private resolveReady() { const resolve = this.readyResolve; this.clearReadyState(); diff --git a/electron/native-bridge/cursor/recording/hyprlandCursorTracker.test.ts b/electron/native-bridge/cursor/recording/hyprlandCursorTracker.test.ts new file mode 100644 index 000000000..7e986108d --- /dev/null +++ b/electron/native-bridge/cursor/recording/hyprlandCursorTracker.test.ts @@ -0,0 +1,326 @@ +import net from "node:net"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { HyprlandCursorTracker, mapHyprlandCursorToCapture } from "./hyprlandCursorTracker"; + +const originalRuntime = process.env.XDG_RUNTIME_DIR; +const originalSignature = process.env.HYPRLAND_INSTANCE_SIGNATURE; + +afterEach(() => { + if (originalRuntime === undefined) delete process.env.XDG_RUNTIME_DIR; + else process.env.XDG_RUNTIME_DIR = originalRuntime; + if (originalSignature === undefined) delete process.env.HYPRLAND_INSTANCE_SIGNATURE; + else process.env.HYPRLAND_INSTANCE_SIGNATURE = originalSignature; + vi.restoreAllMocks(); +}); + +describe("mapHyprlandCursorToCapture", () => { + it("maps a monitor with a non-zero origin and Y offset", () => { + const result = mapHyprlandCursorToCapture( + { x: 5058, y: 673 }, + [ + { x: 0, y: 494, width: 1366, height: 768, scale: 1 }, + { x: 1366, y: 236, width: 2560, height: 1600, scale: 1 }, + { x: 3926, y: 0, width: 3840, height: 2160, scale: 1 }, + ], + { x: 3926, y: 0, width: 3840, height: 2160 }, + ); + + expect(result).toEqual({ cx: 1132 / 3840, cy: 673 / 2160 }); + }); + + it("supports negative origins and fractional scaling", () => { + const result = mapHyprlandCursorToCapture( + { x: -640, y: 100 }, + [{ x: -1280, y: 0, width: 1280, height: 720, scale: 2 }], + { x: -1280, y: 0, width: 1280, height: 720 }, + ); + + expect(result.cx).toBeCloseTo(0.5); + expect(result.cy).toBeCloseTo(100 / 720); + }); + + it("rotates coordinates for transformed monitors", () => { + const result = mapHyprlandCursorToCapture( + { x: 25, y: 50 }, + [{ x: 0, y: 0, width: 100, height: 200, scale: 1, transform: 1 }], + { x: 0, y: 0, width: 100, height: 200 }, + ); + + expect(result).toEqual({ cx: 0.75, cy: 0.25 }); + }); + + it("clamps cursor positions outside a selected recording region", () => { + expect( + mapHyprlandCursorToCapture({ x: 2000, y: -100 }, [], { + x: 0, + y: 0, + width: 1000, + height: 1000, + }), + ).toEqual({ cx: 1, cy: 0 }); + }); + + it("recovers from an IPC disconnect without spawning a process per sample", async () => { + process.env.XDG_RUNTIME_DIR = "/tmp/openscreen-hypr-runtime"; + process.env.HYPRLAND_INSTANCE_SIGNATURE = "test-instance"; + let connections = 0; + vi.spyOn(net, "createConnection").mockImplementation(() => { + connections++; + const listeners = new Map void>(); + const socket = { + writable: true, + destroyed: false, + setTimeout: () => socket, + on: (event: string, listener: (...args: unknown[]) => void) => { + listeners.set(event, listener); + if (event === "connect") queueMicrotask(listener); + return socket; + }, + write: (command: string) => { + if (connections === 1) { + listeners.get("error")?.(new Error("compositor restarted")); + return true; + } + if (command === "j/cursorpos") listeners.get("data")?.('{"x":50,"y":25}'); + if (command === "j/monitors") { + listeners.get("data")?.( + '[{"x":0,"y":0,"width":100,"height":50,"scale":1,"transform":0}]', + ); + } + return true; + }, + destroy: () => { + socket.destroyed = true; + }, + }; + return socket as unknown as net.Socket; + }); + + const tracker = new HyprlandCursorTracker({ sampleIntervalMs: 20, maxSamples: 10 }); + tracker.setCaptureRect({ x: 0, y: 0, width: 100, height: 50 }); + tracker.start(); + await new Promise((resolve) => setTimeout(resolve, 120)); + const result = tracker.stop(); + + expect(connections).toBeGreaterThanOrEqual(2); + expect(result.samples.length).toBeGreaterThan(0); + }); + + it("resolves an omitted portal origin from a matching Hyprland monitor", async () => { + process.env.XDG_RUNTIME_DIR = "/tmp/openscreen-hypr-runtime"; + process.env.HYPRLAND_INSTANCE_SIGNATURE = "geometry-fallback"; + vi.spyOn(net, "createConnection").mockImplementation(() => { + const listeners = new Map void>(); + const socket = { + writable: true, + destroyed: false, + setTimeout: () => socket, + on: (event: string, listener: (...args: unknown[]) => void) => { + listeners.set(event, listener); + if (event === "connect") queueMicrotask(listener); + return socket; + }, + write: (command: string) => { + if (command === "j/cursorpos") listeners.get("data")?.('{"x":1500,"y":500}'); + if (command === "j/monitors") { + listeners.get("data")?.( + '[{"x":0,"y":494,"width":1366,"height":768,"scale":1,"transform":0},{"x":1366,"y":236,"width":2560,"height":1600,"scale":1,"transform":0}]', + ); + } + return true; + }, + destroy: () => { + socket.destroyed = true; + }, + }; + return socket as unknown as net.Socket; + }); + + const tracker = new HyprlandCursorTracker({ sampleIntervalMs: 10, maxSamples: 10 }); + tracker.setCaptureGeometry({ width: 2560, height: 1600 }); + tracker.start(); + await new Promise((resolve) => setTimeout(resolve, 120)); + + const sample = tracker.stop().samples[0]; + expect(sample).toMatchObject({ cx: 134 / 2560, cy: 264 / 1600, visible: true }); + }); + + it("matches a scaled monitor using its logical capture size", async () => { + process.env.XDG_RUNTIME_DIR = "/tmp/openscreen-hypr-runtime"; + process.env.HYPRLAND_INSTANCE_SIGNATURE = "scaled-geometry-fallback"; + vi.spyOn(net, "createConnection").mockImplementation(() => { + const listeners = new Map void>(); + const socket = { + writable: true, + destroyed: false, + setTimeout: () => socket, + on: (event: string, listener: (...args: unknown[]) => void) => { + listeners.set(event, listener); + if (event === "connect") queueMicrotask(listener); + return socket; + }, + write: (command: string) => { + if (command === "j/cursorpos") listeners.get("data")?.('{"x":500,"y":300}'); + if (command === "j/monitors") { + listeners.get("data")?.( + '[{"x":0,"y":0,"width":2560,"height":1600,"scale":2,"transform":0}]', + ); + } + return true; + }, + destroy: () => { + socket.destroyed = true; + }, + }; + return socket as unknown as net.Socket; + }); + + const tracker = new HyprlandCursorTracker({ sampleIntervalMs: 10, maxSamples: 10 }); + tracker.setCaptureGeometry({ width: 1280, height: 800 }); + tracker.start(); + await new Promise((resolve) => setTimeout(resolve, 60)); + + const sample = tracker.stop().samples[0]; + expect(sample).toMatchObject({ cx: 500 / 1280, cy: 300 / 800, visible: true }); + }); + + it("does not invent a window origin when the portal omits it", async () => { + process.env.XDG_RUNTIME_DIR = "/tmp/openscreen-hypr-runtime"; + process.env.HYPRLAND_INSTANCE_SIGNATURE = "window-geometry-fallback"; + vi.spyOn(net, "createConnection").mockImplementation(() => { + const listeners = new Map void>(); + const socket = { + writable: true, + destroyed: false, + setTimeout: () => socket, + on: (event: string, listener: (...args: unknown[]) => void) => { + listeners.set(event, listener); + if (event === "connect") queueMicrotask(listener); + return socket; + }, + write: (command: string) => { + if (command === "j/cursorpos") listeners.get("data")?.('{"x":1500,"y":500}'); + if (command === "j/monitors") { + listeners.get("data")?.( + '[{"x":0,"y":494,"width":1366,"height":768,"scale":1,"transform":0},{"x":1366,"y":236,"width":2560,"height":1600,"scale":1,"transform":0}]', + ); + } + return true; + }, + destroy: () => { + socket.destroyed = true; + }, + }; + return socket as unknown as net.Socket; + }); + + const tracker = new HyprlandCursorTracker({ sampleIntervalMs: 10, maxSamples: 10 }); + tracker.setCaptureGeometry({ width: 1234, height: 720, allowMonitorOriginFallback: false }); + tracker.start(); + await new Promise((resolve) => setTimeout(resolve, 60)); + + expect(tracker.stop().samples).toEqual([]); + }); + + it("corrects a zero portal origin for a uniquely sized monitor", async () => { + process.env.XDG_RUNTIME_DIR = "/tmp/openscreen-hypr-runtime"; + process.env.HYPRLAND_INSTANCE_SIGNATURE = "exact-monitor-origin"; + vi.spyOn(net, "createConnection").mockImplementation(() => { + const listeners = new Map void>(); + const socket = { + writable: true, + destroyed: false, + setTimeout: () => socket, + on: (event: string, listener: (...args: unknown[]) => void) => { + listeners.set(event, listener); + if (event === "connect") queueMicrotask(listener); + return socket; + }, + write: (command: string) => { + if (command === "j/cursorpos") listeners.get("data")?.('{"x":4479,"y":1080}'); + if (command === "j/monitors") { + listeners.get("data")?.( + '[{"x":1366,"y":236,"width":2560,"height":1600},{"x":3926,"y":0,"width":3840,"height":2160}]', + ); + } + return true; + }, + destroy: () => { + socket.destroyed = true; + }, + }; + return socket as unknown as net.Socket; + }); + + const tracker = new HyprlandCursorTracker({ sampleIntervalMs: 10, maxSamples: 5 }); + tracker.setCaptureGeometry({ position: { x: 0, y: 0 }, width: 3840, height: 2160 }); + tracker.start(); + await new Promise((resolve) => setTimeout(resolve, 30)); + const sample = tracker.stop().samples[0]; + + expect(sample?.cx).toBeCloseTo((4479 - 3926) / 3840); + expect(sample?.cy).toBeCloseTo(1080 / 2160); + }); + + it("does not produce samples when the Hyprland socket is unavailable", async () => { + process.env.XDG_RUNTIME_DIR = "/tmp/openscreen-no-such-runtime"; + process.env.HYPRLAND_INSTANCE_SIGNATURE = "missing"; + const tracker = new HyprlandCursorTracker({ sampleIntervalMs: 10, maxSamples: 10 }); + tracker.setCaptureRect({ x: 0, y: 0, width: 100, height: 50 }); + tracker.start(); + await new Promise((resolve) => setTimeout(resolve, 40)); + expect(tracker.stop().samples).toEqual([]); + }); + + it("rebases timestamps and excludes paused time", async () => { + process.env.XDG_RUNTIME_DIR = "/tmp/openscreen-hypr-runtime"; + process.env.HYPRLAND_INSTANCE_SIGNATURE = "timestamp-test"; + vi.spyOn(net, "createConnection").mockImplementation(() => { + const listeners = new Map void>(); + const socket = { + writable: true, + destroyed: false, + setTimeout: () => socket, + on: (event: string, listener: (...args: unknown[]) => void) => { + listeners.set(event, listener); + if (event === "connect") queueMicrotask(listener); + return socket; + }, + write: (command: string) => { + if (command === "j/cursorpos") listeners.get("data")?.('{"x":50,"y":25}'); + if (command === "j/monitors") { + listeners.get("data")?.( + '[{"x":0,"y":0,"width":100,"height":50,"scale":1,"transform":0}]', + ); + } + return true; + }, + destroy: () => { + socket.destroyed = true; + }, + }; + return socket as unknown as net.Socket; + }); + + const tracker = new HyprlandCursorTracker({ sampleIntervalMs: 15, maxSamples: 20 }); + tracker.setCaptureRect({ x: 0, y: 0, width: 100, height: 50 }); + tracker.start(); + await new Promise((resolve) => setTimeout(resolve, 45)); + tracker.rebase(); + await new Promise((resolve) => setTimeout(resolve, 30)); + tracker.pause(); + await new Promise((resolve) => setTimeout(resolve, 80)); + tracker.resume(); + await new Promise((resolve) => setTimeout(resolve, 45)); + + const recording = tracker.stop(); + const samples = recording.samples; + expect(recording.provider).toBe("native"); + expect(samples.length).toBeGreaterThan(1); + expect(samples[0].timeMs).toBeLessThan(30); + expect(samples.at(-1)?.timeMs ?? 0).toBeLessThan(110); + expect( + samples.every((sample, index) => index === 0 || sample.timeMs >= samples[index - 1].timeMs), + ).toBe(true); + }); +}); diff --git a/electron/native-bridge/cursor/recording/hyprlandCursorTracker.ts b/electron/native-bridge/cursor/recording/hyprlandCursorTracker.ts new file mode 100644 index 000000000..473c6c47b --- /dev/null +++ b/electron/native-bridge/cursor/recording/hyprlandCursorTracker.ts @@ -0,0 +1,356 @@ +import { existsSync, readdirSync } from "node:fs"; +import net from "node:net"; +import { hrtime } from "node:process"; +import type { CursorRecordingData, CursorRecordingSample } from "../../../../src/native/contracts"; + +export interface HyprlandMonitorGeometry { + x: number; + y: number; + width: number; + height: number; + scale?: number; + transform?: number; +} + +export interface HyprlandCaptureRect { + x: number; + y: number; + width: number; + height: number; +} + +export interface HyprlandCaptureGeometry { + position?: { x: number; y: number } | null; + width: number; + height: number; + /** False for window streams when the portal omitted the window origin. */ + allowMonitorOriginFallback?: boolean; +} + +function monitorLogicalSize(monitor: HyprlandMonitorGeometry) { + const scale = monitor.scale && monitor.scale > 0 ? monitor.scale : 1; + const swap = (monitor.transform ?? 0) % 2 === 1; + return { + width: (swap ? monitor.height : monitor.width) / scale, + height: (swap ? monitor.width : monitor.height) / scale, + }; +} + +export function mapHyprlandCursorToCapture( + point: { x: number; y: number }, + monitors: HyprlandMonitorGeometry[], + capture: HyprlandCaptureRect, +) { + const monitor = monitors.find((candidate) => { + const { width, height } = monitorLogicalSize(candidate); + return ( + point.x >= candidate.x && + point.y >= candidate.y && + point.x < candidate.x + width && + point.y < candidate.y + height + ); + }); + const transform = monitor?.transform ?? 0; + const logicalX = point.x - capture.x; + const logicalY = point.y - capture.y; + // Hyprland cursor coordinates and portal stream positions/sizes are in the + // compositor's logical coordinate space. The PipeWire frame dimensions are + // physical pixels and must not be used for this ratio, especially on scaled + // monitors. `capture` is populated from the portal's logical `size`. + const logicalWidth = Math.max(1, capture.width); + const logicalHeight = Math.max(1, capture.height); + let x = logicalX / logicalWidth; + let y = logicalY / logicalHeight; + + if (transform === 1) { + [x, y] = [1 - y, x]; + } else if (transform === 2) { + [x, y] = [1 - x, 1 - y]; + } else if (transform === 3) { + [x, y] = [y, 1 - x]; + } + + return { + cx: Math.min(1, Math.max(0, x)), + cy: Math.min(1, Math.max(0, y)), + }; +} + +function hyprlandSocketPath() { + const uid = typeof process.getuid === "function" ? process.getuid() : null; + const runtime = process.env.XDG_RUNTIME_DIR?.trim() || (uid === null ? null : `/run/user/${uid}`); + if (!runtime) return null; + + const configuredSignature = process.env.HYPRLAND_INSTANCE_SIGNATURE?.trim(); + if (configuredSignature) { + return `${runtime}/hypr/${configuredSignature}/.socket.sock`; + } + + // Electron launched from a desktop entry may not inherit Hyprland's + // instance variables even though it runs in the same user session. Discover + // the one live socket rather than disabling position telemetry in that case. + try { + const hyprlandRoot = `${runtime}/hypr`; + const signature = readdirSync(hyprlandRoot, { withFileTypes: true }).find( + (entry) => entry.isDirectory() && existsSync(`${hyprlandRoot}/${entry.name}/.socket.sock`), + )?.name; + return signature ? `${hyprlandRoot}/${signature}/.socket.sock` : null; + } catch { + return null; + } +} + +function monotonicMs() { + return Number(hrtime.bigint() / 1_000_000n); +} + +export class HyprlandCursorTracker { + private sockets = new Set(); + private interval: NodeJS.Timeout | null = null; + private sampleInFlight = false; + private monitorCache: HyprlandMonitorGeometry[] = []; + private monitorCacheAt = 0; + private samples: CursorRecordingSample[] = []; + private startMs = 0; + private pausedAt: number | null = null; + private pausedDurationMs = 0; + private captureRect: HyprlandCaptureRect | null = null; + private captureSize: { width: number; height: number } | null = null; + private allowMonitorOriginFallback = true; + private lastIpcError: string | null = null; + private generation = 0; + + constructor(private readonly options: { sampleIntervalMs: number; maxSamples: number }) {} + + static isAvailable() { + return Boolean(hyprlandSocketPath()); + } + + start() { + this.generation += 1; + this.stopSocket(); + this.samples = []; + this.startMs = monotonicMs(); + this.pausedAt = null; + this.pausedDurationMs = 0; + this.interval = setInterval(() => void this.sample(), this.options.sampleIntervalMs); + this.interval.unref(); + void this.sample(); + } + + stop(): CursorRecordingData { + this.generation += 1; + if (this.interval) clearInterval(this.interval); + this.interval = null; + this.stopSocket(); + return { version: 2, provider: "native", samples: this.samples, assets: [] }; + } + + pause() { + if (this.pausedAt === null) this.pausedAt = monotonicMs(); + } + + resume() { + if (this.pausedAt !== null) { + this.pausedDurationMs += Math.max(0, monotonicMs() - this.pausedAt); + this.pausedAt = null; + } + } + + rebase() { + this.generation += 1; + this.startMs = monotonicMs(); + this.pausedDurationMs = 0; + this.pausedAt = null; + this.samples = []; + } + + setCaptureRect(rect: HyprlandCaptureRect | null) { + this.captureRect = rect; + this.captureSize = rect ? { width: rect.width, height: rect.height } : null; + } + + /** + * Some portal backends omit Stream.position even for a monitor stream. Keep + * sampling in that case and resolve the origin from Hyprland's monitor list + * using the logical stream size. An omitted portal position must not turn a + * valid recording into an empty cursor sidecar. + */ + setCaptureGeometry(geometry: HyprlandCaptureGeometry | null) { + if (!geometry) { + this.captureRect = null; + this.captureSize = null; + this.allowMonitorOriginFallback = true; + return; + } + this.allowMonitorOriginFallback = geometry.allowMonitorOriginFallback ?? true; + this.captureSize = { + width: Math.max(1, geometry.width), + height: Math.max(1, geometry.height), + }; + this.captureRect = geometry.position + ? { + x: geometry.position.x, + y: geometry.position.y, + width: this.captureSize.width, + height: this.captureSize.height, + } + : null; + } + + hasCaptureGeometry() { + return this.captureRect !== null || this.captureSize !== null; + } + + private async sample() { + if (this.sampleInFlight || this.pausedAt !== null || (!this.captureRect && !this.captureSize)) + return; + const generation = this.generation; + this.sampleInFlight = true; + try { + const point = await this.request<{ x: number; y: number }>("j/cursorpos"); + const now = monotonicMs(); + if (now - this.monitorCacheAt > 1000 || this.monitorCacheAt === 0) { + const refreshed = await this.request("j/monitors"); + if (Array.isArray(refreshed)) { + this.monitorCache = refreshed; + this.monitorCacheAt = now; + } + } + this.correctPortalOriginForExactMonitor(point); + if (!this.captureRect && this.captureSize && this.allowMonitorOriginFallback) { + const matches = this.monitorCache.filter((monitor) => { + const scale = monitor.scale && monitor.scale > 0 ? monitor.scale : 1; + const widths = [monitor.width, monitor.width / scale].map(Math.round); + const heights = [monitor.height, monitor.height / scale].map(Math.round); + return ( + widths.includes(this.captureSize?.width ?? -1) && + heights.includes(this.captureSize?.height ?? -1) + ); + }); + const containing = this.monitorCache.find((monitor) => { + const { width, height } = monitorLogicalSize(monitor); + return ( + point.x >= monitor.x && + point.y >= monitor.y && + point.x < monitor.x + width && + point.y < monitor.y + height + ); + }); + // Exact size wins for monitor captures. For a window capture whose + // portal size is smaller than the monitor, the containing monitor is + // the best available origin when the portal omitted Stream.position. + const monitor = matches.length === 1 ? matches[0] : containing; + if (monitor) { + this.captureRect = { + x: monitor.x, + y: monitor.y, + width: this.captureSize.width, + height: this.captureSize.height, + }; + } + } + if (!this.captureRect || generation !== this.generation) return; + const mapped = mapHyprlandCursorToCapture(point, this.monitorCache, this.captureRect); + this.samples.push({ + timeMs: Math.max(0, now - this.startMs - this.pausedDurationMs), + ...mapped, + visible: true, + interactionType: "move", + }); + if (this.samples.length > this.options.maxSamples) this.samples.shift(); + } catch (error) { + // IPC is best-effort. A compositor restart is recovered on the next sample. + const message = error instanceof Error ? error.message : String(error); + if (message !== this.lastIpcError) { + this.lastIpcError = message; + console.warn("[hyprland-cursor] sample failed:", message); + } + } finally { + this.sampleInFlight = false; + } + } + + /** + * Hyprland's portal backend can report a full-monitor stream's origin as + * (0,0), even when the selected monitor has a negative or non-zero virtual + * desktop origin. If the logical stream size uniquely identifies a monitor + * and the pointer is inside it, trust Hyprland's monitor geometry for the + * origin. Window streams remain untouched because their size normally does + * not uniquely identify a monitor. + */ + private correctPortalOriginForExactMonitor(point: { x: number; y: number }) { + if (!this.captureRect) return; + const matches = this.monitorCache.filter((monitor) => { + const scale = monitor.scale && monitor.scale > 0 ? monitor.scale : 1; + const widths = [monitor.width, monitor.width / scale].map(Math.round); + const heights = [monitor.height, monitor.height / scale].map(Math.round); + return ( + widths.includes(Math.round(this.captureRect!.width)) && + heights.includes(Math.round(this.captureRect!.height)) + ); + }); + const monitor = matches.find((candidate) => { + const { width, height } = monitorLogicalSize(candidate); + return ( + point.x >= candidate.x && + point.y >= candidate.y && + point.x < candidate.x + width && + point.y < candidate.y + height + ); + }); + if (matches.length === 1 && monitor) { + this.captureRect.x = monitor.x; + this.captureRect.y = monitor.y; + } + } + + private request(command: string): Promise { + return new Promise((resolve, reject) => { + const socketPath = hyprlandSocketPath(); + if (!socketPath) { + reject(new Error("Hyprland IPC socket is unavailable")); + return; + } + + const socket = net.createConnection(socketPath); + this.sockets.add(socket); + let response = ""; + let settled = false; + const settle = (error?: Error, value?: T) => { + if (settled) return; + settled = true; + this.sockets.delete(socket); + if (error) reject(error); + else resolve(value as T); + }; + + socket.setTimeout(Math.max(250, this.options.sampleIntervalMs * 2)); + socket.on("connect", () => socket.write(command)); + socket.on("data", (chunk) => { + response += chunk.toString(); + try { + settle(undefined, JSON.parse(response) as T); + socket.destroy(); + } catch { + // Wait for the rest of a JSON response. + } + }); + socket.on("error", (error) => settle(error)); + socket.on("timeout", () => { + socket.destroy(); + settle(new Error("Hyprland IPC timeout")); + }); + socket.on("close", () => { + if (!settled) settle(new Error("Hyprland IPC connection closed")); + }); + }); + } + + private stopSocket() { + if (this.interval) clearInterval(this.interval); + this.interval = null; + for (const socket of this.sockets) socket.destroy(); + this.sockets.clear(); + } +} diff --git a/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts b/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts index 9a233c7d2..26c5e98b7 100644 --- a/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts +++ b/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts @@ -31,8 +31,8 @@ export type PipeWireHelperEvent = event: "ready"; timestampMs: number; pipewireVersion?: string | null; - /** The mode really negotiated; absent from helpers older than #1357. */ - cursorMode?: "metadata" | "embedded" | "hidden"; + cursorMetadataSupported?: boolean; + cursorMode?: "auto" | "metadata" | "embedded" | "hidden"; } | { /** The picker has been answered. Fires before any pixel moves. */ @@ -49,6 +49,10 @@ export type PipeWireHelperEvent = nodeId: number; width: number; height: number; + positionX?: number | null; + positionY?: number | null; + logicalWidth?: number | null; + logicalHeight?: number | null; /** What the compositor actually handed over. Absent on backends that * omit it — treat that as unknown, never as a screen. */ sourceKind?: "monitor" | "window" | "virtual" | null; diff --git a/electron/native/pipewire-capture/src/events.rs b/electron/native/pipewire-capture/src/events.rs index 9bc4d4c7a..b8ac85d1b 100644 --- a/electron/native/pipewire-capture/src/events.rs +++ b/electron/native/pipewire-capture/src/events.rs @@ -48,9 +48,8 @@ pub enum Event { pipewire_version: Option, /// Whether the portal advertises METADATA cursor mode at all. cursor_metadata_supported: bool, - /// The cursor mode actually negotiated, which can differ from the one - /// asked for (see `CursorMode::resolve`). - cursor_mode: &'static str, + /// The mode selected for this session after capability negotiation. + cursor_mode: String, }, /// The user has answered the compositor's picker and a source was granted. /// @@ -78,6 +77,10 @@ pub enum Event { /// space, when the portal reports one (monitor streams only). position_x: Option, position_y: Option, + /// Logical compositor-space size. This can differ from the PipeWire + /// frame size on scaled monitors. + logical_width: Option, + logical_height: Option, /// `"monitor"`, `"window"` or `"virtual"` — what the compositor /// actually handed over, straight from the portal's reply. Absent when /// the backend omits it. This is the only honest answer to "what am I @@ -256,7 +259,7 @@ mod tests { timestamp_ms: 42, pipewire_version: Some("1.0.5".to_owned()), cursor_metadata_supported: true, - cursor_mode: "metadata", + cursor_mode: "metadata".to_owned(), }, false, ); @@ -279,13 +282,12 @@ mod tests { timestamp_ms: 7, pipewire_version: None, cursor_metadata_supported: false, - cursor_mode: "embedded", + cursor_mode: "hidden".to_owned(), }); assert_eq!(value["event"], "ready"); assert_eq!(value["timestampMs"], 7); assert!(value["pipewireVersion"].is_null()); assert_eq!(value["cursorMetadataSupported"], false); - assert_eq!(value["cursorMode"], "embedded"); } #[test] @@ -297,6 +299,8 @@ mod tests { height: 1440, position_x: Some(0), position_y: Some(-1080), + logical_width: Some(1920), + logical_height: Some(1080), source_kind: Some("window".to_owned()), }); assert_eq!(value["event"], "stream-started"); diff --git a/electron/native/pipewire-capture/src/main.rs b/electron/native/pipewire-capture/src/main.rs index aac59ec4d..66f238ae9 100644 --- a/electron/native/pipewire-capture/src/main.rs +++ b/electron/native/pipewire-capture/src/main.rs @@ -145,10 +145,11 @@ impl Request { fn cursor_mode(&self) -> Result { match self.cursor_mode.as_deref() { None | Some("") | Some("metadata") => Ok(portal::CursorMode::Metadata), + Some("auto") => Ok(portal::CursorMode::Auto), Some("embedded") => Ok(portal::CursorMode::Embedded), Some("hidden") => Ok(portal::CursorMode::Hidden), Some(other) => Err(format!( - "cursorMode must be one of metadata, embedded, hidden — got {other:?}" + "cursorMode must be one of auto, metadata, embedded, hidden — got {other:?}" )), } } @@ -249,7 +250,7 @@ fn main() { fail(&mut emitter, "invalid-arguments", &message); } }; - let cursor_mode = match request.cursor_mode() { + let requested_cursor_mode = match request.cursor_mode() { Ok(mode) => mode, Err(message) => fail(&mut emitter, "invalid-arguments", &message), }; @@ -280,18 +281,19 @@ fn main() { } // Probed before `ready` so the event can report it, and so an unsupported - // compositor fails immediately instead of after a pointless picker. Only - // fatal when METADATA is the mode actually requested — a recording that - // wants the system cursor painted in has no use for it. - let offered_cursor_modes = match pollster::block_on(portal::offered_cursor_modes()) { - Ok(modes) => modes, + // compositor fails immediately instead of after a pointless picker. Auto + // chooses metadata first, then hidden, then embedded based on the actual + // bit flags advertised by the portal. + let cursor_capabilities = match pollster::block_on(portal::cursor_capabilities()) { + Ok(capabilities) => capabilities, Err(error) => fail(&mut emitter, error.code(), &error.message()), }; - let cursor_metadata_supported = offered_cursor_modes.contains(&portal::CursorMode::Metadata); - // Resolved before anything reads it, so the portal, the click readers and - // `ready` all see the mode that is really negotiated. - let cursor_mode = cursor_mode.resolve(&offered_cursor_modes); - if !cursor_metadata_supported && cursor_mode.reports_cursor() { + let cursor_mode = requested_cursor_mode.resolve( + cursor_capabilities.metadata, + cursor_capabilities.hidden, + cursor_capabilities.embedded, + ); + if !cursor_capabilities.metadata && cursor_mode.reports_cursor() { let error = portal::PortalError::CursorMetadataUnsupported; fail(&mut emitter, error.code(), &error.message()); } @@ -299,8 +301,8 @@ fn main() { let _ = emitter.emit(&Event::Ready { timestamp_ms: timestamp_ms(), pipewire_version: shim::library_version(), - cursor_metadata_supported, - cursor_mode: cursor_mode.as_str(), + cursor_metadata_supported: cursor_capabilities.metadata, + cursor_mode: cursor_mode.as_str().to_owned(), }); if request.probe_only { @@ -578,7 +580,12 @@ fn begin_stream( Ok(started) => { *session = Some(started); *granted_kind = source_kind; - *portal_stream = Some(StreamInfo { node_id, position, source_kind }); + *portal_stream = Some(StreamInfo { + node_id, + position, + size, + source_kind, + }); Ok(()) } Err(message) => { @@ -596,6 +603,7 @@ fn begin_stream( struct StreamInfo { node_id: u32, position: Option<(i32, i32)>, + size: Option<(i32, i32)>, source_kind: Option, } @@ -608,20 +616,6 @@ struct SharedMemoryFallback { source_kind: Option, } -/// Whether the negotiated buffers carry a SPA_META_VideoCrop block, read from -/// the shim's "Header:12,VideoCrop:16" list. -/// -/// Only then is waiting for a window's crop worth anything. mutter negotiates -/// the block and fills it a few frames late; KWin never negotiates it — it sizes -/// a window stream to the window itself — and sends one frame when the stream -/// starts, then only when the window repaints. Waiting on KWin therefore threw -/// away the one frame a still window ever sends, and the start timed out. -fn negotiates_video_crop(metas: &str) -> bool { - metas - .split(',') - .any(|meta| meta.split(':').next() == Some("VideoCrop")) -} - fn should_retry_shared_memory( state: &str, error: Option<&str>, @@ -663,9 +657,6 @@ fn run( let mut granted_kind: Option = None; // Frames skipped while waiting for a window's first usable crop rectangle. let mut frames_awaiting_crop: u32 = 0; - // Set from the first `BufferInfo`, which the shim sends before that same - // buffer's frame. See `negotiates_video_crop`. - let mut crop_meta_negotiated = false; // A crop change is reported once, not once per frame. let mut crop_change_reported = false; // A `deferStart` grant held while the caller runs its countdown. @@ -835,11 +826,8 @@ fn run( // the encoder to that frame would pin a window // recording at monitor size for its whole duration, // which is the very bug being fixed, only intermittent. - // A compositor that never negotiated the crop meta - // (KWin) is not waited on at all. - let expecting_crop = granted_kind == Some(portal::SourceKind::Window) - && crop_meta_negotiated - && !frame.has_crop; + let expecting_crop = + granted_kind == Some(portal::SourceKind::Window) && !frame.has_crop; if expecting_crop && frames_awaiting_crop < MAX_FRAMES_AWAITING_CROP { frames_awaiting_crop += 1; mailbox.recycle(frame.pixels); @@ -1055,9 +1043,9 @@ fn run( Ok(stream) => { // The portal's size is in the compositor's coordinate space // and can differ from the negotiated pixel size on a scaled - // display. Logged rather than used: cursor positions arrive - // in stream pixels, and `content_rect` places them from - // there. + // display. The logical size is forwarded for Hyprland cursor + // normalization; the PipeWire frame size remains the video + // encoder's physical-pixel geometry. let _ = emitter.emit(&Event::Debug { code: "portal-stream".to_owned(), data: json_map([ @@ -1138,6 +1126,8 @@ fn run( height: format.height, position_x: stream.position.map(|(x, _)| x), position_y: stream.position.map(|(_, y)| y), + logical_width: stream.size.map(|(width, _)| width), + logical_height: stream.size.map(|(_, height)| height), source_kind: stream .source_kind .map(|kind| kind.as_str().to_owned()), @@ -1180,7 +1170,6 @@ fn run( // bearing field: a missing Cursor next to a present Header means // our ParamMeta was sent and lost the size intersection, which is // otherwise indistinguishable from never having sent it. - crop_meta_negotiated = negotiates_video_crop(&metas); let _ = emitter.emit(&Event::Debug { code: "buffer-info".to_owned(), data: json_map([ @@ -1862,16 +1851,7 @@ mod microphone_resolution_tests { #[cfg(test)] mod capture_fallback_tests { - use super::{negotiates_video_crop, should_retry_shared_memory, stream_error_is_fatal}; - - #[test] - fn only_a_negotiated_video_crop_block_is_worth_waiting_for() { - // mutter: the block is there, its rect may arrive a few frames late. - assert!(negotiates_video_crop("Header:12,VideoCrop:16,Cursor:589872")); - // KWin: no block, so waiting would discard a still window's only frame. - assert!(!negotiates_video_crop("Header:12,VideoDamage:256,Cursor:589872")); - assert!(!negotiates_video_crop("")); - } + use super::{should_retry_shared_memory, stream_error_is_fatal}; #[test] fn a_stream_error_is_fatal_only_before_capture_starts() { diff --git a/electron/native/pipewire-capture/src/portal.rs b/electron/native/pipewire-capture/src/portal.rs index 68ef7156d..a71f2b7ed 100644 --- a/electron/native/pipewire-capture/src/portal.rs +++ b/electron/native/pipewire-capture/src/portal.rs @@ -26,7 +26,7 @@ use ashpd::desktop::PersistMode; use ashpd::enumflags2::BitFlags; use ashpd::WindowIdentifier; -/// Which of the portal's three cursor modes to ask for. +/// Which cursor mode to ask the portal for. /// /// This is the HUD's cursor-mode toggle, one layer down. `Metadata` is what /// makes the editable cursor possible at all — the compositor keeps the pointer @@ -35,6 +35,7 @@ use ashpd::WindowIdentifier; /// leaves it alone. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum CursorMode { + Auto, Metadata, Embedded, Hidden, @@ -43,43 +44,43 @@ pub enum CursorMode { impl CursorMode { fn to_portal(self) -> PortalCursorMode { match self { + Self::Auto => PortalCursorMode::Metadata, Self::Metadata => PortalCursorMode::Metadata, Self::Embedded => PortalCursorMode::Embedded, Self::Hidden => PortalCursorMode::Hidden, } } - /// Only METADATA yields cursor samples; in the other modes the pointer is - /// either painted into the frames or absent, and there is no metadata to - /// read either way. + /// Only METADATA yields portal cursor samples; in the other modes the + /// pointer is either painted into the frames or absent, and there is no + /// portal metadata to read either way. pub fn reports_cursor(self) -> bool { matches!(self, Self::Metadata) } + pub fn resolve( + self, + metadata_supported: bool, + hidden_supported: bool, + embedded_supported: bool, + ) -> Self { + match self { + Self::Auto if metadata_supported => Self::Metadata, + Self::Auto if hidden_supported => Self::Hidden, + Self::Embedded if !embedded_supported && metadata_supported => Self::Metadata, + Self::Auto => Self::Embedded, + other => other, + } + } + pub fn as_str(self) -> &'static str { match self { + Self::Auto => "auto", Self::Metadata => "metadata", Self::Embedded => "embedded", Self::Hidden => "hidden", } } - - /// The mode to actually negotiate, given what the portal offers. - /// - /// Some portals offer no EMBEDDED at all (niri's, #1357). Refusing to record - /// there leaves the user nothing, so EMBEDDED falls back to METADATA: the - /// pointer stays out of the pixels and the editor draws it from the samples - /// instead. Any other miss is returned unchanged, for the negotiation to - /// refuse with its own error. - pub fn resolve(self, offered: &[CursorMode]) -> CursorMode { - if self == Self::Embedded - && !offered.contains(&Self::Embedded) - && offered.contains(&Self::Metadata) - { - return Self::Metadata; - } - self - } } /// Which kind of source the compositor actually handed over. @@ -192,20 +193,50 @@ fn portal_unavailable(error: impl std::fmt::Display) -> PortalError { )) } -/// Cheap, non-interactive probe: which cursor modes does this portal offer? +/// Cheap, non-interactive probe for the cursor modes needed by auto selection. /// /// Split out from [`negotiate`] so the helper can answer that question — and /// emit `ready` — before anything raises a dialog. -pub async fn offered_cursor_modes() -> Result, PortalError> { +pub struct CursorCapabilities { + pub metadata: bool, + pub hidden: bool, + pub embedded: bool, +} + +pub async fn cursor_capabilities() -> Result { let proxy = Screencast::new().await.map_err(portal_unavailable)?; let cursor_modes = proxy .available_cursor_modes() .await .map_err(|error| failed("AvailableCursorModes", error))?; - Ok([CursorMode::Metadata, CursorMode::Embedded, CursorMode::Hidden] - .into_iter() - .filter(|mode| cursor_modes.contains(mode.to_portal())) - .collect()) + Ok(CursorCapabilities { + metadata: cursor_modes.contains(PortalCursorMode::Metadata), + hidden: cursor_modes.contains(PortalCursorMode::Hidden), + embedded: cursor_modes.contains(PortalCursorMode::Embedded), + }) +} + +#[cfg(test)] +mod tests { + use super::CursorMode; + + #[test] + fn auto_prefers_metadata_then_hidden_then_embedded() { + assert_eq!( + CursorMode::Auto.resolve(true, true, true), + CursorMode::Metadata + ); + assert_eq!(CursorMode::Auto.resolve(false, true, true), CursorMode::Hidden); + assert_eq!(CursorMode::Auto.resolve(false, false, true), CursorMode::Embedded); + assert_eq!( + CursorMode::Embedded.resolve(true, true, false), + CursorMode::Metadata + ); + assert_eq!( + CursorMode::Embedded.resolve(true, true, true), + CursorMode::Embedded + ); + } } /// Runs the negotiation to completion. Blocks on the user for as long as the @@ -223,9 +254,13 @@ pub async fn negotiate(cursor_mode: CursorMode) -> Result PortalError::CursorMetadataUnsupported, @@ -304,25 +339,3 @@ pub async fn negotiate(cursor_mode: CursorMode) -> Result { expect(plan(telemetry)).toEqual([]); }); - it("places no zoom where the pointer only sits still", () => { - expect(plan(park(2000, 0.5, 0.5))).toEqual([]); + it("uses a stable position when the provider has no click telemetry", () => { + const suggestions = plan(park(2000, 0.5, 0.5)); + expect(suggestions.length).toBeGreaterThan(0); + expect(suggestions[0].focus).toEqual({ cx: 0.5, cy: 0.5 }); + }); + + it("does not join position dwells across a hidden interval", () => { + const suggestions = plan( + [ + { timeMs: 1000, cx: 0.5, cy: 0.5, visible: true }, + { timeMs: 1200, cx: 0.5, cy: 0.5, visible: true }, + { timeMs: 1400, cx: 0.5, cy: 0.5, visible: true }, + { timeMs: 1500, cx: 0.5, cy: 0.5, visible: false }, + { timeMs: 8000, cx: 0.5, cy: 0.5, visible: true }, + { timeMs: 8200, cx: 0.5, cy: 0.5, visible: true }, + { timeMs: 8400, cx: 0.5, cy: 0.5, visible: true }, + ], + 10000, + ); + expect(suggestions).toHaveLength(2); + }); + + it("places an irregular position dwell at its temporal midpoint", () => { + const suggestions = plan( + [ + { timeMs: 0, cx: 0.4, cy: 0.6 }, + { timeMs: 100, cx: 0.4, cy: 0.6 }, + { timeMs: 200, cx: 0.4, cy: 0.6 }, + { timeMs: 5000, cx: 0.4, cy: 0.6 }, + ], + 5000, + ); + expect(suggestions[0]?.span).toEqual({ start: 2000, end: 4000 }); }); it("returns nothing without telemetry, or for a lone move or mouseup", () => { diff --git a/src/lib/ai-edition/timeline/zoom-suggestions.ts b/src/lib/ai-edition/timeline/zoom-suggestions.ts index 8df686f48..7257687e8 100644 --- a/src/lib/ai-edition/timeline/zoom-suggestions.ts +++ b/src/lib/ai-edition/timeline/zoom-suggestions.ts @@ -7,9 +7,9 @@ // zoom, which holds while the camera follows the cursor. A zoom never comes straight back after // leaving. The video ends on the whole screen. // -// A still pointer is no reason to zoom: at rest it sits far from where the viewer looks, and -// Screen Studio, Cap and Tella all zoom on clicks alone. A take recorded without clicks gets no -// automatic zoom. +// Clicks remain the strongest signal. Position-only providers such as Hyprland cannot observe +// button presses without privileged input monitoring, so a no-click track falls back to stable +// cursor-position moments instead of disabling auto-zoom altogether. import type { CursorTelemetryPoint, ZoomFocus } from "@/components/video-editor/types"; import { zoomTransitionMs } from "@/lib/zoomMath/constants"; @@ -86,6 +86,9 @@ interface RecordedClick { focus: ZoomFocus; } +const POSITION_CLUSTER_RADIUS = 0.045; +const POSITION_MOMENT_MIN_MS = 350; + /** The recorded interactions that count as a click; `move` and `mouseup` do not. */ const CLICK_INTERACTION_TYPES: ReadonlySet> = new Set(["click", "double-click", "right-click", "middle-click"]); @@ -144,6 +147,76 @@ function visibleClicks( ); } +/** + * Position-only fallback for Wayland providers. A moment is the midpoint of a + * contiguous cursor dwell cluster; movement to a new area starts another one. + * It intentionally runs only when no clicks exist, so metadata-capable systems + * keep their existing click-driven timing and never receive duplicate zooms. + */ +function visiblePositionMoments( + samples: CursorTelemetryPoint[], + totalMs: number, + ignoreFromMs: number, +): RecordedClick[] { + const normalized = normalizeCursorTelemetry(samples, totalMs); + if (normalized.length === 0) return []; + + const moments: RecordedClick[] = []; + let cluster: CursorTelemetryPoint[] = []; + let sumX = 0; + let sumY = 0; + let center = { cx: 0, cy: 0 }; + const reset = () => { + cluster = []; + sumX = 0; + sumY = 0; + center = { cx: 0, cy: 0 }; + }; + const flush = () => { + if (cluster.length < 2) return; + const start = cluster[0].timeMs; + const end = cluster[cluster.length - 1].timeMs; + if (end - start < POSITION_MOMENT_MIN_MS) return; + const timeMs = (start + end) / 2; + const middle = cluster.reduce((nearest, sample) => + Math.abs(sample.timeMs - timeMs) < Math.abs(nearest.timeMs - timeMs) ? sample : nearest, + ); + moments.push({ timeMs, focus: { cx: middle.cx, cy: middle.cy } }); + }; + + for (const point of normalized) { + if (point.timeMs >= ignoreFromMs) break; + if (point.visible === false) { + flush(); + reset(); + continue; + } + const distance = Math.hypot(point.cx - center.cx, point.cy - center.cy); + if (cluster.length > 0 && distance > POSITION_CLUSTER_RADIUS) { + flush(); + reset(); + } + cluster.push(point); + sumX += point.cx; + sumY += point.cy; + center = { + cx: sumX / cluster.length, + cy: sumY / cluster.length, + }; + } + flush(); + return moments; +} + +function recordedZoomMoments( + samples: CursorTelemetryPoint[], + totalMs: number, + ignoreFromMs: number, +): RecordedClick[] { + const clicks = visibleClicks(samples, totalMs, ignoreFromMs); + return clicks.length > 0 ? clicks : visiblePositionMoments(samples, totalMs, ignoreFromMs); +} + /** When the recording stopped: its telemetry runs until the take ends. */ function lastSampleMs(samples: CursorTelemetryPoint[]): number { let last = 0; @@ -342,7 +415,11 @@ export function buildAutoZoomSuggestions(options: { }): AutoZoomSuggestion[] { const { cursorTelemetry, totalMs, existingRegions } = options; if (totalMs <= 0 || cursorTelemetry.length === 0) return []; - const clicks = visibleClicks(cursorTelemetry, totalMs, options.ignoreClicksFromMs ?? Infinity); + const clicks = recordedZoomMoments( + cursorTelemetry, + totalMs, + options.ignoreClicksFromMs ?? Infinity, + ); const shots = planShots( clicks, totalMs, @@ -398,7 +475,7 @@ export function buildAutoZoomSuggestionsForClips(options: { (sample) => sample.timeMs >= sourceOffsetMs && sample.timeMs <= sourceOffsetMs + windowMs, ) .map((sample) => ({ ...sample, timeMs: sample.timeMs - sourceOffsetMs })); - const clicks = visibleClicks( + const clicks = recordedZoomMoments( clipTelemetry, windowMs, lastSampleMs(telemetry) - STOP_CLICK_GUARD_MS - sourceOffsetMs, diff --git a/src/lib/nativeLinuxRecording.test.ts b/src/lib/nativeLinuxRecording.test.ts index ef9342657..c23333fd5 100644 --- a/src/lib/nativeLinuxRecording.test.ts +++ b/src/lib/nativeLinuxRecording.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { portalOwnsSourceSelection } from "./nativeLinuxRecording"; +import { portalCursorMode, portalOwnsSourceSelection } from "./nativeLinuxRecording"; function probe(platform: string, available: boolean | Error) { return { @@ -61,3 +61,14 @@ describe("portalOwnsSourceSelection", () => { ).resolves.toBe(false); }); }); + +describe("portalCursorMode", () => { + it("keeps metadata negotiation off Hyprland", () => { + expect(portalCursorMode("editable-overlay")).toBe("metadata"); + }); + + it("uses capability-aware auto negotiation on Hyprland", () => { + expect(portalCursorMode("editable-overlay", true)).toBe("auto"); + expect(portalCursorMode("system", true)).toBe("embedded"); + }); +}); diff --git a/src/lib/nativeLinuxRecording.ts b/src/lib/nativeLinuxRecording.ts index 3beee7c52..849719ac1 100644 --- a/src/lib/nativeLinuxRecording.ts +++ b/src/lib/nativeLinuxRecording.ts @@ -100,6 +100,7 @@ export type NativeLinuxRecordingStartResult = { */ export function portalCursorMode( mode: import("./recordingSession").CursorCaptureMode, -): "metadata" | "embedded" { - return mode === "system" ? "embedded" : "metadata"; + hyprland = false, +): "auto" | "metadata" | "embedded" { + return mode === "system" ? "embedded" : hyprland ? "auto" : "metadata"; }