diff --git a/CHANGELOG.md b/CHANGELOG.md index 69f72eb00..4b6fdb4ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,44 @@ hashes are given where a change is easiest to understand by reading the commit. ## October 2026 +### VNC, and screen and pointer commands + +- **Built-in VNC server.** `--vnc [ADDR]` or `[vnc] listen` serves the + guest's screen, keyboard and mouse to any VNC viewer (RFB 3.8, Raw and + ZRLE, desktop resizing, QEMU's key-position extension; no password, so it + stays on loopback by default). It serves every machine and board the same + way, with the window open or under `--ci` with no window. It reads the + frame the board composed and finds what changed by comparing frames in + 64x64 tiles; drawing code is untouched. Standalone `iris` only. See + [HELP.md](HELP.md#vnc). (`src/vnc/`) +- **Clicks land where you point.** The guest mouse is relative and X + accelerates it; iris now steers the guest pointer onto an absolute + position by reading back the board's hardware cursor (VC2, VC1, VC3, GBE), + learning the cursor's hotspot from the screen corner. VNC uses it, and so + do new monitor commands for scripts and assistants: `screen + info|shot|pix|wait|cursor`, `ptr move|click|drag|down|up|rel|calibrate`, + `key |type|names`, on every machine + ([debug.md](debug.md#graphics)). The MCP server wraps them as + `screenshot` (returns the image), `click`, `drag`, `move_pointer`, + `type_text`, `press_key`, `wait_screen`, `screen_info`, `screen_pixel`. + (`src/hostinput.rs`, `src/keysym.rs`, `clanker/iris_mcp.py`) +- `GfxDisplay` gained `request_frame`, `set_capture`, `hw_cursor` and a + frame sequence number, so a consumer can follow the composed frame on any + board. The CI socket's `screenshot` uses it and no longer also writes a + `screenshot_NNNN.png` into the working directory on Newport. While a VNC + viewer is connected, Newport's window composes in software, and goes back + to GL when the last one leaves. (`src/gfx_display.rs`, + `src/dev/ng1/rex3.rs`, `src/ci.rs`) +- **Empty GIO slots answer reads with a bus error** (Indy, Indigo2), as the + MC does once the PROM has enabled GIO time-outs. They returned all ones, + so the PROM's probe took an absent board for a Newport and polled its + status 200 000 times on every headless boot (one log line each) before + giving up. Headless Indy and Indigo2 now reach the IRIX serial login + without the flood. (`src/ip22_bus.rs`, `src/dev/mc.rs`) +- **The O2's cursor no longer vanishes at the top and left edges.** GBE's + cursor position fields are 12-bit signed; they were read as unsigned, so a + cursor partly off the edge was drawn 4000 pixels away. (`src/dev/o2gfx.rs`) + ### Host efficiency on small machines - **Threads that have nothing to do now sleep in milliseconds, not diff --git a/Cargo.toml b/Cargo.toml index cd4290f5b..8705735bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -284,6 +284,8 @@ toml = "1.0.3" postcard = { version = "1", features = ["alloc"] } blake3 = "1" png = "0.18" +# The VNC server's ZRLE encoding (zlib). +flate2 = "1" parking_lot = "0.12" spin = "0.9" # rfd >= 0.16 dropped the async-std/tokio runtime features: `xdg-portal` now diff --git a/FEATURES.md b/FEATURES.md index 8d2a7b15e..3b9becfea 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -238,6 +238,13 @@ Omit `[nfs]` to disable the share; the default forward list is empty. | `vino.standard` | `ntsc` | ntsc (60 fields/s) or pal (50 fields/s). | | `vino.camera_index` | `0` | Host camera index; meaningful only for source=camera. | +### [vnc] + +| Key | Default | Values and behavior | +|---|---|---| +| `vnc.listen` | `unset` (no server) | `host:port`, or a port on 127.0.0.1, to serve the screen, keyboard and mouse over VNC (RFB 3.8; Raw and ZRLE; no password). Every board, with a window or `ci`; not `headless`. Standalone `iris` only. See [HELP.md](HELP.md#vnc). | +| `vnc.max_fps` | `30` | Most updates a second sent to a viewer (1–120). | + ### [audio] | Key | Default | Values and behavior | @@ -399,6 +406,7 @@ iris-gui does not parse this standalone option set; use its named-machine UI. | `--graphics` | Graphics board: `newport` (default), `xz`, `extreme`, `solidimpact`, `highimpact`, `maximpact` (or `impact:solid`, `impact:high`, `impact:max`). | | `--gdb-port` | Enable GDB stub on the given TCP port (e.g. --gdb-port 1234). Connect with: target remote localhost: | | `--monitor-port` | Monitor console port on 127.0.0.1 (default 8888). Give each iris running at once its own. | +| `--vnc [ADDR]` | Serve the screen, keyboard and mouse over VNC on ADDR (`host:port` or a port; default 127.0.0.1:5900). Sets `vnc.listen`. | | `--serial-port-a` | Serial channel A (tty2) port on 127.0.0.1 (default 8880). | | `--serial-port-b` | Serial channel B (tty1, the console) port on 127.0.0.1 (default 8881). | | `--test-device` | Map the bare-metal test device into GIO expansion slot 0: SIGNATURE, PUTC (guest console → stdout), DUMP (machine state → JSON) and EXIT (terminate with the guest's exit code). Off by default. | diff --git a/HACKING.md b/HACKING.md index 848a4a6d8..68e8fdfb8 100644 --- a/HACKING.md +++ b/HACKING.md @@ -253,7 +253,8 @@ Every device that has real work to do runs on its own thread: | graphics: `REX3-Processor` / `GR2-HQ2`, `GR2-RE3` / `MGRAS-HQ3`, `MGRAS RSS` ×1-4 | consumers of the graphics FIFOs (section 7) | | `REX3-Refresh`, `GR2-Display`, MGRAS refresh | scanout and composition; the refresh thread owns the GL context | | `hptimer` | host timer callbacks (Compare, PIT ticks, HAL2 audio pacing) | -| listeners | debug console, serial ports, CI socket, GDB stub | +| listeners | debug console, serial ports, CI socket, GDB stub, VNC | +| `VNC-Grab`, `VNC-Pointer`, `VNC-Client`/`VNC-Read` | with `--vnc`: frame diffing, pointer steering, one pair per viewer | | `machine-events` | power-off and reset requests from devices | | `ISP1040-0`, `ISP1040-1` | Octane SCSI request rings and disk transfers | | `IOC3-RX-A`, `IOC3-RX-B`, `IOC3-NVRAM` | Octane serial input per port; the DS1685 NVRAM file writer (IOC3's other deadlines are `hptimer` one-shots) | @@ -418,6 +419,31 @@ without locks and composes it through the board's colour maps and DAC. Everything that touches the host GL context (including teardown) runs on the thread that owns it. +### The composed frame and the hardware cursor + +`GfxDisplay` (`src/gfx_display.rs`) is what the window, the CI socket, the +monitor's `screen`/`ptr` commands and the VNC server see of a board, whichever +board it is. The display thread leaves the composed guest frame in +`screen().rgba` (stride 2048, `0xAABBGGRR`, no status bar) and bumps +`screen().frame_seq` each time it rewrites it. GR2, IMPACT and the O2 compose +on the CPU and do this every frame. Newport composes inside the window's +renderer, so it fills `rgba` only when asked: `request_frame` for one frame, +`set_capture(true)` for every frame while a consumer (VNC) watches. While +capture is on, Newport's window uses the software compositor so the copy is a +slice copy; with no window (`--ci`) the refresh thread composes in software +itself. `copy_frame`/`fresh_frame` are the helpers that read it. + +Nothing downstream looks inside the drawing code. The VNC server +(`src/vnc/`) finds damage by comparing frames in 64x64 tiles, so every board +is served the same way and drawing paths carry no bookkeeping. + +`hw_cursor` reads the board's cursor registers live (VC2, VC1, VC3, GBE): +the glyph's top-left and its opaque pixels. `src/hostinput.rs` uses it to +steer the guest's relative PS/2 mouse onto absolute positions (the X server +accelerates what it reads, so positions cannot be sent), learning each +glyph's hotspot from the top-left screen corner, where X pins the pointer +at 0,0. + ## 8. jitv2: the MIPS JIT `--features jitv2` adds a JIT compiler (`src/cpu/jitv2/`) that turns guest diff --git a/HELP.md b/HELP.md index 03d2b894c..557503049 100644 --- a/HELP.md +++ b/HELP.md @@ -494,6 +494,55 @@ standalone `iris` window if you need them. The window is freely resizable. At 1× and 2× scale the display is rendered with nearest-neighbour pixel-perfect sampling; at all other sizes trilinear filtering with mipmaps is used. `lock_aspect_ratio = false` in `iris.toml` allows free resizing with letterboxing. When the guest changes video mode (PROM, then IRIX during boot) the window snaps to the new resolution times the configured `scale` / `--2x`. +## VNC + +iris has a built-in VNC server. Any VNC viewer (TigerVNC, RealVNC, Remmina, +noVNC, macOS Screen Sharing) then shows the guest's screen and types and +clicks into it: + +```sh +iris --vnc # 127.0.0.1:5900 +iris --ci --vnc # no window at all: VNC is the screen +iris --vnc 0.0.0.0:5901 # reachable from other machines (no password!) +vncviewer localhost:5900 +``` + +or in `iris.toml`: + +```toml +[vnc] +listen = "127.0.0.1:5900" # absent: no VNC server +max_fps = 30 # most updates a second per viewer +``` + +It works on every machine and graphics board (Newport, XZ/Extreme, IMPACT, +the O2), with the normal window open or under `--ci` with no window. Not with +`--headless`, which has no screen to serve. The window, the monitor and any +number of viewers can type and click at the same time. iris-gui has no VNC +server. + +What you see is the guest's screen, without the window's status bar. Only +the parts of the screen that change are sent, so an idle desktop costs next +to nothing. With Newport, the window uses the software compositor while a +viewer is connected (`disp compositor gl` is refused) and switches back when +the last one leaves. + +**The mouse.** The guest's mouse sends movements, not positions, and IRIX +accelerates fast ones. iris steers the guest pointer onto the spot you point +at by watching where the board draws the cursor, so a click lands where you +clicked. The first time you point, the guest pointer jumps to the top-left +corner and back: that teaches iris where the tip of the cursor is. The +viewer's own cursor (a dot or an arrow) runs a little ahead of the guest's; +it catches up within a fraction of a second. + +**The keyboard.** Viewers that send key positions (TigerVNC, noVNC, QEMU's +extension) work with any host layout. Others send characters, which iris +types on a US layout. + +**Security.** There is no password. Leave the server on `127.0.0.1` and use +an SSH tunnel (`ssh -L 5900:localhost:5900 host`) to reach it from +elsewhere. + --- ## Configuration file (`iris.toml`) @@ -716,6 +765,7 @@ Automation and debugging --serial-log Append everything IRIX prints on ttyd1 to FILE --serial-log-timestamps off (raw) or wall: timestamp each completed line --monitor-port Monitor TCP port (default 8888) + --vnc [ADDR] VNC server on ADDR (host:port or port; default 127.0.0.1:5900) --serial-port-a Serial channel A TCP port (default 8880) --serial-port-b Serial channel B TCP port (default 8881) --gdb-port Start the GDB stub diff --git a/clanker/CLANKER.md b/clanker/CLANKER.md index f98a0a9d0..cf14e4eb5 100644 --- a/clanker/CLANKER.md +++ b/clanker/CLANKER.md @@ -60,6 +60,7 @@ found or what to watch out for while changing code, it goes in `clanker/`. | `clanker/INDEX.md` | categorized index of every note | | `clanker/cpu/`, `jitv2/` | CPU architecture findings; the MIPS JIT | | `clanker/devices/`, `scsi/`, `net/`, `video/` | device models, storage, the NAT stack and its services, VINO/IndyCam | +| `clanker/vnc/` | the VNC server, the host input layer (`screen`/`ptr`/`key`) and pointer steering | | `clanker/rex3/`, `gr2/`, `mgras/`, `odyssey/`, `hostgl/` | graphics boards and host GL (`odyssey/` is references only) | | `clanker/gui/`, `macos/` | front-ends and platform packaging | | `clanker/snapshot/`, `perf/`, `testing/`, `build/` | snapshots and `iris-ci`, host performance, test harnesses, builds | diff --git a/clanker/INDEX.md b/clanker/INDEX.md index 0f0ad3e6b..dab49f69e 100644 --- a/clanker/INDEX.md +++ b/clanker/INDEX.md @@ -98,6 +98,7 @@ Non-graphics devices: MC/MEMCFG and RAM banks, HPC3, VDMA, EEPROMs and NVRAM, PS | Note | Kind | Title | |---|---|---| +| [empty-gio-slot-reads-bus-error.md](devices/empty-gio-slot-reads-bus-error.md) | rule | Reads of an empty GIO slot end in a bus error (ABORT_EN); all ones made the PROM poll a phantom Newport 200 000 times headless | | [cpal-alsa-busy-poll.md](devices/cpal-alsa-busy-poll.md) | rule | cpal's ALSA backend can spin a core: `[audio] host = "pulseaudio"` | | [decode-table-is-host-mapped.md](devices/decode-table-is-host-mapped.md) | rule | The physical decode table is a sparse host mapping over 40 bits: change it only through `fill`, set the default first, compare entries with `entry_eq` | | [mc-decodes-32-physical-bits.md](devices/mc-decodes-32-physical-bits.md) | rule | Indy/Indigo2/O2 decode 32 physical bits; IRIX PTE software bits reach PA[35:32]; fold with `MipsExecutor::phys_mask`, never decode them | @@ -170,6 +171,7 @@ Newport (REX3): drawing semantics, HOSTRW, GFIFO, the REX3 JIT, trace and record | Note | Kind | Title | |---|---|---| +| [cursor-drawn-3px-right-of-x-pointer.md](rex3/cursor-drawn-3px-right-of-x-pointer.md) | open | Newport draws the hardware cursor 3 px right of the X pointer (measured by corner pinning) | | [alphahost-fields-and-aline-alpha.md](rex3/alphahost-fields-and-aline-alpha.md) | rule | REX3 source alpha: host fields and antialiased lines | | [batched-hostrw-in-the-jit.md](rex3/batched-hostrw-in-the-jit.md) | rule | Batched HOSTRW in the REX3 JIT — and how the equivalence test lied twice | | [blendalpha-and-alpha-blending.md](rex3/blendalpha-and-alpha-blending.md) | rule | REX3 BLENDALPHA: what it selects, and why the `blast` billboard exposes it | @@ -273,6 +275,15 @@ Host GL pass-through for guest GL programs (`iris-hostgl`). | [windows-silent-exit-0xc000041d.md](gui/windows-silent-exit-0xc000041d.md) | rule | Windows: CLI vanishes with exit 0xC000041D and no message | | [winit-window-calls-must-be-on-event-thread.md](gui/winit-window-calls-must-be-on-event-thread.md) | rule | Window methods must only be called on the event-loop thread (Windows crash) | +## vnc/ + +The VNC server, the host input layer (`screen`/`ptr`/`key`) and pointer steering. + +| Note | Kind | Title | +|---|---|---| +| [pointer-steering-and-hotspots.md](vnc/pointer-steering-and-hotspots.md) | design | Steering the relative mouse by the hardware cursor: X's acceleration curve, hotspots from corner pinning, busy guests | +| [frame-capture-and-damage.md](vnc/frame-capture-and-damage.md) | design | What VNC reads (rgba + frame_seq), Newport capture and forced SW compositor, tile damage | + ## macos/ Mac App Store and sandbox constraints. @@ -387,3 +398,4 @@ Plans, task briefs and reviews written for agents. Each states its own status; m | [networking-tab-redesign.md](plans/networking-tab-redesign.md) | plan | Networking tab redesign — design & plan | | [nfsudp-plan.md](plans/nfsudp-plan.md) | plan | Plan: in-core NFSv3-over-UDP server (`src/net/nfsudp.rs`) | | [pcap-release-plan.md](plans/pcap-release-plan.md) | plan | PCAP release + elevation plan | +| [vnc-server-plan.md](plans/vnc-server-plan.md) | plan | Built-in VNC server: board-agnostic frame tap, damage from frame diffs, pointer steered by the HW cursor, monitor/MCP screenshot + click for agents | diff --git a/clanker/devices/empty-gio-slot-reads-bus-error.md b/clanker/devices/empty-gio-slot-reads-bus-error.md new file mode 100644 index 000000000..180efb615 --- /dev/null +++ b/clanker/devices/empty-gio-slot-reads-bus-error.md @@ -0,0 +1,23 @@ +# Reads of an empty GIO slot end in a bus error + +Status: fixed 2026-10-29 (`GioBusErrorDevice`, `src/ip22_bus.rs`). + +**Symptom.** Indy or Indigo2 with no graphics board (`--headless` with +Newport): the PROM printed `Cannot open video() for output` only after +200 000 `MC: GIO Timeout at 1f0f1338` lines, and IRIX later failed (the +install guide recorded a UTLB miss). The GIO slot probes of a normal boot +also "succeeded" on empty slots. + +**Cause.** Unanswered GIO addresses returned all ones with `BUS_OK`. The +PROM and the kernel find boards with `badaddr()`, which expects a read of an +absent board to fault. With no fault, the PROM took the slot for a Newport, +wrote CONFIG and polled REX3 STATUS (offset 0x1338) for GFXBUSY to clear: +all ones never clears, so it ran its 100 000-iteration timeout, twice. + +**Rule.** With `CPUCTRL1.ABORT_EN` (bit 4, "enable GIO bus time outs"; the +PROM sets it in its startup code) an unanswered GIO read returns `BUS_ERR` +(a data bus error exception) after recording GIO_ERROR_ADDR/STAT. Without +it, all ones as before. Writes are still dropped. Headless Indy and Indigo2 +(IP28 checked) now boot IRIX to the serial login with `hinv` showing no +graphics; the PROM still says `Cannot open video()` because the NVRAM has +`console=g`, and waits in its menu instead of autobooting. diff --git a/clanker/iris_mcp.py b/clanker/iris_mcp.py index 1e11a460a..83c09f3e8 100644 --- a/clanker/iris_mcp.py +++ b/clanker/iris_mcp.py @@ -1,6 +1,8 @@ import asyncio +import os import sys -from mcp.server.fastmcp import FastMCP +import tempfile +from mcp.server.fastmcp import FastMCP, Image # Initialize FastMCP server mcp = FastMCP("iris-monitor") @@ -272,5 +274,89 @@ async def traceback(count: int = 10) -> str: """Show execution traceback (history of executed instructions).""" return await client.send_command(f"dt {count}") +# ---- The guest's screen, pointer and keyboard (any board) ---------------- +# These wrap the monitor's `screen`, `ptr` and `key` commands. Coordinates are +# screen pixels, as in a screenshot. The pointer is steered onto the target +# by reading back the hardware cursor, so X's acceleration does not matter. + +_shot_dir = tempfile.mkdtemp(prefix="iris-mcp-") + +@mcp.tool() +async def screenshot(x: int = 0, y: int = 0, width: int = 0, height: int = 0, max_width: int = 0) -> Image: + """A PNG of the guest's screen as the board composes it (no host status + bar). Optionally cropped to x, y, width, height, and scaled down to + max_width (needs Pillow). Coordinates in the image are screen pixels + unless scaled.""" + path = os.path.join(_shot_dir, "shot.png") + crop = f" {x} {y} {width} {height}" if width and height else "" + out = await client.send_command(f"screen shot {path}{crop}") + if not out.startswith("saved"): + raise RuntimeError(out) + if max_width: + try: + from PIL import Image as PILImage + im = PILImage.open(path) + if im.width > max_width: + im.resize((max_width, im.height * max_width // im.width)).save(path) + except ImportError: + pass + with open(path, "rb") as f: + return Image(data=f.read(), format="png") + +@mcp.tool() +async def screen_info() -> str: + """Screen size, hardware cursor and pointer position, whether the guest + has enabled the mouse, and buttons held.""" + return await client.send_command("screen info") + +@mcp.tool() +async def screen_pixel(x: int, y: int) -> str: + """The colour (rrggbb) of one screen pixel.""" + return await client.send_command(f"screen pix {x} {y}") + +@mcp.tool() +async def wait_screen(quiet_ms: int = 500, timeout_ms: int = 10000) -> str: + """Wait until the screen stops changing for quiet_ms (an app finished + drawing, a window opened), at most timeout_ms.""" + return await client.send_command(f"screen wait {quiet_ms} {timeout_ms}") + +@mcp.tool() +async def click(x: int, y: int, button: str = "left", count: int = 1) -> str: + """Move the pointer to screen pixel (x, y) and click. button: left, + middle or right (IRIX: right opens the toolchest/root menus, middle + pastes/moves). count 2 for a double click.""" + return await client.send_command(f"ptr click {x} {y} {button} {count}") + +@mcp.tool() +async def move_pointer(x: int, y: int) -> str: + """Move the pointer to screen pixel (x, y) without clicking.""" + return await client.send_command(f"ptr move {x} {y}") + +@mcp.tool() +async def drag(x1: int, y1: int, x2: int, y2: int, button: str = "left") -> str: + """Press button at (x1, y1), move to (x2, y2), release.""" + return await client.send_command(f"ptr drag {x1} {y1} {x2} {y2} {button}") + +@mcp.tool() +async def type_text(text: str) -> str: + """Type text on the guest keyboard (US layout, ASCII; newline is Enter). + Runs of spaces are typed as one.""" + out = [] + lines = text.split("\n") + for i, line in enumerate(lines): + if line: + out.append(await client.send_command(f"key type {line}")) + if i < len(lines) - 1: + out.append(await client.send_command("key Return")) + return "\n".join(out) + +@mcp.tool() +async def press_key(keys: str, action: str = "") -> str: + """Press a key or chord: 'Return', 'Escape', 'F1', 'ctrl+c', + 'ctrl+alt+Delete', 'Up'. X keysym names and short aliases (ctrl, alt, + shift, enter, esc, tab, bs, del, pgup...). action: '' to press and + release, 'down' or 'up' to hold or let go.""" + return await client.send_command(f"key {keys} {action}".strip()) + if __name__ == "__main__": mcp.run() diff --git a/clanker/plans/vnc-server-plan.md b/clanker/plans/vnc-server-plan.md new file mode 100644 index 000000000..e12690c56 --- /dev/null +++ b/clanker/plans/vnc-server-plan.md @@ -0,0 +1,194 @@ +# Built-in VNC server — plan + +Status: implemented 2026-10-29 on branch `vnc` (phases 1-5): `src/vnc/`, +`src/hostinput.rs`, `src/keysym.rs`, monitor `screen`/`ptr`/`key`, MCP +tools, docs. Tested on Indy Newport (window and `--ci`), XZ, IP28 IMPACT, +Octane, O2, with TigerVNC and `tools/vnc-check.py`. Not done: VNC password, +second head, CI-socket input commands. Design notes in `clanker/vnc/`. + +## Goal + +An optional RFB (VNC) server in the `iris` binary (not iris-gui) that works: + +- windowed (alongside the winit window; input from both is merged), +- with `--ci` (no window; offscreen rendering already runs), +- for every board: Newport (REX3), GR2, IMPACT (Indigo2 + Octane), O2 GBE. + +It also gives LLM agents and CI a "see the screen, click at x,y" path through +the monitor/MCP/CI socket, reusing the same frame tap and pointer logic. + +Hard constraints: + +- **No instrumentation in drawing code.** No damage tracking in REX3/RE3/RSS + draw paths. Damage comes from diffing composed frames. +- **One path for all boards.** VNC consumes the composed frame + (`Rex3Screen::rgba`, stride 2048, `0xAABBGGRR`) and nothing else. +- REX3 is forced onto the software compositor while VNC is on. +- Off by default; costs nothing when off. + +## Where frames come from today (survey summary) + +- `GfxDisplay` (`src/gfx_display.rs`) is common to all boards; `screen()` + gives `Mutex` with `rgba`, `width`, `height`. +- GR2, IMPACT, O2 compose on their own refresh threads and keep `rgba` + current (`prebuilt = true`). +- REX3 composes inside the installed `Renderer` (`SwCompositor` or + `GlCompositor`) and fills `rgba` only when `need_readback` (screenshot). + With `--ci` there is no renderer, so `ci.rs` composes inline. +- Input: everything goes through `Ps2Controller::push_kb(KeyCode, bool)` and + `push_mouse_input(buttons, dx, dy, dz)`. **Mouse is relative only**, and X + applies pointer acceleration. +- Each board knows its hardware cursor position: REX3 VC2 cursor X/Y regs + (+`cursor_x_adjust`), GR2 `Vc1::cursor_pos`, IMPACT `Dcb::cursor`, O2 GBE + cursor regs. + +## Design + +### 1. Frame tap (core, board-agnostic) + +Extend `GfxDisplay`: + +```rust +fn frame_seq(&self) -> u64; // bumps whenever rgba is rewritten +fn request_frame(&self); // "please make rgba current", no PNG +fn cursor_pos(&self) -> Option<(i32, i32)>; // live HW cursor, screen coords; None if disabled +``` + +- `Rex3Screen` gets `frame_seq: u64`, bumped by whoever writes `rgba`: the + three prebuilt refresh loops (GR2, IMPACT, O2) and the REX3 renderer + readback. These are refresh-loop / present-time sites, not draw code. +- REX3: a separate `frame_wanted: AtomicBool` (the existing + `screenshot_pending` also writes a PNG file, so it can't be reused). The + refresh loop ORs it into `need_readback` and forces a present on the next + tick when content changed. +- REX3 forced SW: while a frame consumer is registered, `GlRenderer` uses + `SwCompositor`; `disp compositor gl` is refused with a message. Readback is + then `read_pixels()`, a slice copy, no GL readback. +- No-window REX3: move iris-gui's `CaptureRenderer` idea into core as + `SwCaptureRenderer` (SwCompositor → `rgba`) and install it when VNC (or CI) + needs frames and no renderer is present. `ci.rs`'s inline compose then goes + away. +- `cursor_pos` reads live device state (VC2/VC1/DCB/GBE), not the 60 Hz + snapshot, so the pointer loop sees updates promptly. + +### 2. Damage from diffs + +One grabber thread per served head: + +- Tick at `max_fps` (default 30). If `frame_seq` unchanged, skip. + Otherwise `request_frame()`, lock `screen`, copy the visible + `width × height` into a tight `cur` buffer, unlock (~1–2 ms at 1280×1024). +- Compare `cur` with `prev` in 64×64 tiles (row slices compared as `[u64]`, + early exit on first mismatch). Changed tiles get `tile_gen[t] = frame_gen`. +- Optionally shrink each dirty tile to its exact bounding box before sending. +- Resolution change → new buffers, every tile dirty, DesktopSize to clients. +- Swap `cur`/`prev`, notify clients (Condvar). + +Clients keep `seen_gen`; an incremental FramebufferUpdateRequest blocks until +some `tile_gen > seen_gen`, then sends dirty tiles inside the requested rect, +merging horizontal runs into wider rects. Non-incremental requests send the +whole rect. Slow clients just skip intermediate frames (no queue). + +### 3. RFB server (`src/vnc/`) + +Own implementation, std::net, thread per client like the monitor/CI socket +(no RFB crate worth depending on; no tokio in the tree). + +- RFB 3.8 (accept 3.3/3.7), security type None. Bind `127.0.0.1:5900` by + default; warn loudly on a non-loopback bind. VNC password auth later. +- Pixel formats: server native 32bpp; honour SetPixelFormat for 32/16/8bpp + true-colour by shift/max conversion. No colour-map mode. +- Encodings: Raw (required), ZRLE (per-connection zlib stream, `flate2` as + a direct dep; supported by TigerVNC, RealVNC and noVNC), pseudo-encodings + DesktopSize and QEMU Extended Key Event. Tight/JPEG and CopyRect later, if + ever. +- Cursor: drawn into the frame by the board compositors, so no cursor + pseudo-encoding. The viewer's local dot cursor works. +- Max clients 8; on disconnect, release that client's held keys/buttons. + +### 4. Input + +**Keyboard.** Prefer the QEMU Extended Key Event (client sends XT scancodes +→ map to `KeyCode`, layout-independent; TigerVNC and noVNC send it). Fallback: +X keysym → `KeyCode` table for a US layout (`src/keysym.rs`), mapping a +shifted symbol to its physical key. Clients already send Shift +themselves. The same table backs monitor `key NAME` and can replace +`ip32_screen::keycode`. + +**Pointer: closed loop on the hardware cursor.** VNC gives absolute +positions; the guest wants relative PS/2 packets that X then accelerates. +Rather than model acceleration, steer: + +- A `PointerSeek` (`src/hostinput.rs`) holds the target. Every ~8 ms: + `err = target - (cursor_pos() + hotspot)`. Send a delta clamped to a + step size that shrinks as `|err|` falls (big accelerated steps far away, + ≤ accel-threshold steps near). Stop at `err == 0` or after N stalled ticks. +- Button/wheel events queue behind the pending move: send them once the move + converges, or after a timeout (~150 ms), so a click lands where it was + aimed. +- **Fallback, open loop**: if `cursor_pos()` is `None` (cursor disabled) or + the cursor stops moving while we push (an app grabbed and warps the + pointer), switch to relative mode: send the VNC position delta as is. +- **Hotspot**: the hardware position is the glyph's top-left, X sets it to + `pointer - hotspot`, and the hotspot isn't in hardware. Start with the + first opaque pixel of the cursor glyph (the arrow tip), then calibrate per + board with `xclick`/`xgetpix` (open item below). +- Both the window and VNC feed the same `Ps2Controller`; the seek only runs + while VNC pointer events arrive, so the two don't fight unless both are + used at once. + +### 5. LLM / CI path (same machinery, no VNC client needed) + +Monitor commands, registered by the host-input module (document in +`debug.md`): + +- `screen shot PATH [x y w h]`, `screen wait [ms]` (block until the frame is + stable or has changed), `screen seq` +- `ptr move X Y`, `ptr click X Y [left|middle|right] [double]`, + `ptr drag X1 Y1 X2 Y2`, `ptr pos` +- `key NAME [down|up]`, `key ctrl+alt+F1`, `key type TEXT` + +MCP (`clanker/iris_mcp.py`): `screenshot` returning image content (optional +crop/scale), `click`, `move_pointer`, `type_text`, `press_key`, +`wait_screen`. CI socket and `iris-ci`: `click`, `key`, `type`, `screenshot +--head`. Skills (`irix-gfx-loop`, `irix-guest-loop`): replace the +corner-pushing and `xclick` guidance with these. + +### 6. Configuration + +```toml +[vnc] +listen = "127.0.0.1:5900" # absent = off +max_fps = 30 +head = 0 # 1 = second Newport head (later: both, 5900/5901) +``` + +CLI: `--vnc [ADDR]`. Runtime option, no Cargo feature (the only new +dependency is `flate2`, already in the lockfile). `--headless` with Newport +has no display: refuse `--vnc` with a hint to use `--ci`. + +## Phases + +1. **Frame tap**: `GfxDisplay` additions, `frame_seq`, REX3 `frame_wanted` + + forced SW, core `SwCaptureRenderer`, `cursor_pos` for all four boards. + ci.rs screenshot moves onto it. Unit tests: seq bumps, rgba current + under `--ci` for REX3. +2. **RFB output**: handshake, Raw, ZRLE, tile diff, DesktopSize. Test with + an in-process minimal RFB client (handshake, full + incremental update, + pixel check) and manually with TigerVNC and noVNC. +3. **Input**: Extended Key Event + keysym table, `PointerSeek`, button + ordering, release on disconnect. Measure click accuracy on each board. +4. **LLM path**: monitor `screen`/`ptr`/`key`, MCP tools, CI socket + commands, skill updates. +5. **Docs**: HELP.md, FEATURES.md, debug.md, HACKING.md (`GfxDisplay` + change, frame tap), CHANGELOG.md. + +## Open items + +- Hotspot offset per board; does IRIX X use the HW cursor on every board + (GR2/IMPACT/O2)? If any uses a software cursor, `cursor_pos` is `None` + there and the seek falls back to open loop. Verify. +- IRIX default `xset m` acceleration/threshold: sets the "near" step size. +- Grab cost at 1600×1200 × 30 fps under the screen lock; if it shows in + profiles, double-buffer `rgba` at the producer instead. +- Two heads: one listener per head, or one with a head switch. diff --git a/clanker/rex3/cursor-drawn-3px-right-of-x-pointer.md b/clanker/rex3/cursor-drawn-3px-right-of-x-pointer.md new file mode 100644 index 000000000..3bff12992 --- /dev/null +++ b/clanker/rex3/cursor-drawn-3px-right-of-x-pointer.md @@ -0,0 +1,22 @@ +# Newport draws the hardware cursor 3 pixels right of the X pointer + +Status: open, measured 2026-10-29 on IRIX 6.5 at 1280x1024 (REX3 decodes +the mode as 1282x1024, `cursor_x_adjust` = 5). + +**How it was seen.** `ptr calibrate` pins the X pointer at 0,0 and reads +the VC2 cursor position as the compositor draws it. xdm's X-shaped cursor +(cursor font hotspot 7,7) pinned with its glyph at (-4,-7); the 4Dwm arrow +(hotspot 1,1) at (2,-1). Both say the glyph is drawn 3 px right of where X +put it and exactly right vertically. On GR2 the arrow pins with its tip on +0,0: no offset there. + +**Effect.** Cosmetic in the window and VNC (the arrow tip is 3 px right of +the real pointer). `ptr`/VNC clicks are unaffected: they learn hotspots in +drawn coordinates, so negative hotspots are allowed. + +**The 1282 width is not ours.** The PROM and IRIX program the VC2 timing +wider than the 1280-pixel X screen, and the X server scissors one column on +the left to black: probably a workaround for a display hardware quirk. So +the decoded width is right and should stay. The cursor offset may be tied to +the same quirk (`cursor_x_adjust` and its -2 bias in `decode_vc2_timings`); +check against a real Indy before changing it. diff --git a/clanker/skills/irix-gfx-loop/SKILL.md b/clanker/skills/irix-gfx-loop/SKILL.md index 1c1a71ba8..95d6cadbf 100644 --- a/clanker/skills/irix-gfx-loop/SKILL.md +++ b/clanker/skills/irix-gfx-loop/SKILL.md @@ -91,9 +91,11 @@ tools/irix-sh.py run 'cd /tmp/gltest && ./glprim --hold 4000' ``` Run the captures while `--hold` keeps the window up (background the guest -command with `&` if you need to sample during the hold). Newport has no -`rex pix`: use `rex fbdump DIR` and read the PNG, `xgetpix X Y [W H]` in the -guest, or `iris-ci screenshot`. +command with `&` if you need to sample during the hold). On every board, +`screen pix ` reads one composed display pixel and `screen shot + [x y w h]` saves the composed screen (or a crop); Newport has no +`rex pix`, so use these there. The MCP `screenshot` tool returns the image +directly. glprim prints its **real window origin** (`window origin OX,OY size WxH`); the window manager moves windows. GL window pixel (gx, gy) is display pixel @@ -164,11 +166,24 @@ Time replays on the host instead. ## 8. Driving input -- **Monitor, any PS/2 machine**: `ps2 mouse [buttons]` (1 left, - 2 right, 4 middle; send the press, then a packet with 0 to release), - `ps2 type `, `ps2 enter`, `ps2 esc`. Moves are relative and go - through X pointer acceleration: push into the top-left corner with a few - large negative moves, then step to the target in small ones. +- **Monitor, any machine and board**: `ptr click [left|middle|right] + [count]`, `ptr move`, `ptr drag`, `ptr down|up`, `key ` (`Return`, + `ctrl+c`, `F1`), `key type `, `screen wait [quiet_ms]` (until the + screen settles), `screen info` (where the pointer is, whether the guest + has enabled the mouse). Coordinates are screen pixels. `ptr` steers the + relative PS/2 mouse by reading back the hardware cursor, so X's + acceleration does not matter; the first move pins the pointer in the + top-left corner once to learn the cursor's hotspot. MCP: `click`, `drag`, + `type_text`, `press_key`, `wait_screen`. +- Windows on the IRIX desktop take keyboard focus on a click, not when the + pointer enters them: click into a winterm before `key type`. +- `ptr` errors with "the hardware cursor is off" before X is up, and "the + guest is not reading the mouse" while it is busy (desktop start-up): + `screen wait`, then retry. +- Raw PS/2 is still there: `ps2 mouse [buttons]`, `ps2 type`. +- **VNC**: start iris with `--vnc` to watch or drive the guest from a VNC + viewer; `tools/vnc-check.py shot|watch|click|key` exercises the server + from scripts. - **In the guest, XTest**: `xclick NAME [BUTTON [X Y]]` or `xclick NAME key KEYSYM` clicks at exact window coordinates in the first window whose name starts with NAME; `xresize NAME W1 H1 W2 H2 [COUNT] [DELAY_MS]` resizes diff --git a/clanker/skills/irix-guest-loop/SKILL.md b/clanker/skills/irix-guest-loop/SKILL.md index 71e5473a4..0fefce014 100644 --- a/clanker/skills/irix-guest-loop/SKILL.md +++ b/clanker/skills/irix-guest-loop/SKILL.md @@ -105,7 +105,11 @@ tools/irix-sh.py run 'echo root > /etc/autologin' 'touch /etc/autologin.on' boot stops at the login window. - The session appears about a minute after telnet comes up. Wait until `ps -ef | grep 4Dwm` shows the window manager (and `toolchest` for the - full desktop), then give it another ~15 s before taking screenshots. + full desktop), then `screen wait 2000 60000` on the monitor (returns once + the screen has been still for 2 s) before taking screenshots. +- Without autologin you can log in on the screen itself: `ptr click X Y` on + the user's icon in the xdm window, then `key Return` (see `screen`, `ptr` + and `key` in debug.md; MCP `screenshot`, `click`, `press_key`). - The screen saver blanks the screen after 10 minutes idle and every pixel probe then reads black. The desktop re-applies its saver settings after login, so run `/usr/bin/X11/xset s off` once the session is up and check diff --git a/clanker/vnc/frame-capture-and-damage.md b/clanker/vnc/frame-capture-and-damage.md new file mode 100644 index 000000000..be01a65a3 --- /dev/null +++ b/clanker/vnc/frame-capture-and-damage.md @@ -0,0 +1,23 @@ +# VNC frame capture and damage + +Status: design notes for `src/vnc/`, 2026-10-29. + +- The server reads only `GfxDisplay::screen().rgba` + `frame_seq`; no board + draw path knows about it. Damage = 64x64 tiles whose pixels differ from + the last published frame (row slices compared; per-tile generation + numbers; viewers remember the generation they saw). +- GR2, IMPACT, O2 rewrite `rgba` every composed frame. Newport only when + asked: `set_capture(true)` makes every full refresh read back, and forces + the window's renderer to `SwCompositor` (`switch_compositor(false)` every + frame is cheap and idempotent; `disp compositor gl` is refused). With no + renderer (`--ci`) the REX3 refresh loop composes in software itself. +- `request_frame` is the one-shot version (CI screenshot, monitor `screen`): + it does not write `screenshot_NNNN.png`, which `request_screenshot` does + on Newport. +- O2 bumps `frame_seq` 60 times a second even when nothing changed; the + tile compare then finds nothing and publishes nothing. +- An idle Newport desktop in `--ci` produces a `frame_seq` bump only on real + change (fb/palette dirty), so a watched idle desktop costs a seq check per + tick. +- `tools/vnc-check.py` is a minimal Python RFB client (Raw + ZRLE decode) + for scripts; its ZRLE decode is slow (pure Python), the server is not. diff --git a/clanker/vnc/pointer-steering-and-hotspots.md b/clanker/vnc/pointer-steering-and-hotspots.md new file mode 100644 index 000000000..389ffd360 --- /dev/null +++ b/clanker/vnc/pointer-steering-and-hotspots.md @@ -0,0 +1,27 @@ +# Pointer steering: what the hardware cursor can and cannot tell + +Status: design notes for `src/hostinput.rs`, 2026-10-29. + +- **X acceleration on IRIX 6.5 (default `xset m`)**: a packet of d mickeys + moves the pointer d when d <= 4, else `2d - 4` (measured: 5->6, 8->12, + 20->36, 40->76). Steps of up to 3 are exact; the steering loop learns a + gain from larger ones. +- **The hotspot is not in the hardware.** The glyph's tip is a bad guess: the + xdm X cursor's hotspot is its centre, and Newport adds a 3 px offset + (`clanker/rex3/cursor-drawn-3px-right-of-x-pointer.md`). The reliable + source is pinning in the top-left corner: X clamps the pointer to 0,0, + so hotspot = -glyph position. The first move does this once + (`calibrate_locked`). +- **Only the top/left edges pin reliably.** The board's visible size can + differ from X's screen (Newport is programmed 1282 wide for X's 1280, as on real hardware), so the + bottom/right pin position is off by that difference. +- **New shapes** are learned when they appear after an exact (<= 3) step from + a known shape; otherwise guessed as tip + the last learned (hotspot - tip). +- **Busy guests**: during desktop start-up X ignores the mouse for seconds + and GR2's X server turns the cursor off for a moment while drawing. The + loop waits for queued mouse bytes to drain before sending more, tolerates + the cursor being off for 400 ms, waits for the cursor to settle 20 ms after + each step (the O2 shows a move a frame late; steering from the old position + oscillated by 2-3 px), and gives up after 3 s without motion. +- **GR2 glyph**: the arrow sits in columns 16..31 of the 32x32 mask; its + hotspot is (16,0). diff --git a/debug.md b/debug.md index 58f1279ec..59d174f09 100644 --- a/debug.md +++ b/debug.md @@ -404,6 +404,34 @@ in the `irix-gfx-loop` skill (`clanker/skills/irix-gfx-loop/SKILL.md`). `iris-ci screenshot` captures the composed screen from any board, also headless. +**Screen, pointer and keyboard (any board)** + +These work the same on Newport, GR2, IMPACT and the O2, with a window, with +`--ci`, or alongside VNC viewers. Coordinates are screen pixels as a +screenshot shows them (0,0 top-left). The guest's mouse is relative and X +accelerates it, so `ptr` steers: it sends a move, reads back where the +board's hardware cursor went, and corrects until the pointer is on the +target. The first `ptr` move pins the pointer in the top-left corner once, +to learn the cursor's hotspot exactly. + +| Command | Description | +|---|---| +| `screen info` | Size, hardware cursor, pointer position, whether the guest has enabled the mouse, buttons held | +| `screen shot [x y w h]` | The composed screen as PNG (no host status bar), optionally cropped | +| `screen pix ` | One pixel, `rrggbb` | +| `screen wait [quiet_ms] [timeout_ms]` | Wait until the screen has not changed for `quiet_ms` (default 500, at most 10000 ms) | +| `screen cursor` | The cursor glyph as text, with its hotspot | +| `ptr pos` | Where the pointer is | +| `ptr move ` | Put the pointer on a pixel | +| `ptr click [left\|middle\|right] [count]` | Move, then click (`count` 2 for a double click) | +| `ptr down\|up [button]` | Press or release a button where the pointer is | +| `ptr drag [button]` | Press at one point, release at the other | +| `ptr rel ` | Raw relative motion, as a mouse sends it | +| `ptr calibrate` | Pin the pointer in the top-left corner and learn the current cursor's hotspot | +| `key [+...] [down\|up]` | Press a key or chord (`Return`, `F1`, `ctrl+c`, `ctrl+alt+Delete`): X keysym names, short aliases (`ctrl`, `alt`, `shift`, `enter`, `esc`, `tab`, `bs`, `del`, `pgup`), single characters, or winit names (`KeyA`) | +| `key type ` | Type ASCII text on a US layout | +| `key names` | Every key name `key` accepts | + ### Hardware devices | Command | Description | @@ -659,12 +687,20 @@ Register it with your client, for example in `.mcp.json`: | `translate_address(address)`, `dump_tlb()` | `translate`, `tlb dump` | | `lookup_symbol(name_or_addr)` | `sym` | | `read_eframe(address, layout)`, `kernel_log(size)` | `eframe`, `putbuf` | +| `screenshot(x, y, width, height, max_width)` (returns the image) | `screen shot` | +| `screen_info()`, `screen_pixel(x, y)`, `wait_screen(quiet_ms, timeout_ms)` | `screen info`, `screen pix`, `screen wait` | +| `click(x, y, button, count)`, `move_pointer(x, y)`, `drag(x1, y1, x2, y2, button)` | `ptr click`, `ptr move`, `ptr drag` | +| `type_text(text)`, `press_key(keys, action)` | `key type`, `key` | Commands without a dedicated tool (graphics, devices, `j2`) go through `run_command`. ## Other introspection +- **VNC**: `--vnc` (or `[vnc] listen`) serves the screen, keyboard and mouse + to any VNC viewer, from any board, with a window or under `--ci`. See + [HELP.md](HELP.md#vnc). + - **`iris-ci`**: the CI control socket drives a running machine from scripts: boot and log in, run guest commands, copy files in and out, screenshots, snapshots, CD changes ([STORAGE.md](STORAGE.md#ci-control-socket-and-iris-ci)). diff --git a/src/ci.rs b/src/ci.rs index 6509c073f..9b04aed54 100644 --- a/src/ci.rs +++ b/src/ci.rs @@ -612,48 +612,15 @@ fn cmd_screenshot(server: &CiServer, args: &Value) -> Response { return Response::err("screenshot: missing 'path' arg"); }; - display.request_screenshot(); - // The board consumes the request when its refresh thread next renders. - // Wait for that (bounded) instead of guessing a frame time; the screen - // lock below then waits for that render to finish copying into `rgba`. - // Boards that do not consume the request report no pending work and fall - // through immediately. - let deadline = std::time::Instant::now() + std::time::Duration::from_millis(250); - while display.screenshot_pending() && std::time::Instant::now() < deadline { - std::thread::sleep(std::time::Duration::from_millis(2)); - } - // Snapshot the framebuffer under the screen lock; unlock before the PNG - // encode so the refresh thread isn't blocked during disk I/O. - let has_renderer = display.renderer_slot().lock().is_some(); - let (width, height, rgba_copy) = { - let screen = display.screen().lock(); - let w = screen.width; - let h = screen.height; - if w == 0 || h == 0 { - return Response::err("screenshot: the display has no video mode yet"); - } - let mut out = Vec::with_capacity(w * h); - if has_renderer { - // The front-end's renderer already composited this frame into - // `rgba`; copy the visible window (row stride 2048). - for y in 0..h { - let base = y * 2048; - out.extend_from_slice(&screen.rgba[base..base + w]); - } - } else { - // `--ci` installs no renderer, so nothing has filled `rgba`. - // Composite the guest screen here from the caches the refresh - // thread just updated. The host status bar and debug overlay are - // deliberately absent: this is the guest's screen. - let mut compositor = crate::compositor::SwCompositor::new(); - compositor.compose_pixels(&screen.compositor_source()); - let pixels = compositor.pixels(); - for y in 0..h { - let base = y * 2048; - out.extend_from_slice(&pixels[base..base + w]); - } - } - (w, h, out) + // The board composes the next frame into `rgba` (Newport with no + // front-end composes in software on its refresh thread). The host + // status bar and debug overlay are never in it: this is the guest's + // screen. + let mut rgba_copy = Vec::new(); + let Some((width, height, _)) = crate::gfx_display::fresh_frame( + &**display, &mut rgba_copy, std::time::Duration::from_millis(250)) + else { + return Response::err("screenshot: the display has no video mode yet"); }; let mut rgb = Vec::with_capacity(width * height * 3); diff --git a/src/config.rs b/src/config.rs index e26d04b43..7457ac4f5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1290,6 +1290,19 @@ impl Default for AudioConfig { } } +/// The built-in VNC server (`[vnc]` section). Off unless `listen` is set. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(deny_unknown_fields)] +pub struct VncConfig { + /// Address to serve on, `host:port` (a bare port means 127.0.0.1). + /// Without a password, keep it on loopback. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub listen: Option, + /// Most frames a second sent to a viewer (default 30). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_fps: Option, +} + /// VINO video-in configuration. #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct VinoConfig { @@ -1516,6 +1529,10 @@ pub struct MachineConfig { #[serde(default)] pub vino: VinoConfig, + /// Built-in VNC server (`[vnc]` section). + #[serde(default)] + pub vnc: VncConfig, + /// Networking backend selection (`[network]` section). Defaults to NAT. #[serde(default)] pub network: NetworkSection, @@ -1713,6 +1730,7 @@ impl Default for MachineConfig { serial_log: None, serial_log_timestamps: Default::default(), vino: VinoConfig::default(), + vnc: VncConfig::default(), network: NetworkSection::default(), mouse_scroll_pixels_per_line: default_scroll_pixels_per_line(), lock_aspect_ratio: default_lock_aspect_ratio(), @@ -2262,6 +2280,12 @@ pub struct Cli { #[arg(long = "monitor-port", value_name = "PORT")] pub monitor_port: Option, + /// Serve the screen, keyboard and mouse over VNC on ADDR (`host:port`, + /// or a port on 127.0.0.1; default 127.0.0.1:5900). Works with a + /// window, with --ci, and on every graphics board. + #[arg(long = "vnc", value_name = "ADDR", num_args = 0..=1, default_missing_value = "127.0.0.1:5900")] + pub vnc: Option, + /// Serial channel A (tty2) port on 127.0.0.1 (default 8880). #[arg(long = "serial-port-a", value_name = "PORT")] pub serial_port_a: Option, @@ -2416,6 +2440,7 @@ impl Cli { if let Some(p) = self.gdb_port { cfg.gdb_port = Some(p); } if let Some(p) = self.monitor_port { cfg.monitor_port = Some(p); } + if let Some(a) = &self.vnc { cfg.vnc.listen = Some(a.clone()); } if let Some(p) = self.serial_port_a { cfg.serial_port_a = Some(p); } if let Some(p) = self.serial_port_b { cfg.serial_port_b = Some(p); } if let Some(k) = self.keyboard { cfg.keyboard = k; } diff --git a/src/dev/gr2/gr2disp.rs b/src/dev/gr2/gr2disp.rs index eb9f08264..03eca97b9 100644 --- a/src/dev/gr2/gr2disp.rs +++ b/src/dev/gr2/gr2disp.rs @@ -63,6 +63,7 @@ impl Gr2 { let row = y * gr2comp::OUT_STRIDE; screen.rgba[row..row + FB_W].copy_from_slice(&screen.fb_rgb[row..row + FB_W]); } + screen.frame_seq = screen.frame_seq.wrapping_add(1); } screen.prebuilt = true; screen.width = FB_W; diff --git a/src/dev/gr2/mod.rs b/src/dev/gr2/mod.rs index 759310044..22f48ae26 100644 --- a/src/dev/gr2/mod.rs +++ b/src/dev/gr2/mod.rs @@ -992,6 +992,18 @@ impl crate::gfx_display::GfxDisplay for Gr2 { fn screen(&self) -> &Mutex { &self.screen } fn request_screenshot(&self) { self.screenshot_pending.store(true, Ordering::Relaxed); } fn cycles(&self) -> CyclesPtr { self.cycles.get() } + fn hw_cursor(&self) -> Option { + // A racy read of the registers, as the display thread's compose does. + let vc1 = &self.regs().vc1; + if !vc1.cursor_visible() { + return None; + } + let (x, y) = vc1.cursor_pos(); + let mask = (0..32).map(|cy| (0..32).fold(0u64, |m, cx| { + m | ((vc1.cursor_pixel(cx, cy) != 0) as u64) << cx + })).collect(); + Some(crate::gfx_display::HwCursor { x, y, size: 32, mask }) + } } impl Resettable for Gr2 { diff --git a/src/dev/mc.rs b/src/dev/mc.rs index 286bbbec3..9c825968e 100644 --- a/src/dev/mc.rs +++ b/src/dev/mc.rs @@ -508,7 +508,9 @@ impl MemoryController { eprintln!("MC: CPU Error at {:08x}", addr); } - pub fn report_gio_timeout(&self, addr: u64) { + /// A GIO access nobody answered. Returns whether the MC aborts it with a + /// bus error (`CPUCTRL1.ABORT_EN`, which the PROM sets at power-on). + pub fn report_gio_timeout(&self, addr: u64) -> bool { let mut state = self.state.lock(); state.regs[(REG_GIO_ERROR_ADDR / 4) as usize] = addr as u32; // TIME bit (10) only set when ABORT_EN (CPUCTRL1 bit 4) is enabled @@ -518,6 +520,7 @@ impl MemoryController { } dlog_dev!(LogModule::Mc, "MC: GIO Timeout at {:08x} (abort_en={})", addr, abort_en); eprintln!("MC: GIO Timeout at {:08x}", addr); + abort_en } // ── Accessors for the DMA engine in mc_vdma.rs ────────────────────────── diff --git a/src/dev/mgras/disp.rs b/src/dev/mgras/disp.rs index b32d1412c..3c9dd0166 100644 --- a/src/dev/mgras/disp.rs +++ b/src/dev/mgras/disp.rs @@ -288,6 +288,7 @@ impl Mgras { let row = y * 2048; screen.rgba[row..row + w].copy_from_slice(&screen.fb_rgb[row..row + w]); } + screen.frame_seq = screen.frame_seq.wrapping_add(1); screen.width = w; screen.height = h; screen.status_bar_only = false; diff --git a/src/dev/mgras/mod.rs b/src/dev/mgras/mod.rs index f12e0cb7c..f6d1b8b79 100644 --- a/src/dev/mgras/mod.rs +++ b/src/dev/mgras/mod.rs @@ -1588,6 +1588,19 @@ impl crate::gfx_display::GfxDisplay for Mgras { fn screen(&self) -> &Mutex { &self.screen } fn request_screenshot(&self) { self.screenshot_pending.store(true, Ordering::Relaxed); } fn cycles(&self) -> crate::cpu::mips_core::CyclesPtr { *self.cycles.lock() } + fn hw_cursor(&self) -> Option { + // A racy read of the VC3, as the display thread's snapshot does. + let vc3 = &self.front_view().dcb.vc3; + let (x, y, size, glyph) = vc3.cursor()?; + // The glyph layout `frame.rs` draws: two planes, MSB leftmost. + let words_per_row = size / 16; + let plane_words = size * words_per_row; + let word = |plane: usize, row: usize, w: usize| vc3.sram[(glyph + plane * plane_words + row * words_per_row + w) & 0x7FFF] as u64; + let mask = (0..size).map(|row| (0..words_per_row).fold(0u64, |m, w| { + m | ((word(0, row, w) | word(1, row, w)).reverse_bits() >> 48) << (w * 16) + })).collect(); + Some(crate::gfx_display::HwCursor { x, y, size, mask }) + } } impl Device for Mgras { diff --git a/src/dev/ng1/rex3.rs b/src/dev/ng1/rex3.rs index 33938d360..62202c1c3 100644 --- a/src/dev/ng1/rex3.rs +++ b/src/dev/ng1/rex3.rs @@ -41,6 +41,8 @@ pub trait Renderer: Send { fn switch_compositor(&mut self, _use_gl: bool) -> &'static str { "sw" } /// Return a short status string: "compositor=gl shader=integer" etc. fn compositor_status(&self) -> String { "compositor=sw shader=n/a".to_string() } + /// Whether the GL compositor is in use, or is about to be. + fn gl_compositor(&self) -> bool { false } } pub const REX3_SIZE: u32 = 0x2000; // 8KB @@ -1471,6 +1473,12 @@ pub struct Rex3 { pub screenshot_pending: AtomicBool, /// Monotonically incrementing screenshot counter for unique filenames. pub screenshot_counter: AtomicU32, + /// One-shot "compose the next frame into `screen.rgba`" (no file), for + /// the CI socket and the monitor's `screen` commands. + frame_request: AtomicBool, + /// Frame consumers (VNC) that want `screen.rgba` kept current; while + /// non-zero every full frame is read back and the compositor is software. + capture_users: AtomicUsize, /// Atomic shadow of MipsCore::count_hz — updated by CPU thread, read by refresh thread. /// Wrapped in Mutex so machine.rs can swap in the real Arc from MipsCore after construction. #[cfg(feature = "developer")] @@ -1631,6 +1639,8 @@ impl Rex3 { show_disp_debug: AtomicBool::new(false), screenshot_pending: AtomicBool::new(false), screenshot_counter: AtomicU32::new(0), + frame_request: AtomicBool::new(false), + capture_users: AtomicUsize::new(0), #[cfg(feature = "developer")] count_hz_atomic: Mutex::new(Arc::new(AtomicU64::new(crate::cpu::mips_core::DEFAULT_COUNT_HZ))), } @@ -2958,6 +2968,12 @@ impl Rex3 { let mut last_topscan: usize = usize::MAX; let mut frames_since_render: u32 = u32::MAX; let mut full_presents: u32 = 0; + // Composes `screen.rgba` when a frame is wanted and no front-end + // renderer is installed (`--ci`); made on first use. + let mut capture_compositor: Option = None; + // While capture forces the renderer onto the software compositor: + // whether it used GL before, to switch back when capture ends. + let mut restore_gl: Option = None; while self.running.load(Ordering::Relaxed) { let start = std::time::Instant::now(); @@ -3052,6 +3068,7 @@ impl Rex3 { || dbg_overlay || topscan != last_topscan || self.screenshot_pending.load(Ordering::Relaxed) + || self.frame_request.load(Ordering::Relaxed) || frames_since_render >= IDLE_HEARTBEAT_FRAMES; if should_render { @@ -3064,7 +3081,8 @@ impl Rex3 { && !fb_was_dirty && !palette_dirty && !dbg_overlay - && !self.screenshot_pending.load(Ordering::Relaxed); + && !self.screenshot_pending.load(Ordering::Relaxed) + && !self.frame_request.load(Ordering::Relaxed); // Push debug state into overlay overlay.show_cmap = self.show_cmap.load(Ordering::Relaxed); @@ -3098,6 +3116,12 @@ impl Rex3 { // Screenshot: grab pixels from the compositor before presenting. let take_screenshot = self.screenshot_pending.swap(false, Ordering::Relaxed); + // Frame consumers: a one-shot request, or capture (VNC) on + // every full frame. Heartbeats change nothing in the picture. + let capture = self.capture_users.load(Ordering::Relaxed) > 0; + let want_frame = self.frame_request.load(Ordering::Relaxed) + || (capture && !screen.status_bar_only); + let need_readback = take_screenshot || want_frame; // Assert VBLANK: device state is now copied into screen caches, so // the CPU gets the maximum window to react before the next refresh. @@ -3125,17 +3149,45 @@ impl Rex3 { crate::pf_scope!(crate::perfetto::events::REX3_PRESENT); self.diag.fetch_or(Self::DIAG_LOOP_GL_RENDER, Ordering::Relaxed); let borrow = screen.fb_borrowed; + // Capture reads every frame back: keep it a slice copy + // out of the software compositor, never a GL readback. + if capture { + restore_gl.get_or_insert_with(|| r.gl_compositor()); + r.switch_compositor(false); + } else if let Some(gl) = restore_gl.take() { + if gl { + r.switch_compositor(true); + } + } r.present( &mut *screen, &mut overlay, &mut status_bar, &mut sbtex, &bar_stats, - take_screenshot, + need_readback, if borrow { Some(&**fb_rgb) } else { None }, if borrow { Some(&**fb_aux) } else { None }, ); self.diag.fetch_and(!Self::DIAG_LOOP_GL_RENDER, Ordering::Relaxed); + } else if need_readback { + // No front-end (`--ci`): compose the guest screen here. + // No status bar or overlay: this is the guest's screen. + let comp = capture_compositor.get_or_insert_with(crate::compositor::SwCompositor::new); + let borrow = screen.fb_borrowed; + comp.compose_pixels(&screen.compositor_source_from( + if borrow { Some(&**fb_rgb) } else { None }, + if borrow { Some(&**fb_aux) } else { None }, + )); + let (w, h) = (screen.width, screen.height); + let px = comp.pixels(); + for y in 0..h { + let row = y * crate::disp::FB_STRIDE; + screen.rgba[row..row + w].copy_from_slice(&px[row..row + w]); + } + } + if need_readback { + screen.frame_seq = screen.frame_seq.wrapping_add(1); } if !screen.status_bar_only { @@ -3157,6 +3209,9 @@ impl Rex3 { } } + // Cleared after the copy above, under the screen lock, so a + // waiter that sees it clear then reads the finished frame. + self.frame_request.store(false, Ordering::Relaxed); self.diag.fetch_and(!(Self::DIAG_LOCK_SCREEN | Self::DIAG_LOCK_RENDERER), Ordering::Relaxed); } else { // Nothing visible changed — skip the full refresh + GL upload and @@ -4001,6 +4056,9 @@ impl Device for Rex3 { Some("sw") => false, _ => return Err("Usage: disp compositor ".to_string()), }; + if use_gl && self.capture_users.load(Ordering::Relaxed) > 0 { + return Err("the VNC server needs the software compositor while it runs".to_string()); + } let active = if let Some(renderer) = self.renderer.lock().as_mut() { renderer.switch_compositor(use_gl) } else { @@ -4878,6 +4936,49 @@ impl crate::gfx_display::GfxDisplay for Rex3 { fn request_screenshot(&self) { self.screenshot_pending.store(true, Ordering::Relaxed); } fn screenshot_pending(&self) -> bool { self.screenshot_pending.load(Ordering::Relaxed) } fn cycles(&self) -> crate::cpu::mips_core::CyclesPtr { self.cycles.get() } + fn request_frame(&self) { self.frame_request.store(true, Ordering::Relaxed); } + fn frame_pending(&self) -> bool { self.frame_request.load(Ordering::Relaxed) } + fn set_capture(&self, on: bool) { + if on { + self.capture_users.fetch_add(1, Ordering::Relaxed); + // Repaint at once so the new consumer gets a frame. + self.fb_dirty.store(true, Ordering::Release); + } else { + let _ = self.capture_users.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |n| n.checked_sub(1)); + } + } + fn hw_cursor(&self) -> Option { + use crate::dev::ng1::vc2::{VC2_REG_CURRENT_CURSOR_X, VC2_REG_CURSOR_Y_LOC, + VC2_REG_CURSOR_ENTRY_PTR, VC2_REG_DISPLAY_CONTROL, VC2_CTRL_CURSOR_EN, VC2_CTRL_CURSOR_SIZE}; + let x_adjust = self.screen.lock().cursor_x_adjust; + let vc2 = self.vc2.lock(); + let ctrl = vc2.regs[VC2_REG_DISPLAY_CONTROL as usize]; + if ctrl & VC2_CTRL_CURSOR_EN == 0 { + return None; + } + let entry = vc2.regs[VC2_REG_CURSOR_ENTRY_PTR as usize] as usize; + let ram = &vc2.ram; + let word = |a: usize| ram.get(a).copied().unwrap_or(0) as u64; + // The same glyph layout the compositor draws (compositor.rs). + let (size, mask): (usize, Vec) = if ctrl & VC2_CTRL_CURSOR_SIZE != 0 { + (64, (0..64).map(|y| (0..4).fold(0u64, |m, w| { + m | (word(entry + y * 4 + w).reverse_bits() >> 48) << (w * 16) + })).collect()) + } else { + (32, (0..32).map(|y| (0..2).fold(0u64, |m, w| { + let a = entry + y * 2 + w; + m | ((word(a) | word(a + 64)).reverse_bits() >> 48) << (w * 16) + })).collect()) + }; + Some(crate::gfx_display::HwCursor { + // The compositor latches Y at retrace; the live location register + // is where it will be drawn next. + x: vc2.regs[VC2_REG_CURRENT_CURSOR_X as usize] as i32 - 31 + x_adjust, + y: vc2.regs[VC2_REG_CURSOR_Y_LOC as usize] as i32 - 31, + size, + mask, + }) + } } impl Resettable for Rex3 { diff --git a/src/dev/o2gfx.rs b/src/dev/o2gfx.rs index b2e22a089..16b5ac434 100644 --- a/src/dev/o2gfx.rs +++ b/src/dev/o2gfx.rs @@ -420,6 +420,38 @@ impl Gbe { snap } + /// CRS_POS: x in bits 11:0 and y in bits 27:16, each a 12-bit signed + /// number (the X server puts the glyph at -6 to keep a hotspot on the + /// edge). + fn cursor_xy(crs: u32) -> (i32, i32) { + let sext12 = |v: u32| ((v & 0xfff) as i32) << 20 >> 20; + (sext12(crs), sext12(crs >> 16)) + } + + /// The hardware cursor when enabled: its glyph's top-left on screen and + /// one opaque-pixel mask word per row (bit `c` = column `c`), 32x32. + pub fn cursor(&self) -> Option<(i32, i32, Vec)> { + let r = { + let mut s = self.s.lock().unwrap(); + self.latch(&mut s); + [Self::get(&s.r, CRS_CTL), Self::get(&s.r, CRS_POS)] + .into_iter() + .chain((0..64).map(|i| Self::get(&s.r, CRS_GLYPH + 4 * i))) + .collect::>() + }; + if r[0] & 1 == 0 { + return None; + } + let (x, y) = Self::cursor_xy(r[1]); + // Two bits a pixel, 16 pixels a word, leftmost in the top bits; as + // `render` draws it. + let mask = (0..32usize).map(|dy| (0..32usize).fold(0u64, |m, dx| { + let word = r[2 + 2 * dy + dx / 16]; + m | (((word >> (30 - 2 * (dx % 16))) & 3 != 0) as u64) << dx + })).collect(); + Some((x, y, mask)) + } + /// Scan the frame out of RAM into `out` (R, G, B, A bytes, row 0 at the /// top), as GBE would. Returns the size drawn, or None when the display /// is off. `out` must hold the largest mode (2048 x 2048 pixels). @@ -538,7 +570,7 @@ impl Gbe { let crs_on = g(CRS_CTL) & 1 != 0; let crs = g(CRS_POS); let probe_at = self.probe.lock().unwrap().0; - let (cx, cy) = ((crs & 0xffff) as i16 as i32, (crs >> 16) as i16 as i32); + let (cx, cy) = Self::cursor_xy(crs); for y in 0..h { if use_did { diff --git a/src/dev/ps2.rs b/src/dev/ps2.rs index 34b4cebbc..f81633a86 100644 --- a/src/dev/ps2.rs +++ b/src/dev/ps2.rs @@ -519,6 +519,18 @@ impl Ps2Controller { Some(byte) } + /// Whether the guest has turned the mouse on: until it has, moves and + /// clicks are dropped. + pub fn mouse_enabled(&self) -> bool { + let state = self.state.lock(); + state.mouse_enabled && state.config & 0x20 == 0 + } + + /// Mouse bytes queued that the guest has not read yet. + pub fn mouse_pending(&self) -> usize { + self.state.lock().mouse_queue_bytes + } + /// Push a scancode byte to the keyboard queue (called from UI) pub fn push_kb(&self, key: KeyCode, pressed: bool) { if !self.running.load(Ordering::Relaxed) { return; } diff --git a/src/disp.rs b/src/disp.rs index f3747f3c6..44a97df8e 100644 --- a/src/disp.rs +++ b/src/disp.rs @@ -71,6 +71,9 @@ pub struct Rex3Screen { /// board's own compositor (GR2's `gr2comp`). The renderer's compositor /// then presents it as-is instead of decoding Newport state. pub prebuilt: bool, + /// Bumped each time `rgba` is rewritten with a composed frame, so frame + /// consumers (VNC) can skip unchanged ones without comparing pixels. + pub frame_seq: u64, } impl Rex3Screen { @@ -95,6 +98,7 @@ impl Rex3Screen { status_bar_only: false, fb_borrowed: false, prebuilt: false, + frame_seq: 0, } } diff --git a/src/gfx_display.rs b/src/gfx_display.rs index 71c579d8a..3163f9dce 100644 --- a/src/gfx_display.rs +++ b/src/gfx_display.rs @@ -3,7 +3,8 @@ //! //! Each board owns its display thread and composition; this trait only exposes //! the hand-off points: the renderer slot the front-end installs into, the -//! last-presented screen (screenshots), and the CPU cycle counter. +//! last-presented screen (screenshots, VNC), the hardware cursor, and the CPU +//! cycle counter. use parking_lot::Mutex; @@ -31,4 +32,84 @@ pub trait GfxDisplay: Send + Sync { /// CPU cycle counter the board was wired to (GUI's live MIPS estimate). fn cycles(&self) -> CyclesPtr; + + /// Ask for the next composed frame in `screen().rgba`, without the + /// screenshot file [`request_screenshot`](Self::request_screenshot) also + /// writes. Boards that compose into `rgba` every frame ignore it. + fn request_frame(&self) {} + + /// True while a [`request_frame`](Self::request_frame) is still pending. + fn frame_pending(&self) -> bool { false } + + /// Keep `screen().rgba` current with every composed frame while at least + /// one consumer (VNC) has turned capture on. Calls nest: each `true` is + /// matched by a `false`. Newport then composes in software, so the copy is + /// a slice copy and not a GL readback. Boards that always compose into + /// `rgba` ignore it. + fn set_capture(&self, _on: bool) {} + + /// The hardware cursor as the guest has set it, read live from the + /// board's registers (not the last composed frame). + fn hw_cursor(&self) -> Option { None } +} + +/// The hardware cursor: where the board draws its glyph and which of the +/// glyph's pixels are opaque. The glyph's top-left is the pointer position +/// minus the hotspot, which the hardware does not know. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct HwCursor { + /// Glyph top-left in display coordinates; negative near the top/left edge. + pub x: i32, + pub y: i32, + /// Glyph side, 32 or 64. + pub size: usize, + /// One word per glyph row, bit `c` set where column `c` is opaque. + pub mask: Vec, +} + +impl HwCursor { + /// The hotspot when nothing better is known: the first opaque pixel of + /// the topmost opaque row, the tip of an arrow. (0, 0) for an empty glyph. + pub fn tip(&self) -> (i32, i32) { + self.mask.iter().enumerate() + .find(|(_, &m)| m != 0) + .map(|(y, &m)| (m.trailing_zeros() as i32, y as i32)) + .unwrap_or((0, 0)) + } + + /// A key that tells glyphs apart, to remember a learned hotspot per shape. + pub fn shape_key(&self) -> u64 { + // FNV-1a over the mask words. + self.mask.iter().fold(0xcbf2_9ce4_8422_2325u64, |h, &m| (h ^ m).wrapping_mul(0x100_0000_01b3)) + ^ self.size as u64 + } +} + +/// The visible part of the last composed frame, tight (`width` pixels a row, +/// `0xAABBGGRR`, R in the low byte), with its sequence number. `None` until +/// the display has a video mode. Holds the screen lock only for the copy. +pub fn copy_frame(display: &dyn GfxDisplay, out: &mut Vec) -> Option<(usize, usize, u64)> { + let screen = display.screen().lock(); + let (w, h) = (screen.width, screen.height); + if w == 0 || h == 0 { + return None; + } + out.clear(); + out.reserve(w * h); + for y in 0..h { + let row = y * crate::disp::FB_STRIDE; + out.extend_from_slice(&screen.rgba[row..row + w]); + } + Some((w, h, screen.frame_seq)) +} + +/// Request a fresh frame and wait (at most `timeout`) for the board to +/// compose it, then copy it as [`copy_frame`] does. +pub fn fresh_frame(display: &dyn GfxDisplay, out: &mut Vec, timeout: std::time::Duration) -> Option<(usize, usize, u64)> { + display.request_frame(); + let deadline = std::time::Instant::now() + timeout; + while display.frame_pending() && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(2)); + } + copy_frame(display, out) } diff --git a/src/hostinput.rs b/src/hostinput.rs new file mode 100644 index 000000000..7ecaee0ac --- /dev/null +++ b/src/hostinput.rs @@ -0,0 +1,718 @@ +//! The host's hands on the guest: look at the screen, put the pointer on a +//! pixel, click, press keys. Shared by the VNC server and the monitor's +//! `screen`, `ptr` and `key` commands, for every machine and board. +//! +//! The guest's mouse is relative (PS/2), and the X server accelerates what +//! it reads, so a position cannot be sent. Instead the pointer is steered: +//! send a move, read back where the board's hardware cursor went (IRIX always +//! uses it), and correct until the pointer sits on the target. The hardware +//! knows the glyph's top-left, not its hotspot. A hotspot is known exactly +//! when the pointer is pinned in the top-left corner (X keeps the pointer on +//! screen, so it is at 0,0), which the first move does once, or when a new +//! shape appears after a move small enough that X does not accelerate it +//! (the pointer then moved exactly that far). Until then a new shape's +//! hotspot is guessed as its tip plus the offset the known shapes showed. +//! Only the top and left edges pin reliably: the size X uses can differ +//! from the board's visible size by a few pixels. + +use std::collections::HashMap; +use std::io::Write; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use parking_lot::Mutex; +use winit::keyboard::KeyCode; + +use crate::dev::ps2::Ps2Controller; +use crate::gfx_display::{GfxDisplay, HwCursor}; + +/// PS/2 button bits. +pub const BUTTON_LEFT: u8 = 1; +pub const BUTTON_RIGHT: u8 = 2; +pub const BUTTON_MIDDLE: u8 = 4; + +/// How long a move waits for the cursor to answer one packet. +const MOVE_ANSWER: Duration = Duration::from_millis(250); +/// How long the cursor must stay put after a move before the next step. +const SETTLE: Duration = Duration::from_millis(20); +/// How long the cursor may be off before a move gives up on it. +const CURSOR_GAP: Duration = Duration::from_millis(400); +/// How long the pointer may ignore the mouse (the guest busy) before a move +/// gives up. +const STALL_LIMIT: Duration = Duration::from_millis(3000); +/// Moves this small (mickeys) are sent exactly: under X's acceleration +/// threshold, they move the pointer as far as they say. +const EXACT_STEP: i32 = 3; + +pub struct HostInput { + display: Option>, + ps2: Arc, + /// Buttons held now (PS/2 bits), sent with every move. + buttons: AtomicU8, + /// Hotspots seen exactly, by glyph shape. + hotspots: Mutex>, + /// Hotspot minus tip of the last shape learnt: the guess for a new one. + bias: Mutex<(i32, i32)>, + /// Pixels the pointer moves per mickey on a large move (X's + /// acceleration), learnt as moves are made. + gain: Mutex, + /// One steering loop at a time: two would fight over the pointer. + seek: Mutex<()>, +} + +/// Where a move ended. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MoveResult { + /// The pointer is on the target. + Reached, + /// Time ran out this far (x, y) from the target. + Short(i32, i32), +} + +impl HostInput { + pub fn new(display: Option>, ps2: Arc) -> Self { + Self { + display, + ps2, + buttons: AtomicU8::new(0), + hotspots: Mutex::new(HashMap::new()), + bias: Mutex::new((0, 0)), + gain: Mutex::new(1.0), + seek: Mutex::new(()), + } + } + + /// Build one for a machine and add its commands to the machine's monitor. + pub fn attach(machine: &dyn crate::workstation::Workstation) -> Arc { + let hi = Arc::new(Self::new(machine.get_display(), machine.get_ps2())); + machine.register_monitor_device(hi.clone()); + hi + } + + pub fn display(&self) -> Option<&Arc> { + self.display.as_ref() + } + + pub fn ps2(&self) -> &Arc { + &self.ps2 + } + + /// The display's visible size, (0, 0) before it has a mode. + pub fn screen_size(&self) -> (usize, usize) { + self.display.as_ref().map(|d| { + let s = d.screen().lock(); + (s.width, s.height) + }).unwrap_or((0, 0)) + } + + fn cursor(&self) -> Option { + self.display.as_ref()?.hw_cursor() + } + + /// The cursor, waiting out a short gap: X servers turn the hardware + /// cursor off for a moment while they draw under it or change it. + fn cursor_settled(&self) -> Option { + let start = Instant::now(); + loop { + if let Some(c) = self.cursor() { + return Some(c); + } + if start.elapsed() >= CURSOR_GAP { + return None; + } + std::thread::sleep(Duration::from_millis(5)); + } + } + + /// The hotspot of a glyph: seen exactly, or else guessed. + pub fn hotspot(&self, c: &HwCursor) -> (i32, i32) { + self.known_hotspot(c).unwrap_or_else(|| { + let (tx, ty) = c.tip(); + let (bx, by) = *self.bias.lock(); + (tx + bx, ty + by) + }) + } + + fn known_hotspot(&self, c: &HwCursor) -> Option<(i32, i32)> { + self.hotspots.lock().get(&c.shape_key()).copied() + } + + fn learn(&self, c: &HwCursor, hot: (i32, i32)) { + if !hotspot_plausible(c, hot) { + return; + } + self.hotspots.lock().insert(c.shape_key(), hot); + let (tx, ty) = c.tip(); + *self.bias.lock() = (hot.0 - tx, hot.1 - ty); + } + + /// Where the pointer is, by the hardware cursor; None while it is off. + pub fn pointer(&self) -> Option<(i32, i32)> { + let c = self.cursor()?; + let (hx, hy) = self.hotspot(&c); + Some((c.x + hx, c.y + hy)) + } + + /// Steer the pointer onto (x, y), clamped to the screen. `retarget` is + /// asked between steps for a newer target (a VNC viewer still moving). + /// Errors when there is no cursor to steer by or it does not move. + pub fn move_to(&self, x: i32, y: i32, timeout: Duration, + retarget: &mut dyn FnMut() -> Option<(i32, i32)>) -> Result { + let _one = self.seek.lock(); + if self.hotspots.lock().is_empty() { + // Once: pin the pointer in the corner to know a hotspot exactly. + self.calibrate_locked()?; + } + let deadline = Instant::now() + timeout; + let (mut tx, mut ty) = (x, y); + let mut stalls = [0u32; 2]; + let mut last_motion = Instant::now(); + // How far right/down the pointer goes, once seen pinned there: a + // board that draws the cursor a pixel off leaves the last column or + // row out of reach, and a move there ends at the edge. + let mut far_edge: [Option; 2] = [None; 2]; + loop { + if let Some(t) = retarget() { + (tx, ty) = t; + stalls = [0; 2]; + } + let (w, h) = self.screen_size(); + if w == 0 { + return Err("the display has no video mode yet".into()); + } + let (tx, ty) = (tx.clamp(0, w as i32 - 1), ty.clamp(0, h as i32 - 1)); + let (tx, ty) = (far_edge[0].map_or(tx, |e| tx.min(e)), far_edge[1].map_or(ty, |e| ty.min(e))); + let c = self.cursor_settled().ok_or("the hardware cursor is off (no X server yet?)")?; + let (hx, hy) = self.hotspot(&c); + let (ex, ey) = (tx - (c.x + hx), ty - (c.y + hy)); + if ex == 0 && ey == 0 { + return Ok(MoveResult::Reached); + } + if Instant::now() >= deadline { + return Ok(MoveResult::Short(ex, ey)); + } + let gain = *self.gain.lock(); + let step = |e: i32| -> i32 { + if e.abs() <= EXACT_STEP { + e + } else { + match ((e as f32 / gain).round() as i32).clamp(-127, 127) { + 0 => e.signum(), + s => s, + } + } + }; + let (sx, sy) = (step(ex), step(ey)); + // Steer from where the cursor is, not from moves still queued: + // let a busy guest catch up before sending more. + let queued = Instant::now(); + while self.ps2.mouse_pending() > 0 && queued.elapsed() < MOVE_ANSWER { + std::thread::sleep(Duration::from_millis(1)); + } + if self.ps2.mouse_pending() > 0 { + if last_motion.elapsed() >= STALL_LIMIT { + return Err("the guest is not reading the mouse".into()); + } + continue; + } + self.ps2.push_mouse_input(self.buttons.load(Ordering::Relaxed), sx, sy, 0); + + // Wait for the X server to read it and move the cursor, then + // for the cursor to settle: a board may show a move a frame late, + // and steering from a position before the last step oscillates. + let sent = Instant::now(); + let mut now_at: Option = None; + let mut changed_at = sent; + while sent.elapsed() < MOVE_ANSWER { + std::thread::sleep(Duration::from_millis(1)); + match self.cursor() { + Some(n) if (n.x, n.y) != (c.x, c.y) || n.mask != c.mask => { + if now_at.as_ref().is_none_or(|m| (m.x, m.y, &m.mask) != (n.x, n.y, &n.mask)) { + changed_at = Instant::now(); + now_at = Some(n); + } else if changed_at.elapsed() >= SETTLE { + break; + } + } + Some(_) | None => {} + } + } + let (mx, my) = match &now_at { + Some(n) if n.mask == c.mask => (n.x - c.x, n.y - c.y), + Some(n) => { + // A new shape. After an unaccelerated step from a known + // shape the pointer moved exactly (sx, sy): that gives + // the new hotspot. + if sx.abs() <= EXACT_STEP && sy.abs() <= EXACT_STEP && self.known_hotspot(n).is_none() { + if let Some((ohx, ohy)) = self.known_hotspot(&c) { + // Let X finish the move before reading the glyph. + std::thread::sleep(Duration::from_millis(5)); + if let Some(n) = self.cursor().filter(|m| m.mask == n.mask) { + let (px, py) = (c.x + ohx + sx, c.y + ohy + sy); + self.learn(&n, (px - n.x, py - n.y)); + } + } + } + stalls = [0; 2]; + continue; + } + None => (0, 0), + }; + // Learn the acceleration from large steps. + for (m, s) in [(mx, sx), (my, sy)] { + if s.abs() > EXACT_STEP * 2 && m != 0 && m.signum() == s.signum() { + let mut g = self.gain.lock(); + *g = (*g * 0.5 + (m as f32 / s as f32) * 0.5).clamp(0.25, 8.0); + } + } + // An axis that will not move although pushed: pinned at an edge + // (X keeps the pointer on screen). At the left or top edge the + // pointer is at 0, which tells the hotspot. Or the guest is not + // reading the mouse. + let size = c.size as i32; + let mut learnt = false; + for (axis, (m, s, pos)) in [(mx, sx, c.x), (my, sy, c.y)].into_iter().enumerate() { + if s == 0 || m != 0 { + stalls[axis] = 0; + continue; + } + stalls[axis] += 1; + if stalls[axis] < 2 { + continue; + } + if s > 0 { + // Within a glyph of the right or bottom edge: that is + // the edge. + let p = pos + [hx, hy][axis]; + if p >= [w, h][axis] as i32 - size { + far_edge[axis] = Some(p); + learnt = true; + } + continue; + } + let hot = -pos; + if (-size..size).contains(&hot) && hot != [hx, hy][axis] { + let mut both = (hx, hy); + if axis == 0 { both.0 = hot } else { both.1 = hot } + self.learn(&c, both); + stalls[axis] = 0; + learnt = true; + } + } + if mx != 0 || my != 0 || learnt { + last_motion = Instant::now(); + } + if last_motion.elapsed() >= STALL_LIMIT { + return Err(if self.ps2.mouse_enabled() { + "the cursor does not follow the mouse (guest busy, or no X server?)".into() + } else { + "the guest has not enabled the mouse".into() + }); + } + } + } + + /// Move, then check the move arrived. + pub fn move_to_exact(&self, x: i32, y: i32) -> Result<(), String> { + match self.move_to(x, y, Duration::from_secs(3), &mut || None)? { + MoveResult::Reached => Ok(()), + MoveResult::Short(ex, ey) => Err(format!("pointer stopped {ex},{ey} short of {x},{y}")), + } + } + + /// Push the pointer into the top-left corner and learn the current + /// glyph's hotspot from where the cursor stops. Returns the hotspot. + pub fn calibrate(&self) -> Result<(i32, i32), String> { + let _one = self.seek.lock(); + self.calibrate_locked() + } + + fn calibrate_locked(&self) -> Result<(i32, i32), String> { + // Keep pushing up-left until the cursor has stayed put for a while + // although the guest read every packet: then X has it pinned. + const PINNED: Duration = Duration::from_millis(150); + let start = Instant::now(); + let mut still: Option<((i32, i32), Instant)> = None; + while start.elapsed() < Duration::from_secs(10) { + let drained = Instant::now(); + while self.ps2.mouse_pending() > 0 && drained.elapsed() < Duration::from_millis(500) { + std::thread::sleep(Duration::from_millis(2)); + } + let c = self.cursor_settled().ok_or("the hardware cursor is off")?; + match still { + Some((at, since)) if at == (c.x, c.y) && self.ps2.mouse_pending() == 0 => { + if since.elapsed() >= PINNED { + let hot = (-c.x, -c.y); + if !hotspot_plausible(&c, hot) { + return Err(format!("the cursor stopped at {},{}, not in the corner", c.x, c.y)); + } + self.learn(&c, hot); + return Ok(hot); + } + } + _ => still = Some(((c.x, c.y), Instant::now())), + } + self.ps2.push_mouse_input(self.buttons.load(Ordering::Relaxed), -100, -100, 0); + std::thread::sleep(Duration::from_millis(20)); + } + Err("the cursor never settled in the corner".into()) + } + + /// A raw relative move, as a mouse would send it. + pub fn nudge(&self, dx: i32, dy: i32) { + let b = self.buttons.load(Ordering::Relaxed); + // Packets carry 9-bit deltas. + let (mut dx, mut dy) = (dx, dy); + while dx != 0 || dy != 0 { + let (sx, sy) = (dx.clamp(-200, 200), dy.clamp(-200, 200)); + self.ps2.push_mouse_input(b, sx, sy, 0); + (dx, dy) = (dx - sx, dy - sy); + } + } + + /// Hold exactly `mask` (PS/2 bits) from now on. + pub fn set_buttons(&self, mask: u8) { + if self.buttons.swap(mask, Ordering::Relaxed) != mask { + self.ps2.push_mouse_input(mask, 0, 0, 0); + } + } + + pub fn buttons(&self) -> u8 { + self.buttons.load(Ordering::Relaxed) + } + + /// Wheel clicks (positive away from the user); only reaches a guest + /// that has switched the mouse to its wheel mode. + pub fn wheel(&self, dz: i32) { + self.ps2.push_mouse_input(self.buttons.load(Ordering::Relaxed), 0, 0, dz); + } + + /// Press and release `button` `count` times where the pointer is. + pub fn click(&self, button: u8, count: u32) { + let held = self.buttons.load(Ordering::Relaxed); + for i in 0..count { + if i > 0 { + std::thread::sleep(Duration::from_millis(60)); + } + self.set_buttons(held | button); + std::thread::sleep(Duration::from_millis(40)); + self.set_buttons(held & !button); + } + } + + pub fn key(&self, code: KeyCode, down: bool) { + self.ps2.push_kb(code, down); + } + + /// Press a chord such as `ctrl+alt+F1`, or with `down` = Some, only + /// press or only release it. + pub fn chord(&self, chord: &str, down: Option) -> Result<(), String> { + let keys = chord.split('+').filter(|s| !s.is_empty()) + .map(|n| crate::keysym::from_name(n).ok_or_else(|| format!("unknown key '{n}'"))) + .collect::, _>>()?; + if keys.is_empty() { + return Err("no key named".into()); + } + if down != Some(false) { + for &k in &keys { + self.key(k, true); + } + } + if down.is_none() { + std::thread::sleep(Duration::from_millis(30)); + } + if down != Some(true) { + for &k in keys.iter().rev() { + self.key(k, false); + } + } + Ok(()) + } + + /// Type text on a US keyboard; returns how many characters had a key. + pub fn type_text(&self, text: &str) -> usize { + text.chars().filter(|&ch| match crate::keysym::char_key(ch) { + Some((k, shifted)) => { + if shifted { self.key(KeyCode::ShiftLeft, true); } + self.key(k, true); + self.key(k, false); + if shifted { self.key(KeyCode::ShiftLeft, false); } + true + } + None => self.ps2.type_char(ch), + }).count() + } + + /// A fresh frame, tight, with its size. + pub fn frame(&self) -> Result<(usize, usize, Vec), String> { + let d = self.display.as_ref().ok_or("no display (headless)")?; + let mut px = Vec::new(); + let (w, h, _) = crate::gfx_display::fresh_frame(&**d, &mut px, Duration::from_millis(250)) + .ok_or("the display has no video mode yet")?; + Ok((w, h, px)) + } + + /// Wait until the screen has not changed for `quiet`, at most `timeout`. + /// Returns whether it settled. + pub fn wait_still(&self, quiet: Duration, timeout: Duration) -> Result { + let start = Instant::now(); + let mut last = self.frame()?; + let mut since = Instant::now(); + while start.elapsed() < timeout { + std::thread::sleep(Duration::from_millis(50)); + let f = self.frame()?; + if f != last { + last = f; + since = Instant::now(); + } else if since.elapsed() >= quiet { + return Ok(true); + } + } + Ok(false) + } +} + +/// A hotspot is inside its glyph, give or take how far the board draws the +/// cursor from where the guest put it: a board whose cursor offset we have +/// slightly wrong shows a hotspot a few pixels outside. +fn hotspot_plausible(c: &HwCursor, (hx, hy): (i32, i32)) -> bool { + let size = c.size as i32; + (-8..size + 8).contains(&hx) && (-8..size + 8).contains(&hy) +} + +/// Save tight `0xAABBGGRR` pixels (row length `w`) as an RGB PNG, cropped to +/// `crop` (x, y, w, h) when given. +pub fn save_png(path: &str, px: &[u32], w: usize, h: usize, crop: Option<(usize, usize, usize, usize)>) -> Result<(usize, usize), String> { + let (cx, cy, cw, ch) = crop.unwrap_or((0, 0, w, h)); + if cw == 0 || ch == 0 || cx + cw > w || cy + ch > h { + return Err(format!("crop {cx},{cy} {cw}x{ch} is outside the {w}x{h} screen")); + } + let mut rgb = Vec::with_capacity(cw * ch * 3); + for y in cy..cy + ch { + for &p in &px[y * w + cx..y * w + cx + cw] { + rgb.extend_from_slice(&[p as u8, (p >> 8) as u8, (p >> 16) as u8]); + } + } + let tmp = format!("{path}.tmp"); + { + let f = std::fs::File::create(&tmp).map_err(|e| format!("{path}: {e}"))?; + let mut enc = png::Encoder::new(std::io::BufWriter::new(f), cw as u32, ch as u32); + enc.set_color(png::ColorType::Rgb); + enc.set_depth(png::BitDepth::Eight); + let mut wr = enc.write_header().map_err(|e| e.to_string())?; + wr.write_image_data(&rgb).map_err(|e| e.to_string())?; + } + std::fs::rename(&tmp, path).map_err(|e| format!("{path}: {e}"))?; + Ok((cw, ch)) +} + +fn parse_button(s: Option<&&str>) -> Result { + Ok(match s.copied().unwrap_or("left") { + "left" | "l" | "1" => BUTTON_LEFT, + "middle" | "m" | "2" => BUTTON_MIDDLE, + "right" | "r" | "3" => BUTTON_RIGHT, + b => return Err(format!("unknown button '{b}' (left, middle, right)")), + }) +} + +fn num(args: &[&str], i: usize, what: &str) -> Result { + args.get(i).ok_or(format!("missing {what}"))?.parse().map_err(|_| format!("bad {what} '{}'", args[i])) +} + +const SCREEN_HELP: &str = "screen info | screen cursor | screen shot [x y w h] | screen pix | screen wait [quiet_ms] [timeout_ms]"; +const PTR_HELP: &str = "ptr pos | ptr move | ptr click [left|middle|right] [count] | ptr down|up [button] | ptr drag [button] | ptr rel | ptr calibrate"; +const KEY_HELP: &str = "key [+...] [down|up] | key type | key names"; + +impl crate::traits::Device for HostInput { + fn step(&self, _cycles: u64) {} + fn stop(&self) {} + fn start(&self) {} + fn is_running(&self) -> bool { true } + fn get_clock(&self) -> u64 { 0 } + + fn register_commands(&self) -> Vec<(String, String)> { + vec![ + ("screen".into(), format!("The guest's screen, any board: {SCREEN_HELP}")), + ("ptr".into(), format!("Pointer at screen pixels, steered by the hardware cursor: {PTR_HELP}")), + ("key".into(), format!("Keyboard by key name (X keysym names, ctrl/alt/shift/enter/esc..., KeyA): {KEY_HELP}")), + ] + } + + fn execute_command(&self, cmd: &str, args: &[&str], mut w: Box) -> Result<(), String> { + let sub = args.first().copied().unwrap_or(""); + let out = |w: &mut Box, s: String| { let _ = writeln!(w, "{s}"); }; + match (cmd, sub) { + ("screen", "info") | ("screen", "") => { + let (sw, sh) = self.screen_size(); + let seq = self.display.as_ref().map(|d| d.screen().lock().frame_seq).unwrap_or(0); + out(&mut w, format!("screen {sw}x{sh} frame {seq}")); + match self.cursor() { + Some(c) => { + let (hx, hy) = self.hotspot(&c); + out(&mut w, format!("cursor glyph at {},{} size {} hotspot {hx},{hy} -> pointer {},{}", + c.x, c.y, c.size, c.x + hx, c.y + hy)); + } + None => out(&mut w, "cursor off".into()), + } + out(&mut w, format!("mouse {}, buttons held {:#x}", + if self.ps2.mouse_enabled() { "enabled" } else { "not enabled by the guest" }, self.buttons())); + Ok(()) + } + ("screen", "cursor") => { + let c = self.cursor().ok_or("cursor off")?; + let (hx, hy) = self.hotspot(&c); + out(&mut w, format!("glyph at {},{} size {} shape {:016x} hotspot {hx},{hy} (tip {:?})", + c.x, c.y, c.size, c.shape_key(), c.tip())); + for (y, m) in c.mask.iter().enumerate() { + let row: String = (0..c.size).map(|x| match (m >> x & 1 != 0, (x as i32, y as i32) == (hx, hy)) { + (_, true) => '@', + (true, _) => '#', + _ => '.', + }).collect(); + out(&mut w, row); + } + Ok(()) + } + ("screen", "shot") => { + let path = args.get(1).ok_or("Usage: screen shot [x y w h]")?; + let crop = if args.len() >= 6 { + Some((num(args, 2, "x")?, num(args, 3, "y")?, num(args, 4, "w")?, num(args, 5, "h")?)) + } else { None }; + let (fw, fh, px) = self.frame()?; + let (cw, ch) = save_png(path, &px, fw, fh, crop)?; + out(&mut w, format!("saved {path} {cw}x{ch} (screen {fw}x{fh})")); + Ok(()) + } + ("screen", "pix") => { + let (x, y): (usize, usize) = (num(args, 1, "x")?, num(args, 2, "y")?); + let (fw, fh, px) = self.frame()?; + if x >= fw || y >= fh { + return Err(format!("{x},{y} is outside the {fw}x{fh} screen")); + } + let p = px[y * fw + x]; + out(&mut w, format!("{x},{y} = {:02x}{:02x}{:02x}", p & 0xff, (p >> 8) & 0xff, (p >> 16) & 0xff)); + Ok(()) + } + ("screen", "wait") => { + let quiet = args.get(1).map(|_| num::(args, 1, "quiet_ms")).transpose()?.unwrap_or(500); + let timeout = args.get(2).map(|_| num::(args, 2, "timeout_ms")).transpose()?.unwrap_or(10_000); + let t = Instant::now(); + let still = self.wait_still(Duration::from_millis(quiet), Duration::from_millis(timeout))?; + out(&mut w, if still { + format!("still for {quiet} ms after {} ms", t.elapsed().as_millis()) + } else { + format!("still changing after {timeout} ms") + }); + Ok(()) + } + ("screen", _) => Err(format!("Usage: {SCREEN_HELP}")), + + ("ptr", "pos") | ("ptr", "") => { + match self.pointer() { + Some((x, y)) => out(&mut w, format!("{x} {y}")), + None => out(&mut w, "cursor off".into()), + } + Ok(()) + } + ("ptr", "move") => { + let (x, y) = (num(args, 1, "x")?, num(args, 2, "y")?); + self.move_to_exact(x, y)?; + let (px, py) = self.pointer().unwrap_or((x, y)); + out(&mut w, format!("pointer at {px} {py}")); + Ok(()) + } + ("ptr", "click") => { + let (x, y) = (num(args, 1, "x")?, num(args, 2, "y")?); + let b = parse_button(args.get(3))?; + let n = args.get(4).map(|_| num::(args, 4, "count")).transpose()?.unwrap_or(1); + self.move_to_exact(x, y)?; + self.click(b, n.clamp(1, 3)); + let (px, py) = self.pointer().unwrap_or((x, y)); + out(&mut w, format!("clicked at {px} {py}")); + Ok(()) + } + ("ptr", "down") | ("ptr", "up") => { + let b = parse_button(args.get(1))?; + let held = self.buttons(); + self.set_buttons(if sub == "down" { held | b } else { held & !b }); + out(&mut w, format!("buttons held {:#x}", self.buttons())); + Ok(()) + } + ("ptr", "drag") => { + let (x1, y1, x2, y2) = (num(args, 1, "x1")?, num(args, 2, "y1")?, num(args, 3, "x2")?, num(args, 4, "y2")?); + let b = parse_button(args.get(5))?; + self.move_to_exact(x1, y1)?; + let held = self.buttons(); + self.set_buttons(held | b); + std::thread::sleep(Duration::from_millis(40)); + let r = self.move_to_exact(x2, y2); + std::thread::sleep(Duration::from_millis(40)); + self.set_buttons(held); + r?; + out(&mut w, format!("dragged {x1} {y1} -> {x2} {y2}")); + Ok(()) + } + ("ptr", "rel") => { + let (dx, dy) = (num(args, 1, "dx")?, num(args, 2, "dy")?); + self.nudge(dx, dy); + out(&mut w, format!("sent {dx} {dy}")); + Ok(()) + } + ("ptr", "calibrate") => { + let (hx, hy) = self.calibrate()?; + out(&mut w, format!("hotspot {hx},{hy}; pointer at 0 0")); + Ok(()) + } + ("ptr", _) => Err(format!("Usage: {PTR_HELP}")), + + ("key", "type") => { + let text = args[1..].join(" "); + let n = self.type_text(&text); + out(&mut w, format!("typed {n} of {} chars", text.chars().count())); + Ok(()) + } + ("key", "names") => { + out(&mut w, crate::keysym::names().collect::>().join(" ")); + Ok(()) + } + ("key", "") => Err(format!("Usage: {KEY_HELP}")), + ("key", chord) => { + let down = match args.get(1).copied() { + None => None, + Some("down") => Some(true), + Some("up") => Some(false), + Some(o) => return Err(format!("'{o}': down or up")), + }; + self.chord(chord, down)?; + out(&mut w, format!("key {chord}{}", match down { Some(true) => " down", Some(false) => " up", None => "" })); + Ok(()) + } + _ => Err("Command not found".into()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tip_and_shape() { + let mut mask = vec![0u64; 32]; + mask[2] = 0b1100; + mask[3] = 0b1110; + let c = HwCursor { x: 10, y: 20, size: 32, mask: mask.clone() }; + assert_eq!(c.tip(), (2, 2)); + let mut other = mask; + other[4] = 1; + let d = HwCursor { x: 0, y: 0, size: 32, mask: other }; + assert_ne!(c.shape_key(), d.shape_key()); + } + + #[test] + fn png_crop_bounds() { + let px = vec![0u32; 4 * 3]; + assert!(save_png("/nonexistent/x.png", &px, 4, 3, Some((3, 0, 2, 1))).is_err()); + } +} diff --git a/src/ip22_bus.rs b/src/ip22_bus.rs index 9527aabc9..ecb160685 100644 --- a/src/ip22_bus.rs +++ b/src/ip22_bus.rs @@ -42,20 +42,32 @@ impl BusDevice for CpuBusErrorDevice { fn write64(&self, addr: u64, _v: u64) -> u32 { self.mc.report_cpu_error(addr); BUS_OK } } -/// GIO bus timeout sink: reports to MC (GIO_ERROR_STAT bit 10 TIME) then returns 0/Ready. -/// Covers GIO space 0x18000000..0x1FA00000 (reserved future GIO + empty expansion slots). +/// GIO bus timeout sink: an address in GIO space no device answers (reserved +/// GIO space and empty expansion or graphics slots, 0x18000000..0x1FA00000). +/// The MC records the timeout in GIO_ERROR_ADDR/STAT and, with time-outs +/// enabled (CPUCTRL1.ABORT_EN, set by the PROM at power-on), aborts a read +/// with a bus error: that is how the PROM's and the kernel's `badaddr` +/// probes find a slot empty. Writes are dropped. struct GioBusErrorDevice { mc: MemoryController, } impl BusDevice for GioBusErrorDevice { - fn read8(&self, addr: u64) -> BusRead8 { self.mc.report_gio_timeout(addr); BusRead8::ok(0xFF) } - fn write8(&self, addr: u64, _v: u8) -> u32 { self.mc.report_gio_timeout(addr); BUS_OK } - fn read16(&self, addr: u64) -> BusRead16 { self.mc.report_gio_timeout(addr); BusRead16::ok(0xFFFF) } + fn read8(&self, addr: u64) -> BusRead8 { + if self.mc.report_gio_timeout(addr) { BusRead8::err() } else { BusRead8::ok(0xFF) } + } + fn write8(&self, addr: u64, _v: u8) -> u32 { self.mc.report_gio_timeout(addr); BUS_OK } + fn read16(&self, addr: u64) -> BusRead16 { + if self.mc.report_gio_timeout(addr) { BusRead16::err() } else { BusRead16::ok(0xFFFF) } + } fn write16(&self, addr: u64, _v: u16) -> u32 { self.mc.report_gio_timeout(addr); BUS_OK } - fn read32(&self, addr: u64) -> BusRead32 { self.mc.report_gio_timeout(addr); BusRead32::ok(0xFFFFFFFF) } + fn read32(&self, addr: u64) -> BusRead32 { + if self.mc.report_gio_timeout(addr) { BusRead32::err() } else { BusRead32::ok(0xFFFF_FFFF) } + } fn write32(&self, addr: u64, _v: u32) -> u32 { self.mc.report_gio_timeout(addr); BUS_OK } - fn read64(&self, addr: u64) -> BusRead64 { self.mc.report_gio_timeout(addr); BusRead64::ok(0xFFFFFFFFFFFFFFFF) } + fn read64(&self, addr: u64) -> BusRead64 { + if self.mc.report_gio_timeout(addr) { BusRead64::err() } else { BusRead64::ok(u64::MAX) } + } fn write64(&self, addr: u64, _v: u64) -> u32 { self.mc.report_gio_timeout(addr); BUS_OK } } diff --git a/src/ip30_machine.rs b/src/ip30_machine.rs index 07a71254a..30ee42328 100644 --- a/src/ip30_machine.rs +++ b/src/ip30_machine.rs @@ -296,7 +296,7 @@ impl OctaneMachine { // HEART's levels 0 to 4 are the CPU's IP2 to IP6. heart.set_interrupts(cpu.interrupts_ptr()); - let mut monitor = crate::monitor::Monitor::new(); + let monitor = crate::monitor::Monitor::new(); monitor.register_device(crate::devlog::init_devlog()); monitor.register_device(cpu.clone().as_device()); monitor.register_device(heart.clone()); @@ -629,6 +629,7 @@ impl Workstation for OctaneMachine { fn get_display(&self) -> Option> { self.gfx.as_ref().map(|g| g.display()) } + fn register_monitor_device(&self, device: Arc) { self.monitor.register_device(device); } fn get_ps2(&self) -> Arc { self.ioc3.ps2() } diff --git a/src/ip32_machine.rs b/src/ip32_machine.rs index 546495074..baaf525ab 100644 --- a/src/ip32_machine.rs +++ b/src/ip32_machine.rs @@ -403,7 +403,7 @@ impl O2Machine { } // The shared monitor, as the other machines have it: the CPU's // commands (registers, breakpoints, memory) and the O2's devices'. - let mut monitor = crate::monitor::Monitor::new(); + let monitor = crate::monitor::Monitor::new(); monitor.register_device(crate::devlog::init_devlog()); monitor.register_device(cpu.clone().as_device()); monitor.register_device(bus.clone()); @@ -920,6 +920,7 @@ impl Workstation for O2Machine { fn get_display(&self) -> Option> { Some(O2Machine::get_display(self)) } fn get_ps2(&self) -> Arc { O2Machine::get_ps2(self) } + fn register_monitor_device(&self, device: Arc) { self.monitor.register_device(device); } fn get_timer_manager(&self) -> Arc { O2Machine::get_timer_manager(self) } fn get_cpu_debug(&self) -> Arc { self.cpu.clone().debug_adapter() } fn fasttick_count(&self) -> Arc { self.cpu.fasttick_count() } @@ -1006,6 +1007,7 @@ impl O2Display { screen.rgba[y * stride + x] = v; } } + screen.frame_seq = screen.frame_seq.wrapping_add(1); screen.prebuilt = true; screen.width = w; screen.height = h; @@ -1064,6 +1066,10 @@ impl crate::gfx_display::GfxDisplay for O2Display { fn cycles(&self) -> CyclesPtr { self.cycles.clone() } + fn hw_cursor(&self) -> Option { + let (x, y, mask) = self.bus.gbe.cursor()?; + Some(crate::gfx_display::HwCursor { x, y, size: 32, mask }) + } } /// The control port: one command per line, one reply line each (`ok ...` or diff --git a/src/iris_ci_main.rs b/src/iris_ci_main.rs index 437b670c2..4e685dfc2 100644 --- a/src/iris_ci_main.rs +++ b/src/iris_ci_main.rs @@ -109,7 +109,7 @@ enum Cmd { #[arg(short = 'n', long, default_value_t = 1_000_000)] n: u64, }, - /// Save the REX3 framebuffer to a PNG. + /// Save the guest screen (any graphics board) to a PNG. Screenshot { path: PathBuf }, /// Send keystrokes to the IRIX serial console. diff --git a/src/keysym.rs b/src/keysym.rs new file mode 100644 index 000000000..316ea3358 --- /dev/null +++ b/src/keysym.rs @@ -0,0 +1,268 @@ +//! One table of the keys the host can press on the guest's keyboard: the +//! physical key (winit's `KeyCode`, what `Ps2Controller::push_kb` takes), its +//! X keysyms unshifted and shifted on a US layout, its XT (scancode set 1) +//! number as QEMU's extended key event sends it (`0xE0 nn` as `0x80 | nn`), +//! and its X keysym name. +//! +//! VNC viewers send keysyms, or with the QEMU extension the scancode of the +//! key itself, which needs no layout. The monitor's `key` command takes the +//! keysym names, a few short aliases, and winit's names (`KeyA`). + +use winit::keyboard::KeyCode; +use winit::keyboard::KeyCode::*; + +struct Key { + code: KeyCode, + xt: u16, + name: &'static str, + sym: u32, + shifted: u32, +} + +const fn k(code: KeyCode, xt: u16, name: &'static str, sym: u32, shifted: u32) -> Key { + Key { code, xt, name, sym, shifted } +} + +#[rustfmt::skip] +static KEYS: &[Key] = &[ + k(Escape, 0x01, "Escape", 0xff1b, 0), + k(Digit1, 0x02, "1", b'1' as u32, b'!' as u32), + k(Digit2, 0x03, "2", b'2' as u32, b'@' as u32), + k(Digit3, 0x04, "3", b'3' as u32, b'#' as u32), + k(Digit4, 0x05, "4", b'4' as u32, b'$' as u32), + k(Digit5, 0x06, "5", b'5' as u32, b'%' as u32), + k(Digit6, 0x07, "6", b'6' as u32, b'^' as u32), + k(Digit7, 0x08, "7", b'7' as u32, b'&' as u32), + k(Digit8, 0x09, "8", b'8' as u32, b'*' as u32), + k(Digit9, 0x0a, "9", b'9' as u32, b'(' as u32), + k(Digit0, 0x0b, "0", b'0' as u32, b')' as u32), + k(Minus, 0x0c, "minus", b'-' as u32, b'_' as u32), + k(Equal, 0x0d, "equal", b'=' as u32, b'+' as u32), + k(Backspace, 0x0e, "BackSpace", 0xff08, 0), + k(Tab, 0x0f, "Tab", 0xff09, 0xfe20), + k(KeyQ, 0x10, "q", b'q' as u32, b'Q' as u32), + k(KeyW, 0x11, "w", b'w' as u32, b'W' as u32), + k(KeyE, 0x12, "e", b'e' as u32, b'E' as u32), + k(KeyR, 0x13, "r", b'r' as u32, b'R' as u32), + k(KeyT, 0x14, "t", b't' as u32, b'T' as u32), + k(KeyY, 0x15, "y", b'y' as u32, b'Y' as u32), + k(KeyU, 0x16, "u", b'u' as u32, b'U' as u32), + k(KeyI, 0x17, "i", b'i' as u32, b'I' as u32), + k(KeyO, 0x18, "o", b'o' as u32, b'O' as u32), + k(KeyP, 0x19, "p", b'p' as u32, b'P' as u32), + k(BracketLeft, 0x1a, "bracketleft", b'[' as u32, b'{' as u32), + k(BracketRight, 0x1b, "bracketright", b']' as u32, b'}' as u32), + k(Enter, 0x1c, "Return", 0xff0d, 0), + k(ControlLeft, 0x1d, "Control_L", 0xffe3, 0), + k(KeyA, 0x1e, "a", b'a' as u32, b'A' as u32), + k(KeyS, 0x1f, "s", b's' as u32, b'S' as u32), + k(KeyD, 0x20, "d", b'd' as u32, b'D' as u32), + k(KeyF, 0x21, "f", b'f' as u32, b'F' as u32), + k(KeyG, 0x22, "g", b'g' as u32, b'G' as u32), + k(KeyH, 0x23, "h", b'h' as u32, b'H' as u32), + k(KeyJ, 0x24, "j", b'j' as u32, b'J' as u32), + k(KeyK, 0x25, "k", b'k' as u32, b'K' as u32), + k(KeyL, 0x26, "l", b'l' as u32, b'L' as u32), + k(Semicolon, 0x27, "semicolon", b';' as u32, b':' as u32), + k(Quote, 0x28, "apostrophe", b'\'' as u32, b'"' as u32), + k(Backquote, 0x29, "grave", b'`' as u32, b'~' as u32), + k(ShiftLeft, 0x2a, "Shift_L", 0xffe1, 0), + k(Backslash, 0x2b, "backslash", b'\\' as u32, b'|' as u32), + k(KeyZ, 0x2c, "z", b'z' as u32, b'Z' as u32), + k(KeyX, 0x2d, "x", b'x' as u32, b'X' as u32), + k(KeyC, 0x2e, "c", b'c' as u32, b'C' as u32), + k(KeyV, 0x2f, "v", b'v' as u32, b'V' as u32), + k(KeyB, 0x30, "b", b'b' as u32, b'B' as u32), + k(KeyN, 0x31, "n", b'n' as u32, b'N' as u32), + k(KeyM, 0x32, "m", b'm' as u32, b'M' as u32), + k(Comma, 0x33, "comma", b',' as u32, b'<' as u32), + k(Period, 0x34, "period", b'.' as u32, b'>' as u32), + k(Slash, 0x35, "slash", b'/' as u32, b'?' as u32), + k(ShiftRight, 0x36, "Shift_R", 0xffe2, 0), + k(NumpadMultiply, 0x37, "KP_Multiply", 0xffaa, 0), + k(AltLeft, 0x38, "Alt_L", 0xffe9, 0xffe7), + k(Space, 0x39, "space", b' ' as u32, 0), + k(CapsLock, 0x3a, "Caps_Lock", 0xffe5, 0), + k(F1, 0x3b, "F1", 0xffbe, 0), + k(F2, 0x3c, "F2", 0xffbf, 0), + k(F3, 0x3d, "F3", 0xffc0, 0), + k(F4, 0x3e, "F4", 0xffc1, 0), + k(F5, 0x3f, "F5", 0xffc2, 0), + k(F6, 0x40, "F6", 0xffc3, 0), + k(F7, 0x41, "F7", 0xffc4, 0), + k(F8, 0x42, "F8", 0xffc5, 0), + k(F9, 0x43, "F9", 0xffc6, 0), + k(F10, 0x44, "F10", 0xffc7, 0), + k(NumLock, 0x45, "Num_Lock", 0xff7f, 0), + k(ScrollLock, 0x46, "Scroll_Lock", 0xff14, 0), + k(Numpad7, 0x47, "KP_7", 0xffb7, 0xff95), + k(Numpad8, 0x48, "KP_8", 0xffb8, 0xff97), + k(Numpad9, 0x49, "KP_9", 0xffb9, 0xff9a), + k(NumpadSubtract, 0x4a, "KP_Subtract", 0xffad, 0), + k(Numpad4, 0x4b, "KP_4", 0xffb4, 0xff96), + k(Numpad5, 0x4c, "KP_5", 0xffb5, 0xff9d), + k(Numpad6, 0x4d, "KP_6", 0xffb6, 0xff98), + k(NumpadAdd, 0x4e, "KP_Add", 0xffab, 0), + k(Numpad1, 0x4f, "KP_1", 0xffb1, 0xff9c), + k(Numpad2, 0x50, "KP_2", 0xffb2, 0xff99), + k(Numpad3, 0x51, "KP_3", 0xffb3, 0xff9b), + k(Numpad0, 0x52, "KP_0", 0xffb0, 0xff9e), + k(NumpadDecimal, 0x53, "KP_Decimal", 0xffae, 0xff9f), + k(IntlBackslash, 0x56, "less", b'<' as u32 | 0x1_0000, 0), + k(F11, 0x57, "F11", 0xffc8, 0), + k(F12, 0x58, "F12", 0xffc9, 0), + k(NumpadEnter, 0x9c, "KP_Enter", 0xff8d, 0), + k(ControlRight, 0x9d, "Control_R", 0xffe4, 0), + k(NumpadDivide, 0xb5, "KP_Divide", 0xffaf, 0), + k(PrintScreen, 0xb7, "Print", 0xff61, 0), + k(AltRight, 0xb8, "Alt_R", 0xffea, 0xfe03), + k(Home, 0xc7, "Home", 0xff50, 0), + k(ArrowUp, 0xc8, "Up", 0xff52, 0), + k(PageUp, 0xc9, "Prior", 0xff55, 0), + k(ArrowLeft, 0xcb, "Left", 0xff51, 0), + k(ArrowRight, 0xcd, "Right", 0xff53, 0), + k(End, 0xcf, "End", 0xff57, 0), + k(ArrowDown, 0xd0, "Down", 0xff54, 0), + k(PageDown, 0xd1, "Next", 0xff56, 0), + k(Insert, 0xd2, "Insert", 0xff63, 0), + k(Delete, 0xd3, "Delete", 0xffff, 0), + k(SuperLeft, 0xdb, "Super_L", 0xffeb, 0), + k(SuperRight, 0xdc, "Super_R", 0xffec, 0), + k(ContextMenu, 0xdd, "Menu", 0xff67, 0), +]; + +/// Short names people type, beside the keysym names. +static ALIASES: &[(&str, KeyCode)] = &[ + ("ctrl", ControlLeft), ("control", ControlLeft), ("shift", ShiftLeft), + ("alt", AltLeft), ("meta", AltLeft), ("super", SuperLeft), ("win", SuperLeft), + ("enter", Enter), ("ret", Enter), ("esc", Escape), ("tab", Tab), + ("bs", Backspace), ("backspace", Backspace), ("del", Delete), ("ins", Insert), + ("pgup", PageUp), ("pageup", PageUp), ("pgdn", PageDown), ("pagedown", PageDown), + ("up", ArrowUp), ("down", ArrowDown), ("left", ArrowLeft), ("right", ArrowRight), + ("home", Home), ("end", End), ("caps", CapsLock), ("menu", ContextMenu), + ("Page_Up", PageUp), ("Page_Down", PageDown), ("KP_Separator", NumpadDecimal), +]; + +/// The physical key for an X keysym. A shifted symbol (`!`) gives the key it +/// is on (`1`); the viewer sends Shift itself. Upper and lower case letters +/// are the same key. +pub fn from_keysym(sym: u32) -> Option { + KEYS.iter().find(|e| e.sym == sym || (e.shifted != 0 && e.shifted == sym)).map(|e| e.code) +} + +/// The key that types an ASCII character on a US layout, and whether it +/// needs Shift. Newline and carriage return are Enter, tab is Tab. +pub fn char_key(ch: char) -> Option<(KeyCode, bool)> { + match ch { + '\n' | '\r' => return Some((Enter, false)), + '\t' => return Some((Tab, false)), + '\x08' => return Some((Backspace, false)), + '\x1b' => return Some((Escape, false)), + _ => {} + } + if !(' '..='~').contains(&ch) { + return None; + } + let sym = ch as u32; + KEYS.iter().find_map(|e| { + if e.sym == sym && e.xt != 0x56 { + Some((e.code, false)) + } else if e.shifted == sym { + Some((e.code, true)) + } else { + None + } + }) +} + +/// The physical key for an XT scancode as QEMU's extended key event numbers +/// it (`0xE0 nn` sent as `0x80 | nn`). +pub fn from_xt(xt: u32) -> Option { + KEYS.iter().find(|e| e.xt as u32 == xt).map(|e| e.code) +} + +/// The physical key for a name: an X keysym name (`Return`, `F1`, `a`, +/// `Control_L`), an alias (`ctrl`, `enter`, `esc`, `pgup`), a single +/// character (`a`, `5`, `/`), or a winit name (`KeyA`). Case is ignored +/// where it does not change the key. +pub fn from_name(name: &str) -> Option { + if let Some(e) = KEYS.iter().find(|e| e.name == name) { + return Some(e.code); + } + let mut chars = name.chars(); + if let (Some(c), None) = (chars.next(), chars.next()) { + if let Some(code) = from_keysym(c as u32) { + return Some(code); + } + } + if let Some(e) = KEYS.iter().find(|e| e.name.eq_ignore_ascii_case(name)) { + return Some(e.code); + } + if let Some((_, code)) = ALIASES.iter().find(|(a, _)| a.eq_ignore_ascii_case(name)) { + return Some(*code); + } + KEYS.iter().find(|e| format!("{:?}", e.code) == name).map(|e| e.code) +} + +/// Every key name `from_name` knows, for help text. +pub fn names() -> impl Iterator { + KEYS.iter().map(|e| e.name).chain(ALIASES.iter().map(|(a, _)| *a)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keysyms() { + assert_eq!(from_keysym(b'a' as u32), Some(KeyA)); + assert_eq!(from_keysym(b'A' as u32), Some(KeyA)); + assert_eq!(from_keysym(b'!' as u32), Some(Digit1)); + assert_eq!(from_keysym(0xff0d), Some(Enter)); + assert_eq!(from_keysym(0xffe3), Some(ControlLeft)); + assert_eq!(from_keysym(0x1234), None); + } + + #[test] + fn characters() { + assert_eq!(char_key('a'), Some((KeyA, false))); + assert_eq!(char_key('A'), Some((KeyA, true))); + assert_eq!(char_key('~'), Some((Backquote, true))); + assert_eq!(char_key('|'), Some((Backslash, true))); + assert_eq!(char_key('<'), Some((Comma, true))); + assert_eq!(char_key(' '), Some((Space, false))); + assert_eq!(char_key('\n'), Some((Enter, false))); + for c in ' '..='~' { + assert!(char_key(c).is_some(), "{c:?}"); + } + } + + #[test] + fn scancodes() { + assert_eq!(from_xt(0x1e), Some(KeyA)); + assert_eq!(from_xt(0xc8), Some(ArrowUp)); + assert_eq!(from_xt(0x9d), Some(ControlRight)); + } + + #[test] + fn names() { + assert_eq!(from_name("Return"), Some(Enter)); + assert_eq!(from_name("return"), Some(Enter)); + assert_eq!(from_name("ctrl"), Some(ControlLeft)); + assert_eq!(from_name("F12"), Some(F12)); + assert_eq!(from_name("KeyQ"), Some(KeyQ)); + assert_eq!(from_name("/"), Some(Slash)); + assert_eq!(from_name("A"), Some(KeyA)); + assert_eq!(from_name("nope"), None); + } + + #[test] + fn scancodes_unique() { + for (i, a) in KEYS.iter().enumerate() { + for b in &KEYS[i + 1..] { + assert_ne!(a.xt, b.xt, "{} and {}", a.name, b.name); + assert_ne!(a.code, b.code, "{} and {}", a.name, b.name); + } + } + } +} diff --git a/src/lib.rs b/src/lib.rs index aca7eef68..9c51549c5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -204,6 +204,9 @@ pub mod dev; pub mod gfifo; pub mod trace_writer; pub mod gfx_display; +pub mod hostinput; +pub mod keysym; +pub mod vnc; pub mod compositor; pub mod gl_compositor; pub mod headless_gl; diff --git a/src/machine.rs b/src/machine.rs index b9d90680a..f60a38ee4 100644 --- a/src/machine.rs +++ b/src/machine.rs @@ -922,7 +922,7 @@ impl Machine { let devlog = crate::devlog::init_devlog(); // Setup Monitor - let mut monitor = Monitor::new(); + let monitor = Monitor::new(); monitor.register_device(devlog.clone()); monitor.register_device(cpu.clone()); monitor.register_device(Arc::new(mc.clone())); @@ -2099,6 +2099,7 @@ impl Workstation for Machine { fn get_display(&self) -> Option> { Machine::get_display(self) } fn get_display_head1(&self) -> Option> { Machine::get_display_head1(self) } fn get_ps2(&self) -> Arc { Machine::get_ps2(self) } + fn register_monitor_device(&self, device: Arc) { self.monitor.register_device(device); } fn get_timer_manager(&self) -> Arc { Machine::get_timer_manager(self) } fn get_cpu_debug(&self) -> Arc { Machine::get_cpu_debug(self) } fn fasttick_count(&self) -> Arc { Machine::fasttick_count(self) } diff --git a/src/main.rs b/src/main.rs index 17f1c6e40..06a668a2f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -44,6 +44,7 @@ fn main() { let ci_display = cfg.ci_display; let ci_socket_path = cfg.ci_socket.clone(); let load_elf = cfg.load_elf.clone(); + let vnc = cfg.vnc.clone(); let perfetto = (cfg.perfetto.clone(), cfg.perfetto_cats.clone(), cfg.perfetto_clock.clone()); // Apply [debug] and [jitv2] from TOML (env vars still override if set externally). @@ -138,6 +139,17 @@ fn main() { None }; + // The monitor's `screen`, `ptr` and `key` commands, and the VNC server's + // hands on the guest. + let host_input = iris::hostinput::HostInput::attach(&*machine); + let _vnc_server = vnc.listen.as_deref().map(|listen| { + iris::vnc::VncServer::start(host_input.clone(), listen, vnc.max_fps, "IRIS") + .unwrap_or_else(|e| { + eprintln!("iris: {e}"); + std::process::exit(1); + }) + }); + // The terminal console talks to this process's own monitor only. Without // one (its port taken by another iris), there is no console: attached to // the other iris's monitor, it would exit along with that iris. diff --git a/src/monitor.rs b/src/monitor.rs index b2e8ed349..987dcce4b 100644 --- a/src/monitor.rs +++ b/src/monitor.rs @@ -45,7 +45,7 @@ impl Monitor { *self.bound.lock() } - pub fn register_device(&mut self, device: Arc) { + pub fn register_device(&self, device: Arc) { self.devices.lock().push(device); } diff --git a/src/ui.rs b/src/ui.rs index b23b8877e..cbc412a0b 100644 --- a/src/ui.rs +++ b/src/ui.rs @@ -631,7 +631,11 @@ impl Renderer for GlRenderer { // the GL framebuffer back would only replace it (it came back black). if need_readback && !screen.prebuilt { if let Some(pixels) = self.compositor.read_pixels() { - screen.rgba.copy_from_slice(pixels); + // Visible rows only: VNC capture reads every frame back. + for y in 0..height { + let row = y * crate::disp::FB_STRIDE; + screen.rgba[row..row + width].copy_from_slice(&pixels[row..row + width]); + } } else { self.compositor.readback_to_screen(&mut screen.rgba, width, height, gl); } @@ -721,6 +725,10 @@ impl Renderer for GlRenderer { self.current_win_h = 0; } + fn gl_compositor(&self) -> bool { + self.pending_compositor.unwrap_or(self.use_gl_compositor) + } + fn compositor_status(&self) -> String { let comp = if self.use_gl_compositor { "gl" } else { "sw" }; format!("compositor={} shader=integer+fallback", comp) diff --git a/src/vnc/encode.rs b/src/vnc/encode.rs new file mode 100644 index 000000000..907b2bdf8 --- /dev/null +++ b/src/vnc/encode.rs @@ -0,0 +1,448 @@ +//! RFB pixel formats and rectangle encodings: Raw and ZRLE. +//! +//! Frames arrive as `0xAABBGGRR` (R in the low byte), the compositors' +//! format. Our own format is 32 bpp little-endian with red at shift 0, so +//! Raw to a viewer that keeps it is a straight copy. + +use flate2::{Compress, Compression, FlushCompress}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PixelFormat { + pub bpp: u8, + pub depth: u8, + pub big_endian: bool, + pub true_colour: bool, + pub max: [u16; 3], + pub shift: [u8; 3], +} + +impl PixelFormat { + /// What the server offers: the compositors' own layout. + pub const NATIVE: PixelFormat = PixelFormat { + bpp: 32, depth: 24, big_endian: false, true_colour: true, + max: [255, 255, 255], shift: [0, 8, 16], + }; + + pub fn to_bytes(&self) -> [u8; 16] { + let mut b = [0u8; 16]; + b[0] = self.bpp; + b[1] = self.depth; + b[2] = self.big_endian as u8; + b[3] = self.true_colour as u8; + for i in 0..3 { + b[4 + 2 * i..6 + 2 * i].copy_from_slice(&self.max[i].to_be_bytes()); + b[10 + i] = self.shift[i]; + } + b + } + + pub fn from_bytes(b: &[u8; 16]) -> Result { + let pf = PixelFormat { + bpp: b[0], + depth: b[1], + big_endian: b[2] != 0, + true_colour: b[3] != 0, + max: [u16::from_be_bytes([b[4], b[5]]), u16::from_be_bytes([b[6], b[7]]), u16::from_be_bytes([b[8], b[9]])], + shift: [b[10], b[11], b[12]], + }; + if !matches!(pf.bpp, 8 | 16 | 32) { + return Err(format!("{} bits per pixel", pf.bpp)); + } + if !pf.true_colour { + return Err("colour-map pixel formats are not supported".into()); + } + if pf.max.iter().any(|&m| m == 0 || m & (m + 1) != 0) { + return Err(format!("colour maxima {:?} are not 2^n-1", pf.max)); + } + Ok(pf) + } + + fn bytes(&self) -> usize { + self.bpp as usize / 8 + } + + /// A compositor pixel in this format. + #[inline] + pub fn convert(&self, p: u32) -> u32 { + if *self == Self::NATIVE { + return p & 0x00ff_ffff; + } + let c = [p & 0xff, (p >> 8) & 0xff, (p >> 16) & 0xff]; + let mut v = 0u32; + for i in 0..3 { + let m = self.max[i] as u32; + v |= ((c[i] * m + 127) / 255) << self.shift[i]; + } + v + } + + #[inline] + fn put(&self, v: u32, out: &mut Vec) { + match (self.bytes(), self.big_endian) { + (4, false) => out.extend_from_slice(&v.to_le_bytes()), + (4, true) => out.extend_from_slice(&v.to_be_bytes()), + (2, false) => out.extend_from_slice(&(v as u16).to_le_bytes()), + (2, true) => out.extend_from_slice(&(v as u16).to_be_bytes()), + _ => out.push(v as u8), + } + } + + /// ZRLE's compact pixel: 32 bpp formats whose colour fits in three bytes + /// send only those three. + fn cpixel(&self) -> CPixel { + if self.bpp != 32 || self.depth > 24 { + return CPixel::Full; + } + let used = (0..3).fold(0u32, |m, i| m | (self.max[i] as u32) << self.shift[i]); + // Byte offsets in the pixel as sent. + let (low3, high3) = if self.big_endian { (1, 0) } else { (0, 1) }; + if used & 0xff00_0000 == 0 { + CPixel::Three(low3) + } else if used & 0xff == 0 { + CPixel::Three(high3) + } else { + CPixel::Full + } + } +} + +#[derive(Clone, Copy)] +enum CPixel { + Full, + /// Three bytes starting at this offset of the full pixel. + Three(usize), +} + +impl CPixel { + fn len(self, pf: &PixelFormat) -> usize { + match self { + CPixel::Full => pf.bytes(), + CPixel::Three(_) => 3, + } + } + #[inline] + fn put(self, pf: &PixelFormat, v: u32, out: &mut Vec) { + match self { + CPixel::Full => pf.put(v, out), + CPixel::Three(o) => { + let b = if pf.big_endian { v.to_be_bytes() } else { v.to_le_bytes() }; + out.extend_from_slice(&b[o..o + 3]); + } + } + } +} + +/// A rectangle of a tight frame `px` (row length `stride`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Rect { + pub x: usize, + pub y: usize, + pub w: usize, + pub h: usize, +} + +pub fn raw(pf: &PixelFormat, px: &[u32], stride: usize, r: Rect, out: &mut Vec) { + out.reserve(r.w * r.h * pf.bytes()); + for y in r.y..r.y + r.h { + for &p in &px[y * stride + r.x..y * stride + r.x + r.w] { + pf.put(pf.convert(p), out); + } + } +} + +/// One viewer's ZRLE state: its zlib stream runs for the whole connection. +pub struct Zrle { + z: Compress, + tile: Vec, + plain: Vec, +} + +impl Zrle { + pub fn new() -> Self { + Self { z: Compress::new(Compression::new(3), true), tile: Vec::new(), plain: Vec::new() } + } + + /// The rectangle's ZRLE data (length word and zlib bytes) onto `out`. + pub fn encode(&mut self, pf: &PixelFormat, px: &[u32], stride: usize, r: Rect, out: &mut Vec) -> Result<(), String> { + let cp = pf.cpixel(); + self.plain.clear(); + for ty in (r.y..r.y + r.h).step_by(64) { + let th = 64.min(r.y + r.h - ty); + for tx in (r.x..r.x + r.w).step_by(64) { + let tw = 64.min(r.x + r.w - tx); + self.tile.clear(); + for y in ty..ty + th { + self.tile.extend(px[y * stride + tx..y * stride + tx + tw].iter().map(|&p| pf.convert(p))); + } + encode_tile(pf, cp, &self.tile, tw, &mut self.plain); + } + } + // Deflate with a sync flush so the viewer can decode this rectangle + // now; worst case a little over the input. + let len_at = out.len(); + out.extend_from_slice(&[0; 4]); + let start = out.len(); + let mut input = &self.plain[..]; + loop { + out.reserve(input.len() / 2 + 1024); + let before_in = self.z.total_in(); + self.z.compress_vec(input, out, FlushCompress::Sync).map_err(|e| e.to_string())?; + let used = (self.z.total_in() - before_in) as usize; + input = &input[used..]; + // Done when all input went in and zlib had room to spare. + if input.is_empty() && out.len() < out.capacity() { + break; + } + } + let n = (out.len() - start) as u32; + out[len_at..len_at + 4].copy_from_slice(&n.to_be_bytes()); + Ok(()) + } +} + +fn run_length(mut n: usize, out: &mut Vec) { + n -= 1; + while n >= 255 { + out.push(255); + n -= 255; + } + out.push(n as u8); +} + +fn run_length_size(n: usize) -> usize { + (n - 1) / 255 + 1 +} + +/// One ZRLE tile, whichever subencoding comes out smallest. +fn encode_tile(pf: &PixelFormat, cp: CPixel, t: &[u32], tw: usize, out: &mut Vec) { + let cpl = cp.len(pf); + let n = t.len(); + let th = n / tw; + + // Palette (up to 127 colours) and runs. + let mut pal: Vec = Vec::with_capacity(16); + let mut runs = 0usize; + let mut plain_rle = 0usize; + let mut pal_rle = 0usize; + let mut i = 0; + while i < n { + let v = t[i]; + let mut j = i + 1; + while j < n && t[j] == v { + j += 1; + } + let len = j - i; + runs += 1; + plain_rle += cpl + run_length_size(len); + pal_rle += if len == 1 { 1 } else { 1 + run_length_size(len) }; + if pal.len() <= 127 && !pal.contains(&v) { + pal.push(v); + } + i = j; + } + let _ = runs; + + if pal.len() == 1 { + out.push(1); + cp.put(pf, pal[0], out); + return; + } + let raw_size = n * cpl; + let packed_bits = match pal.len() { 2 => 1, 3..=4 => 2, 5..=16 => 4, _ => 0 }; + let packed_size = if packed_bits > 0 { pal.len() * cpl + th * (tw * packed_bits).div_ceil(8) } else { usize::MAX }; + let pal_rle_size = if pal.len() <= 127 { pal.len() * cpl + pal_rle } else { usize::MAX }; + let best = raw_size.min(packed_size).min(plain_rle).min(pal_rle_size); + let index = |v: u32| pal.iter().position(|&p| p == v).unwrap() as u8; + + if best == packed_size { + out.push(pal.len() as u8); + for &p in &pal { + cp.put(pf, p, out); + } + for row in t.chunks(tw) { + let (mut byte, mut nbits) = (0u8, 0); + for &v in row { + byte = (byte << packed_bits) | index(v); + nbits += packed_bits; + if nbits == 8 { + out.push(byte); + (byte, nbits) = (0, 0); + } + } + if nbits > 0 { + out.push(byte << (8 - nbits)); + } + } + } else if best == pal_rle_size { + out.push(128 + pal.len() as u8); + for &p in &pal { + cp.put(pf, p, out); + } + for_runs(t, |v, len| { + if len == 1 { + out.push(index(v)); + } else { + out.push(index(v) | 128); + run_length(len, out); + } + }); + } else if best == plain_rle { + out.push(128); + for_runs(t, |v, len| { + cp.put(pf, v, out); + run_length(len, out); + }); + } else { + out.push(0); + for &v in t { + cp.put(pf, v, out); + } + } +} + +fn for_runs(t: &[u32], mut f: impl FnMut(u32, usize)) { + let mut i = 0; + while i < t.len() { + let v = t[i]; + let mut j = i + 1; + while j < t.len() && t[j] == v { + j += 1; + } + f(v, j - i); + i = j; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use flate2::{Decompress, FlushDecompress}; + + /// Decode one ZRLE rectangle back to pixels (format values). + fn unzrle(pf: &PixelFormat, z: &mut Decompress, data: &[u8], r: Rect) -> Vec { + let n = u32::from_be_bytes(data[..4].try_into().unwrap()) as usize; + assert_eq!(n + 4, data.len()); + let mut plain = Vec::with_capacity(1 << 20); + z.decompress_vec(&data[4..], &mut plain, FlushDecompress::Sync).unwrap(); + let cp = pf.cpixel(); + let cpl = cp.len(pf); + let rd = |b: &[u8]| -> u32 { + let mut full = [0u8; 4]; + match cp { + CPixel::Full => full[..cpl].copy_from_slice(b), + CPixel::Three(o) => full[o..o + 3].copy_from_slice(b), + } + if pf.big_endian { u32::from_be_bytes(full) } else { u32::from_le_bytes(full) } + }; + let mut out = vec![0u32; r.w * r.h]; + let mut p = 0; + for ty in (0..r.h).step_by(64) { + let th = 64.min(r.h - ty); + for tx in (0..r.w).step_by(64) { + let tw = 64.min(r.w - tx); + let sub = plain[p]; + p += 1; + let mut tile = Vec::new(); + let mut pal = Vec::new(); + let pal_n = match sub { 2..=16 => sub as usize, 130..=255 => sub as usize - 128, _ => 0 }; + for _ in 0..pal_n { + pal.push(rd(&plain[p..p + cpl])); + p += cpl; + } + let rl = |p: &mut usize| { let mut n = 1; loop { let b = plain[*p]; *p += 1; n += b as usize; if b != 255 { return n; } } }; + match sub { + 0 => for _ in 0..tw * th { tile.push(rd(&plain[p..p + cpl])); p += cpl; }, + 1 => { let v = rd(&plain[p..p + cpl]); p += cpl; tile = vec![v; tw * th]; } + 2..=16 => { + let bits = match sub { 2 => 1, 3 | 4 => 2, _ => 4 }; + for _ in 0..th { + let mut taken = 0; + let row_bytes = (tw * bits).div_ceil(8); + for x in 0..tw { + let bit = x * bits; + let b = plain[p + bit / 8]; + let idx = (b >> (8 - bits - bit % 8)) & ((1 << bits) - 1); + tile.push(pal[idx as usize]); + taken = row_bytes; + } + p += taken; + } + } + 128 => while tile.len() < tw * th { let v = rd(&plain[p..p + cpl]); p += cpl; let n = rl(&mut p); tile.extend(std::iter::repeat_n(v, n)); }, + 130..=255 => while tile.len() < tw * th { + let b = plain[p]; p += 1; + let v = pal[(b & 127) as usize]; + let n = if b & 128 != 0 { rl(&mut p) } else { 1 }; + tile.extend(std::iter::repeat_n(v, n)); + }, + s => panic!("subencoding {s}"), + } + assert_eq!(tile.len(), tw * th); + for y in 0..th { + out[(ty + y) * r.w + tx..(ty + y) * r.w + tx + tw].copy_from_slice(&tile[y * tw..(y + 1) * tw]); + } + } + } + assert_eq!(p, plain.len()); + out + } + + fn picture(w: usize, h: usize) -> Vec { + let mut px = vec![0xff20_4060u32; w * h]; + for y in 0..h { + for x in 0..w { + px[y * w + x] = match (x / 40, y / 30) { + (0, _) => 0xff00_0000 | ((x * 7 + y * 13) as u32 * 2_654_435_761) & 0xff_ffff, // noise + (1, _) => if (x + y) % 2 == 0 { 0xffff_ffff } else { 0xff00_0000 }, // 2 colours + (2, _) => [0xff11_2233, 0xff44_5566, 0xff77_8899][(x / 3 + y) % 3], // 3 colours + (3, _) => 0xff00_0000 | ((x / 10) as u32 * 0x30_1020), // runs + _ => 0xffab_cdef, // solid + } + } + } + px + } + + #[test] + fn zrle_round_trips() { + let (w, h) = (230, 150); + let px = picture(w, h); + let formats = [ + PixelFormat::NATIVE, + PixelFormat { big_endian: true, shift: [16, 8, 0], ..PixelFormat::NATIVE }, + PixelFormat { shift: [24, 16, 8], depth: 24, ..PixelFormat::NATIVE }, + PixelFormat { bpp: 16, depth: 16, big_endian: false, true_colour: true, max: [31, 63, 31], shift: [11, 5, 0] }, + PixelFormat { bpp: 8, depth: 8, big_endian: false, true_colour: true, max: [7, 7, 3], shift: [0, 3, 6] }, + ]; + for pf in formats { + let mut enc = Zrle::new(); + let mut dec = Decompress::new(true); + // Two rectangles on one stream, as a connection sends them. + for r in [Rect { x: 0, y: 0, w, h }, Rect { x: 17, y: 9, w: 150, h: 70 }] { + let mut out = Vec::new(); + enc.encode(&pf, &px, w, r, &mut out).unwrap(); + let got = unzrle(&pf, &mut dec, &out, r); + for y in 0..r.h { + for x in 0..r.w { + assert_eq!(got[y * r.w + x], pf.convert(px[(r.y + y) * w + r.x + x]), "{pf:?} at {x},{y}"); + } + } + } + } + } + + #[test] + fn raw_native_is_a_copy() { + let px = vec![0xff01_0203u32, 0xff04_0506]; + let mut out = Vec::new(); + raw(&PixelFormat::NATIVE, &px, 2, Rect { x: 0, y: 0, w: 2, h: 1 }, &mut out); + assert_eq!(out, [3, 2, 1, 0, 6, 5, 4, 0]); + } + + #[test] + fn convert_16bpp() { + let pf = PixelFormat { bpp: 16, depth: 16, big_endian: false, true_colour: true, max: [31, 63, 31], shift: [11, 5, 0] }; + assert_eq!(pf.convert(0xff00_00ff), 31 << 11); // red + assert_eq!(pf.convert(0xffff_0000), 31); // blue + } +} diff --git a/src/vnc/mod.rs b/src/vnc/mod.rs new file mode 100644 index 000000000..0df1fe995 --- /dev/null +++ b/src/vnc/mod.rs @@ -0,0 +1,318 @@ +//! The built-in VNC (RFB 3.8) server. +//! +//! It serves whatever the board composes, for every board, and knows nothing +//! about drawing: a grabber thread copies the composed frame (`screen.rgba`) +//! when its sequence number moves, compares it with the last one in 64x64 +//! tiles, and stamps each changed tile with the frame's generation. Each +//! viewer remembers the generation it last saw and is sent the tiles newer +//! than that, so a slow viewer skips frames instead of queueing them. +//! +//! Input goes to the guest's PS/2 keyboard and mouse through +//! [`HostInput`]: absolute pointer positions are steered onto the hardware +//! cursor. A window, the monitor and VNC viewers can all type at once. +//! +//! Threads: one accept thread, one grabber, one pointer driver, and two per +//! viewer (reading its messages; sending its updates). All std::net. + +pub mod encode; +mod session; + +use std::collections::VecDeque; +use std::net::{SocketAddr, TcpListener}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use parking_lot::{Condvar, Mutex}; + +use crate::gfx_display::GfxDisplay; +use crate::hostinput::{HostInput, MoveResult}; + +/// Tile side for damage tracking. +pub const TILE: usize = 64; +/// Viewers served at once. +const MAX_CLIENTS: usize = 8; + +/// The last frame and which of its tiles changed when. +#[derive(Clone)] +pub struct Frame { + pub w: usize, + pub h: usize, + pub px: Arc>, + /// Per tile (row-major, `tiles_x` a row), the generation it last changed in. + pub tile_gen: Arc>, + pub tiles_x: usize, + /// Generation of this frame; bumped when any tile changes. + pub gen: u64, + /// Bumped when the size changes. + pub size_gen: u64, +} + +pub(crate) struct Shared { + pub input: Arc, + display: Arc, + pub frame: Mutex, + /// Signalled on a new frame, and by viewers' readers when they want + /// their sender to look again. + pub changed: Condvar, + pub clients: AtomicUsize, + pub stop: AtomicBool, + max_fps: u32, + pub pointer: PointerDriver, + pub name: String, +} + +pub struct VncServer { + shared: Arc, + addr: SocketAddr, +} + +impl VncServer { + /// Bind `listen` and start serving `input`'s display. + pub fn start(input: Arc, listen: &str, max_fps: Option, name: &str) -> Result { + let display = input.display().cloned() + .ok_or("VNC needs a display: this machine runs without one (--headless, or no graphics board)")?; + let listen = if listen.parse::().is_ok() { format!("127.0.0.1:{listen}") } else { listen.to_string() }; + let listener = TcpListener::bind(&listen).map_err(|e| format!("VNC: bind {listen}: {e}"))?; + let addr = listener.local_addr().map_err(|e| e.to_string())?; + if !addr.ip().is_loopback() { + eprintln!("iris: WARNING: the VNC server on {addr} has no password: anyone who can reach it controls the guest"); + } + let shared = Arc::new(Shared { + input: input.clone(), + display, + frame: Mutex::new(Frame { + w: 0, h: 0, px: Arc::new(Vec::new()), tile_gen: Arc::new(Vec::new()), + tiles_x: 0, gen: 0, size_gen: 0, + }), + changed: Condvar::new(), + clients: AtomicUsize::new(0), + stop: AtomicBool::new(false), + max_fps: max_fps.unwrap_or(30).clamp(1, 120), + pointer: PointerDriver::new(), + name: name.to_string(), + }); + { + let s = shared.clone(); + std::thread::Builder::new().name("VNC-Grab".into()).spawn(move || grab_loop(&s)).unwrap(); + } + { + let s = shared.clone(); + std::thread::Builder::new().name("VNC-Pointer".into()).spawn(move || s.pointer.run(&s.input, &s.stop)).unwrap(); + } + { + let s = shared.clone(); + std::thread::Builder::new().name("VNC-Accept".into()).spawn(move || { + for conn in listener.incoming() { + if s.stop.load(Ordering::Relaxed) { + break; + } + let Ok(stream) = conn else { continue }; + if s.clients.load(Ordering::Relaxed) >= MAX_CLIENTS { + drop(stream); + continue; + } + let s = s.clone(); + let _ = std::thread::Builder::new().name("VNC-Client".into()).spawn(move || { + let peer = stream.peer_addr().map(|a| a.to_string()).unwrap_or_default(); + s.clients.fetch_add(1, Ordering::SeqCst); + s.changed.notify_all(); + let r = session::serve(&s, stream); + s.clients.fetch_sub(1, Ordering::SeqCst); + if let Err(e) = r { + eprintln!("iris: VNC {peer}: {e}"); + } + }); + } + }).unwrap(); + } + eprintln!("iris: VNC server on {addr}"); + Ok(VncServer { shared, addr }) + } + + pub fn addr(&self) -> SocketAddr { + self.addr + } + + pub fn clients(&self) -> usize { + self.shared.clients.load(Ordering::Relaxed) + } +} + +impl Drop for VncServer { + fn drop(&mut self) { + self.shared.stop.store(true, Ordering::SeqCst); + self.shared.changed.notify_all(); + let _ = std::net::TcpStream::connect(self.addr); // wake accept() + } +} + +/// Follow the composed frame while anyone watches. +fn grab_loop(s: &Shared) { + let period = Duration::from_micros(1_000_000 / s.max_fps as u64); + let mut capturing = false; + let mut last_seq = None; + let mut cur = Vec::new(); + while !s.stop.load(Ordering::Relaxed) { + let start = Instant::now(); + let watching = s.clients.load(Ordering::Relaxed) > 0; + if watching != capturing { + s.display.set_capture(watching); + capturing = watching; + last_seq = None; + } + if !watching { + // Sleep until a viewer connects (or a poll period passes). + let mut f = s.frame.lock(); + s.changed.wait_for(&mut f, Duration::from_millis(200)); + continue; + } + let seq = s.display.screen().lock().frame_seq; + if last_seq != Some(seq) { + if let Some((w, h, seq)) = crate::gfx_display::copy_frame(&*s.display, &mut cur) { + last_seq = Some(seq); + publish(s, w, h, &mut cur); + } + } + let spent = start.elapsed(); + if spent < period { + std::thread::sleep(period - spent); + } + } + if capturing { + s.display.set_capture(false); + } +} + +/// Compare `cur` with the published frame and publish it if anything moved. +/// `cur` comes back holding a spare buffer. +fn publish(s: &Shared, w: usize, h: usize, cur: &mut Vec) { + let old = s.frame.lock().clone(); + let tiles_x = w.div_ceil(TILE); + let tiles_y = h.div_ceil(TILE); + let resized = (old.w, old.h) != (w, h); + let mut dirty = vec![resized; tiles_x * tiles_y]; + if !resized { + let prev = &old.px; + for ty in 0..tiles_y { + let row_tiles = &mut dirty[ty * tiles_x..(ty + 1) * tiles_x]; + for y in ty * TILE..(ty * TILE + TILE).min(h) { + let (a, b) = (&cur[y * w..(y + 1) * w], &prev[y * w..(y + 1) * w]); + if a == b { + continue; + } + for (tx, d) in row_tiles.iter_mut().enumerate() { + if !*d { + let x0 = tx * TILE; + let x1 = (x0 + TILE).min(w); + *d = a[x0..x1] != b[x0..x1]; + } + } + } + } + if !dirty.iter().any(|&d| d) { + return; + } + } + let gen = old.gen + 1; + let tile_gen: Vec = if resized { + vec![gen; tiles_x * tiles_y] + } else { + old.tile_gen.iter().zip(&dirty).map(|(&g, &d)| if d { gen } else { g }).collect() + }; + let px = Arc::new(std::mem::take(cur)); + { + let mut f = s.frame.lock(); + *f = Frame { + w, h, px, tile_gen: Arc::new(tile_gen), tiles_x, gen, + size_gen: old.size_gen + resized as u64, + }; + } + // Reuse the frame before last when no viewer still holds it. + if let Ok(v) = Arc::try_unwrap(old.px) { + *cur = v; + } + s.changed.notify_all(); +} + +/// Turns viewers' pointer events into steered moves and button changes, in +/// order, on one thread: a click is sent once the pointer is where the +/// viewer clicked. +pub(crate) struct PointerDriver { + queue: Mutex>, + ready: Condvar, +} + +impl PointerDriver { + fn new() -> Self { + Self { queue: Mutex::new(VecDeque::new()), ready: Condvar::new() } + } + + /// A viewer's pointer event: position and PS/2 buttons. + pub fn push(&self, x: i32, y: i32, buttons: u8) { + let mut q = self.queue.lock(); + // Moves with no button change only matter as the latest target. + if let Some(last) = q.back_mut() { + if last.2 == buttons && q.len() > 1 && q[q.len() - 2].2 == buttons { + *q.back_mut().unwrap() = (x, y, buttons); + drop(q); + self.ready.notify_one(); + return; + } + } + if q.len() < 1024 { + q.push_back((x, y, buttons)); + } + drop(q); + self.ready.notify_one(); + } + + fn run(&self, input: &HostInput, stop: &AtomicBool) { + // Where the viewer last put the pointer: for moving by difference + // when there is no cursor to steer by. + let mut last: Option<(i32, i32)> = None; + while !stop.load(Ordering::Relaxed) { + let (x, y, buttons) = { + let mut q = self.queue.lock(); + if q.is_empty() { + self.ready.wait_for(&mut q, Duration::from_millis(200)); + } + match q.pop_front() { + Some(e) => e, + None => continue, + } + }; + let held = input.buttons(); + // Take newer moves with the same buttons as the new target; not + // while a press or release waits for this move to arrive. + let mut target = (x, y); + let mut retarget = || { + let mut q = self.queue.lock(); + match q.front() { + Some(&(nx, ny, b)) if b == buttons && buttons == held => { + q.pop_front(); + target = (nx, ny); + Some((nx, ny)) + } + _ => None, + } + }; + let steered = input.move_to(x, y, Duration::from_millis(400), &mut retarget); + let target = target; + match steered { + Ok(MoveResult::Reached) | Ok(MoveResult::Short(..)) => {} + Err(_) => { + // No cursor to steer by (the X server is not up, or a + // program hid it): move by the viewer's difference. + if let Some((lx, ly)) = last { + input.nudge(target.0 - lx, target.1 - ly); + } + } + } + last = Some(target); + if buttons != held { + input.set_buttons(buttons); + } + } + } +} diff --git a/src/vnc/session.rs b/src/vnc/session.rs new file mode 100644 index 000000000..7434c8411 --- /dev/null +++ b/src/vnc/session.rs @@ -0,0 +1,402 @@ +//! One viewer: the RFB handshake, its messages in, its updates out. + +use std::collections::HashSet; +use std::io::{Read, Write}; +use std::net::{Shutdown, TcpStream}; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use parking_lot::Mutex; +use winit::keyboard::KeyCode; + +use super::encode::{self, PixelFormat, Rect}; +use super::{Frame, Shared, TILE}; +use crate::hostinput::{BUTTON_LEFT, BUTTON_MIDDLE, BUTTON_RIGHT}; + +const ENC_RAW: i32 = 0; +const ENC_ZRLE: i32 = 16; +const ENC_DESKTOP_SIZE: i32 = -223; +const ENC_QEMU_EXT_KEY: i32 = -258; + +#[derive(Clone, Copy)] +struct Request { + incremental: bool, + rect: Rect, +} + +struct Client { + pf: PixelFormat, + zrle: bool, + desktop_size: bool, + ext_key: bool, + ext_key_acked: bool, + request: Option, + seen_gen: u64, + size_gen: u64, + dead: Option, + held_keys: HashSet, + /// Viewer buttons (RFB bits) last seen, for wheel edges. + vnc_buttons: u8, +} + +pub(crate) fn serve(s: &Arc, mut stream: TcpStream) -> Result<(), String> { + stream.set_nodelay(true).ok(); + stream.set_read_timeout(Some(Duration::from_secs(10))).ok(); + let io = |e: std::io::Error| e.to_string(); + + // Version, security (None), ClientInit. + stream.write_all(b"RFB 003.008\n").map_err(io)?; + let mut ver = [0u8; 12]; + stream.read_exact(&mut ver).map_err(io)?; + let minor = match &ver { + b"RFB 003.003\n" => 3, + b"RFB 003.007\n" => 7, + v if v.starts_with(b"RFB 003.") => 8, + _ => return Err(format!("not an RFB viewer ({:?})", String::from_utf8_lossy(&ver))), + }; + if minor == 3 { + stream.write_all(&1u32.to_be_bytes()).map_err(io)?; + } else { + stream.write_all(&[1, 1]).map_err(io)?; + let mut choice = [0u8]; + stream.read_exact(&mut choice).map_err(io)?; + if choice[0] != 1 { + return Err(format!("viewer chose security type {}", choice[0])); + } + if minor >= 8 { + stream.write_all(&0u32.to_be_bytes()).map_err(io)?; + } + } + let mut shared_flag = [0u8]; + stream.read_exact(&mut shared_flag).map_err(io)?; + + // The size to announce: wait a little for the first frame. + let frame = { + let deadline = Instant::now() + Duration::from_secs(3); + let mut f = s.frame.lock(); + while f.w == 0 && Instant::now() < deadline { + s.changed.wait_for(&mut f, Duration::from_millis(100)); + } + f.clone() + }; + let (w, h) = if frame.w > 0 { (frame.w, frame.h) } else { (1024, 768) }; + let mut init = Vec::new(); + init.extend_from_slice(&(w as u16).to_be_bytes()); + init.extend_from_slice(&(h as u16).to_be_bytes()); + init.extend_from_slice(&PixelFormat::NATIVE.to_bytes()); + init.extend_from_slice(&(s.name.len() as u32).to_be_bytes()); + init.extend_from_slice(s.name.as_bytes()); + stream.write_all(&init).map_err(io)?; + stream.set_read_timeout(None).ok(); + + let client = Arc::new(Mutex::new(Client { + pf: PixelFormat::NATIVE, + zrle: false, + desktop_size: false, + ext_key: false, + ext_key_acked: false, + request: None, + seen_gen: 0, + // A placeholder size is a size change still to send. + size_gen: if frame.w > 0 { frame.size_gen } else { u64::MAX }, + dead: None, + held_keys: HashSet::new(), + vnc_buttons: 0, + })); + + let reader = { + let (s, client) = (s.clone(), client.clone()); + let mut rd = stream.try_clone().map_err(io)?; + std::thread::Builder::new().name("VNC-Read".into()).spawn(move || { + let why = read_loop(&s, &client, &mut rd).err().unwrap_or_else(|| "closed".into()); + client.lock().dead = Some(why); + s.changed.notify_all(); + }).map_err(io)? + }; + + let result = send_loop(s, &client, &mut stream); + let _ = stream.shutdown(Shutdown::Both); + let _ = reader.join(); + + // Let go of whatever this viewer was holding. + let c = client.lock(); + for &k in &c.held_keys { + s.input.key(k, false); + } + if c.vnc_buttons & 7 != 0 { + s.input.set_buttons(0); + } + match result { + Err(e) => Err(e), + Ok(()) => match &c.dead { + Some(why) if why != "closed" => Err(why.clone()), + _ => Ok(()), + }, + } +} + +fn read_loop(s: &Shared, client: &Mutex, rd: &mut TcpStream) -> Result<(), String> { + let mut b = [0u8; 20]; + loop { + let mut ty = [0u8]; + if rd.read(&mut ty).map_err(|e| e.to_string())? == 0 { + return Ok(()); + } + match ty[0] { + 0 => { + let m = take(rd, &mut b, 19)?; + let pf = PixelFormat::from_bytes(m[3..19].try_into().unwrap())?; + client.lock().pf = pf; + } + 2 => { + let m = take(rd, &mut b, 3)?; + let n = u16::from_be_bytes([m[1], m[2]]) as usize; + let mut list = vec![0u8; n * 4]; + rd.read_exact(&mut list).map_err(|e| e.to_string())?; + let encs: Vec = list.chunks(4).map(|c| i32::from_be_bytes(c.try_into().unwrap())).collect(); + let mut c = client.lock(); + c.zrle = encs.contains(&ENC_ZRLE); + c.desktop_size = encs.contains(&ENC_DESKTOP_SIZE); + let ext = encs.contains(&ENC_QEMU_EXT_KEY); + if ext && !c.ext_key { + c.ext_key_acked = false; + } + c.ext_key = ext; + } + 3 => { + let m = take(rd, &mut b, 9)?; + let v = |i: usize| u16::from_be_bytes([m[i], m[i + 1]]) as usize; + client.lock().request = Some(Request { + incremental: m[0] != 0, + rect: Rect { x: v(1), y: v(3), w: v(5), h: v(7) }, + }); + s.changed.notify_all(); + } + 4 => { + let m = take(rd, &mut b, 7)?; + let sym = u32::from_be_bytes(m[3..7].try_into().unwrap()); + key(s, client, m[0] != 0, crate::keysym::from_keysym(sym)); + } + 5 => { + let m = take(rd, &mut b, 5)?; + let (mask, x, y) = (m[0], u16::from_be_bytes([m[1], m[2]]) as i32, u16::from_be_bytes([m[3], m[4]]) as i32); + let rising = { + let mut c = client.lock(); + let r = mask & !c.vnc_buttons; + c.vnc_buttons = mask; + r + }; + // Wheel up and down come as presses of buttons 4 and 5. + if rising & 8 != 0 { + s.input.wheel(1); + } + if rising & 16 != 0 { + s.input.wheel(-1); + } + let mut ps2 = 0; + if mask & 1 != 0 { ps2 |= BUTTON_LEFT; } + if mask & 2 != 0 { ps2 |= BUTTON_MIDDLE; } + if mask & 4 != 0 { ps2 |= BUTTON_RIGHT; } + s.pointer.push(x, y, ps2); + } + 6 => { + let m = take(rd, &mut b, 7)?; + let n = u32::from_be_bytes(m[3..7].try_into().unwrap()) as u64; + // Clipboard: not passed on. + std::io::copy(&mut Read::by_ref(rd).take(n), &mut std::io::sink()).map_err(|e| e.to_string())?; + } + 255 => { + let m = take(rd, &mut b, 1)?; + if m[0] != 0 { + return Err(format!("unknown QEMU message {}", m[0])); + } + let m = take(rd, &mut b, 10)?; + let down = u16::from_be_bytes([m[0], m[1]]) != 0; + let sym = u32::from_be_bytes(m[2..6].try_into().unwrap()); + let xt = u32::from_be_bytes(m[6..10].try_into().unwrap()); + let code = crate::keysym::from_xt(xt).or_else(|| crate::keysym::from_keysym(sym)); + key(s, client, down, code); + } + t => return Err(format!("unknown message type {t}")), + } + } +} + +fn take<'a>(rd: &mut TcpStream, b: &'a mut [u8; 20], n: usize) -> Result<&'a [u8], String> { + rd.read_exact(&mut b[..n]).map_err(|e| e.to_string())?; + Ok(&b[..n]) +} + +fn key(s: &Shared, client: &Mutex, down: bool, code: Option) { + let Some(code) = code else { return }; + let mut c = client.lock(); + if down { + c.held_keys.insert(code); + } else { + c.held_keys.remove(&code); + } + drop(c); + s.input.key(code, down); +} + +/// What to send next, or None to keep waiting. +fn next_update(f: &Frame, c: &mut Client) -> Result, Vec)>, String> { + let Some(req) = c.request else { return Ok(None) }; + if f.w == 0 { + return Ok(None); + } + let mut pseudo = Vec::new(); + if c.ext_key && !c.ext_key_acked { + pseudo.push((Rect { x: 0, y: 0, w: 0, h: 0 }, ENC_QEMU_EXT_KEY)); + c.ext_key_acked = true; + } + let full = Rect { x: 0, y: 0, w: f.w, h: f.h }; + let rects = if c.size_gen != f.size_gen { + if !c.desktop_size { + return Err(format!("the screen is now {}x{} and the viewer cannot resize: reconnect", f.w, f.h)); + } + pseudo.push((full, ENC_DESKTOP_SIZE)); + vec![full] + } else if !req.incremental { + clip(req.rect, f).into_iter().collect() + } else if f.gen > c.seen_gen { + dirty_rects(f, c.seen_gen, req.rect) + } else { + Vec::new() + }; + c.seen_gen = f.gen; + c.size_gen = f.size_gen; + if rects.is_empty() && pseudo.is_empty() { + // Nothing in the requested area changed; the request stands. + return Ok(None); + } + c.request = None; + Ok(Some((pseudo, rects))) +} + +fn clip(r: Rect, f: &Frame) -> Option { + let (x1, y1) = ((r.x + r.w).min(f.w), (r.y + r.h).min(f.h)); + (r.x < x1 && r.y < y1).then(|| Rect { x: r.x, y: r.y, w: x1 - r.x, h: y1 - r.y }) +} + +/// Tiles changed since `seen` inside `area`, merged into runs across and +/// stacked runs down. +fn dirty_rects(f: &Frame, seen: u64, area: Rect) -> Vec { + let tiles_y = f.h.div_ceil(TILE); + let mut out: Vec = Vec::new(); + // Rects that ended on the previous tile row, by index into `out`. + let mut open: Vec = Vec::new(); + for ty in 0..tiles_y { + let mut next_open = Vec::new(); + let mut tx = 0; + while tx < f.tiles_x { + if f.tile_gen[ty * f.tiles_x + tx] <= seen { + tx += 1; + continue; + } + let start = tx; + while tx < f.tiles_x && f.tile_gen[ty * f.tiles_x + tx] > seen { + tx += 1; + } + let r = Rect { + x: start * TILE, + y: ty * TILE, + w: (tx * TILE).min(f.w) - start * TILE, + h: ((ty + 1) * TILE).min(f.h) - ty * TILE, + }; + match open.iter().position(|&i| out[i].x == r.x && out[i].w == r.w) { + Some(p) => { + let i = open.remove(p); + out[i].h += r.h; + next_open.push(i); + } + None => { + out.push(r); + next_open.push(out.len() - 1); + } + } + } + open = next_open; + } + out.into_iter().filter_map(|r| { + let x0 = r.x.max(area.x); + let y0 = r.y.max(area.y); + let x1 = (r.x + r.w).min(area.x + area.w); + let y1 = (r.y + r.h).min(area.y + area.h); + (x0 < x1 && y0 < y1).then(|| Rect { x: x0, y: y0, w: x1 - x0, h: y1 - y0 }) + }).collect() +} + +fn send_loop(s: &Shared, client: &Mutex, stream: &mut TcpStream) -> Result<(), String> { + let mut zrle = encode::Zrle::new(); + let mut buf = Vec::new(); + loop { + let (frame, pseudo, rects, pf, use_zrle) = { + let mut f = s.frame.lock(); + loop { + if s.stop.load(Ordering::Relaxed) { + return Ok(()); + } + let mut c = client.lock(); + if c.dead.is_some() { + return Ok(()); + } + if let Some((pseudo, rects)) = next_update(&f, &mut c)? { + break (f.clone(), pseudo, rects, c.pf, c.zrle); + } + drop(c); + s.changed.wait_for(&mut f, Duration::from_millis(100)); + } + }; + buf.clear(); + buf.extend_from_slice(&[0, 0]); + buf.extend_from_slice(&((pseudo.len() + rects.len()) as u16).to_be_bytes()); + let header = |buf: &mut Vec, r: Rect, enc: i32| { + for v in [r.x, r.y, r.w, r.h] { + buf.extend_from_slice(&(v as u16).to_be_bytes()); + } + buf.extend_from_slice(&enc.to_be_bytes()); + }; + for &(r, enc) in &pseudo { + header(&mut buf, r, enc); + } + for &r in &rects { + if use_zrle { + header(&mut buf, r, ENC_ZRLE); + zrle.encode(&pf, &frame.px, frame.w, r, &mut buf)?; + } else { + header(&mut buf, r, ENC_RAW); + encode::raw(&pf, &frame.px, frame.w, r, &mut buf); + } + } + stream.write_all(&buf).map_err(|e| e.to_string())?; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn frame(w: usize, h: usize, tile_gen: Vec) -> Frame { + Frame { + w, h, px: Arc::new(vec![0; w * h]), tiles_x: w.div_ceil(TILE), + tile_gen: Arc::new(tile_gen), gen: 5, size_gen: 0, + } + } + + #[test] + fn dirty_tiles_merge() { + // 3x2 tiles, 150x100: a 2-tile run on both rows (stacks), and one tile. + let f = frame(150, 100, vec![5, 5, 0, 5, 5, 4]); + let all = Rect { x: 0, y: 0, w: 150, h: 100 }; + let r = dirty_rects(&f, 4, all); + assert_eq!(r, vec![Rect { x: 0, y: 0, w: 128, h: 100 }]); + let r = dirty_rects(&f, 3, all); + assert_eq!(r, vec![Rect { x: 0, y: 0, w: 128, h: 64 }, Rect { x: 0, y: 64, w: 150, h: 36 }]); + // Clipped to the requested area. + let r = dirty_rects(&f, 4, Rect { x: 10, y: 10, w: 20, h: 20 }); + assert_eq!(r, vec![Rect { x: 10, y: 10, w: 20, h: 20 }]); + assert!(dirty_rects(&f, 5, all).is_empty()); + } +} diff --git a/src/workstation.rs b/src/workstation.rs index a2f4277ae..1c9b1ccd9 100644 --- a/src/workstation.rs +++ b/src/workstation.rs @@ -155,6 +155,9 @@ pub trait Workstation: Send { } /// The keyboard and mouse the host window types into. fn get_ps2(&self) -> Arc; + /// Add a host-side command set (the screen, pointer and key commands) + /// to this machine's monitor. + fn register_monitor_device(&self, device: Arc); fn get_timer_manager(&self) -> Arc; /// The CPU's debug interface, for the GDB stub. fn get_cpu_debug(&self) -> Arc; diff --git a/tools/vnc-check.py b/tools/vnc-check.py new file mode 100644 index 000000000..833597481 --- /dev/null +++ b/tools/vnc-check.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""Minimal RFB client for checking IRIS's VNC server from scripts. + + tools/vnc-check.py [--addr 127.0.0.1:5900] [--zrle] shot OUT.png + tools/vnc-check.py ... watch SECONDS # incremental updates: rects, pixels + # (--no-decode with --zrle times the server) + tools/vnc-check.py ... click X Y [BUTTON] # 1 left, 2 middle, 4 right + tools/vnc-check.py ... move X Y + tools/vnc-check.py ... key KEYSYM # e.g. 0xff0d (Return), 0x61 (a) + +Needs Pillow for `shot`. Decodes Raw and ZRLE (with --zrle). +""" +import argparse +import socket +import struct +import sys +import time +import zlib + + +def recvn(s, n): + b = bytearray() + while len(b) < n: + c = s.recv(n - len(b)) + if not c: + raise EOFError("server closed") + b += c + return bytes(b) + + +class Rfb: + def __init__(self, addr, zrle, decode=True): + self.decode = decode + host, port = addr.rsplit(":", 1) + self.s = socket.create_connection((host, int(port)), timeout=20) + ver = recvn(self.s, 12) + assert ver.startswith(b"RFB "), ver + self.s.sendall(b"RFB 003.008\n") + n = recvn(self.s, 1)[0] + types = recvn(self.s, n) + assert 1 in types, types + self.s.sendall(b"\x01") + assert struct.unpack(">I", recvn(self.s, 4))[0] == 0 + self.s.sendall(b"\x01") # shared + self.w, self.h = struct.unpack(">HH", recvn(self.s, 4)) + recvn(self.s, 16) + (nl,) = struct.unpack(">I", recvn(self.s, 4)) + self.name = recvn(self.s, nl).decode() + # 32 bpp, R at shift 16 (the usual viewer format) to exercise conversion. + pf = struct.pack(">BBBBHHHBBBxxx", 32, 24, 0, 1, 255, 255, 255, 16, 8, 0) + self.s.sendall(b"\x00\x00\x00\x00" + pf) + encs = ([16] if zrle else []) + [0, -223, -258] + self.s.sendall(struct.pack(">BxH", 2, len(encs)) + b"".join(struct.pack(">i", e) for e in encs)) + self.fb = bytearray(self.w * self.h * 4) + self.z = zlib.decompressobj() + + def request(self, incremental, x=0, y=0, w=None, h=None): + self.s.sendall(struct.pack(">BBHHHH", 3, incremental, x, y, w or self.w, h or self.h)) + + def pointer(self, x, y, mask): + self.s.sendall(struct.pack(">BBHH", 5, mask, x, y)) + + def key(self, sym, down): + self.s.sendall(struct.pack(">BBxxI", 4, down, sym)) + + def read_update(self): + """One server message; returns list of (x, y, w, h, enc) for an update.""" + t = recvn(self.s, 1)[0] + if t != 0: + raise RuntimeError(f"message type {t}") + (n,) = struct.unpack(">xH", recvn(self.s, 3)) + rects = [] + for _ in range(n): + x, y, w, h, enc = struct.unpack(">HHHHi", recvn(self.s, 12)) + rects.append((x, y, w, h, enc)) + if enc == 0: + data = recvn(self.s, w * h * 4) + for r in range(h): + o = ((y + r) * self.w + x) * 4 + self.fb[o:o + w * 4] = data[r * w * 4:(r + 1) * w * 4] + elif enc == 16: + (ln,) = struct.unpack(">I", recvn(self.s, 4)) + d = self.z.decompress(recvn(self.s, ln)) + if self.decode: + self.unzrle(d, x, y, w, h) + elif enc == -223: + self.w, self.h = w, h + self.fb = bytearray(w * h * 4) + elif enc == -258: + pass + else: + raise RuntimeError(f"encoding {enc}") + return rects + + def unzrle(self, d, x, y, w, h): + p = 0 + + def cpix(): + nonlocal p + v = d[p:p + 3] + b"\x00" + p += 3 + return v + + def rl(): + nonlocal p + n = 1 + while True: + b = d[p] + p += 1 + n += b + if b != 255: + return n + + for ty in range(y, y + h, 64): + th = min(64, y + h - ty) + for tx in range(x, x + w, 64): + tw = min(64, x + w - tx) + sub = d[p] + p += 1 + pal = [] + if 2 <= sub <= 16 or sub >= 130: + pal = [cpix() for _ in range(sub if sub <= 16 else sub - 128)] + tile = [] + if sub == 0: + tile = [cpix() for _ in range(tw * th)] + elif sub == 1: + tile = [cpix()] * (tw * th) + elif sub <= 16: + bits = 1 if sub == 2 else 2 if sub <= 4 else 4 + rowb = (tw * bits + 7) // 8 + for _ in range(th): + for i in range(tw): + b = d[p + (i * bits) // 8] + idx = (b >> (8 - bits - (i * bits) % 8)) & ((1 << bits) - 1) + tile.append(pal[idx]) + p += rowb + elif sub == 128: + while len(tile) < tw * th: + v = cpix() + tile += [v] * rl() + else: + while len(tile) < tw * th: + b = d[p] + p += 1 + tile += [pal[b & 127]] * (rl() if b & 128 else 1) + for r in range(th): + for c in range(tw): + o = ((ty + r) * self.w + tx + c) * 4 + self.fb[o:o + 4] = tile[r * tw + c] + assert p == len(d), (p, len(d)) + + def save(self, path): + from PIL import Image + # Pixels are little-endian with R at shift 16: bytes B, G, R, 0. + Image.frombytes("RGBX", (self.w, self.h), bytes(self.fb), "raw", "BGRX").convert("RGB").save(path) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--addr", default="127.0.0.1:5900") + ap.add_argument("--zrle", action="store_true") + ap.add_argument("--no-decode", action="store_true", help="watch: inflate ZRLE but do not decode tiles (times the server)") + ap.add_argument("cmd") + ap.add_argument("args", nargs="*") + a = ap.parse_args() + c = Rfb(a.addr, a.zrle, not a.no_decode) + print(f"connected: {c.name} {c.w}x{c.h}") + if a.cmd == "shot": + c.request(0) + t = time.time() + rects = c.read_update() + # The ext-key ack or a resize may come first; make sure we have pixels. + while not any(r[4] in (0, 16) for r in rects): + c.request(0) + rects = c.read_update() + c.save(a.args[0]) + print(f"saved {a.args[0]} {c.w}x{c.h} in {time.time() - t:.2f}s") + elif a.cmd == "watch": + end = time.time() + float(a.args[0]) + c.request(0) + c.read_update() + c.request(1) + n = 0 + c.s.settimeout(max(0.1, end - time.time())) + try: + while time.time() < end: + rects = c.read_update() + n += 1 + px = sum(r[2] * r[3] for r in rects) + print(f"update {n}: {len(rects)} rects, {px} px: {rects[:4]}") + c.request(1) + except socket.timeout: + pass + print(f"{n} updates") + elif a.cmd in ("click", "move"): + x, y = int(a.args[0]), int(a.args[1]) + b = int(a.args[2]) if len(a.args) > 2 else 1 + c.pointer(x, y, 0) + if a.cmd == "click": + time.sleep(0.05) + c.pointer(x, y, b) + time.sleep(0.05) + c.pointer(x, y, 0) + time.sleep(0.8) # let the server steer before we hang up + elif a.cmd == "key": + sym = int(a.args[0], 0) + c.key(sym, 1) + c.key(sym, 0) + time.sleep(0.2) + else: + sys.exit(f"unknown command {a.cmd}") + + +if __name__ == "__main__": + main()