From 8df32183bf8b470627a23f9e63b7b87d17e95e69 Mon Sep 17 00:00:00 2001 From: technomancer Date: Thu, 29 Oct 2026 05:55:23 -0700 Subject: [PATCH] vnc: built-in VNC server, and screen, pointer and keyboard commands An optional RFB 3.8 server in the iris binary (--vnc [ADDR], [vnc] listen), for every board, windowed or under --ci. It reads only the frame each board has already composed and finds damage by diffing 64x64 tiles, so no drawing code is touched. Raw and ZRLE, DesktopSize and the QEMU extended key event; no authentication, loopback by default. While a viewer is connected Newport uses the software compositor. The guest mouse is relative and X accelerates it, so pointer moves are steered in a closed loop on each board's hardware cursor (VC2, VC1, VC3, GBE), with hotspots learned by pinning the pointer in a corner. The same machinery backs new monitor commands (screen, ptr, key) and MCP tools (screenshot, click, drag, type_text, press_key, wait_screen), and iris-ci screenshot now works on every board. Also: - ip22: a read from an empty GIO slot is a bus error once the PROM sets CPUCTRL1.ABORT_EN, so the PROM no longer takes it for a Newport and polls it 200,000 times on headless boots. - o2: the GBE cursor position is 12-bit signed; the cursor no longer vanishes at the top and left edges. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 38 + Cargo.toml | 2 + FEATURES.md | 8 + HACKING.md | 28 +- HELP.md | 50 ++ clanker/CLANKER.md | 1 + clanker/INDEX.md | 12 + .../devices/empty-gio-slot-reads-bus-error.md | 23 + clanker/iris_mcp.py | 88 ++- clanker/plans/vnc-server-plan.md | 194 +++++ .../cursor-drawn-3px-right-of-x-pointer.md | 22 + clanker/skills/irix-gfx-loop/SKILL.md | 31 +- clanker/skills/irix-guest-loop/SKILL.md | 6 +- clanker/vnc/frame-capture-and-damage.md | 23 + clanker/vnc/pointer-steering-and-hotspots.md | 27 + debug.md | 36 + src/ci.rs | 51 +- src/config.rs | 25 + src/dev/gr2/gr2disp.rs | 1 + src/dev/gr2/mod.rs | 12 + src/dev/mc.rs | 5 +- src/dev/mgras/disp.rs | 1 + src/dev/mgras/mod.rs | 13 + src/dev/ng1/rex3.rs | 105 ++- src/dev/o2gfx.rs | 34 +- src/dev/ps2.rs | 12 + src/disp.rs | 4 + src/gfx_display.rs | 83 +- src/hostinput.rs | 718 ++++++++++++++++++ src/ip22_bus.rs | 26 +- src/ip30_machine.rs | 3 +- src/ip32_machine.rs | 8 +- src/iris_ci_main.rs | 2 +- src/keysym.rs | 268 +++++++ src/lib.rs | 3 + src/machine.rs | 3 +- src/main.rs | 12 + src/monitor.rs | 2 +- src/ui.rs | 10 +- src/vnc/encode.rs | 448 +++++++++++ src/vnc/mod.rs | 318 ++++++++ src/vnc/session.rs | 402 ++++++++++ src/workstation.rs | 3 + tools/vnc-check.py | 215 ++++++ 44 files changed, 3305 insertions(+), 71 deletions(-) create mode 100644 clanker/devices/empty-gio-slot-reads-bus-error.md create mode 100644 clanker/plans/vnc-server-plan.md create mode 100644 clanker/rex3/cursor-drawn-3px-right-of-x-pointer.md create mode 100644 clanker/vnc/frame-capture-and-damage.md create mode 100644 clanker/vnc/pointer-steering-and-hotspots.md create mode 100644 src/hostinput.rs create mode 100644 src/keysym.rs create mode 100644 src/vnc/encode.rs create mode 100644 src/vnc/mod.rs create mode 100644 src/vnc/session.rs create mode 100644 tools/vnc-check.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 69f72eb00..4b6fdb4ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,44 @@ hashes are given where a change is easiest to understand by reading the commit. ## October 2026 +### VNC, and screen and pointer commands + +- **Built-in VNC server.** `--vnc [ADDR]` or `[vnc] listen` serves the + guest's screen, keyboard and mouse to any VNC viewer (RFB 3.8, Raw and + ZRLE, desktop resizing, QEMU's key-position extension; no password, so it + stays on loopback by default). It serves every machine and board the same + way, with the window open or under `--ci` with no window. It reads the + frame the board composed and finds what changed by comparing frames in + 64x64 tiles; drawing code is untouched. Standalone `iris` only. See + [HELP.md](HELP.md#vnc). (`src/vnc/`) +- **Clicks land where you point.** The guest mouse is relative and X + accelerates it; iris now steers the guest pointer onto an absolute + position by reading back the board's hardware cursor (VC2, VC1, VC3, GBE), + learning the cursor's hotspot from the screen corner. VNC uses it, and so + do new monitor commands for scripts and assistants: `screen + info|shot|pix|wait|cursor`, `ptr move|click|drag|down|up|rel|calibrate`, + `key |type|names`, on every machine + ([debug.md](debug.md#graphics)). The MCP server wraps them as + `screenshot` (returns the image), `click`, `drag`, `move_pointer`, + `type_text`, `press_key`, `wait_screen`, `screen_info`, `screen_pixel`. + (`src/hostinput.rs`, `src/keysym.rs`, `clanker/iris_mcp.py`) +- `GfxDisplay` gained `request_frame`, `set_capture`, `hw_cursor` and a + frame sequence number, so a consumer can follow the composed frame on any + board. The CI socket's `screenshot` uses it and no longer also writes a + `screenshot_NNNN.png` into the working directory on Newport. While a VNC + viewer is connected, Newport's window composes in software, and goes back + to GL when the last one leaves. (`src/gfx_display.rs`, + `src/dev/ng1/rex3.rs`, `src/ci.rs`) +- **Empty GIO slots answer reads with a bus error** (Indy, Indigo2), as the + MC does once the PROM has enabled GIO time-outs. They returned all ones, + so the PROM's probe took an absent board for a Newport and polled its + status 200 000 times on every headless boot (one log line each) before + giving up. Headless Indy and Indigo2 now reach the IRIX serial login + without the flood. (`src/ip22_bus.rs`, `src/dev/mc.rs`) +- **The O2's cursor no longer vanishes at the top and left edges.** GBE's + cursor position fields are 12-bit signed; they were read as unsigned, so a + cursor partly off the edge was drawn 4000 pixels away. (`src/dev/o2gfx.rs`) + ### Host efficiency on small machines - **Threads that have nothing to do now sleep in milliseconds, not diff --git a/Cargo.toml b/Cargo.toml index cd4290f5b..8705735bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -284,6 +284,8 @@ toml = "1.0.3" postcard = { version = "1", features = ["alloc"] } blake3 = "1" png = "0.18" +# The VNC server's ZRLE encoding (zlib). +flate2 = "1" parking_lot = "0.12" spin = "0.9" # rfd >= 0.16 dropped the async-std/tokio runtime features: `xdg-portal` now diff --git a/FEATURES.md b/FEATURES.md index 8d2a7b15e..3b9becfea 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -238,6 +238,13 @@ Omit `[nfs]` to disable the share; the default forward list is empty. | `vino.standard` | `ntsc` | ntsc (60 fields/s) or pal (50 fields/s). | | `vino.camera_index` | `0` | Host camera index; meaningful only for source=camera. | +### [vnc] + +| Key | Default | Values and behavior | +|---|---|---| +| `vnc.listen` | `unset` (no server) | `host:port`, or a port on 127.0.0.1, to serve the screen, keyboard and mouse over VNC (RFB 3.8; Raw and ZRLE; no password). Every board, with a window or `ci`; not `headless`. Standalone `iris` only. See [HELP.md](HELP.md#vnc). | +| `vnc.max_fps` | `30` | Most updates a second sent to a viewer (1–120). | + ### [audio] | Key | Default | Values and behavior | @@ -399,6 +406,7 @@ iris-gui does not parse this standalone option set; use its named-machine UI. | `--graphics` | Graphics board: `newport` (default), `xz`, `extreme`, `solidimpact`, `highimpact`, `maximpact` (or `impact:solid`, `impact:high`, `impact:max`). | | `--gdb-port` | Enable GDB stub on the given TCP port (e.g. --gdb-port 1234). Connect with: target remote localhost: | | `--monitor-port` | Monitor console port on 127.0.0.1 (default 8888). Give each iris running at once its own. | +| `--vnc [ADDR]` | Serve the screen, keyboard and mouse over VNC on ADDR (`host:port` or a port; default 127.0.0.1:5900). Sets `vnc.listen`. | | `--serial-port-a` | Serial channel A (tty2) port on 127.0.0.1 (default 8880). | | `--serial-port-b` | Serial channel B (tty1, the console) port on 127.0.0.1 (default 8881). | | `--test-device` | Map the bare-metal test device into GIO expansion slot 0: SIGNATURE, PUTC (guest console → stdout), DUMP (machine state → JSON) and EXIT (terminate with the guest's exit code). Off by default. | diff --git a/HACKING.md b/HACKING.md index 848a4a6d8..68e8fdfb8 100644 --- a/HACKING.md +++ b/HACKING.md @@ -253,7 +253,8 @@ Every device that has real work to do runs on its own thread: | graphics: `REX3-Processor` / `GR2-HQ2`, `GR2-RE3` / `MGRAS-HQ3`, `MGRAS RSS` ×1-4 | consumers of the graphics FIFOs (section 7) | | `REX3-Refresh`, `GR2-Display`, MGRAS refresh | scanout and composition; the refresh thread owns the GL context | | `hptimer` | host timer callbacks (Compare, PIT ticks, HAL2 audio pacing) | -| listeners | debug console, serial ports, CI socket, GDB stub | +| listeners | debug console, serial ports, CI socket, GDB stub, VNC | +| `VNC-Grab`, `VNC-Pointer`, `VNC-Client`/`VNC-Read` | with `--vnc`: frame diffing, pointer steering, one pair per viewer | | `machine-events` | power-off and reset requests from devices | | `ISP1040-0`, `ISP1040-1` | Octane SCSI request rings and disk transfers | | `IOC3-RX-A`, `IOC3-RX-B`, `IOC3-NVRAM` | Octane serial input per port; the DS1685 NVRAM file writer (IOC3's other deadlines are `hptimer` one-shots) | @@ -418,6 +419,31 @@ without locks and composes it through the board's colour maps and DAC. Everything that touches the host GL context (including teardown) runs on the thread that owns it. +### The composed frame and the hardware cursor + +`GfxDisplay` (`src/gfx_display.rs`) is what the window, the CI socket, the +monitor's `screen`/`ptr` commands and the VNC server see of a board, whichever +board it is. The display thread leaves the composed guest frame in +`screen().rgba` (stride 2048, `0xAABBGGRR`, no status bar) and bumps +`screen().frame_seq` each time it rewrites it. GR2, IMPACT and the O2 compose +on the CPU and do this every frame. Newport composes inside the window's +renderer, so it fills `rgba` only when asked: `request_frame` for one frame, +`set_capture(true)` for every frame while a consumer (VNC) watches. While +capture is on, Newport's window uses the software compositor so the copy is a +slice copy; with no window (`--ci`) the refresh thread composes in software +itself. `copy_frame`/`fresh_frame` are the helpers that read it. + +Nothing downstream looks inside the drawing code. The VNC server +(`src/vnc/`) finds damage by comparing frames in 64x64 tiles, so every board +is served the same way and drawing paths carry no bookkeeping. + +`hw_cursor` reads the board's cursor registers live (VC2, VC1, VC3, GBE): +the glyph's top-left and its opaque pixels. `src/hostinput.rs` uses it to +steer the guest's relative PS/2 mouse onto absolute positions (the X server +accelerates what it reads, so positions cannot be sent), learning each +glyph's hotspot from the top-left screen corner, where X pins the pointer +at 0,0. + ## 8. jitv2: the MIPS JIT `--features jitv2` adds a JIT compiler (`src/cpu/jitv2/`) that turns guest diff --git a/HELP.md b/HELP.md index 03d2b894c..557503049 100644 --- a/HELP.md +++ b/HELP.md @@ -494,6 +494,55 @@ standalone `iris` window if you need them. The window is freely resizable. At 1× and 2× scale the display is rendered with nearest-neighbour pixel-perfect sampling; at all other sizes trilinear filtering with mipmaps is used. `lock_aspect_ratio = false` in `iris.toml` allows free resizing with letterboxing. When the guest changes video mode (PROM, then IRIX during boot) the window snaps to the new resolution times the configured `scale` / `--2x`. +## VNC + +iris has a built-in VNC server. Any VNC viewer (TigerVNC, RealVNC, Remmina, +noVNC, macOS Screen Sharing) then shows the guest's screen and types and +clicks into it: + +```sh +iris --vnc # 127.0.0.1:5900 +iris --ci --vnc # no window at all: VNC is the screen +iris --vnc 0.0.0.0:5901 # reachable from other machines (no password!) +vncviewer localhost:5900 +``` + +or in `iris.toml`: + +```toml +[vnc] +listen = "127.0.0.1:5900" # absent: no VNC server +max_fps = 30 # most updates a second per viewer +``` + +It works on every machine and graphics board (Newport, XZ/Extreme, IMPACT, +the O2), with the normal window open or under `--ci` with no window. Not with +`--headless`, which has no screen to serve. The window, the monitor and any +number of viewers can type and click at the same time. iris-gui has no VNC +server. + +What you see is the guest's screen, without the window's status bar. Only +the parts of the screen that change are sent, so an idle desktop costs next +to nothing. With Newport, the window uses the software compositor while a +viewer is connected (`disp compositor gl` is refused) and switches back when +the last one leaves. + +**The mouse.** The guest's mouse sends movements, not positions, and IRIX +accelerates fast ones. iris steers the guest pointer onto the spot you point +at by watching where the board draws the cursor, so a click lands where you +clicked. The first time you point, the guest pointer jumps to the top-left +corner and back: that teaches iris where the tip of the cursor is. The +viewer's own cursor (a dot or an arrow) runs a little ahead of the guest's; +it catches up within a fraction of a second. + +**The keyboard.** Viewers that send key positions (TigerVNC, noVNC, QEMU's +extension) work with any host layout. Others send characters, which iris +types on a US layout. + +**Security.** There is no password. Leave the server on `127.0.0.1` and use +an SSH tunnel (`ssh -L 5900:localhost:5900 host`) to reach it from +elsewhere. + --- ## Configuration file (`iris.toml`) @@ -716,6 +765,7 @@ Automation and debugging --serial-log Append everything IRIX prints on ttyd1 to FILE --serial-log-timestamps off (raw) or wall: timestamp each completed line --monitor-port Monitor TCP port (default 8888) + --vnc [ADDR] VNC server on ADDR (host:port or port; default 127.0.0.1:5900) --serial-port-a Serial channel A TCP port (default 8880) --serial-port-b Serial channel B TCP port (default 8881) --gdb-port Start the GDB stub diff --git a/clanker/CLANKER.md b/clanker/CLANKER.md index f98a0a9d0..cf14e4eb5 100644 --- a/clanker/CLANKER.md +++ b/clanker/CLANKER.md @@ -60,6 +60,7 @@ found or what to watch out for while changing code, it goes in `clanker/`. | `clanker/INDEX.md` | categorized index of every note | | `clanker/cpu/`, `jitv2/` | CPU architecture findings; the MIPS JIT | | `clanker/devices/`, `scsi/`, `net/`, `video/` | device models, storage, the NAT stack and its services, VINO/IndyCam | +| `clanker/vnc/` | the VNC server, the host input layer (`screen`/`ptr`/`key`) and pointer steering | | `clanker/rex3/`, `gr2/`, `mgras/`, `odyssey/`, `hostgl/` | graphics boards and host GL (`odyssey/` is references only) | | `clanker/gui/`, `macos/` | front-ends and platform packaging | | `clanker/snapshot/`, `perf/`, `testing/`, `build/` | snapshots and `iris-ci`, host performance, test harnesses, builds | diff --git a/clanker/INDEX.md b/clanker/INDEX.md index 0f0ad3e6b..dab49f69e 100644 --- a/clanker/INDEX.md +++ b/clanker/INDEX.md @@ -98,6 +98,7 @@ Non-graphics devices: MC/MEMCFG and RAM banks, HPC3, VDMA, EEPROMs and NVRAM, PS | Note | Kind | Title | |---|---|---| +| [empty-gio-slot-reads-bus-error.md](devices/empty-gio-slot-reads-bus-error.md) | rule | Reads of an empty GIO slot end in a bus error (ABORT_EN); all ones made the PROM poll a phantom Newport 200 000 times headless | | [cpal-alsa-busy-poll.md](devices/cpal-alsa-busy-poll.md) | rule | cpal's ALSA backend can spin a core: `[audio] host = "pulseaudio"` | | [decode-table-is-host-mapped.md](devices/decode-table-is-host-mapped.md) | rule | The physical decode table is a sparse host mapping over 40 bits: change it only through `fill`, set the default first, compare entries with `entry_eq` | | [mc-decodes-32-physical-bits.md](devices/mc-decodes-32-physical-bits.md) | rule | Indy/Indigo2/O2 decode 32 physical bits; IRIX PTE software bits reach PA[35:32]; fold with `MipsExecutor::phys_mask`, never decode them | @@ -170,6 +171,7 @@ Newport (REX3): drawing semantics, HOSTRW, GFIFO, the REX3 JIT, trace and record | Note | Kind | Title | |---|---|---| +| [cursor-drawn-3px-right-of-x-pointer.md](rex3/cursor-drawn-3px-right-of-x-pointer.md) | open | Newport draws the hardware cursor 3 px right of the X pointer (measured by corner pinning) | | [alphahost-fields-and-aline-alpha.md](rex3/alphahost-fields-and-aline-alpha.md) | rule | REX3 source alpha: host fields and antialiased lines | | [batched-hostrw-in-the-jit.md](rex3/batched-hostrw-in-the-jit.md) | rule | Batched HOSTRW in the REX3 JIT — and how the equivalence test lied twice | | [blendalpha-and-alpha-blending.md](rex3/blendalpha-and-alpha-blending.md) | rule | REX3 BLENDALPHA: what it selects, and why the `blast` billboard exposes it | @@ -273,6 +275,15 @@ Host GL pass-through for guest GL programs (`iris-hostgl`). | [windows-silent-exit-0xc000041d.md](gui/windows-silent-exit-0xc000041d.md) | rule | Windows: CLI vanishes with exit 0xC000041D and no message | | [winit-window-calls-must-be-on-event-thread.md](gui/winit-window-calls-must-be-on-event-thread.md) | rule | Window methods must only be called on the event-loop thread (Windows crash) | +## vnc/ + +The VNC server, the host input layer (`screen`/`ptr`/`key`) and pointer steering. + +| Note | Kind | Title | +|---|---|---| +| [pointer-steering-and-hotspots.md](vnc/pointer-steering-and-hotspots.md) | design | Steering the relative mouse by the hardware cursor: X's acceleration curve, hotspots from corner pinning, busy guests | +| [frame-capture-and-damage.md](vnc/frame-capture-and-damage.md) | design | What VNC reads (rgba + frame_seq), Newport capture and forced SW compositor, tile damage | + ## macos/ Mac App Store and sandbox constraints. @@ -387,3 +398,4 @@ Plans, task briefs and reviews written for agents. Each states its own status; m | [networking-tab-redesign.md](plans/networking-tab-redesign.md) | plan | Networking tab redesign — design & plan | | [nfsudp-plan.md](plans/nfsudp-plan.md) | plan | Plan: in-core NFSv3-over-UDP server (`src/net/nfsudp.rs`) | | [pcap-release-plan.md](plans/pcap-release-plan.md) | plan | PCAP release + elevation plan | +| [vnc-server-plan.md](plans/vnc-server-plan.md) | plan | Built-in VNC server: board-agnostic frame tap, damage from frame diffs, pointer steered by the HW cursor, monitor/MCP screenshot + click for agents | diff --git a/clanker/devices/empty-gio-slot-reads-bus-error.md b/clanker/devices/empty-gio-slot-reads-bus-error.md new file mode 100644 index 000000000..180efb615 --- /dev/null +++ b/clanker/devices/empty-gio-slot-reads-bus-error.md @@ -0,0 +1,23 @@ +# Reads of an empty GIO slot end in a bus error + +Status: fixed 2026-10-29 (`GioBusErrorDevice`, `src/ip22_bus.rs`). + +**Symptom.** Indy or Indigo2 with no graphics board (`--headless` with +Newport): the PROM printed `Cannot open video() for output` only after +200 000 `MC: GIO Timeout at 1f0f1338` lines, and IRIX later failed (the +install guide recorded a UTLB miss). The GIO slot probes of a normal boot +also "succeeded" on empty slots. + +**Cause.** Unanswered GIO addresses returned all ones with `BUS_OK`. The +PROM and the kernel find boards with `badaddr()`, which expects a read of an +absent board to fault. With no fault, the PROM took the slot for a Newport, +wrote CONFIG and polled REX3 STATUS (offset 0x1338) for GFXBUSY to clear: +all ones never clears, so it ran its 100 000-iteration timeout, twice. + +**Rule.** With `CPUCTRL1.ABORT_EN` (bit 4, "enable GIO bus time outs"; the +PROM sets it in its startup code) an unanswered GIO read returns `BUS_ERR` +(a data bus error exception) after recording GIO_ERROR_ADDR/STAT. Without +it, all ones as before. Writes are still dropped. Headless Indy and Indigo2 +(IP28 checked) now boot IRIX to the serial login with `hinv` showing no +graphics; the PROM still says `Cannot open video()` because the NVRAM has +`console=g`, and waits in its menu instead of autobooting. diff --git a/clanker/iris_mcp.py b/clanker/iris_mcp.py index 1e11a460a..83c09f3e8 100644 --- a/clanker/iris_mcp.py +++ b/clanker/iris_mcp.py @@ -1,6 +1,8 @@ import asyncio +import os import sys -from mcp.server.fastmcp import FastMCP +import tempfile +from mcp.server.fastmcp import FastMCP, Image # Initialize FastMCP server mcp = FastMCP("iris-monitor") @@ -272,5 +274,89 @@ async def traceback(count: int = 10) -> str: """Show execution traceback (history of executed instructions).""" return await client.send_command(f"dt {count}") +# ---- The guest's screen, pointer and keyboard (any board) ---------------- +# These wrap the monitor's `screen`, `ptr` and `key` commands. Coordinates are +# screen pixels, as in a screenshot. The pointer is steered onto the target +# by reading back the hardware cursor, so X's acceleration does not matter. + +_shot_dir = tempfile.mkdtemp(prefix="iris-mcp-") + +@mcp.tool() +async def screenshot(x: int = 0, y: int = 0, width: int = 0, height: int = 0, max_width: int = 0) -> Image: + """A PNG of the guest's screen as the board composes it (no host status + bar). Optionally cropped to x, y, width, height, and scaled down to + max_width (needs Pillow). Coordinates in the image are screen pixels + unless scaled.""" + path = os.path.join(_shot_dir, "shot.png") + crop = f" {x} {y} {width} {height}" if width and height else "" + out = await client.send_command(f"screen shot {path}{crop}") + if not out.startswith("saved"): + raise RuntimeError(out) + if max_width: + try: + from PIL import Image as PILImage + im = PILImage.open(path) + if im.width > max_width: + im.resize((max_width, im.height * max_width // im.width)).save(path) + except ImportError: + pass + with open(path, "rb") as f: + return Image(data=f.read(), format="png") + +@mcp.tool() +async def screen_info() -> str: + """Screen size, hardware cursor and pointer position, whether the guest + has enabled the mouse, and buttons held.""" + return await client.send_command("screen info") + +@mcp.tool() +async def screen_pixel(x: int, y: int) -> str: + """The colour (rrggbb) of one screen pixel.""" + return await client.send_command(f"screen pix {x} {y}") + +@mcp.tool() +async def wait_screen(quiet_ms: int = 500, timeout_ms: int = 10000) -> str: + """Wait until the screen stops changing for quiet_ms (an app finished + drawing, a window opened), at most timeout_ms.""" + return await client.send_command(f"screen wait {quiet_ms} {timeout_ms}") + +@mcp.tool() +async def click(x: int, y: int, button: str = "left", count: int = 1) -> str: + """Move the pointer to screen pixel (x, y) and click. button: left, + middle or right (IRIX: right opens the toolchest/root menus, middle + pastes/moves). count 2 for a double click.""" + return await client.send_command(f"ptr click {x} {y} {button} {count}") + +@mcp.tool() +async def move_pointer(x: int, y: int) -> str: + """Move the pointer to screen pixel (x, y) without clicking.""" + return await client.send_command(f"ptr move {x} {y}") + +@mcp.tool() +async def drag(x1: int, y1: int, x2: int, y2: int, button: str = "left") -> str: + """Press button at (x1, y1), move to (x2, y2), release.""" + return await client.send_command(f"ptr drag {x1} {y1} {x2} {y2} {button}") + +@mcp.tool() +async def type_text(text: str) -> str: + """Type text on the guest keyboard (US layout, ASCII; newline is Enter). + Runs of spaces are typed as one.""" + out = [] + lines = text.split("\n") + for i, line in enumerate(lines): + if line: + out.append(await client.send_command(f"key type {line}")) + if i < len(lines) - 1: + out.append(await client.send_command("key Return")) + return "\n".join(out) + +@mcp.tool() +async def press_key(keys: str, action: str = "") -> str: + """Press a key or chord: 'Return', 'Escape', 'F1', 'ctrl+c', + 'ctrl+alt+Delete', 'Up'. X keysym names and short aliases (ctrl, alt, + shift, enter, esc, tab, bs, del, pgup...). action: '' to press and + release, 'down' or 'up' to hold or let go.""" + return await client.send_command(f"key {keys} {action}".strip()) + if __name__ == "__main__": mcp.run() diff --git a/clanker/plans/vnc-server-plan.md b/clanker/plans/vnc-server-plan.md new file mode 100644 index 000000000..e12690c56 --- /dev/null +++ b/clanker/plans/vnc-server-plan.md @@ -0,0 +1,194 @@ +# Built-in VNC server — plan + +Status: implemented 2026-10-29 on branch `vnc` (phases 1-5): `src/vnc/`, +`src/hostinput.rs`, `src/keysym.rs`, monitor `screen`/`ptr`/`key`, MCP +tools, docs. Tested on Indy Newport (window and `--ci`), XZ, IP28 IMPACT, +Octane, O2, with TigerVNC and `tools/vnc-check.py`. Not done: VNC password, +second head, CI-socket input commands. Design notes in `clanker/vnc/`. + +## Goal + +An optional RFB (VNC) server in the `iris` binary (not iris-gui) that works: + +- windowed (alongside the winit window; input from both is merged), +- with `--ci` (no window; offscreen rendering already runs), +- for every board: Newport (REX3), GR2, IMPACT (Indigo2 + Octane), O2 GBE. + +It also gives LLM agents and CI a "see the screen, click at x,y" path through +the monitor/MCP/CI socket, reusing the same frame tap and pointer logic. + +Hard constraints: + +- **No instrumentation in drawing code.** No damage tracking in REX3/RE3/RSS + draw paths. Damage comes from diffing composed frames. +- **One path for all boards.** VNC consumes the composed frame + (`Rex3Screen::rgba`, stride 2048, `0xAABBGGRR`) and nothing else. +- REX3 is forced onto the software compositor while VNC is on. +- Off by default; costs nothing when off. + +## Where frames come from today (survey summary) + +- `GfxDisplay` (`src/gfx_display.rs`) is common to all boards; `screen()` + gives `Mutex` with `rgba`, `width`, `height`. +- GR2, IMPACT, O2 compose on their own refresh threads and keep `rgba` + current (`prebuilt = true`). +- REX3 composes inside the installed `Renderer` (`SwCompositor` or + `GlCompositor`) and fills `rgba` only when `need_readback` (screenshot). + With `--ci` there is no renderer, so `ci.rs` composes inline. +- Input: everything goes through `Ps2Controller::push_kb(KeyCode, bool)` and + `push_mouse_input(buttons, dx, dy, dz)`. **Mouse is relative only**, and X + applies pointer acceleration. +- Each board knows its hardware cursor position: REX3 VC2 cursor X/Y regs + (+`cursor_x_adjust`), GR2 `Vc1::cursor_pos`, IMPACT `Dcb::cursor`, O2 GBE + cursor regs. + +## Design + +### 1. Frame tap (core, board-agnostic) + +Extend `GfxDisplay`: + +```rust +fn frame_seq(&self) -> u64; // bumps whenever rgba is rewritten +fn request_frame(&self); // "please make rgba current", no PNG +fn cursor_pos(&self) -> Option<(i32, i32)>; // live HW cursor, screen coords; None if disabled +``` + +- `Rex3Screen` gets `frame_seq: u64`, bumped by whoever writes `rgba`: the + three prebuilt refresh loops (GR2, IMPACT, O2) and the REX3 renderer + readback. These are refresh-loop / present-time sites, not draw code. +- REX3: a separate `frame_wanted: AtomicBool` (the existing + `screenshot_pending` also writes a PNG file, so it can't be reused). The + refresh loop ORs it into `need_readback` and forces a present on the next + tick when content changed. +- REX3 forced SW: while a frame consumer is registered, `GlRenderer` uses + `SwCompositor`; `disp compositor gl` is refused with a message. Readback is + then `read_pixels()`, a slice copy, no GL readback. +- No-window REX3: move iris-gui's `CaptureRenderer` idea into core as + `SwCaptureRenderer` (SwCompositor → `rgba`) and install it when VNC (or CI) + needs frames and no renderer is present. `ci.rs`'s inline compose then goes + away. +- `cursor_pos` reads live device state (VC2/VC1/DCB/GBE), not the 60 Hz + snapshot, so the pointer loop sees updates promptly. + +### 2. Damage from diffs + +One grabber thread per served head: + +- Tick at `max_fps` (default 30). If `frame_seq` unchanged, skip. + Otherwise `request_frame()`, lock `screen`, copy the visible + `width × height` into a tight `cur` buffer, unlock (~1–2 ms at 1280×1024). +- Compare `cur` with `prev` in 64×64 tiles (row slices compared as `[u64]`, + early exit on first mismatch). Changed tiles get `tile_gen[t] = frame_gen`. +- Optionally shrink each dirty tile to its exact bounding box before sending. +- Resolution change → new buffers, every tile dirty, DesktopSize to clients. +- Swap `cur`/`prev`, notify clients (Condvar). + +Clients keep `seen_gen`; an incremental FramebufferUpdateRequest blocks until +some `tile_gen > seen_gen`, then sends dirty tiles inside the requested rect, +merging horizontal runs into wider rects. Non-incremental requests send the +whole rect. Slow clients just skip intermediate frames (no queue). + +### 3. RFB server (`src/vnc/`) + +Own implementation, std::net, thread per client like the monitor/CI socket +(no RFB crate worth depending on; no tokio in the tree). + +- RFB 3.8 (accept 3.3/3.7), security type None. Bind `127.0.0.1:5900` by + default; warn loudly on a non-loopback bind. VNC password auth later. +- Pixel formats: server native 32bpp; honour SetPixelFormat for 32/16/8bpp + true-colour by shift/max conversion. No colour-map mode. +- Encodings: Raw (required), ZRLE (per-connection zlib stream, `flate2` as + a direct dep; supported by TigerVNC, RealVNC and noVNC), pseudo-encodings + DesktopSize and QEMU Extended Key Event. Tight/JPEG and CopyRect later, if + ever. +- Cursor: drawn into the frame by the board compositors, so no cursor + pseudo-encoding. The viewer's local dot cursor works. +- Max clients 8; on disconnect, release that client's held keys/buttons. + +### 4. Input + +**Keyboard.** Prefer the QEMU Extended Key Event (client sends XT scancodes +→ map to `KeyCode`, layout-independent; TigerVNC and noVNC send it). Fallback: +X keysym → `KeyCode` table for a US layout (`src/keysym.rs`), mapping a +shifted symbol to its physical key. Clients already send Shift +themselves. The same table backs monitor `key NAME` and can replace +`ip32_screen::keycode`. + +**Pointer: closed loop on the hardware cursor.** VNC gives absolute +positions; the guest wants relative PS/2 packets that X then accelerates. +Rather than model acceleration, steer: + +- A `PointerSeek` (`src/hostinput.rs`) holds the target. Every ~8 ms: + `err = target - (cursor_pos() + hotspot)`. Send a delta clamped to a + step size that shrinks as `|err|` falls (big accelerated steps far away, + ≤ accel-threshold steps near). Stop at `err == 0` or after N stalled ticks. +- Button/wheel events queue behind the pending move: send them once the move + converges, or after a timeout (~150 ms), so a click lands where it was + aimed. +- **Fallback, open loop**: if `cursor_pos()` is `None` (cursor disabled) or + the cursor stops moving while we push (an app grabbed and warps the + pointer), switch to relative mode: send the VNC position delta as is. +- **Hotspot**: the hardware position is the glyph's top-left, X sets it to + `pointer - hotspot`, and the hotspot isn't in hardware. Start with the + first opaque pixel of the cursor glyph (the arrow tip), then calibrate per + board with `xclick`/`xgetpix` (open item below). +- Both the window and VNC feed the same `Ps2Controller`; the seek only runs + while VNC pointer events arrive, so the two don't fight unless both are + used at once. + +### 5. LLM / CI path (same machinery, no VNC client needed) + +Monitor commands, registered by the host-input module (document in +`debug.md`): + +- `screen shot PATH [x y w h]`, `screen wait [ms]` (block until the frame is + stable or has changed), `screen seq` +- `ptr move X Y`, `ptr click X Y [left|middle|right] [double]`, + `ptr drag X1 Y1 X2 Y2`, `ptr pos` +- `key NAME [down|up]`, `key ctrl+alt+F1`, `key type TEXT` + +MCP (`clanker/iris_mcp.py`): `screenshot` returning image content (optional +crop/scale), `click`, `move_pointer`, `type_text`, `press_key`, +`wait_screen`. CI socket and `iris-ci`: `click`, `key`, `type`, `screenshot +--head`. Skills (`irix-gfx-loop`, `irix-guest-loop`): replace the +corner-pushing and `xclick` guidance with these. + +### 6. Configuration + +```toml +[vnc] +listen = "127.0.0.1:5900" # absent = off +max_fps = 30 +head = 0 # 1 = second Newport head (later: both, 5900/5901) +``` + +CLI: `--vnc [ADDR]`. Runtime option, no Cargo feature (the only new +dependency is `flate2`, already in the lockfile). `--headless` with Newport +has no display: refuse `--vnc` with a hint to use `--ci`. + +## Phases + +1. **Frame tap**: `GfxDisplay` additions, `frame_seq`, REX3 `frame_wanted` + + forced SW, core `SwCaptureRenderer`, `cursor_pos` for all four boards. + ci.rs screenshot moves onto it. Unit tests: seq bumps, rgba current + under `--ci` for REX3. +2. **RFB output**: handshake, Raw, ZRLE, tile diff, DesktopSize. Test with + an in-process minimal RFB client (handshake, full + incremental update, + pixel check) and manually with TigerVNC and noVNC. +3. **Input**: Extended Key Event + keysym table, `PointerSeek`, button + ordering, release on disconnect. Measure click accuracy on each board. +4. **LLM path**: monitor `screen`/`ptr`/`key`, MCP tools, CI socket + commands, skill updates. +5. **Docs**: HELP.md, FEATURES.md, debug.md, HACKING.md (`GfxDisplay` + change, frame tap), CHANGELOG.md. + +## Open items + +- Hotspot offset per board; does IRIX X use the HW cursor on every board + (GR2/IMPACT/O2)? If any uses a software cursor, `cursor_pos` is `None` + there and the seek falls back to open loop. Verify. +- IRIX default `xset m` acceleration/threshold: sets the "near" step size. +- Grab cost at 1600×1200 × 30 fps under the screen lock; if it shows in + profiles, double-buffer `rgba` at the producer instead. +- Two heads: one listener per head, or one with a head switch. diff --git a/clanker/rex3/cursor-drawn-3px-right-of-x-pointer.md b/clanker/rex3/cursor-drawn-3px-right-of-x-pointer.md new file mode 100644 index 000000000..3bff12992 --- /dev/null +++ b/clanker/rex3/cursor-drawn-3px-right-of-x-pointer.md @@ -0,0 +1,22 @@ +# Newport draws the hardware cursor 3 pixels right of the X pointer + +Status: open, measured 2026-10-29 on IRIX 6.5 at 1280x1024 (REX3 decodes +the mode as 1282x1024, `cursor_x_adjust` = 5). + +**How it was seen.** `ptr calibrate` pins the X pointer at 0,0 and reads +the VC2 cursor position as the compositor draws it. xdm's X-shaped cursor +(cursor font hotspot 7,7) pinned with its glyph at (-4,-7); the 4Dwm arrow +(hotspot 1,1) at (2,-1). Both say the glyph is drawn 3 px right of where X +put it and exactly right vertically. On GR2 the arrow pins with its tip on +0,0: no offset there. + +**Effect.** Cosmetic in the window and VNC (the arrow tip is 3 px right of +the real pointer). `ptr`/VNC clicks are unaffected: they learn hotspots in +drawn coordinates, so negative hotspots are allowed. + +**The 1282 width is not ours.** The PROM and IRIX program the VC2 timing +wider than the 1280-pixel X screen, and the X server scissors one column on +the left to black: probably a workaround for a display hardware quirk. So +the decoded width is right and should stay. The cursor offset may be tied to +the same quirk (`cursor_x_adjust` and its -2 bias in `decode_vc2_timings`); +check against a real Indy before changing it. diff --git a/clanker/skills/irix-gfx-loop/SKILL.md b/clanker/skills/irix-gfx-loop/SKILL.md index 1c1a71ba8..95d6cadbf 100644 --- a/clanker/skills/irix-gfx-loop/SKILL.md +++ b/clanker/skills/irix-gfx-loop/SKILL.md @@ -91,9 +91,11 @@ tools/irix-sh.py run 'cd /tmp/gltest && ./glprim --hold 4000' ``` Run the captures while `--hold` keeps the window up (background the guest -command with `&` if you need to sample during the hold). Newport has no -`rex pix`: use `rex fbdump DIR` and read the PNG, `xgetpix X Y [W H]` in the -guest, or `iris-ci screenshot`. +command with `&` if you need to sample during the hold). On every board, +`screen pix ` reads one composed display pixel and `screen shot + [x y w h]` saves the composed screen (or a crop); Newport has no +`rex pix`, so use these there. The MCP `screenshot` tool returns the image +directly. glprim prints its **real window origin** (`window origin OX,OY size WxH`); the window manager moves windows. GL window pixel (gx, gy) is display pixel @@ -164,11 +166,24 @@ Time replays on the host instead. ## 8. Driving input -- **Monitor, any PS/2 machine**: `ps2 mouse [buttons]` (1 left, - 2 right, 4 middle; send the press, then a packet with 0 to release), - `ps2 type `, `ps2 enter`, `ps2 esc`. Moves are relative and go - through X pointer acceleration: push into the top-left corner with a few - large negative moves, then step to the target in small ones. +- **Monitor, any machine and board**: `ptr click [left|middle|right] + [count]`, `ptr move`, `ptr drag`, `ptr down|up`, `key ` (`Return`, + `ctrl+c`, `F1`), `key type `, `screen wait [quiet_ms]` (until the + screen settles), `screen info` (where the pointer is, whether the guest + has enabled the mouse). Coordinates are screen pixels. `ptr` steers the + relative PS/2 mouse by reading back the hardware cursor, so X's + acceleration does not matter; the first move pins the pointer in the + top-left corner once to learn the cursor's hotspot. MCP: `click`, `drag`, + `type_text`, `press_key`, `wait_screen`. +- Windows on the IRIX desktop take keyboard focus on a click, not when the + pointer enters them: click into a winterm before `key type`. +- `ptr` errors with "the hardware cursor is off" before X is up, and "the + guest is not reading the mouse" while it is busy (desktop start-up): + `screen wait`, then retry. +- Raw PS/2 is still there: `ps2 mouse [buttons]`, `ps2 type`. +- **VNC**: start iris with `--vnc` to watch or drive the guest from a VNC + viewer; `tools/vnc-check.py shot|watch|click|key` exercises the server + from scripts. - **In the guest, XTest**: `xclick NAME [BUTTON [X Y]]` or `xclick NAME key KEYSYM` clicks at exact window coordinates in the first window whose name starts with NAME; `xresize NAME W1 H1 W2 H2 [COUNT] [DELAY_MS]` resizes diff --git a/clanker/skills/irix-guest-loop/SKILL.md b/clanker/skills/irix-guest-loop/SKILL.md index 71e5473a4..0fefce014 100644 --- a/clanker/skills/irix-guest-loop/SKILL.md +++ b/clanker/skills/irix-guest-loop/SKILL.md @@ -105,7 +105,11 @@ tools/irix-sh.py run 'echo root > /etc/autologin' 'touch /etc/autologin.on' boot stops at the login window. - The session appears about a minute after telnet comes up. Wait until `ps -ef | grep 4Dwm` shows the window manager (and `toolchest` for the - full desktop), then give it another ~15 s before taking screenshots. + full desktop), then `screen wait 2000 60000` on the monitor (returns once + the screen has been still for 2 s) before taking screenshots. +- Without autologin you can log in on the screen itself: `ptr click X Y` on + the user's icon in the xdm window, then `key Return` (see `screen`, `ptr` + and `key` in debug.md; MCP `screenshot`, `click`, `press_key`). - The screen saver blanks the screen after 10 minutes idle and every pixel probe then reads black. The desktop re-applies its saver settings after login, so run `/usr/bin/X11/xset s off` once the session is up and check diff --git a/clanker/vnc/frame-capture-and-damage.md b/clanker/vnc/frame-capture-and-damage.md new file mode 100644 index 000000000..be01a65a3 --- /dev/null +++ b/clanker/vnc/frame-capture-and-damage.md @@ -0,0 +1,23 @@ +# VNC frame capture and damage + +Status: design notes for `src/vnc/`, 2026-10-29. + +- The server reads only `GfxDisplay::screen().rgba` + `frame_seq`; no board + draw path knows about it. Damage = 64x64 tiles whose pixels differ from + the last published frame (row slices compared; per-tile generation + numbers; viewers remember the generation they saw). +- GR2, IMPACT, O2 rewrite `rgba` every composed frame. Newport only when + asked: `set_capture(true)` makes every full refresh read back, and forces + the window's renderer to `SwCompositor` (`switch_compositor(false)` every + frame is cheap and idempotent; `disp compositor gl` is refused). With no + renderer (`--ci`) the REX3 refresh loop composes in software itself. +- `request_frame` is the one-shot version (CI screenshot, monitor `screen`): + it does not write `screenshot_NNNN.png`, which `request_screenshot` does + on Newport. +- O2 bumps `frame_seq` 60 times a second even when nothing changed; the + tile compare then finds nothing and publishes nothing. +- An idle Newport desktop in `--ci` produces a `frame_seq` bump only on real + change (fb/palette dirty), so a watched idle desktop costs a seq check per + tick. +- `tools/vnc-check.py` is a minimal Python RFB client (Raw + ZRLE decode) + for scripts; its ZRLE decode is slow (pure Python), the server is not. diff --git a/clanker/vnc/pointer-steering-and-hotspots.md b/clanker/vnc/pointer-steering-and-hotspots.md new file mode 100644 index 000000000..389ffd360 --- /dev/null +++ b/clanker/vnc/pointer-steering-and-hotspots.md @@ -0,0 +1,27 @@ +# Pointer steering: what the hardware cursor can and cannot tell + +Status: design notes for `src/hostinput.rs`, 2026-10-29. + +- **X acceleration on IRIX 6.5 (default `xset m`)**: a packet of d mickeys + moves the pointer d when d <= 4, else `2d - 4` (measured: 5->6, 8->12, + 20->36, 40->76). Steps of up to 3 are exact; the steering loop learns a + gain from larger ones. +- **The hotspot is not in the hardware.** The glyph's tip is a bad guess: the + xdm X cursor's hotspot is its centre, and Newport adds a 3 px offset + (`clanker/rex3/cursor-drawn-3px-right-of-x-pointer.md`). The reliable + source is pinning in the top-left corner: X clamps the pointer to 0,0, + so hotspot = -glyph position. The first move does this once + (`calibrate_locked`). +- **Only the top/left edges pin reliably.** The board's visible size can + differ from X's screen (Newport is programmed 1282 wide for X's 1280, as on real hardware), so the + bottom/right pin position is off by that difference. +- **New shapes** are learned when they appear after an exact (<= 3) step from + a known shape; otherwise guessed as tip + the last learned (hotspot - tip). +- **Busy guests**: during desktop start-up X ignores the mouse for seconds + and GR2's X server turns the cursor off for a moment while drawing. The + loop waits for queued mouse bytes to drain before sending more, tolerates + the cursor being off for 400 ms, waits for the cursor to settle 20 ms after + each step (the O2 shows a move a frame late; steering from the old position + oscillated by 2-3 px), and gives up after 3 s without motion. +- **GR2 glyph**: the arrow sits in columns 16..31 of the 32x32 mask; its + hotspot is (16,0). diff --git a/debug.md b/debug.md index 58f1279ec..59d174f09 100644 --- a/debug.md +++ b/debug.md @@ -404,6 +404,34 @@ in the `irix-gfx-loop` skill (`clanker/skills/irix-gfx-loop/SKILL.md`). `iris-ci screenshot` captures the composed screen from any board, also headless. +**Screen, pointer and keyboard (any board)** + +These work the same on Newport, GR2, IMPACT and the O2, with a window, with +`--ci`, or alongside VNC viewers. Coordinates are screen pixels as a +screenshot shows them (0,0 top-left). The guest's mouse is relative and X +accelerates it, so `ptr` steers: it sends a move, reads back where the +board's hardware cursor went, and corrects until the pointer is on the +target. The first `ptr` move pins the pointer in the top-left corner once, +to learn the cursor's hotspot exactly. + +| Command | Description | +|---|---| +| `screen info` | Size, hardware cursor, pointer position, whether the guest has enabled the mouse, buttons held | +| `screen shot [x y w h]` | The composed screen as PNG (no host status bar), optionally cropped | +| `screen pix ` | One pixel, `rrggbb` | +| `screen wait [quiet_ms] [timeout_ms]` | Wait until the screen has not changed for `quiet_ms` (default 500, at most 10000 ms) | +| `screen cursor` | The cursor glyph as text, with its hotspot | +| `ptr pos` | Where the pointer is | +| `ptr move ` | Put the pointer on a pixel | +| `ptr click [left\|middle\|right] [count]` | Move, then click (`count` 2 for a double click) | +| `ptr down\|up [button]` | Press or release a button where the pointer is | +| `ptr drag [button]` | Press at one point, release at the other | +| `ptr rel ` | Raw relative motion, as a mouse sends it | +| `ptr calibrate` | Pin the pointer in the top-left corner and learn the current cursor's hotspot | +| `key [+...] [down\|up]` | Press a key or chord (`Return`, `F1`, `ctrl+c`, `ctrl+alt+Delete`): X keysym names, short aliases (`ctrl`, `alt`, `shift`, `enter`, `esc`, `tab`, `bs`, `del`, `pgup`), single characters, or winit names (`KeyA`) | +| `key type ` | Type ASCII text on a US layout | +| `key names` | Every key name `key` accepts | + ### Hardware devices | Command | Description | @@ -659,12 +687,20 @@ Register it with your client, for example in `.mcp.json`: | `translate_address(address)`, `dump_tlb()` | `translate`, `tlb dump` | | `lookup_symbol(name_or_addr)` | `sym` | | `read_eframe(address, layout)`, `kernel_log(size)` | `eframe`, `putbuf` | +| `screenshot(x, y, width, height, max_width)` (returns the image) | `screen shot` | +| `screen_info()`, `screen_pixel(x, y)`, `wait_screen(quiet_ms, timeout_ms)` | `screen info`, `screen pix`, `screen wait` | +| `click(x, y, button, count)`, `move_pointer(x, y)`, `drag(x1, y1, x2, y2, button)` | `ptr click`, `ptr move`, `ptr drag` | +| `type_text(text)`, `press_key(keys, action)` | `key type`, `key` | Commands without a dedicated tool (graphics, devices, `j2`) go through `run_command`. ## Other introspection +- **VNC**: `--vnc` (or `[vnc] listen`) serves the screen, keyboard and mouse + to any VNC viewer, from any board, with a window or under `--ci`. See + [HELP.md](HELP.md#vnc). + - **`iris-ci`**: the CI control socket drives a running machine from scripts: boot and log in, run guest commands, copy files in and out, screenshots, snapshots, CD changes ([STORAGE.md](STORAGE.md#ci-control-socket-and-iris-ci)). diff --git a/src/ci.rs b/src/ci.rs index 6509c073f..9b04aed54 100644 --- a/src/ci.rs +++ b/src/ci.rs @@ -612,48 +612,15 @@ fn cmd_screenshot(server: &CiServer, args: &Value) -> Response { return Response::err("screenshot: missing 'path' arg"); }; - display.request_screenshot(); - // The board consumes the request when its refresh thread next renders. - // Wait for that (bounded) instead of guessing a frame time; the screen - // lock below then waits for that render to finish copying into `rgba`. - // Boards that do not consume the request report no pending work and fall - // through immediately. - let deadline = std::time::Instant::now() + std::time::Duration::from_millis(250); - while display.screenshot_pending() && std::time::Instant::now() < deadline { - std::thread::sleep(std::time::Duration::from_millis(2)); - } - // Snapshot the framebuffer under the screen lock; unlock before the PNG - // encode so the refresh thread isn't blocked during disk I/O. - let has_renderer = display.renderer_slot().lock().is_some(); - let (width, height, rgba_copy) = { - let screen = display.screen().lock(); - let w = screen.width; - let h = screen.height; - if w == 0 || h == 0 { - return Response::err("screenshot: the display has no video mode yet"); - } - let mut out = Vec::with_capacity(w * h); - if has_renderer { - // The front-end's renderer already composited this frame into - // `rgba`; copy the visible window (row stride 2048). - for y in 0..h { - let base = y * 2048; - out.extend_from_slice(&screen.rgba[base..base + w]); - } - } else { - // `--ci` installs no renderer, so nothing has filled `rgba`. - // Composite the guest screen here from the caches the refresh - // thread just updated. The host status bar and debug overlay are - // deliberately absent: this is the guest's screen. - let mut compositor = crate::compositor::SwCompositor::new(); - compositor.compose_pixels(&screen.compositor_source()); - let pixels = compositor.pixels(); - for y in 0..h { - let base = y * 2048; - out.extend_from_slice(&pixels[base..base + w]); - } - } - (w, h, out) + // The board composes the next frame into `rgba` (Newport with no + // front-end composes in software on its refresh thread). The host + // status bar and debug overlay are never in it: this is the guest's + // screen. + let mut rgba_copy = Vec::new(); + let Some((width, height, _)) = crate::gfx_display::fresh_frame( + &**display, &mut rgba_copy, std::time::Duration::from_millis(250)) + else { + return Response::err("screenshot: the display has no video mode yet"); }; let mut rgb = Vec::with_capacity(width * height * 3); diff --git a/src/config.rs b/src/config.rs index e26d04b43..7457ac4f5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1290,6 +1290,19 @@ impl Default for AudioConfig { } } +/// The built-in VNC server (`[vnc]` section). Off unless `listen` is set. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(deny_unknown_fields)] +pub struct VncConfig { + /// Address to serve on, `host:port` (a bare port means 127.0.0.1). + /// Without a password, keep it on loopback. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub listen: Option, + /// Most frames a second sent to a viewer (default 30). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_fps: Option, +} + /// VINO video-in configuration. #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct VinoConfig { @@ -1516,6 +1529,10 @@ pub struct MachineConfig { #[serde(default)] pub vino: VinoConfig, + /// Built-in VNC server (`[vnc]` section). + #[serde(default)] + pub vnc: VncConfig, + /// Networking backend selection (`[network]` section). Defaults to NAT. #[serde(default)] pub network: NetworkSection, @@ -1713,6 +1730,7 @@ impl Default for MachineConfig { serial_log: None, serial_log_timestamps: Default::default(), vino: VinoConfig::default(), + vnc: VncConfig::default(), network: NetworkSection::default(), mouse_scroll_pixels_per_line: default_scroll_pixels_per_line(), lock_aspect_ratio: default_lock_aspect_ratio(), @@ -2262,6 +2280,12 @@ pub struct Cli { #[arg(long = "monitor-port", value_name = "PORT")] pub monitor_port: Option, + /// Serve the screen, keyboard and mouse over VNC on ADDR (`host:port`, + /// or a port on 127.0.0.1; default 127.0.0.1:5900). Works with a + /// window, with --ci, and on every graphics board. + #[arg(long = "vnc", value_name = "ADDR", num_args = 0..=1, default_missing_value = "127.0.0.1:5900")] + pub vnc: Option, + /// Serial channel A (tty2) port on 127.0.0.1 (default 8880). #[arg(long = "serial-port-a", value_name = "PORT")] pub serial_port_a: Option, @@ -2416,6 +2440,7 @@ impl Cli { if let Some(p) = self.gdb_port { cfg.gdb_port = Some(p); } if let Some(p) = self.monitor_port { cfg.monitor_port = Some(p); } + if let Some(a) = &self.vnc { cfg.vnc.listen = Some(a.clone()); } if let Some(p) = self.serial_port_a { cfg.serial_port_a = Some(p); } if let Some(p) = self.serial_port_b { cfg.serial_port_b = Some(p); } if let Some(k) = self.keyboard { cfg.keyboard = k; } diff --git a/src/dev/gr2/gr2disp.rs b/src/dev/gr2/gr2disp.rs index eb9f08264..03eca97b9 100644 --- a/src/dev/gr2/gr2disp.rs +++ b/src/dev/gr2/gr2disp.rs @@ -63,6 +63,7 @@ impl Gr2 { let row = y * gr2comp::OUT_STRIDE; screen.rgba[row..row + FB_W].copy_from_slice(&screen.fb_rgb[row..row + FB_W]); } + screen.frame_seq = screen.frame_seq.wrapping_add(1); } screen.prebuilt = true; screen.width = FB_W; diff --git a/src/dev/gr2/mod.rs b/src/dev/gr2/mod.rs index 759310044..22f48ae26 100644 --- a/src/dev/gr2/mod.rs +++ b/src/dev/gr2/mod.rs @@ -992,6 +992,18 @@ impl crate::gfx_display::GfxDisplay for Gr2 { fn screen(&self) -> &Mutex { &self.screen } fn request_screenshot(&self) { self.screenshot_pending.store(true, Ordering::Relaxed); } fn cycles(&self) -> CyclesPtr { self.cycles.get() } + fn hw_cursor(&self) -> Option { + // A racy read of the registers, as the display thread's compose does. + let vc1 = &self.regs().vc1; + if !vc1.cursor_visible() { + return None; + } + let (x, y) = vc1.cursor_pos(); + let mask = (0..32).map(|cy| (0..32).fold(0u64, |m, cx| { + m | ((vc1.cursor_pixel(cx, cy) != 0) as u64) << cx + })).collect(); + Some(crate::gfx_display::HwCursor { x, y, size: 32, mask }) + } } impl Resettable for Gr2 { diff --git a/src/dev/mc.rs b/src/dev/mc.rs index 286bbbec3..9c825968e 100644 --- a/src/dev/mc.rs +++ b/src/dev/mc.rs @@ -508,7 +508,9 @@ impl MemoryController { eprintln!("MC: CPU Error at {:08x}", addr); } - pub fn report_gio_timeout(&self, addr: u64) { + /// A GIO access nobody answered. Returns whether the MC aborts it with a + /// bus error (`CPUCTRL1.ABORT_EN`, which the PROM sets at power-on). + pub fn report_gio_timeout(&self, addr: u64) -> bool { let mut state = self.state.lock(); state.regs[(REG_GIO_ERROR_ADDR / 4) as usize] = addr as u32; // TIME bit (10) only set when ABORT_EN (CPUCTRL1 bit 4) is enabled @@ -518,6 +520,7 @@ impl MemoryController { } dlog_dev!(LogModule::Mc, "MC: GIO Timeout at {:08x} (abort_en={})", addr, abort_en); eprintln!("MC: GIO Timeout at {:08x}", addr); + abort_en } // ── Accessors for the DMA engine in mc_vdma.rs ────────────────────────── diff --git a/src/dev/mgras/disp.rs b/src/dev/mgras/disp.rs index b32d1412c..3c9dd0166 100644 --- a/src/dev/mgras/disp.rs +++ b/src/dev/mgras/disp.rs @@ -288,6 +288,7 @@ impl Mgras { let row = y * 2048; screen.rgba[row..row + w].copy_from_slice(&screen.fb_rgb[row..row + w]); } + screen.frame_seq = screen.frame_seq.wrapping_add(1); screen.width = w; screen.height = h; screen.status_bar_only = false; diff --git a/src/dev/mgras/mod.rs b/src/dev/mgras/mod.rs index f12e0cb7c..f6d1b8b79 100644 --- a/src/dev/mgras/mod.rs +++ b/src/dev/mgras/mod.rs @@ -1588,6 +1588,19 @@ impl crate::gfx_display::GfxDisplay for Mgras { fn screen(&self) -> &Mutex { &self.screen } fn request_screenshot(&self) { self.screenshot_pending.store(true, Ordering::Relaxed); } fn cycles(&self) -> crate::cpu::mips_core::CyclesPtr { *self.cycles.lock() } + fn hw_cursor(&self) -> Option { + // A racy read of the VC3, as the display thread's snapshot does. + let vc3 = &self.front_view().dcb.vc3; + let (x, y, size, glyph) = vc3.cursor()?; + // The glyph layout `frame.rs` draws: two planes, MSB leftmost. + let words_per_row = size / 16; + let plane_words = size * words_per_row; + let word = |plane: usize, row: usize, w: usize| vc3.sram[(glyph + plane * plane_words + row * words_per_row + w) & 0x7FFF] as u64; + let mask = (0..size).map(|row| (0..words_per_row).fold(0u64, |m, w| { + m | ((word(0, row, w) | word(1, row, w)).reverse_bits() >> 48) << (w * 16) + })).collect(); + Some(crate::gfx_display::HwCursor { x, y, size, mask }) + } } impl Device for Mgras { diff --git a/src/dev/ng1/rex3.rs b/src/dev/ng1/rex3.rs index 33938d360..62202c1c3 100644 --- a/src/dev/ng1/rex3.rs +++ b/src/dev/ng1/rex3.rs @@ -41,6 +41,8 @@ pub trait Renderer: Send { fn switch_compositor(&mut self, _use_gl: bool) -> &'static str { "sw" } /// Return a short status string: "compositor=gl shader=integer" etc. fn compositor_status(&self) -> String { "compositor=sw shader=n/a".to_string() } + /// Whether the GL compositor is in use, or is about to be. + fn gl_compositor(&self) -> bool { false } } pub const REX3_SIZE: u32 = 0x2000; // 8KB @@ -1471,6 +1473,12 @@ pub struct Rex3 { pub screenshot_pending: AtomicBool, /// Monotonically incrementing screenshot counter for unique filenames. pub screenshot_counter: AtomicU32, + /// One-shot "compose the next frame into `screen.rgba`" (no file), for + /// the CI socket and the monitor's `screen` commands. + frame_request: AtomicBool, + /// Frame consumers (VNC) that want `screen.rgba` kept current; while + /// non-zero every full frame is read back and the compositor is software. + capture_users: AtomicUsize, /// Atomic shadow of MipsCore::count_hz — updated by CPU thread, read by refresh thread. /// Wrapped in Mutex so machine.rs can swap in the real Arc from MipsCore after construction. #[cfg(feature = "developer")] @@ -1631,6 +1639,8 @@ impl Rex3 { show_disp_debug: AtomicBool::new(false), screenshot_pending: AtomicBool::new(false), screenshot_counter: AtomicU32::new(0), + frame_request: AtomicBool::new(false), + capture_users: AtomicUsize::new(0), #[cfg(feature = "developer")] count_hz_atomic: Mutex::new(Arc::new(AtomicU64::new(crate::cpu::mips_core::DEFAULT_COUNT_HZ))), } @@ -2958,6 +2968,12 @@ impl Rex3 { let mut last_topscan: usize = usize::MAX; let mut frames_since_render: u32 = u32::MAX; let mut full_presents: u32 = 0; + // Composes `screen.rgba` when a frame is wanted and no front-end + // renderer is installed (`--ci`); made on first use. + let mut capture_compositor: Option = None; + // While capture forces the renderer onto the software compositor: + // whether it used GL before, to switch back when capture ends. + let mut restore_gl: Option = None; while self.running.load(Ordering::Relaxed) { let start = std::time::Instant::now(); @@ -3052,6 +3068,7 @@ impl Rex3 { || dbg_overlay || topscan != last_topscan || self.screenshot_pending.load(Ordering::Relaxed) + || self.frame_request.load(Ordering::Relaxed) || frames_since_render >= IDLE_HEARTBEAT_FRAMES; if should_render { @@ -3064,7 +3081,8 @@ impl Rex3 { && !fb_was_dirty && !palette_dirty && !dbg_overlay - && !self.screenshot_pending.load(Ordering::Relaxed); + && !self.screenshot_pending.load(Ordering::Relaxed) + && !self.frame_request.load(Ordering::Relaxed); // Push debug state into overlay overlay.show_cmap = self.show_cmap.load(Ordering::Relaxed); @@ -3098,6 +3116,12 @@ impl Rex3 { // Screenshot: grab pixels from the compositor before presenting. let take_screenshot = self.screenshot_pending.swap(false, Ordering::Relaxed); + // Frame consumers: a one-shot request, or capture (VNC) on + // every full frame. Heartbeats change nothing in the picture. + let capture = self.capture_users.load(Ordering::Relaxed) > 0; + let want_frame = self.frame_request.load(Ordering::Relaxed) + || (capture && !screen.status_bar_only); + let need_readback = take_screenshot || want_frame; // Assert VBLANK: device state is now copied into screen caches, so // the CPU gets the maximum window to react before the next refresh. @@ -3125,17 +3149,45 @@ impl Rex3 { crate::pf_scope!(crate::perfetto::events::REX3_PRESENT); self.diag.fetch_or(Self::DIAG_LOOP_GL_RENDER, Ordering::Relaxed); let borrow = screen.fb_borrowed; + // Capture reads every frame back: keep it a slice copy + // out of the software compositor, never a GL readback. + if capture { + restore_gl.get_or_insert_with(|| r.gl_compositor()); + r.switch_compositor(false); + } else if let Some(gl) = restore_gl.take() { + if gl { + r.switch_compositor(true); + } + } r.present( &mut *screen, &mut overlay, &mut status_bar, &mut sbtex, &bar_stats, - take_screenshot, + need_readback, if borrow { Some(&**fb_rgb) } else { None }, if borrow { Some(&**fb_aux) } else { None }, ); self.diag.fetch_and(!Self::DIAG_LOOP_GL_RENDER, Ordering::Relaxed); + } else if need_readback { + // No front-end (`--ci`): compose the guest screen here. + // No status bar or overlay: this is the guest's screen. + let comp = capture_compositor.get_or_insert_with(crate::compositor::SwCompositor::new); + let borrow = screen.fb_borrowed; + comp.compose_pixels(&screen.compositor_source_from( + if borrow { Some(&**fb_rgb) } else { None }, + if borrow { Some(&**fb_aux) } else { None }, + )); + let (w, h) = (screen.width, screen.height); + let px = comp.pixels(); + for y in 0..h { + let row = y * crate::disp::FB_STRIDE; + screen.rgba[row..row + w].copy_from_slice(&px[row..row + w]); + } + } + if need_readback { + screen.frame_seq = screen.frame_seq.wrapping_add(1); } if !screen.status_bar_only { @@ -3157,6 +3209,9 @@ impl Rex3 { } } + // Cleared after the copy above, under the screen lock, so a + // waiter that sees it clear then reads the finished frame. + self.frame_request.store(false, Ordering::Relaxed); self.diag.fetch_and(!(Self::DIAG_LOCK_SCREEN | Self::DIAG_LOCK_RENDERER), Ordering::Relaxed); } else { // Nothing visible changed — skip the full refresh + GL upload and @@ -4001,6 +4056,9 @@ impl Device for Rex3 { Some("sw") => false, _ => return Err("Usage: disp compositor ".to_string()), }; + if use_gl && self.capture_users.load(Ordering::Relaxed) > 0 { + return Err("the VNC server needs the software compositor while it runs".to_string()); + } let active = if let Some(renderer) = self.renderer.lock().as_mut() { renderer.switch_compositor(use_gl) } else { @@ -4878,6 +4936,49 @@ impl crate::gfx_display::GfxDisplay for Rex3 { fn request_screenshot(&self) { self.screenshot_pending.store(true, Ordering::Relaxed); } fn screenshot_pending(&self) -> bool { self.screenshot_pending.load(Ordering::Relaxed) } fn cycles(&self) -> crate::cpu::mips_core::CyclesPtr { self.cycles.get() } + fn request_frame(&self) { self.frame_request.store(true, Ordering::Relaxed); } + fn frame_pending(&self) -> bool { self.frame_request.load(Ordering::Relaxed) } + fn set_capture(&self, on: bool) { + if on { + self.capture_users.fetch_add(1, Ordering::Relaxed); + // Repaint at once so the new consumer gets a frame. + self.fb_dirty.store(true, Ordering::Release); + } else { + let _ = self.capture_users.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |n| n.checked_sub(1)); + } + } + fn hw_cursor(&self) -> Option { + use crate::dev::ng1::vc2::{VC2_REG_CURRENT_CURSOR_X, VC2_REG_CURSOR_Y_LOC, + VC2_REG_CURSOR_ENTRY_PTR, VC2_REG_DISPLAY_CONTROL, VC2_CTRL_CURSOR_EN, VC2_CTRL_CURSOR_SIZE}; + let x_adjust = self.screen.lock().cursor_x_adjust; + let vc2 = self.vc2.lock(); + let ctrl = vc2.regs[VC2_REG_DISPLAY_CONTROL as usize]; + if ctrl & VC2_CTRL_CURSOR_EN == 0 { + return None; + } + let entry = vc2.regs[VC2_REG_CURSOR_ENTRY_PTR as usize] as usize; + let ram = &vc2.ram; + let word = |a: usize| ram.get(a).copied().unwrap_or(0) as u64; + // The same glyph layout the compositor draws (compositor.rs). + let (size, mask): (usize, Vec) = if ctrl & VC2_CTRL_CURSOR_SIZE != 0 { + (64, (0..64).map(|y| (0..4).fold(0u64, |m, w| { + m | (word(entry + y * 4 + w).reverse_bits() >> 48) << (w * 16) + })).collect()) + } else { + (32, (0..32).map(|y| (0..2).fold(0u64, |m, w| { + let a = entry + y * 2 + w; + m | ((word(a) | word(a + 64)).reverse_bits() >> 48) << (w * 16) + })).collect()) + }; + Some(crate::gfx_display::HwCursor { + // The compositor latches Y at retrace; the live location register + // is where it will be drawn next. + x: vc2.regs[VC2_REG_CURRENT_CURSOR_X as usize] as i32 - 31 + x_adjust, + y: vc2.regs[VC2_REG_CURSOR_Y_LOC as usize] as i32 - 31, + size, + mask, + }) + } } impl Resettable for Rex3 { diff --git a/src/dev/o2gfx.rs b/src/dev/o2gfx.rs index b2e22a089..16b5ac434 100644 --- a/src/dev/o2gfx.rs +++ b/src/dev/o2gfx.rs @@ -420,6 +420,38 @@ impl Gbe { snap } + /// CRS_POS: x in bits 11:0 and y in bits 27:16, each a 12-bit signed + /// number (the X server puts the glyph at -6 to keep a hotspot on the + /// edge). + fn cursor_xy(crs: u32) -> (i32, i32) { + let sext12 = |v: u32| ((v & 0xfff) as i32) << 20 >> 20; + (sext12(crs), sext12(crs >> 16)) + } + + /// The hardware cursor when enabled: its glyph's top-left on screen and + /// one opaque-pixel mask word per row (bit `c` = column `c`), 32x32. + pub fn cursor(&self) -> Option<(i32, i32, Vec)> { + let r = { + let mut s = self.s.lock().unwrap(); + self.latch(&mut s); + [Self::get(&s.r, CRS_CTL), Self::get(&s.r, CRS_POS)] + .into_iter() + .chain((0..64).map(|i| Self::get(&s.r, CRS_GLYPH + 4 * i))) + .collect::>() + }; + if r[0] & 1 == 0 { + return None; + } + let (x, y) = Self::cursor_xy(r[1]); + // Two bits a pixel, 16 pixels a word, leftmost in the top bits; as + // `render` draws it. + let mask = (0..32usize).map(|dy| (0..32usize).fold(0u64, |m, dx| { + let word = r[2 + 2 * dy + dx / 16]; + m | (((word >> (30 - 2 * (dx % 16))) & 3 != 0) as u64) << dx + })).collect(); + Some((x, y, mask)) + } + /// Scan the frame out of RAM into `out` (R, G, B, A bytes, row 0 at the /// top), as GBE would. Returns the size drawn, or None when the display /// is off. `out` must hold the largest mode (2048 x 2048 pixels). @@ -538,7 +570,7 @@ impl Gbe { let crs_on = g(CRS_CTL) & 1 != 0; let crs = g(CRS_POS); let probe_at = self.probe.lock().unwrap().0; - let (cx, cy) = ((crs & 0xffff) as i16 as i32, (crs >> 16) as i16 as i32); + let (cx, cy) = Self::cursor_xy(crs); for y in 0..h { if use_did { diff --git a/src/dev/ps2.rs b/src/dev/ps2.rs index 34b4cebbc..f81633a86 100644 --- a/src/dev/ps2.rs +++ b/src/dev/ps2.rs @@ -519,6 +519,18 @@ impl Ps2Controller { Some(byte) } + /// Whether the guest has turned the mouse on: until it has, moves and + /// clicks are dropped. + pub fn mouse_enabled(&self) -> bool { + let state = self.state.lock(); + state.mouse_enabled && state.config & 0x20 == 0 + } + + /// Mouse bytes queued that the guest has not read yet. + pub fn mouse_pending(&self) -> usize { + self.state.lock().mouse_queue_bytes + } + /// Push a scancode byte to the keyboard queue (called from UI) pub fn push_kb(&self, key: KeyCode, pressed: bool) { if !self.running.load(Ordering::Relaxed) { return; } diff --git a/src/disp.rs b/src/disp.rs index f3747f3c6..44a97df8e 100644 --- a/src/disp.rs +++ b/src/disp.rs @@ -71,6 +71,9 @@ pub struct Rex3Screen { /// board's own compositor (GR2's `gr2comp`). The renderer's compositor /// then presents it as-is instead of decoding Newport state. pub prebuilt: bool, + /// Bumped each time `rgba` is rewritten with a composed frame, so frame + /// consumers (VNC) can skip unchanged ones without comparing pixels. + pub frame_seq: u64, } impl Rex3Screen { @@ -95,6 +98,7 @@ impl Rex3Screen { status_bar_only: false, fb_borrowed: false, prebuilt: false, + frame_seq: 0, } } diff --git a/src/gfx_display.rs b/src/gfx_display.rs index 71c579d8a..3163f9dce 100644 --- a/src/gfx_display.rs +++ b/src/gfx_display.rs @@ -3,7 +3,8 @@ //! //! Each board owns its display thread and composition; this trait only exposes //! the hand-off points: the renderer slot the front-end installs into, the -//! last-presented screen (screenshots), and the CPU cycle counter. +//! last-presented screen (screenshots, VNC), the hardware cursor, and the CPU +//! cycle counter. use parking_lot::Mutex; @@ -31,4 +32,84 @@ pub trait GfxDisplay: Send + Sync { /// CPU cycle counter the board was wired to (GUI's live MIPS estimate). fn cycles(&self) -> CyclesPtr; + + /// Ask for the next composed frame in `screen().rgba`, without the + /// screenshot file [`request_screenshot`](Self::request_screenshot) also + /// writes. Boards that compose into `rgba` every frame ignore it. + fn request_frame(&self) {} + + /// True while a [`request_frame`](Self::request_frame) is still pending. + fn frame_pending(&self) -> bool { false } + + /// Keep `screen().rgba` current with every composed frame while at least + /// one consumer (VNC) has turned capture on. Calls nest: each `true` is + /// matched by a `false`. Newport then composes in software, so the copy is + /// a slice copy and not a GL readback. Boards that always compose into + /// `rgba` ignore it. + fn set_capture(&self, _on: bool) {} + + /// The hardware cursor as the guest has set it, read live from the + /// board's registers (not the last composed frame). + fn hw_cursor(&self) -> Option { None } +} + +/// The hardware cursor: where the board draws its glyph and which of the +/// glyph's pixels are opaque. The glyph's top-left is the pointer position +/// minus the hotspot, which the hardware does not know. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct HwCursor { + /// Glyph top-left in display coordinates; negative near the top/left edge. + pub x: i32, + pub y: i32, + /// Glyph side, 32 or 64. + pub size: usize, + /// One word per glyph row, bit `c` set where column `c` is opaque. + pub mask: Vec, +} + +impl HwCursor { + /// The hotspot when nothing better is known: the first opaque pixel of + /// the topmost opaque row, the tip of an arrow. (0, 0) for an empty glyph. + pub fn tip(&self) -> (i32, i32) { + self.mask.iter().enumerate() + .find(|(_, &m)| m != 0) + .map(|(y, &m)| (m.trailing_zeros() as i32, y as i32)) + .unwrap_or((0, 0)) + } + + /// A key that tells glyphs apart, to remember a learned hotspot per shape. + pub fn shape_key(&self) -> u64 { + // FNV-1a over the mask words. + self.mask.iter().fold(0xcbf2_9ce4_8422_2325u64, |h, &m| (h ^ m).wrapping_mul(0x100_0000_01b3)) + ^ self.size as u64 + } +} + +/// The visible part of the last composed frame, tight (`width` pixels a row, +/// `0xAABBGGRR`, R in the low byte), with its sequence number. `None` until +/// the display has a video mode. Holds the screen lock only for the copy. +pub fn copy_frame(display: &dyn GfxDisplay, out: &mut Vec) -> Option<(usize, usize, u64)> { + let screen = display.screen().lock(); + let (w, h) = (screen.width, screen.height); + if w == 0 || h == 0 { + return None; + } + out.clear(); + out.reserve(w * h); + for y in 0..h { + let row = y * crate::disp::FB_STRIDE; + out.extend_from_slice(&screen.rgba[row..row + w]); + } + Some((w, h, screen.frame_seq)) +} + +/// Request a fresh frame and wait (at most `timeout`) for the board to +/// compose it, then copy it as [`copy_frame`] does. +pub fn fresh_frame(display: &dyn GfxDisplay, out: &mut Vec, timeout: std::time::Duration) -> Option<(usize, usize, u64)> { + display.request_frame(); + let deadline = std::time::Instant::now() + timeout; + while display.frame_pending() && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(2)); + } + copy_frame(display, out) } diff --git a/src/hostinput.rs b/src/hostinput.rs new file mode 100644 index 000000000..7ecaee0ac --- /dev/null +++ b/src/hostinput.rs @@ -0,0 +1,718 @@ +//! The host's hands on the guest: look at the screen, put the pointer on a +//! pixel, click, press keys. Shared by the VNC server and the monitor's +//! `screen`, `ptr` and `key` commands, for every machine and board. +//! +//! The guest's mouse is relative (PS/2), and the X server accelerates what +//! it reads, so a position cannot be sent. Instead the pointer is steered: +//! send a move, read back where the board's hardware cursor went (IRIX always +//! uses it), and correct until the pointer sits on the target. The hardware +//! knows the glyph's top-left, not its hotspot. A hotspot is known exactly +//! when the pointer is pinned in the top-left corner (X keeps the pointer on +//! screen, so it is at 0,0), which the first move does once, or when a new +//! shape appears after a move small enough that X does not accelerate it +//! (the pointer then moved exactly that far). Until then a new shape's +//! hotspot is guessed as its tip plus the offset the known shapes showed. +//! Only the top and left edges pin reliably: the size X uses can differ +//! from the board's visible size by a few pixels. + +use std::collections::HashMap; +use std::io::Write; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use parking_lot::Mutex; +use winit::keyboard::KeyCode; + +use crate::dev::ps2::Ps2Controller; +use crate::gfx_display::{GfxDisplay, HwCursor}; + +/// PS/2 button bits. +pub const BUTTON_LEFT: u8 = 1; +pub const BUTTON_RIGHT: u8 = 2; +pub const BUTTON_MIDDLE: u8 = 4; + +/// How long a move waits for the cursor to answer one packet. +const MOVE_ANSWER: Duration = Duration::from_millis(250); +/// How long the cursor must stay put after a move before the next step. +const SETTLE: Duration = Duration::from_millis(20); +/// How long the cursor may be off before a move gives up on it. +const CURSOR_GAP: Duration = Duration::from_millis(400); +/// How long the pointer may ignore the mouse (the guest busy) before a move +/// gives up. +const STALL_LIMIT: Duration = Duration::from_millis(3000); +/// Moves this small (mickeys) are sent exactly: under X's acceleration +/// threshold, they move the pointer as far as they say. +const EXACT_STEP: i32 = 3; + +pub struct HostInput { + display: Option>, + ps2: Arc, + /// Buttons held now (PS/2 bits), sent with every move. + buttons: AtomicU8, + /// Hotspots seen exactly, by glyph shape. + hotspots: Mutex>, + /// Hotspot minus tip of the last shape learnt: the guess for a new one. + bias: Mutex<(i32, i32)>, + /// Pixels the pointer moves per mickey on a large move (X's + /// acceleration), learnt as moves are made. + gain: Mutex, + /// One steering loop at a time: two would fight over the pointer. + seek: Mutex<()>, +} + +/// Where a move ended. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MoveResult { + /// The pointer is on the target. + Reached, + /// Time ran out this far (x, y) from the target. + Short(i32, i32), +} + +impl HostInput { + pub fn new(display: Option>, ps2: Arc) -> Self { + Self { + display, + ps2, + buttons: AtomicU8::new(0), + hotspots: Mutex::new(HashMap::new()), + bias: Mutex::new((0, 0)), + gain: Mutex::new(1.0), + seek: Mutex::new(()), + } + } + + /// Build one for a machine and add its commands to the machine's monitor. + pub fn attach(machine: &dyn crate::workstation::Workstation) -> Arc { + let hi = Arc::new(Self::new(machine.get_display(), machine.get_ps2())); + machine.register_monitor_device(hi.clone()); + hi + } + + pub fn display(&self) -> Option<&Arc> { + self.display.as_ref() + } + + pub fn ps2(&self) -> &Arc { + &self.ps2 + } + + /// The display's visible size, (0, 0) before it has a mode. + pub fn screen_size(&self) -> (usize, usize) { + self.display.as_ref().map(|d| { + let s = d.screen().lock(); + (s.width, s.height) + }).unwrap_or((0, 0)) + } + + fn cursor(&self) -> Option { + self.display.as_ref()?.hw_cursor() + } + + /// The cursor, waiting out a short gap: X servers turn the hardware + /// cursor off for a moment while they draw under it or change it. + fn cursor_settled(&self) -> Option { + let start = Instant::now(); + loop { + if let Some(c) = self.cursor() { + return Some(c); + } + if start.elapsed() >= CURSOR_GAP { + return None; + } + std::thread::sleep(Duration::from_millis(5)); + } + } + + /// The hotspot of a glyph: seen exactly, or else guessed. + pub fn hotspot(&self, c: &HwCursor) -> (i32, i32) { + self.known_hotspot(c).unwrap_or_else(|| { + let (tx, ty) = c.tip(); + let (bx, by) = *self.bias.lock(); + (tx + bx, ty + by) + }) + } + + fn known_hotspot(&self, c: &HwCursor) -> Option<(i32, i32)> { + self.hotspots.lock().get(&c.shape_key()).copied() + } + + fn learn(&self, c: &HwCursor, hot: (i32, i32)) { + if !hotspot_plausible(c, hot) { + return; + } + self.hotspots.lock().insert(c.shape_key(), hot); + let (tx, ty) = c.tip(); + *self.bias.lock() = (hot.0 - tx, hot.1 - ty); + } + + /// Where the pointer is, by the hardware cursor; None while it is off. + pub fn pointer(&self) -> Option<(i32, i32)> { + let c = self.cursor()?; + let (hx, hy) = self.hotspot(&c); + Some((c.x + hx, c.y + hy)) + } + + /// Steer the pointer onto (x, y), clamped to the screen. `retarget` is + /// asked between steps for a newer target (a VNC viewer still moving). + /// Errors when there is no cursor to steer by or it does not move. + pub fn move_to(&self, x: i32, y: i32, timeout: Duration, + retarget: &mut dyn FnMut() -> Option<(i32, i32)>) -> Result { + let _one = self.seek.lock(); + if self.hotspots.lock().is_empty() { + // Once: pin the pointer in the corner to know a hotspot exactly. + self.calibrate_locked()?; + } + let deadline = Instant::now() + timeout; + let (mut tx, mut ty) = (x, y); + let mut stalls = [0u32; 2]; + let mut last_motion = Instant::now(); + // How far right/down the pointer goes, once seen pinned there: a + // board that draws the cursor a pixel off leaves the last column or + // row out of reach, and a move there ends at the edge. + let mut far_edge: [Option; 2] = [None; 2]; + loop { + if let Some(t) = retarget() { + (tx, ty) = t; + stalls = [0; 2]; + } + let (w, h) = self.screen_size(); + if w == 0 { + return Err("the display has no video mode yet".into()); + } + let (tx, ty) = (tx.clamp(0, w as i32 - 1), ty.clamp(0, h as i32 - 1)); + let (tx, ty) = (far_edge[0].map_or(tx, |e| tx.min(e)), far_edge[1].map_or(ty, |e| ty.min(e))); + let c = self.cursor_settled().ok_or("the hardware cursor is off (no X server yet?)")?; + let (hx, hy) = self.hotspot(&c); + let (ex, ey) = (tx - (c.x + hx), ty - (c.y + hy)); + if ex == 0 && ey == 0 { + return Ok(MoveResult::Reached); + } + if Instant::now() >= deadline { + return Ok(MoveResult::Short(ex, ey)); + } + let gain = *self.gain.lock(); + let step = |e: i32| -> i32 { + if e.abs() <= EXACT_STEP { + e + } else { + match ((e as f32 / gain).round() as i32).clamp(-127, 127) { + 0 => e.signum(), + s => s, + } + } + }; + let (sx, sy) = (step(ex), step(ey)); + // Steer from where the cursor is, not from moves still queued: + // let a busy guest catch up before sending more. + let queued = Instant::now(); + while self.ps2.mouse_pending() > 0 && queued.elapsed() < MOVE_ANSWER { + std::thread::sleep(Duration::from_millis(1)); + } + if self.ps2.mouse_pending() > 0 { + if last_motion.elapsed() >= STALL_LIMIT { + return Err("the guest is not reading the mouse".into()); + } + continue; + } + self.ps2.push_mouse_input(self.buttons.load(Ordering::Relaxed), sx, sy, 0); + + // Wait for the X server to read it and move the cursor, then + // for the cursor to settle: a board may show a move a frame late, + // and steering from a position before the last step oscillates. + let sent = Instant::now(); + let mut now_at: Option = None; + let mut changed_at = sent; + while sent.elapsed() < MOVE_ANSWER { + std::thread::sleep(Duration::from_millis(1)); + match self.cursor() { + Some(n) if (n.x, n.y) != (c.x, c.y) || n.mask != c.mask => { + if now_at.as_ref().is_none_or(|m| (m.x, m.y, &m.mask) != (n.x, n.y, &n.mask)) { + changed_at = Instant::now(); + now_at = Some(n); + } else if changed_at.elapsed() >= SETTLE { + break; + } + } + Some(_) | None => {} + } + } + let (mx, my) = match &now_at { + Some(n) if n.mask == c.mask => (n.x - c.x, n.y - c.y), + Some(n) => { + // A new shape. After an unaccelerated step from a known + // shape the pointer moved exactly (sx, sy): that gives + // the new hotspot. + if sx.abs() <= EXACT_STEP && sy.abs() <= EXACT_STEP && self.known_hotspot(n).is_none() { + if let Some((ohx, ohy)) = self.known_hotspot(&c) { + // Let X finish the move before reading the glyph. + std::thread::sleep(Duration::from_millis(5)); + if let Some(n) = self.cursor().filter(|m| m.mask == n.mask) { + let (px, py) = (c.x + ohx + sx, c.y + ohy + sy); + self.learn(&n, (px - n.x, py - n.y)); + } + } + } + stalls = [0; 2]; + continue; + } + None => (0, 0), + }; + // Learn the acceleration from large steps. + for (m, s) in [(mx, sx), (my, sy)] { + if s.abs() > EXACT_STEP * 2 && m != 0 && m.signum() == s.signum() { + let mut g = self.gain.lock(); + *g = (*g * 0.5 + (m as f32 / s as f32) * 0.5).clamp(0.25, 8.0); + } + } + // An axis that will not move although pushed: pinned at an edge + // (X keeps the pointer on screen). At the left or top edge the + // pointer is at 0, which tells the hotspot. Or the guest is not + // reading the mouse. + let size = c.size as i32; + let mut learnt = false; + for (axis, (m, s, pos)) in [(mx, sx, c.x), (my, sy, c.y)].into_iter().enumerate() { + if s == 0 || m != 0 { + stalls[axis] = 0; + continue; + } + stalls[axis] += 1; + if stalls[axis] < 2 { + continue; + } + if s > 0 { + // Within a glyph of the right or bottom edge: that is + // the edge. + let p = pos + [hx, hy][axis]; + if p >= [w, h][axis] as i32 - size { + far_edge[axis] = Some(p); + learnt = true; + } + continue; + } + let hot = -pos; + if (-size..size).contains(&hot) && hot != [hx, hy][axis] { + let mut both = (hx, hy); + if axis == 0 { both.0 = hot } else { both.1 = hot } + self.learn(&c, both); + stalls[axis] = 0; + learnt = true; + } + } + if mx != 0 || my != 0 || learnt { + last_motion = Instant::now(); + } + if last_motion.elapsed() >= STALL_LIMIT { + return Err(if self.ps2.mouse_enabled() { + "the cursor does not follow the mouse (guest busy, or no X server?)".into() + } else { + "the guest has not enabled the mouse".into() + }); + } + } + } + + /// Move, then check the move arrived. + pub fn move_to_exact(&self, x: i32, y: i32) -> Result<(), String> { + match self.move_to(x, y, Duration::from_secs(3), &mut || None)? { + MoveResult::Reached => Ok(()), + MoveResult::Short(ex, ey) => Err(format!("pointer stopped {ex},{ey} short of {x},{y}")), + } + } + + /// Push the pointer into the top-left corner and learn the current + /// glyph's hotspot from where the cursor stops. Returns the hotspot. + pub fn calibrate(&self) -> Result<(i32, i32), String> { + let _one = self.seek.lock(); + self.calibrate_locked() + } + + fn calibrate_locked(&self) -> Result<(i32, i32), String> { + // Keep pushing up-left until the cursor has stayed put for a while + // although the guest read every packet: then X has it pinned. + const PINNED: Duration = Duration::from_millis(150); + let start = Instant::now(); + let mut still: Option<((i32, i32), Instant)> = None; + while start.elapsed() < Duration::from_secs(10) { + let drained = Instant::now(); + while self.ps2.mouse_pending() > 0 && drained.elapsed() < Duration::from_millis(500) { + std::thread::sleep(Duration::from_millis(2)); + } + let c = self.cursor_settled().ok_or("the hardware cursor is off")?; + match still { + Some((at, since)) if at == (c.x, c.y) && self.ps2.mouse_pending() == 0 => { + if since.elapsed() >= PINNED { + let hot = (-c.x, -c.y); + if !hotspot_plausible(&c, hot) { + return Err(format!("the cursor stopped at {},{}, not in the corner", c.x, c.y)); + } + self.learn(&c, hot); + return Ok(hot); + } + } + _ => still = Some(((c.x, c.y), Instant::now())), + } + self.ps2.push_mouse_input(self.buttons.load(Ordering::Relaxed), -100, -100, 0); + std::thread::sleep(Duration::from_millis(20)); + } + Err("the cursor never settled in the corner".into()) + } + + /// A raw relative move, as a mouse would send it. + pub fn nudge(&self, dx: i32, dy: i32) { + let b = self.buttons.load(Ordering::Relaxed); + // Packets carry 9-bit deltas. + let (mut dx, mut dy) = (dx, dy); + while dx != 0 || dy != 0 { + let (sx, sy) = (dx.clamp(-200, 200), dy.clamp(-200, 200)); + self.ps2.push_mouse_input(b, sx, sy, 0); + (dx, dy) = (dx - sx, dy - sy); + } + } + + /// Hold exactly `mask` (PS/2 bits) from now on. + pub fn set_buttons(&self, mask: u8) { + if self.buttons.swap(mask, Ordering::Relaxed) != mask { + self.ps2.push_mouse_input(mask, 0, 0, 0); + } + } + + pub fn buttons(&self) -> u8 { + self.buttons.load(Ordering::Relaxed) + } + + /// Wheel clicks (positive away from the user); only reaches a guest + /// that has switched the mouse to its wheel mode. + pub fn wheel(&self, dz: i32) { + self.ps2.push_mouse_input(self.buttons.load(Ordering::Relaxed), 0, 0, dz); + } + + /// Press and release `button` `count` times where the pointer is. + pub fn click(&self, button: u8, count: u32) { + let held = self.buttons.load(Ordering::Relaxed); + for i in 0..count { + if i > 0 { + std::thread::sleep(Duration::from_millis(60)); + } + self.set_buttons(held | button); + std::thread::sleep(Duration::from_millis(40)); + self.set_buttons(held & !button); + } + } + + pub fn key(&self, code: KeyCode, down: bool) { + self.ps2.push_kb(code, down); + } + + /// Press a chord such as `ctrl+alt+F1`, or with `down` = Some, only + /// press or only release it. + pub fn chord(&self, chord: &str, down: Option) -> Result<(), String> { + let keys = chord.split('+').filter(|s| !s.is_empty()) + .map(|n| crate::keysym::from_name(n).ok_or_else(|| format!("unknown key '{n}'"))) + .collect::, _>>()?; + if keys.is_empty() { + return Err("no key named".into()); + } + if down != Some(false) { + for &k in &keys { + self.key(k, true); + } + } + if down.is_none() { + std::thread::sleep(Duration::from_millis(30)); + } + if down != Some(true) { + for &k in keys.iter().rev() { + self.key(k, false); + } + } + Ok(()) + } + + /// Type text on a US keyboard; returns how many characters had a key. + pub fn type_text(&self, text: &str) -> usize { + text.chars().filter(|&ch| match crate::keysym::char_key(ch) { + Some((k, shifted)) => { + if shifted { self.key(KeyCode::ShiftLeft, true); } + self.key(k, true); + self.key(k, false); + if shifted { self.key(KeyCode::ShiftLeft, false); } + true + } + None => self.ps2.type_char(ch), + }).count() + } + + /// A fresh frame, tight, with its size. + pub fn frame(&self) -> Result<(usize, usize, Vec), String> { + let d = self.display.as_ref().ok_or("no display (headless)")?; + let mut px = Vec::new(); + let (w, h, _) = crate::gfx_display::fresh_frame(&**d, &mut px, Duration::from_millis(250)) + .ok_or("the display has no video mode yet")?; + Ok((w, h, px)) + } + + /// Wait until the screen has not changed for `quiet`, at most `timeout`. + /// Returns whether it settled. + pub fn wait_still(&self, quiet: Duration, timeout: Duration) -> Result { + let start = Instant::now(); + let mut last = self.frame()?; + let mut since = Instant::now(); + while start.elapsed() < timeout { + std::thread::sleep(Duration::from_millis(50)); + let f = self.frame()?; + if f != last { + last = f; + since = Instant::now(); + } else if since.elapsed() >= quiet { + return Ok(true); + } + } + Ok(false) + } +} + +/// A hotspot is inside its glyph, give or take how far the board draws the +/// cursor from where the guest put it: a board whose cursor offset we have +/// slightly wrong shows a hotspot a few pixels outside. +fn hotspot_plausible(c: &HwCursor, (hx, hy): (i32, i32)) -> bool { + let size = c.size as i32; + (-8..size + 8).contains(&hx) && (-8..size + 8).contains(&hy) +} + +/// Save tight `0xAABBGGRR` pixels (row length `w`) as an RGB PNG, cropped to +/// `crop` (x, y, w, h) when given. +pub fn save_png(path: &str, px: &[u32], w: usize, h: usize, crop: Option<(usize, usize, usize, usize)>) -> Result<(usize, usize), String> { + let (cx, cy, cw, ch) = crop.unwrap_or((0, 0, w, h)); + if cw == 0 || ch == 0 || cx + cw > w || cy + ch > h { + return Err(format!("crop {cx},{cy} {cw}x{ch} is outside the {w}x{h} screen")); + } + let mut rgb = Vec::with_capacity(cw * ch * 3); + for y in cy..cy + ch { + for &p in &px[y * w + cx..y * w + cx + cw] { + rgb.extend_from_slice(&[p as u8, (p >> 8) as u8, (p >> 16) as u8]); + } + } + let tmp = format!("{path}.tmp"); + { + let f = std::fs::File::create(&tmp).map_err(|e| format!("{path}: {e}"))?; + let mut enc = png::Encoder::new(std::io::BufWriter::new(f), cw as u32, ch as u32); + enc.set_color(png::ColorType::Rgb); + enc.set_depth(png::BitDepth::Eight); + let mut wr = enc.write_header().map_err(|e| e.to_string())?; + wr.write_image_data(&rgb).map_err(|e| e.to_string())?; + } + std::fs::rename(&tmp, path).map_err(|e| format!("{path}: {e}"))?; + Ok((cw, ch)) +} + +fn parse_button(s: Option<&&str>) -> Result { + Ok(match s.copied().unwrap_or("left") { + "left" | "l" | "1" => BUTTON_LEFT, + "middle" | "m" | "2" => BUTTON_MIDDLE, + "right" | "r" | "3" => BUTTON_RIGHT, + b => return Err(format!("unknown button '{b}' (left, middle, right)")), + }) +} + +fn num(args: &[&str], i: usize, what: &str) -> Result { + args.get(i).ok_or(format!("missing {what}"))?.parse().map_err(|_| format!("bad {what} '{}'", args[i])) +} + +const SCREEN_HELP: &str = "screen info | screen cursor | screen shot [x y w h] | screen pix | screen wait [quiet_ms] [timeout_ms]"; +const PTR_HELP: &str = "ptr pos | ptr move | ptr click [left|middle|right] [count] | ptr down|up [button] | ptr drag [button] | ptr rel | ptr calibrate"; +const KEY_HELP: &str = "key [+...] [down|up] | key type | key names"; + +impl crate::traits::Device for HostInput { + fn step(&self, _cycles: u64) {} + fn stop(&self) {} + fn start(&self) {} + fn is_running(&self) -> bool { true } + fn get_clock(&self) -> u64 { 0 } + + fn register_commands(&self) -> Vec<(String, String)> { + vec![ + ("screen".into(), format!("The guest's screen, any board: {SCREEN_HELP}")), + ("ptr".into(), format!("Pointer at screen pixels, steered by the hardware cursor: {PTR_HELP}")), + ("key".into(), format!("Keyboard by key name (X keysym names, ctrl/alt/shift/enter/esc..., KeyA): {KEY_HELP}")), + ] + } + + fn execute_command(&self, cmd: &str, args: &[&str], mut w: Box) -> Result<(), String> { + let sub = args.first().copied().unwrap_or(""); + let out = |w: &mut Box, s: String| { let _ = writeln!(w, "{s}"); }; + match (cmd, sub) { + ("screen", "info") | ("screen", "") => { + let (sw, sh) = self.screen_size(); + let seq = self.display.as_ref().map(|d| d.screen().lock().frame_seq).unwrap_or(0); + out(&mut w, format!("screen {sw}x{sh} frame {seq}")); + match self.cursor() { + Some(c) => { + let (hx, hy) = self.hotspot(&c); + out(&mut w, format!("cursor glyph at {},{} size {} hotspot {hx},{hy} -> pointer {},{}", + c.x, c.y, c.size, c.x + hx, c.y + hy)); + } + None => out(&mut w, "cursor off".into()), + } + out(&mut w, format!("mouse {}, buttons held {:#x}", + if self.ps2.mouse_enabled() { "enabled" } else { "not enabled by the guest" }, self.buttons())); + Ok(()) + } + ("screen", "cursor") => { + let c = self.cursor().ok_or("cursor off")?; + let (hx, hy) = self.hotspot(&c); + out(&mut w, format!("glyph at {},{} size {} shape {:016x} hotspot {hx},{hy} (tip {:?})", + c.x, c.y, c.size, c.shape_key(), c.tip())); + for (y, m) in c.mask.iter().enumerate() { + let row: String = (0..c.size).map(|x| match (m >> x & 1 != 0, (x as i32, y as i32) == (hx, hy)) { + (_, true) => '@', + (true, _) => '#', + _ => '.', + }).collect(); + out(&mut w, row); + } + Ok(()) + } + ("screen", "shot") => { + let path = args.get(1).ok_or("Usage: screen shot [x y w h]")?; + let crop = if args.len() >= 6 { + Some((num(args, 2, "x")?, num(args, 3, "y")?, num(args, 4, "w")?, num(args, 5, "h")?)) + } else { None }; + let (fw, fh, px) = self.frame()?; + let (cw, ch) = save_png(path, &px, fw, fh, crop)?; + out(&mut w, format!("saved {path} {cw}x{ch} (screen {fw}x{fh})")); + Ok(()) + } + ("screen", "pix") => { + let (x, y): (usize, usize) = (num(args, 1, "x")?, num(args, 2, "y")?); + let (fw, fh, px) = self.frame()?; + if x >= fw || y >= fh { + return Err(format!("{x},{y} is outside the {fw}x{fh} screen")); + } + let p = px[y * fw + x]; + out(&mut w, format!("{x},{y} = {:02x}{:02x}{:02x}", p & 0xff, (p >> 8) & 0xff, (p >> 16) & 0xff)); + Ok(()) + } + ("screen", "wait") => { + let quiet = args.get(1).map(|_| num::(args, 1, "quiet_ms")).transpose()?.unwrap_or(500); + let timeout = args.get(2).map(|_| num::(args, 2, "timeout_ms")).transpose()?.unwrap_or(10_000); + let t = Instant::now(); + let still = self.wait_still(Duration::from_millis(quiet), Duration::from_millis(timeout))?; + out(&mut w, if still { + format!("still for {quiet} ms after {} ms", t.elapsed().as_millis()) + } else { + format!("still changing after {timeout} ms") + }); + Ok(()) + } + ("screen", _) => Err(format!("Usage: {SCREEN_HELP}")), + + ("ptr", "pos") | ("ptr", "") => { + match self.pointer() { + Some((x, y)) => out(&mut w, format!("{x} {y}")), + None => out(&mut w, "cursor off".into()), + } + Ok(()) + } + ("ptr", "move") => { + let (x, y) = (num(args, 1, "x")?, num(args, 2, "y")?); + self.move_to_exact(x, y)?; + let (px, py) = self.pointer().unwrap_or((x, y)); + out(&mut w, format!("pointer at {px} {py}")); + Ok(()) + } + ("ptr", "click") => { + let (x, y) = (num(args, 1, "x")?, num(args, 2, "y")?); + let b = parse_button(args.get(3))?; + let n = args.get(4).map(|_| num::(args, 4, "count")).transpose()?.unwrap_or(1); + self.move_to_exact(x, y)?; + self.click(b, n.clamp(1, 3)); + let (px, py) = self.pointer().unwrap_or((x, y)); + out(&mut w, format!("clicked at {px} {py}")); + Ok(()) + } + ("ptr", "down") | ("ptr", "up") => { + let b = parse_button(args.get(1))?; + let held = self.buttons(); + self.set_buttons(if sub == "down" { held | b } else { held & !b }); + out(&mut w, format!("buttons held {:#x}", self.buttons())); + Ok(()) + } + ("ptr", "drag") => { + let (x1, y1, x2, y2) = (num(args, 1, "x1")?, num(args, 2, "y1")?, num(args, 3, "x2")?, num(args, 4, "y2")?); + let b = parse_button(args.get(5))?; + self.move_to_exact(x1, y1)?; + let held = self.buttons(); + self.set_buttons(held | b); + std::thread::sleep(Duration::from_millis(40)); + let r = self.move_to_exact(x2, y2); + std::thread::sleep(Duration::from_millis(40)); + self.set_buttons(held); + r?; + out(&mut w, format!("dragged {x1} {y1} -> {x2} {y2}")); + Ok(()) + } + ("ptr", "rel") => { + let (dx, dy) = (num(args, 1, "dx")?, num(args, 2, "dy")?); + self.nudge(dx, dy); + out(&mut w, format!("sent {dx} {dy}")); + Ok(()) + } + ("ptr", "calibrate") => { + let (hx, hy) = self.calibrate()?; + out(&mut w, format!("hotspot {hx},{hy}; pointer at 0 0")); + Ok(()) + } + ("ptr", _) => Err(format!("Usage: {PTR_HELP}")), + + ("key", "type") => { + let text = args[1..].join(" "); + let n = self.type_text(&text); + out(&mut w, format!("typed {n} of {} chars", text.chars().count())); + Ok(()) + } + ("key", "names") => { + out(&mut w, crate::keysym::names().collect::>().join(" ")); + Ok(()) + } + ("key", "") => Err(format!("Usage: {KEY_HELP}")), + ("key", chord) => { + let down = match args.get(1).copied() { + None => None, + Some("down") => Some(true), + Some("up") => Some(false), + Some(o) => return Err(format!("'{o}': down or up")), + }; + self.chord(chord, down)?; + out(&mut w, format!("key {chord}{}", match down { Some(true) => " down", Some(false) => " up", None => "" })); + Ok(()) + } + _ => Err("Command not found".into()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tip_and_shape() { + let mut mask = vec![0u64; 32]; + mask[2] = 0b1100; + mask[3] = 0b1110; + let c = HwCursor { x: 10, y: 20, size: 32, mask: mask.clone() }; + assert_eq!(c.tip(), (2, 2)); + let mut other = mask; + other[4] = 1; + let d = HwCursor { x: 0, y: 0, size: 32, mask: other }; + assert_ne!(c.shape_key(), d.shape_key()); + } + + #[test] + fn png_crop_bounds() { + let px = vec![0u32; 4 * 3]; + assert!(save_png("/nonexistent/x.png", &px, 4, 3, Some((3, 0, 2, 1))).is_err()); + } +} diff --git a/src/ip22_bus.rs b/src/ip22_bus.rs index 9527aabc9..ecb160685 100644 --- a/src/ip22_bus.rs +++ b/src/ip22_bus.rs @@ -42,20 +42,32 @@ impl BusDevice for CpuBusErrorDevice { fn write64(&self, addr: u64, _v: u64) -> u32 { self.mc.report_cpu_error(addr); BUS_OK } } -/// GIO bus timeout sink: reports to MC (GIO_ERROR_STAT bit 10 TIME) then returns 0/Ready. -/// Covers GIO space 0x18000000..0x1FA00000 (reserved future GIO + empty expansion slots). +/// GIO bus timeout sink: an address in GIO space no device answers (reserved +/// GIO space and empty expansion or graphics slots, 0x18000000..0x1FA00000). +/// The MC records the timeout in GIO_ERROR_ADDR/STAT and, with time-outs +/// enabled (CPUCTRL1.ABORT_EN, set by the PROM at power-on), aborts a read +/// with a bus error: that is how the PROM's and the kernel's `badaddr` +/// probes find a slot empty. Writes are dropped. struct GioBusErrorDevice { mc: MemoryController, } impl BusDevice for GioBusErrorDevice { - fn read8(&self, addr: u64) -> BusRead8 { self.mc.report_gio_timeout(addr); BusRead8::ok(0xFF) } - fn write8(&self, addr: u64, _v: u8) -> u32 { self.mc.report_gio_timeout(addr); BUS_OK } - fn read16(&self, addr: u64) -> BusRead16 { self.mc.report_gio_timeout(addr); BusRead16::ok(0xFFFF) } + fn read8(&self, addr: u64) -> BusRead8 { + if self.mc.report_gio_timeout(addr) { BusRead8::err() } else { BusRead8::ok(0xFF) } + } + fn write8(&self, addr: u64, _v: u8) -> u32 { self.mc.report_gio_timeout(addr); BUS_OK } + fn read16(&self, addr: u64) -> BusRead16 { + if self.mc.report_gio_timeout(addr) { BusRead16::err() } else { BusRead16::ok(0xFFFF) } + } fn write16(&self, addr: u64, _v: u16) -> u32 { self.mc.report_gio_timeout(addr); BUS_OK } - fn read32(&self, addr: u64) -> BusRead32 { self.mc.report_gio_timeout(addr); BusRead32::ok(0xFFFFFFFF) } + fn read32(&self, addr: u64) -> BusRead32 { + if self.mc.report_gio_timeout(addr) { BusRead32::err() } else { BusRead32::ok(0xFFFF_FFFF) } + } fn write32(&self, addr: u64, _v: u32) -> u32 { self.mc.report_gio_timeout(addr); BUS_OK } - fn read64(&self, addr: u64) -> BusRead64 { self.mc.report_gio_timeout(addr); BusRead64::ok(0xFFFFFFFFFFFFFFFF) } + fn read64(&self, addr: u64) -> BusRead64 { + if self.mc.report_gio_timeout(addr) { BusRead64::err() } else { BusRead64::ok(u64::MAX) } + } fn write64(&self, addr: u64, _v: u64) -> u32 { self.mc.report_gio_timeout(addr); BUS_OK } } diff --git a/src/ip30_machine.rs b/src/ip30_machine.rs index 07a71254a..30ee42328 100644 --- a/src/ip30_machine.rs +++ b/src/ip30_machine.rs @@ -296,7 +296,7 @@ impl OctaneMachine { // HEART's levels 0 to 4 are the CPU's IP2 to IP6. heart.set_interrupts(cpu.interrupts_ptr()); - let mut monitor = crate::monitor::Monitor::new(); + let monitor = crate::monitor::Monitor::new(); monitor.register_device(crate::devlog::init_devlog()); monitor.register_device(cpu.clone().as_device()); monitor.register_device(heart.clone()); @@ -629,6 +629,7 @@ impl Workstation for OctaneMachine { fn get_display(&self) -> Option> { self.gfx.as_ref().map(|g| g.display()) } + fn register_monitor_device(&self, device: Arc) { self.monitor.register_device(device); } fn get_ps2(&self) -> Arc { self.ioc3.ps2() } diff --git a/src/ip32_machine.rs b/src/ip32_machine.rs index 546495074..baaf525ab 100644 --- a/src/ip32_machine.rs +++ b/src/ip32_machine.rs @@ -403,7 +403,7 @@ impl O2Machine { } // The shared monitor, as the other machines have it: the CPU's // commands (registers, breakpoints, memory) and the O2's devices'. - let mut monitor = crate::monitor::Monitor::new(); + let monitor = crate::monitor::Monitor::new(); monitor.register_device(crate::devlog::init_devlog()); monitor.register_device(cpu.clone().as_device()); monitor.register_device(bus.clone()); @@ -920,6 +920,7 @@ impl Workstation for O2Machine { fn get_display(&self) -> Option> { Some(O2Machine::get_display(self)) } fn get_ps2(&self) -> Arc { O2Machine::get_ps2(self) } + fn register_monitor_device(&self, device: Arc) { self.monitor.register_device(device); } fn get_timer_manager(&self) -> Arc { O2Machine::get_timer_manager(self) } fn get_cpu_debug(&self) -> Arc { self.cpu.clone().debug_adapter() } fn fasttick_count(&self) -> Arc { self.cpu.fasttick_count() } @@ -1006,6 +1007,7 @@ impl O2Display { screen.rgba[y * stride + x] = v; } } + screen.frame_seq = screen.frame_seq.wrapping_add(1); screen.prebuilt = true; screen.width = w; screen.height = h; @@ -1064,6 +1066,10 @@ impl crate::gfx_display::GfxDisplay for O2Display { fn cycles(&self) -> CyclesPtr { self.cycles.clone() } + fn hw_cursor(&self) -> Option { + let (x, y, mask) = self.bus.gbe.cursor()?; + Some(crate::gfx_display::HwCursor { x, y, size: 32, mask }) + } } /// The control port: one command per line, one reply line each (`ok ...` or diff --git a/src/iris_ci_main.rs b/src/iris_ci_main.rs index 437b670c2..4e685dfc2 100644 --- a/src/iris_ci_main.rs +++ b/src/iris_ci_main.rs @@ -109,7 +109,7 @@ enum Cmd { #[arg(short = 'n', long, default_value_t = 1_000_000)] n: u64, }, - /// Save the REX3 framebuffer to a PNG. + /// Save the guest screen (any graphics board) to a PNG. Screenshot { path: PathBuf }, /// Send keystrokes to the IRIX serial console. diff --git a/src/keysym.rs b/src/keysym.rs new file mode 100644 index 000000000..316ea3358 --- /dev/null +++ b/src/keysym.rs @@ -0,0 +1,268 @@ +//! One table of the keys the host can press on the guest's keyboard: the +//! physical key (winit's `KeyCode`, what `Ps2Controller::push_kb` takes), its +//! X keysyms unshifted and shifted on a US layout, its XT (scancode set 1) +//! number as QEMU's extended key event sends it (`0xE0 nn` as `0x80 | nn`), +//! and its X keysym name. +//! +//! VNC viewers send keysyms, or with the QEMU extension the scancode of the +//! key itself, which needs no layout. The monitor's `key` command takes the +//! keysym names, a few short aliases, and winit's names (`KeyA`). + +use winit::keyboard::KeyCode; +use winit::keyboard::KeyCode::*; + +struct Key { + code: KeyCode, + xt: u16, + name: &'static str, + sym: u32, + shifted: u32, +} + +const fn k(code: KeyCode, xt: u16, name: &'static str, sym: u32, shifted: u32) -> Key { + Key { code, xt, name, sym, shifted } +} + +#[rustfmt::skip] +static KEYS: &[Key] = &[ + k(Escape, 0x01, "Escape", 0xff1b, 0), + k(Digit1, 0x02, "1", b'1' as u32, b'!' as u32), + k(Digit2, 0x03, "2", b'2' as u32, b'@' as u32), + k(Digit3, 0x04, "3", b'3' as u32, b'#' as u32), + k(Digit4, 0x05, "4", b'4' as u32, b'$' as u32), + k(Digit5, 0x06, "5", b'5' as u32, b'%' as u32), + k(Digit6, 0x07, "6", b'6' as u32, b'^' as u32), + k(Digit7, 0x08, "7", b'7' as u32, b'&' as u32), + k(Digit8, 0x09, "8", b'8' as u32, b'*' as u32), + k(Digit9, 0x0a, "9", b'9' as u32, b'(' as u32), + k(Digit0, 0x0b, "0", b'0' as u32, b')' as u32), + k(Minus, 0x0c, "minus", b'-' as u32, b'_' as u32), + k(Equal, 0x0d, "equal", b'=' as u32, b'+' as u32), + k(Backspace, 0x0e, "BackSpace", 0xff08, 0), + k(Tab, 0x0f, "Tab", 0xff09, 0xfe20), + k(KeyQ, 0x10, "q", b'q' as u32, b'Q' as u32), + k(KeyW, 0x11, "w", b'w' as u32, b'W' as u32), + k(KeyE, 0x12, "e", b'e' as u32, b'E' as u32), + k(KeyR, 0x13, "r", b'r' as u32, b'R' as u32), + k(KeyT, 0x14, "t", b't' as u32, b'T' as u32), + k(KeyY, 0x15, "y", b'y' as u32, b'Y' as u32), + k(KeyU, 0x16, "u", b'u' as u32, b'U' as u32), + k(KeyI, 0x17, "i", b'i' as u32, b'I' as u32), + k(KeyO, 0x18, "o", b'o' as u32, b'O' as u32), + k(KeyP, 0x19, "p", b'p' as u32, b'P' as u32), + k(BracketLeft, 0x1a, "bracketleft", b'[' as u32, b'{' as u32), + k(BracketRight, 0x1b, "bracketright", b']' as u32, b'}' as u32), + k(Enter, 0x1c, "Return", 0xff0d, 0), + k(ControlLeft, 0x1d, "Control_L", 0xffe3, 0), + k(KeyA, 0x1e, "a", b'a' as u32, b'A' as u32), + k(KeyS, 0x1f, "s", b's' as u32, b'S' as u32), + k(KeyD, 0x20, "d", b'd' as u32, b'D' as u32), + k(KeyF, 0x21, "f", b'f' as u32, b'F' as u32), + k(KeyG, 0x22, "g", b'g' as u32, b'G' as u32), + k(KeyH, 0x23, "h", b'h' as u32, b'H' as u32), + k(KeyJ, 0x24, "j", b'j' as u32, b'J' as u32), + k(KeyK, 0x25, "k", b'k' as u32, b'K' as u32), + k(KeyL, 0x26, "l", b'l' as u32, b'L' as u32), + k(Semicolon, 0x27, "semicolon", b';' as u32, b':' as u32), + k(Quote, 0x28, "apostrophe", b'\'' as u32, b'"' as u32), + k(Backquote, 0x29, "grave", b'`' as u32, b'~' as u32), + k(ShiftLeft, 0x2a, "Shift_L", 0xffe1, 0), + k(Backslash, 0x2b, "backslash", b'\\' as u32, b'|' as u32), + k(KeyZ, 0x2c, "z", b'z' as u32, b'Z' as u32), + k(KeyX, 0x2d, "x", b'x' as u32, b'X' as u32), + k(KeyC, 0x2e, "c", b'c' as u32, b'C' as u32), + k(KeyV, 0x2f, "v", b'v' as u32, b'V' as u32), + k(KeyB, 0x30, "b", b'b' as u32, b'B' as u32), + k(KeyN, 0x31, "n", b'n' as u32, b'N' as u32), + k(KeyM, 0x32, "m", b'm' as u32, b'M' as u32), + k(Comma, 0x33, "comma", b',' as u32, b'<' as u32), + k(Period, 0x34, "period", b'.' as u32, b'>' as u32), + k(Slash, 0x35, "slash", b'/' as u32, b'?' as u32), + k(ShiftRight, 0x36, "Shift_R", 0xffe2, 0), + k(NumpadMultiply, 0x37, "KP_Multiply", 0xffaa, 0), + k(AltLeft, 0x38, "Alt_L", 0xffe9, 0xffe7), + k(Space, 0x39, "space", b' ' as u32, 0), + k(CapsLock, 0x3a, "Caps_Lock", 0xffe5, 0), + k(F1, 0x3b, "F1", 0xffbe, 0), + k(F2, 0x3c, "F2", 0xffbf, 0), + k(F3, 0x3d, "F3", 0xffc0, 0), + k(F4, 0x3e, "F4", 0xffc1, 0), + k(F5, 0x3f, "F5", 0xffc2, 0), + k(F6, 0x40, "F6", 0xffc3, 0), + k(F7, 0x41, "F7", 0xffc4, 0), + k(F8, 0x42, "F8", 0xffc5, 0), + k(F9, 0x43, "F9", 0xffc6, 0), + k(F10, 0x44, "F10", 0xffc7, 0), + k(NumLock, 0x45, "Num_Lock", 0xff7f, 0), + k(ScrollLock, 0x46, "Scroll_Lock", 0xff14, 0), + k(Numpad7, 0x47, "KP_7", 0xffb7, 0xff95), + k(Numpad8, 0x48, "KP_8", 0xffb8, 0xff97), + k(Numpad9, 0x49, "KP_9", 0xffb9, 0xff9a), + k(NumpadSubtract, 0x4a, "KP_Subtract", 0xffad, 0), + k(Numpad4, 0x4b, "KP_4", 0xffb4, 0xff96), + k(Numpad5, 0x4c, "KP_5", 0xffb5, 0xff9d), + k(Numpad6, 0x4d, "KP_6", 0xffb6, 0xff98), + k(NumpadAdd, 0x4e, "KP_Add", 0xffab, 0), + k(Numpad1, 0x4f, "KP_1", 0xffb1, 0xff9c), + k(Numpad2, 0x50, "KP_2", 0xffb2, 0xff99), + k(Numpad3, 0x51, "KP_3", 0xffb3, 0xff9b), + k(Numpad0, 0x52, "KP_0", 0xffb0, 0xff9e), + k(NumpadDecimal, 0x53, "KP_Decimal", 0xffae, 0xff9f), + k(IntlBackslash, 0x56, "less", b'<' as u32 | 0x1_0000, 0), + k(F11, 0x57, "F11", 0xffc8, 0), + k(F12, 0x58, "F12", 0xffc9, 0), + k(NumpadEnter, 0x9c, "KP_Enter", 0xff8d, 0), + k(ControlRight, 0x9d, "Control_R", 0xffe4, 0), + k(NumpadDivide, 0xb5, "KP_Divide", 0xffaf, 0), + k(PrintScreen, 0xb7, "Print", 0xff61, 0), + k(AltRight, 0xb8, "Alt_R", 0xffea, 0xfe03), + k(Home, 0xc7, "Home", 0xff50, 0), + k(ArrowUp, 0xc8, "Up", 0xff52, 0), + k(PageUp, 0xc9, "Prior", 0xff55, 0), + k(ArrowLeft, 0xcb, "Left", 0xff51, 0), + k(ArrowRight, 0xcd, "Right", 0xff53, 0), + k(End, 0xcf, "End", 0xff57, 0), + k(ArrowDown, 0xd0, "Down", 0xff54, 0), + k(PageDown, 0xd1, "Next", 0xff56, 0), + k(Insert, 0xd2, "Insert", 0xff63, 0), + k(Delete, 0xd3, "Delete", 0xffff, 0), + k(SuperLeft, 0xdb, "Super_L", 0xffeb, 0), + k(SuperRight, 0xdc, "Super_R", 0xffec, 0), + k(ContextMenu, 0xdd, "Menu", 0xff67, 0), +]; + +/// Short names people type, beside the keysym names. +static ALIASES: &[(&str, KeyCode)] = &[ + ("ctrl", ControlLeft), ("control", ControlLeft), ("shift", ShiftLeft), + ("alt", AltLeft), ("meta", AltLeft), ("super", SuperLeft), ("win", SuperLeft), + ("enter", Enter), ("ret", Enter), ("esc", Escape), ("tab", Tab), + ("bs", Backspace), ("backspace", Backspace), ("del", Delete), ("ins", Insert), + ("pgup", PageUp), ("pageup", PageUp), ("pgdn", PageDown), ("pagedown", PageDown), + ("up", ArrowUp), ("down", ArrowDown), ("left", ArrowLeft), ("right", ArrowRight), + ("home", Home), ("end", End), ("caps", CapsLock), ("menu", ContextMenu), + ("Page_Up", PageUp), ("Page_Down", PageDown), ("KP_Separator", NumpadDecimal), +]; + +/// The physical key for an X keysym. A shifted symbol (`!`) gives the key it +/// is on (`1`); the viewer sends Shift itself. Upper and lower case letters +/// are the same key. +pub fn from_keysym(sym: u32) -> Option { + KEYS.iter().find(|e| e.sym == sym || (e.shifted != 0 && e.shifted == sym)).map(|e| e.code) +} + +/// The key that types an ASCII character on a US layout, and whether it +/// needs Shift. Newline and carriage return are Enter, tab is Tab. +pub fn char_key(ch: char) -> Option<(KeyCode, bool)> { + match ch { + '\n' | '\r' => return Some((Enter, false)), + '\t' => return Some((Tab, false)), + '\x08' => return Some((Backspace, false)), + '\x1b' => return Some((Escape, false)), + _ => {} + } + if !(' '..='~').contains(&ch) { + return None; + } + let sym = ch as u32; + KEYS.iter().find_map(|e| { + if e.sym == sym && e.xt != 0x56 { + Some((e.code, false)) + } else if e.shifted == sym { + Some((e.code, true)) + } else { + None + } + }) +} + +/// The physical key for an XT scancode as QEMU's extended key event numbers +/// it (`0xE0 nn` sent as `0x80 | nn`). +pub fn from_xt(xt: u32) -> Option { + KEYS.iter().find(|e| e.xt as u32 == xt).map(|e| e.code) +} + +/// The physical key for a name: an X keysym name (`Return`, `F1`, `a`, +/// `Control_L`), an alias (`ctrl`, `enter`, `esc`, `pgup`), a single +/// character (`a`, `5`, `/`), or a winit name (`KeyA`). Case is ignored +/// where it does not change the key. +pub fn from_name(name: &str) -> Option { + if let Some(e) = KEYS.iter().find(|e| e.name == name) { + return Some(e.code); + } + let mut chars = name.chars(); + if let (Some(c), None) = (chars.next(), chars.next()) { + if let Some(code) = from_keysym(c as u32) { + return Some(code); + } + } + if let Some(e) = KEYS.iter().find(|e| e.name.eq_ignore_ascii_case(name)) { + return Some(e.code); + } + if let Some((_, code)) = ALIASES.iter().find(|(a, _)| a.eq_ignore_ascii_case(name)) { + return Some(*code); + } + KEYS.iter().find(|e| format!("{:?}", e.code) == name).map(|e| e.code) +} + +/// Every key name `from_name` knows, for help text. +pub fn names() -> impl Iterator { + KEYS.iter().map(|e| e.name).chain(ALIASES.iter().map(|(a, _)| *a)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keysyms() { + assert_eq!(from_keysym(b'a' as u32), Some(KeyA)); + assert_eq!(from_keysym(b'A' as u32), Some(KeyA)); + assert_eq!(from_keysym(b'!' as u32), Some(Digit1)); + assert_eq!(from_keysym(0xff0d), Some(Enter)); + assert_eq!(from_keysym(0xffe3), Some(ControlLeft)); + assert_eq!(from_keysym(0x1234), None); + } + + #[test] + fn characters() { + assert_eq!(char_key('a'), Some((KeyA, false))); + assert_eq!(char_key('A'), Some((KeyA, true))); + assert_eq!(char_key('~'), Some((Backquote, true))); + assert_eq!(char_key('|'), Some((Backslash, true))); + assert_eq!(char_key('<'), Some((Comma, true))); + assert_eq!(char_key(' '), Some((Space, false))); + assert_eq!(char_key('\n'), Some((Enter, false))); + for c in ' '..='~' { + assert!(char_key(c).is_some(), "{c:?}"); + } + } + + #[test] + fn scancodes() { + assert_eq!(from_xt(0x1e), Some(KeyA)); + assert_eq!(from_xt(0xc8), Some(ArrowUp)); + assert_eq!(from_xt(0x9d), Some(ControlRight)); + } + + #[test] + fn names() { + assert_eq!(from_name("Return"), Some(Enter)); + assert_eq!(from_name("return"), Some(Enter)); + assert_eq!(from_name("ctrl"), Some(ControlLeft)); + assert_eq!(from_name("F12"), Some(F12)); + assert_eq!(from_name("KeyQ"), Some(KeyQ)); + assert_eq!(from_name("/"), Some(Slash)); + assert_eq!(from_name("A"), Some(KeyA)); + assert_eq!(from_name("nope"), None); + } + + #[test] + fn scancodes_unique() { + for (i, a) in KEYS.iter().enumerate() { + for b in &KEYS[i + 1..] { + assert_ne!(a.xt, b.xt, "{} and {}", a.name, b.name); + assert_ne!(a.code, b.code, "{} and {}", a.name, b.name); + } + } + } +} diff --git a/src/lib.rs b/src/lib.rs index aca7eef68..9c51549c5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -204,6 +204,9 @@ pub mod dev; pub mod gfifo; pub mod trace_writer; pub mod gfx_display; +pub mod hostinput; +pub mod keysym; +pub mod vnc; pub mod compositor; pub mod gl_compositor; pub mod headless_gl; diff --git a/src/machine.rs b/src/machine.rs index b9d90680a..f60a38ee4 100644 --- a/src/machine.rs +++ b/src/machine.rs @@ -922,7 +922,7 @@ impl Machine { let devlog = crate::devlog::init_devlog(); // Setup Monitor - let mut monitor = Monitor::new(); + let monitor = Monitor::new(); monitor.register_device(devlog.clone()); monitor.register_device(cpu.clone()); monitor.register_device(Arc::new(mc.clone())); @@ -2099,6 +2099,7 @@ impl Workstation for Machine { fn get_display(&self) -> Option> { Machine::get_display(self) } fn get_display_head1(&self) -> Option> { Machine::get_display_head1(self) } fn get_ps2(&self) -> Arc { Machine::get_ps2(self) } + fn register_monitor_device(&self, device: Arc) { self.monitor.register_device(device); } fn get_timer_manager(&self) -> Arc { Machine::get_timer_manager(self) } fn get_cpu_debug(&self) -> Arc { Machine::get_cpu_debug(self) } fn fasttick_count(&self) -> Arc { Machine::fasttick_count(self) } diff --git a/src/main.rs b/src/main.rs index 17f1c6e40..06a668a2f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -44,6 +44,7 @@ fn main() { let ci_display = cfg.ci_display; let ci_socket_path = cfg.ci_socket.clone(); let load_elf = cfg.load_elf.clone(); + let vnc = cfg.vnc.clone(); let perfetto = (cfg.perfetto.clone(), cfg.perfetto_cats.clone(), cfg.perfetto_clock.clone()); // Apply [debug] and [jitv2] from TOML (env vars still override if set externally). @@ -138,6 +139,17 @@ fn main() { None }; + // The monitor's `screen`, `ptr` and `key` commands, and the VNC server's + // hands on the guest. + let host_input = iris::hostinput::HostInput::attach(&*machine); + let _vnc_server = vnc.listen.as_deref().map(|listen| { + iris::vnc::VncServer::start(host_input.clone(), listen, vnc.max_fps, "IRIS") + .unwrap_or_else(|e| { + eprintln!("iris: {e}"); + std::process::exit(1); + }) + }); + // The terminal console talks to this process's own monitor only. Without // one (its port taken by another iris), there is no console: attached to // the other iris's monitor, it would exit along with that iris. diff --git a/src/monitor.rs b/src/monitor.rs index b2e8ed349..987dcce4b 100644 --- a/src/monitor.rs +++ b/src/monitor.rs @@ -45,7 +45,7 @@ impl Monitor { *self.bound.lock() } - pub fn register_device(&mut self, device: Arc) { + pub fn register_device(&self, device: Arc) { self.devices.lock().push(device); } diff --git a/src/ui.rs b/src/ui.rs index b23b8877e..cbc412a0b 100644 --- a/src/ui.rs +++ b/src/ui.rs @@ -631,7 +631,11 @@ impl Renderer for GlRenderer { // the GL framebuffer back would only replace it (it came back black). if need_readback && !screen.prebuilt { if let Some(pixels) = self.compositor.read_pixels() { - screen.rgba.copy_from_slice(pixels); + // Visible rows only: VNC capture reads every frame back. + for y in 0..height { + let row = y * crate::disp::FB_STRIDE; + screen.rgba[row..row + width].copy_from_slice(&pixels[row..row + width]); + } } else { self.compositor.readback_to_screen(&mut screen.rgba, width, height, gl); } @@ -721,6 +725,10 @@ impl Renderer for GlRenderer { self.current_win_h = 0; } + fn gl_compositor(&self) -> bool { + self.pending_compositor.unwrap_or(self.use_gl_compositor) + } + fn compositor_status(&self) -> String { let comp = if self.use_gl_compositor { "gl" } else { "sw" }; format!("compositor={} shader=integer+fallback", comp) diff --git a/src/vnc/encode.rs b/src/vnc/encode.rs new file mode 100644 index 000000000..907b2bdf8 --- /dev/null +++ b/src/vnc/encode.rs @@ -0,0 +1,448 @@ +//! RFB pixel formats and rectangle encodings: Raw and ZRLE. +//! +//! Frames arrive as `0xAABBGGRR` (R in the low byte), the compositors' +//! format. Our own format is 32 bpp little-endian with red at shift 0, so +//! Raw to a viewer that keeps it is a straight copy. + +use flate2::{Compress, Compression, FlushCompress}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PixelFormat { + pub bpp: u8, + pub depth: u8, + pub big_endian: bool, + pub true_colour: bool, + pub max: [u16; 3], + pub shift: [u8; 3], +} + +impl PixelFormat { + /// What the server offers: the compositors' own layout. + pub const NATIVE: PixelFormat = PixelFormat { + bpp: 32, depth: 24, big_endian: false, true_colour: true, + max: [255, 255, 255], shift: [0, 8, 16], + }; + + pub fn to_bytes(&self) -> [u8; 16] { + let mut b = [0u8; 16]; + b[0] = self.bpp; + b[1] = self.depth; + b[2] = self.big_endian as u8; + b[3] = self.true_colour as u8; + for i in 0..3 { + b[4 + 2 * i..6 + 2 * i].copy_from_slice(&self.max[i].to_be_bytes()); + b[10 + i] = self.shift[i]; + } + b + } + + pub fn from_bytes(b: &[u8; 16]) -> Result { + let pf = PixelFormat { + bpp: b[0], + depth: b[1], + big_endian: b[2] != 0, + true_colour: b[3] != 0, + max: [u16::from_be_bytes([b[4], b[5]]), u16::from_be_bytes([b[6], b[7]]), u16::from_be_bytes([b[8], b[9]])], + shift: [b[10], b[11], b[12]], + }; + if !matches!(pf.bpp, 8 | 16 | 32) { + return Err(format!("{} bits per pixel", pf.bpp)); + } + if !pf.true_colour { + return Err("colour-map pixel formats are not supported".into()); + } + if pf.max.iter().any(|&m| m == 0 || m & (m + 1) != 0) { + return Err(format!("colour maxima {:?} are not 2^n-1", pf.max)); + } + Ok(pf) + } + + fn bytes(&self) -> usize { + self.bpp as usize / 8 + } + + /// A compositor pixel in this format. + #[inline] + pub fn convert(&self, p: u32) -> u32 { + if *self == Self::NATIVE { + return p & 0x00ff_ffff; + } + let c = [p & 0xff, (p >> 8) & 0xff, (p >> 16) & 0xff]; + let mut v = 0u32; + for i in 0..3 { + let m = self.max[i] as u32; + v |= ((c[i] * m + 127) / 255) << self.shift[i]; + } + v + } + + #[inline] + fn put(&self, v: u32, out: &mut Vec) { + match (self.bytes(), self.big_endian) { + (4, false) => out.extend_from_slice(&v.to_le_bytes()), + (4, true) => out.extend_from_slice(&v.to_be_bytes()), + (2, false) => out.extend_from_slice(&(v as u16).to_le_bytes()), + (2, true) => out.extend_from_slice(&(v as u16).to_be_bytes()), + _ => out.push(v as u8), + } + } + + /// ZRLE's compact pixel: 32 bpp formats whose colour fits in three bytes + /// send only those three. + fn cpixel(&self) -> CPixel { + if self.bpp != 32 || self.depth > 24 { + return CPixel::Full; + } + let used = (0..3).fold(0u32, |m, i| m | (self.max[i] as u32) << self.shift[i]); + // Byte offsets in the pixel as sent. + let (low3, high3) = if self.big_endian { (1, 0) } else { (0, 1) }; + if used & 0xff00_0000 == 0 { + CPixel::Three(low3) + } else if used & 0xff == 0 { + CPixel::Three(high3) + } else { + CPixel::Full + } + } +} + +#[derive(Clone, Copy)] +enum CPixel { + Full, + /// Three bytes starting at this offset of the full pixel. + Three(usize), +} + +impl CPixel { + fn len(self, pf: &PixelFormat) -> usize { + match self { + CPixel::Full => pf.bytes(), + CPixel::Three(_) => 3, + } + } + #[inline] + fn put(self, pf: &PixelFormat, v: u32, out: &mut Vec) { + match self { + CPixel::Full => pf.put(v, out), + CPixel::Three(o) => { + let b = if pf.big_endian { v.to_be_bytes() } else { v.to_le_bytes() }; + out.extend_from_slice(&b[o..o + 3]); + } + } + } +} + +/// A rectangle of a tight frame `px` (row length `stride`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Rect { + pub x: usize, + pub y: usize, + pub w: usize, + pub h: usize, +} + +pub fn raw(pf: &PixelFormat, px: &[u32], stride: usize, r: Rect, out: &mut Vec) { + out.reserve(r.w * r.h * pf.bytes()); + for y in r.y..r.y + r.h { + for &p in &px[y * stride + r.x..y * stride + r.x + r.w] { + pf.put(pf.convert(p), out); + } + } +} + +/// One viewer's ZRLE state: its zlib stream runs for the whole connection. +pub struct Zrle { + z: Compress, + tile: Vec, + plain: Vec, +} + +impl Zrle { + pub fn new() -> Self { + Self { z: Compress::new(Compression::new(3), true), tile: Vec::new(), plain: Vec::new() } + } + + /// The rectangle's ZRLE data (length word and zlib bytes) onto `out`. + pub fn encode(&mut self, pf: &PixelFormat, px: &[u32], stride: usize, r: Rect, out: &mut Vec) -> Result<(), String> { + let cp = pf.cpixel(); + self.plain.clear(); + for ty in (r.y..r.y + r.h).step_by(64) { + let th = 64.min(r.y + r.h - ty); + for tx in (r.x..r.x + r.w).step_by(64) { + let tw = 64.min(r.x + r.w - tx); + self.tile.clear(); + for y in ty..ty + th { + self.tile.extend(px[y * stride + tx..y * stride + tx + tw].iter().map(|&p| pf.convert(p))); + } + encode_tile(pf, cp, &self.tile, tw, &mut self.plain); + } + } + // Deflate with a sync flush so the viewer can decode this rectangle + // now; worst case a little over the input. + let len_at = out.len(); + out.extend_from_slice(&[0; 4]); + let start = out.len(); + let mut input = &self.plain[..]; + loop { + out.reserve(input.len() / 2 + 1024); + let before_in = self.z.total_in(); + self.z.compress_vec(input, out, FlushCompress::Sync).map_err(|e| e.to_string())?; + let used = (self.z.total_in() - before_in) as usize; + input = &input[used..]; + // Done when all input went in and zlib had room to spare. + if input.is_empty() && out.len() < out.capacity() { + break; + } + } + let n = (out.len() - start) as u32; + out[len_at..len_at + 4].copy_from_slice(&n.to_be_bytes()); + Ok(()) + } +} + +fn run_length(mut n: usize, out: &mut Vec) { + n -= 1; + while n >= 255 { + out.push(255); + n -= 255; + } + out.push(n as u8); +} + +fn run_length_size(n: usize) -> usize { + (n - 1) / 255 + 1 +} + +/// One ZRLE tile, whichever subencoding comes out smallest. +fn encode_tile(pf: &PixelFormat, cp: CPixel, t: &[u32], tw: usize, out: &mut Vec) { + let cpl = cp.len(pf); + let n = t.len(); + let th = n / tw; + + // Palette (up to 127 colours) and runs. + let mut pal: Vec = Vec::with_capacity(16); + let mut runs = 0usize; + let mut plain_rle = 0usize; + let mut pal_rle = 0usize; + let mut i = 0; + while i < n { + let v = t[i]; + let mut j = i + 1; + while j < n && t[j] == v { + j += 1; + } + let len = j - i; + runs += 1; + plain_rle += cpl + run_length_size(len); + pal_rle += if len == 1 { 1 } else { 1 + run_length_size(len) }; + if pal.len() <= 127 && !pal.contains(&v) { + pal.push(v); + } + i = j; + } + let _ = runs; + + if pal.len() == 1 { + out.push(1); + cp.put(pf, pal[0], out); + return; + } + let raw_size = n * cpl; + let packed_bits = match pal.len() { 2 => 1, 3..=4 => 2, 5..=16 => 4, _ => 0 }; + let packed_size = if packed_bits > 0 { pal.len() * cpl + th * (tw * packed_bits).div_ceil(8) } else { usize::MAX }; + let pal_rle_size = if pal.len() <= 127 { pal.len() * cpl + pal_rle } else { usize::MAX }; + let best = raw_size.min(packed_size).min(plain_rle).min(pal_rle_size); + let index = |v: u32| pal.iter().position(|&p| p == v).unwrap() as u8; + + if best == packed_size { + out.push(pal.len() as u8); + for &p in &pal { + cp.put(pf, p, out); + } + for row in t.chunks(tw) { + let (mut byte, mut nbits) = (0u8, 0); + for &v in row { + byte = (byte << packed_bits) | index(v); + nbits += packed_bits; + if nbits == 8 { + out.push(byte); + (byte, nbits) = (0, 0); + } + } + if nbits > 0 { + out.push(byte << (8 - nbits)); + } + } + } else if best == pal_rle_size { + out.push(128 + pal.len() as u8); + for &p in &pal { + cp.put(pf, p, out); + } + for_runs(t, |v, len| { + if len == 1 { + out.push(index(v)); + } else { + out.push(index(v) | 128); + run_length(len, out); + } + }); + } else if best == plain_rle { + out.push(128); + for_runs(t, |v, len| { + cp.put(pf, v, out); + run_length(len, out); + }); + } else { + out.push(0); + for &v in t { + cp.put(pf, v, out); + } + } +} + +fn for_runs(t: &[u32], mut f: impl FnMut(u32, usize)) { + let mut i = 0; + while i < t.len() { + let v = t[i]; + let mut j = i + 1; + while j < t.len() && t[j] == v { + j += 1; + } + f(v, j - i); + i = j; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use flate2::{Decompress, FlushDecompress}; + + /// Decode one ZRLE rectangle back to pixels (format values). + fn unzrle(pf: &PixelFormat, z: &mut Decompress, data: &[u8], r: Rect) -> Vec { + let n = u32::from_be_bytes(data[..4].try_into().unwrap()) as usize; + assert_eq!(n + 4, data.len()); + let mut plain = Vec::with_capacity(1 << 20); + z.decompress_vec(&data[4..], &mut plain, FlushDecompress::Sync).unwrap(); + let cp = pf.cpixel(); + let cpl = cp.len(pf); + let rd = |b: &[u8]| -> u32 { + let mut full = [0u8; 4]; + match cp { + CPixel::Full => full[..cpl].copy_from_slice(b), + CPixel::Three(o) => full[o..o + 3].copy_from_slice(b), + } + if pf.big_endian { u32::from_be_bytes(full) } else { u32::from_le_bytes(full) } + }; + let mut out = vec![0u32; r.w * r.h]; + let mut p = 0; + for ty in (0..r.h).step_by(64) { + let th = 64.min(r.h - ty); + for tx in (0..r.w).step_by(64) { + let tw = 64.min(r.w - tx); + let sub = plain[p]; + p += 1; + let mut tile = Vec::new(); + let mut pal = Vec::new(); + let pal_n = match sub { 2..=16 => sub as usize, 130..=255 => sub as usize - 128, _ => 0 }; + for _ in 0..pal_n { + pal.push(rd(&plain[p..p + cpl])); + p += cpl; + } + let rl = |p: &mut usize| { let mut n = 1; loop { let b = plain[*p]; *p += 1; n += b as usize; if b != 255 { return n; } } }; + match sub { + 0 => for _ in 0..tw * th { tile.push(rd(&plain[p..p + cpl])); p += cpl; }, + 1 => { let v = rd(&plain[p..p + cpl]); p += cpl; tile = vec![v; tw * th]; } + 2..=16 => { + let bits = match sub { 2 => 1, 3 | 4 => 2, _ => 4 }; + for _ in 0..th { + let mut taken = 0; + let row_bytes = (tw * bits).div_ceil(8); + for x in 0..tw { + let bit = x * bits; + let b = plain[p + bit / 8]; + let idx = (b >> (8 - bits - bit % 8)) & ((1 << bits) - 1); + tile.push(pal[idx as usize]); + taken = row_bytes; + } + p += taken; + } + } + 128 => while tile.len() < tw * th { let v = rd(&plain[p..p + cpl]); p += cpl; let n = rl(&mut p); tile.extend(std::iter::repeat_n(v, n)); }, + 130..=255 => while tile.len() < tw * th { + let b = plain[p]; p += 1; + let v = pal[(b & 127) as usize]; + let n = if b & 128 != 0 { rl(&mut p) } else { 1 }; + tile.extend(std::iter::repeat_n(v, n)); + }, + s => panic!("subencoding {s}"), + } + assert_eq!(tile.len(), tw * th); + for y in 0..th { + out[(ty + y) * r.w + tx..(ty + y) * r.w + tx + tw].copy_from_slice(&tile[y * tw..(y + 1) * tw]); + } + } + } + assert_eq!(p, plain.len()); + out + } + + fn picture(w: usize, h: usize) -> Vec { + let mut px = vec![0xff20_4060u32; w * h]; + for y in 0..h { + for x in 0..w { + px[y * w + x] = match (x / 40, y / 30) { + (0, _) => 0xff00_0000 | ((x * 7 + y * 13) as u32 * 2_654_435_761) & 0xff_ffff, // noise + (1, _) => if (x + y) % 2 == 0 { 0xffff_ffff } else { 0xff00_0000 }, // 2 colours + (2, _) => [0xff11_2233, 0xff44_5566, 0xff77_8899][(x / 3 + y) % 3], // 3 colours + (3, _) => 0xff00_0000 | ((x / 10) as u32 * 0x30_1020), // runs + _ => 0xffab_cdef, // solid + } + } + } + px + } + + #[test] + fn zrle_round_trips() { + let (w, h) = (230, 150); + let px = picture(w, h); + let formats = [ + PixelFormat::NATIVE, + PixelFormat { big_endian: true, shift: [16, 8, 0], ..PixelFormat::NATIVE }, + PixelFormat { shift: [24, 16, 8], depth: 24, ..PixelFormat::NATIVE }, + PixelFormat { bpp: 16, depth: 16, big_endian: false, true_colour: true, max: [31, 63, 31], shift: [11, 5, 0] }, + PixelFormat { bpp: 8, depth: 8, big_endian: false, true_colour: true, max: [7, 7, 3], shift: [0, 3, 6] }, + ]; + for pf in formats { + let mut enc = Zrle::new(); + let mut dec = Decompress::new(true); + // Two rectangles on one stream, as a connection sends them. + for r in [Rect { x: 0, y: 0, w, h }, Rect { x: 17, y: 9, w: 150, h: 70 }] { + let mut out = Vec::new(); + enc.encode(&pf, &px, w, r, &mut out).unwrap(); + let got = unzrle(&pf, &mut dec, &out, r); + for y in 0..r.h { + for x in 0..r.w { + assert_eq!(got[y * r.w + x], pf.convert(px[(r.y + y) * w + r.x + x]), "{pf:?} at {x},{y}"); + } + } + } + } + } + + #[test] + fn raw_native_is_a_copy() { + let px = vec![0xff01_0203u32, 0xff04_0506]; + let mut out = Vec::new(); + raw(&PixelFormat::NATIVE, &px, 2, Rect { x: 0, y: 0, w: 2, h: 1 }, &mut out); + assert_eq!(out, [3, 2, 1, 0, 6, 5, 4, 0]); + } + + #[test] + fn convert_16bpp() { + let pf = PixelFormat { bpp: 16, depth: 16, big_endian: false, true_colour: true, max: [31, 63, 31], shift: [11, 5, 0] }; + assert_eq!(pf.convert(0xff00_00ff), 31 << 11); // red + assert_eq!(pf.convert(0xffff_0000), 31); // blue + } +} diff --git a/src/vnc/mod.rs b/src/vnc/mod.rs new file mode 100644 index 000000000..0df1fe995 --- /dev/null +++ b/src/vnc/mod.rs @@ -0,0 +1,318 @@ +//! The built-in VNC (RFB 3.8) server. +//! +//! It serves whatever the board composes, for every board, and knows nothing +//! about drawing: a grabber thread copies the composed frame (`screen.rgba`) +//! when its sequence number moves, compares it with the last one in 64x64 +//! tiles, and stamps each changed tile with the frame's generation. Each +//! viewer remembers the generation it last saw and is sent the tiles newer +//! than that, so a slow viewer skips frames instead of queueing them. +//! +//! Input goes to the guest's PS/2 keyboard and mouse through +//! [`HostInput`]: absolute pointer positions are steered onto the hardware +//! cursor. A window, the monitor and VNC viewers can all type at once. +//! +//! Threads: one accept thread, one grabber, one pointer driver, and two per +//! viewer (reading its messages; sending its updates). All std::net. + +pub mod encode; +mod session; + +use std::collections::VecDeque; +use std::net::{SocketAddr, TcpListener}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use parking_lot::{Condvar, Mutex}; + +use crate::gfx_display::GfxDisplay; +use crate::hostinput::{HostInput, MoveResult}; + +/// Tile side for damage tracking. +pub const TILE: usize = 64; +/// Viewers served at once. +const MAX_CLIENTS: usize = 8; + +/// The last frame and which of its tiles changed when. +#[derive(Clone)] +pub struct Frame { + pub w: usize, + pub h: usize, + pub px: Arc>, + /// Per tile (row-major, `tiles_x` a row), the generation it last changed in. + pub tile_gen: Arc>, + pub tiles_x: usize, + /// Generation of this frame; bumped when any tile changes. + pub gen: u64, + /// Bumped when the size changes. + pub size_gen: u64, +} + +pub(crate) struct Shared { + pub input: Arc, + display: Arc, + pub frame: Mutex, + /// Signalled on a new frame, and by viewers' readers when they want + /// their sender to look again. + pub changed: Condvar, + pub clients: AtomicUsize, + pub stop: AtomicBool, + max_fps: u32, + pub pointer: PointerDriver, + pub name: String, +} + +pub struct VncServer { + shared: Arc, + addr: SocketAddr, +} + +impl VncServer { + /// Bind `listen` and start serving `input`'s display. + pub fn start(input: Arc, listen: &str, max_fps: Option, name: &str) -> Result { + let display = input.display().cloned() + .ok_or("VNC needs a display: this machine runs without one (--headless, or no graphics board)")?; + let listen = if listen.parse::().is_ok() { format!("127.0.0.1:{listen}") } else { listen.to_string() }; + let listener = TcpListener::bind(&listen).map_err(|e| format!("VNC: bind {listen}: {e}"))?; + let addr = listener.local_addr().map_err(|e| e.to_string())?; + if !addr.ip().is_loopback() { + eprintln!("iris: WARNING: the VNC server on {addr} has no password: anyone who can reach it controls the guest"); + } + let shared = Arc::new(Shared { + input: input.clone(), + display, + frame: Mutex::new(Frame { + w: 0, h: 0, px: Arc::new(Vec::new()), tile_gen: Arc::new(Vec::new()), + tiles_x: 0, gen: 0, size_gen: 0, + }), + changed: Condvar::new(), + clients: AtomicUsize::new(0), + stop: AtomicBool::new(false), + max_fps: max_fps.unwrap_or(30).clamp(1, 120), + pointer: PointerDriver::new(), + name: name.to_string(), + }); + { + let s = shared.clone(); + std::thread::Builder::new().name("VNC-Grab".into()).spawn(move || grab_loop(&s)).unwrap(); + } + { + let s = shared.clone(); + std::thread::Builder::new().name("VNC-Pointer".into()).spawn(move || s.pointer.run(&s.input, &s.stop)).unwrap(); + } + { + let s = shared.clone(); + std::thread::Builder::new().name("VNC-Accept".into()).spawn(move || { + for conn in listener.incoming() { + if s.stop.load(Ordering::Relaxed) { + break; + } + let Ok(stream) = conn else { continue }; + if s.clients.load(Ordering::Relaxed) >= MAX_CLIENTS { + drop(stream); + continue; + } + let s = s.clone(); + let _ = std::thread::Builder::new().name("VNC-Client".into()).spawn(move || { + let peer = stream.peer_addr().map(|a| a.to_string()).unwrap_or_default(); + s.clients.fetch_add(1, Ordering::SeqCst); + s.changed.notify_all(); + let r = session::serve(&s, stream); + s.clients.fetch_sub(1, Ordering::SeqCst); + if let Err(e) = r { + eprintln!("iris: VNC {peer}: {e}"); + } + }); + } + }).unwrap(); + } + eprintln!("iris: VNC server on {addr}"); + Ok(VncServer { shared, addr }) + } + + pub fn addr(&self) -> SocketAddr { + self.addr + } + + pub fn clients(&self) -> usize { + self.shared.clients.load(Ordering::Relaxed) + } +} + +impl Drop for VncServer { + fn drop(&mut self) { + self.shared.stop.store(true, Ordering::SeqCst); + self.shared.changed.notify_all(); + let _ = std::net::TcpStream::connect(self.addr); // wake accept() + } +} + +/// Follow the composed frame while anyone watches. +fn grab_loop(s: &Shared) { + let period = Duration::from_micros(1_000_000 / s.max_fps as u64); + let mut capturing = false; + let mut last_seq = None; + let mut cur = Vec::new(); + while !s.stop.load(Ordering::Relaxed) { + let start = Instant::now(); + let watching = s.clients.load(Ordering::Relaxed) > 0; + if watching != capturing { + s.display.set_capture(watching); + capturing = watching; + last_seq = None; + } + if !watching { + // Sleep until a viewer connects (or a poll period passes). + let mut f = s.frame.lock(); + s.changed.wait_for(&mut f, Duration::from_millis(200)); + continue; + } + let seq = s.display.screen().lock().frame_seq; + if last_seq != Some(seq) { + if let Some((w, h, seq)) = crate::gfx_display::copy_frame(&*s.display, &mut cur) { + last_seq = Some(seq); + publish(s, w, h, &mut cur); + } + } + let spent = start.elapsed(); + if spent < period { + std::thread::sleep(period - spent); + } + } + if capturing { + s.display.set_capture(false); + } +} + +/// Compare `cur` with the published frame and publish it if anything moved. +/// `cur` comes back holding a spare buffer. +fn publish(s: &Shared, w: usize, h: usize, cur: &mut Vec) { + let old = s.frame.lock().clone(); + let tiles_x = w.div_ceil(TILE); + let tiles_y = h.div_ceil(TILE); + let resized = (old.w, old.h) != (w, h); + let mut dirty = vec![resized; tiles_x * tiles_y]; + if !resized { + let prev = &old.px; + for ty in 0..tiles_y { + let row_tiles = &mut dirty[ty * tiles_x..(ty + 1) * tiles_x]; + for y in ty * TILE..(ty * TILE + TILE).min(h) { + let (a, b) = (&cur[y * w..(y + 1) * w], &prev[y * w..(y + 1) * w]); + if a == b { + continue; + } + for (tx, d) in row_tiles.iter_mut().enumerate() { + if !*d { + let x0 = tx * TILE; + let x1 = (x0 + TILE).min(w); + *d = a[x0..x1] != b[x0..x1]; + } + } + } + } + if !dirty.iter().any(|&d| d) { + return; + } + } + let gen = old.gen + 1; + let tile_gen: Vec = if resized { + vec![gen; tiles_x * tiles_y] + } else { + old.tile_gen.iter().zip(&dirty).map(|(&g, &d)| if d { gen } else { g }).collect() + }; + let px = Arc::new(std::mem::take(cur)); + { + let mut f = s.frame.lock(); + *f = Frame { + w, h, px, tile_gen: Arc::new(tile_gen), tiles_x, gen, + size_gen: old.size_gen + resized as u64, + }; + } + // Reuse the frame before last when no viewer still holds it. + if let Ok(v) = Arc::try_unwrap(old.px) { + *cur = v; + } + s.changed.notify_all(); +} + +/// Turns viewers' pointer events into steered moves and button changes, in +/// order, on one thread: a click is sent once the pointer is where the +/// viewer clicked. +pub(crate) struct PointerDriver { + queue: Mutex>, + ready: Condvar, +} + +impl PointerDriver { + fn new() -> Self { + Self { queue: Mutex::new(VecDeque::new()), ready: Condvar::new() } + } + + /// A viewer's pointer event: position and PS/2 buttons. + pub fn push(&self, x: i32, y: i32, buttons: u8) { + let mut q = self.queue.lock(); + // Moves with no button change only matter as the latest target. + if let Some(last) = q.back_mut() { + if last.2 == buttons && q.len() > 1 && q[q.len() - 2].2 == buttons { + *q.back_mut().unwrap() = (x, y, buttons); + drop(q); + self.ready.notify_one(); + return; + } + } + if q.len() < 1024 { + q.push_back((x, y, buttons)); + } + drop(q); + self.ready.notify_one(); + } + + fn run(&self, input: &HostInput, stop: &AtomicBool) { + // Where the viewer last put the pointer: for moving by difference + // when there is no cursor to steer by. + let mut last: Option<(i32, i32)> = None; + while !stop.load(Ordering::Relaxed) { + let (x, y, buttons) = { + let mut q = self.queue.lock(); + if q.is_empty() { + self.ready.wait_for(&mut q, Duration::from_millis(200)); + } + match q.pop_front() { + Some(e) => e, + None => continue, + } + }; + let held = input.buttons(); + // Take newer moves with the same buttons as the new target; not + // while a press or release waits for this move to arrive. + let mut target = (x, y); + let mut retarget = || { + let mut q = self.queue.lock(); + match q.front() { + Some(&(nx, ny, b)) if b == buttons && buttons == held => { + q.pop_front(); + target = (nx, ny); + Some((nx, ny)) + } + _ => None, + } + }; + let steered = input.move_to(x, y, Duration::from_millis(400), &mut retarget); + let target = target; + match steered { + Ok(MoveResult::Reached) | Ok(MoveResult::Short(..)) => {} + Err(_) => { + // No cursor to steer by (the X server is not up, or a + // program hid it): move by the viewer's difference. + if let Some((lx, ly)) = last { + input.nudge(target.0 - lx, target.1 - ly); + } + } + } + last = Some(target); + if buttons != held { + input.set_buttons(buttons); + } + } + } +} diff --git a/src/vnc/session.rs b/src/vnc/session.rs new file mode 100644 index 000000000..7434c8411 --- /dev/null +++ b/src/vnc/session.rs @@ -0,0 +1,402 @@ +//! One viewer: the RFB handshake, its messages in, its updates out. + +use std::collections::HashSet; +use std::io::{Read, Write}; +use std::net::{Shutdown, TcpStream}; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use parking_lot::Mutex; +use winit::keyboard::KeyCode; + +use super::encode::{self, PixelFormat, Rect}; +use super::{Frame, Shared, TILE}; +use crate::hostinput::{BUTTON_LEFT, BUTTON_MIDDLE, BUTTON_RIGHT}; + +const ENC_RAW: i32 = 0; +const ENC_ZRLE: i32 = 16; +const ENC_DESKTOP_SIZE: i32 = -223; +const ENC_QEMU_EXT_KEY: i32 = -258; + +#[derive(Clone, Copy)] +struct Request { + incremental: bool, + rect: Rect, +} + +struct Client { + pf: PixelFormat, + zrle: bool, + desktop_size: bool, + ext_key: bool, + ext_key_acked: bool, + request: Option, + seen_gen: u64, + size_gen: u64, + dead: Option, + held_keys: HashSet, + /// Viewer buttons (RFB bits) last seen, for wheel edges. + vnc_buttons: u8, +} + +pub(crate) fn serve(s: &Arc, mut stream: TcpStream) -> Result<(), String> { + stream.set_nodelay(true).ok(); + stream.set_read_timeout(Some(Duration::from_secs(10))).ok(); + let io = |e: std::io::Error| e.to_string(); + + // Version, security (None), ClientInit. + stream.write_all(b"RFB 003.008\n").map_err(io)?; + let mut ver = [0u8; 12]; + stream.read_exact(&mut ver).map_err(io)?; + let minor = match &ver { + b"RFB 003.003\n" => 3, + b"RFB 003.007\n" => 7, + v if v.starts_with(b"RFB 003.") => 8, + _ => return Err(format!("not an RFB viewer ({:?})", String::from_utf8_lossy(&ver))), + }; + if minor == 3 { + stream.write_all(&1u32.to_be_bytes()).map_err(io)?; + } else { + stream.write_all(&[1, 1]).map_err(io)?; + let mut choice = [0u8]; + stream.read_exact(&mut choice).map_err(io)?; + if choice[0] != 1 { + return Err(format!("viewer chose security type {}", choice[0])); + } + if minor >= 8 { + stream.write_all(&0u32.to_be_bytes()).map_err(io)?; + } + } + let mut shared_flag = [0u8]; + stream.read_exact(&mut shared_flag).map_err(io)?; + + // The size to announce: wait a little for the first frame. + let frame = { + let deadline = Instant::now() + Duration::from_secs(3); + let mut f = s.frame.lock(); + while f.w == 0 && Instant::now() < deadline { + s.changed.wait_for(&mut f, Duration::from_millis(100)); + } + f.clone() + }; + let (w, h) = if frame.w > 0 { (frame.w, frame.h) } else { (1024, 768) }; + let mut init = Vec::new(); + init.extend_from_slice(&(w as u16).to_be_bytes()); + init.extend_from_slice(&(h as u16).to_be_bytes()); + init.extend_from_slice(&PixelFormat::NATIVE.to_bytes()); + init.extend_from_slice(&(s.name.len() as u32).to_be_bytes()); + init.extend_from_slice(s.name.as_bytes()); + stream.write_all(&init).map_err(io)?; + stream.set_read_timeout(None).ok(); + + let client = Arc::new(Mutex::new(Client { + pf: PixelFormat::NATIVE, + zrle: false, + desktop_size: false, + ext_key: false, + ext_key_acked: false, + request: None, + seen_gen: 0, + // A placeholder size is a size change still to send. + size_gen: if frame.w > 0 { frame.size_gen } else { u64::MAX }, + dead: None, + held_keys: HashSet::new(), + vnc_buttons: 0, + })); + + let reader = { + let (s, client) = (s.clone(), client.clone()); + let mut rd = stream.try_clone().map_err(io)?; + std::thread::Builder::new().name("VNC-Read".into()).spawn(move || { + let why = read_loop(&s, &client, &mut rd).err().unwrap_or_else(|| "closed".into()); + client.lock().dead = Some(why); + s.changed.notify_all(); + }).map_err(io)? + }; + + let result = send_loop(s, &client, &mut stream); + let _ = stream.shutdown(Shutdown::Both); + let _ = reader.join(); + + // Let go of whatever this viewer was holding. + let c = client.lock(); + for &k in &c.held_keys { + s.input.key(k, false); + } + if c.vnc_buttons & 7 != 0 { + s.input.set_buttons(0); + } + match result { + Err(e) => Err(e), + Ok(()) => match &c.dead { + Some(why) if why != "closed" => Err(why.clone()), + _ => Ok(()), + }, + } +} + +fn read_loop(s: &Shared, client: &Mutex, rd: &mut TcpStream) -> Result<(), String> { + let mut b = [0u8; 20]; + loop { + let mut ty = [0u8]; + if rd.read(&mut ty).map_err(|e| e.to_string())? == 0 { + return Ok(()); + } + match ty[0] { + 0 => { + let m = take(rd, &mut b, 19)?; + let pf = PixelFormat::from_bytes(m[3..19].try_into().unwrap())?; + client.lock().pf = pf; + } + 2 => { + let m = take(rd, &mut b, 3)?; + let n = u16::from_be_bytes([m[1], m[2]]) as usize; + let mut list = vec![0u8; n * 4]; + rd.read_exact(&mut list).map_err(|e| e.to_string())?; + let encs: Vec = list.chunks(4).map(|c| i32::from_be_bytes(c.try_into().unwrap())).collect(); + let mut c = client.lock(); + c.zrle = encs.contains(&ENC_ZRLE); + c.desktop_size = encs.contains(&ENC_DESKTOP_SIZE); + let ext = encs.contains(&ENC_QEMU_EXT_KEY); + if ext && !c.ext_key { + c.ext_key_acked = false; + } + c.ext_key = ext; + } + 3 => { + let m = take(rd, &mut b, 9)?; + let v = |i: usize| u16::from_be_bytes([m[i], m[i + 1]]) as usize; + client.lock().request = Some(Request { + incremental: m[0] != 0, + rect: Rect { x: v(1), y: v(3), w: v(5), h: v(7) }, + }); + s.changed.notify_all(); + } + 4 => { + let m = take(rd, &mut b, 7)?; + let sym = u32::from_be_bytes(m[3..7].try_into().unwrap()); + key(s, client, m[0] != 0, crate::keysym::from_keysym(sym)); + } + 5 => { + let m = take(rd, &mut b, 5)?; + let (mask, x, y) = (m[0], u16::from_be_bytes([m[1], m[2]]) as i32, u16::from_be_bytes([m[3], m[4]]) as i32); + let rising = { + let mut c = client.lock(); + let r = mask & !c.vnc_buttons; + c.vnc_buttons = mask; + r + }; + // Wheel up and down come as presses of buttons 4 and 5. + if rising & 8 != 0 { + s.input.wheel(1); + } + if rising & 16 != 0 { + s.input.wheel(-1); + } + let mut ps2 = 0; + if mask & 1 != 0 { ps2 |= BUTTON_LEFT; } + if mask & 2 != 0 { ps2 |= BUTTON_MIDDLE; } + if mask & 4 != 0 { ps2 |= BUTTON_RIGHT; } + s.pointer.push(x, y, ps2); + } + 6 => { + let m = take(rd, &mut b, 7)?; + let n = u32::from_be_bytes(m[3..7].try_into().unwrap()) as u64; + // Clipboard: not passed on. + std::io::copy(&mut Read::by_ref(rd).take(n), &mut std::io::sink()).map_err(|e| e.to_string())?; + } + 255 => { + let m = take(rd, &mut b, 1)?; + if m[0] != 0 { + return Err(format!("unknown QEMU message {}", m[0])); + } + let m = take(rd, &mut b, 10)?; + let down = u16::from_be_bytes([m[0], m[1]]) != 0; + let sym = u32::from_be_bytes(m[2..6].try_into().unwrap()); + let xt = u32::from_be_bytes(m[6..10].try_into().unwrap()); + let code = crate::keysym::from_xt(xt).or_else(|| crate::keysym::from_keysym(sym)); + key(s, client, down, code); + } + t => return Err(format!("unknown message type {t}")), + } + } +} + +fn take<'a>(rd: &mut TcpStream, b: &'a mut [u8; 20], n: usize) -> Result<&'a [u8], String> { + rd.read_exact(&mut b[..n]).map_err(|e| e.to_string())?; + Ok(&b[..n]) +} + +fn key(s: &Shared, client: &Mutex, down: bool, code: Option) { + let Some(code) = code else { return }; + let mut c = client.lock(); + if down { + c.held_keys.insert(code); + } else { + c.held_keys.remove(&code); + } + drop(c); + s.input.key(code, down); +} + +/// What to send next, or None to keep waiting. +fn next_update(f: &Frame, c: &mut Client) -> Result, Vec)>, String> { + let Some(req) = c.request else { return Ok(None) }; + if f.w == 0 { + return Ok(None); + } + let mut pseudo = Vec::new(); + if c.ext_key && !c.ext_key_acked { + pseudo.push((Rect { x: 0, y: 0, w: 0, h: 0 }, ENC_QEMU_EXT_KEY)); + c.ext_key_acked = true; + } + let full = Rect { x: 0, y: 0, w: f.w, h: f.h }; + let rects = if c.size_gen != f.size_gen { + if !c.desktop_size { + return Err(format!("the screen is now {}x{} and the viewer cannot resize: reconnect", f.w, f.h)); + } + pseudo.push((full, ENC_DESKTOP_SIZE)); + vec![full] + } else if !req.incremental { + clip(req.rect, f).into_iter().collect() + } else if f.gen > c.seen_gen { + dirty_rects(f, c.seen_gen, req.rect) + } else { + Vec::new() + }; + c.seen_gen = f.gen; + c.size_gen = f.size_gen; + if rects.is_empty() && pseudo.is_empty() { + // Nothing in the requested area changed; the request stands. + return Ok(None); + } + c.request = None; + Ok(Some((pseudo, rects))) +} + +fn clip(r: Rect, f: &Frame) -> Option { + let (x1, y1) = ((r.x + r.w).min(f.w), (r.y + r.h).min(f.h)); + (r.x < x1 && r.y < y1).then(|| Rect { x: r.x, y: r.y, w: x1 - r.x, h: y1 - r.y }) +} + +/// Tiles changed since `seen` inside `area`, merged into runs across and +/// stacked runs down. +fn dirty_rects(f: &Frame, seen: u64, area: Rect) -> Vec { + let tiles_y = f.h.div_ceil(TILE); + let mut out: Vec = Vec::new(); + // Rects that ended on the previous tile row, by index into `out`. + let mut open: Vec = Vec::new(); + for ty in 0..tiles_y { + let mut next_open = Vec::new(); + let mut tx = 0; + while tx < f.tiles_x { + if f.tile_gen[ty * f.tiles_x + tx] <= seen { + tx += 1; + continue; + } + let start = tx; + while tx < f.tiles_x && f.tile_gen[ty * f.tiles_x + tx] > seen { + tx += 1; + } + let r = Rect { + x: start * TILE, + y: ty * TILE, + w: (tx * TILE).min(f.w) - start * TILE, + h: ((ty + 1) * TILE).min(f.h) - ty * TILE, + }; + match open.iter().position(|&i| out[i].x == r.x && out[i].w == r.w) { + Some(p) => { + let i = open.remove(p); + out[i].h += r.h; + next_open.push(i); + } + None => { + out.push(r); + next_open.push(out.len() - 1); + } + } + } + open = next_open; + } + out.into_iter().filter_map(|r| { + let x0 = r.x.max(area.x); + let y0 = r.y.max(area.y); + let x1 = (r.x + r.w).min(area.x + area.w); + let y1 = (r.y + r.h).min(area.y + area.h); + (x0 < x1 && y0 < y1).then(|| Rect { x: x0, y: y0, w: x1 - x0, h: y1 - y0 }) + }).collect() +} + +fn send_loop(s: &Shared, client: &Mutex, stream: &mut TcpStream) -> Result<(), String> { + let mut zrle = encode::Zrle::new(); + let mut buf = Vec::new(); + loop { + let (frame, pseudo, rects, pf, use_zrle) = { + let mut f = s.frame.lock(); + loop { + if s.stop.load(Ordering::Relaxed) { + return Ok(()); + } + let mut c = client.lock(); + if c.dead.is_some() { + return Ok(()); + } + if let Some((pseudo, rects)) = next_update(&f, &mut c)? { + break (f.clone(), pseudo, rects, c.pf, c.zrle); + } + drop(c); + s.changed.wait_for(&mut f, Duration::from_millis(100)); + } + }; + buf.clear(); + buf.extend_from_slice(&[0, 0]); + buf.extend_from_slice(&((pseudo.len() + rects.len()) as u16).to_be_bytes()); + let header = |buf: &mut Vec, r: Rect, enc: i32| { + for v in [r.x, r.y, r.w, r.h] { + buf.extend_from_slice(&(v as u16).to_be_bytes()); + } + buf.extend_from_slice(&enc.to_be_bytes()); + }; + for &(r, enc) in &pseudo { + header(&mut buf, r, enc); + } + for &r in &rects { + if use_zrle { + header(&mut buf, r, ENC_ZRLE); + zrle.encode(&pf, &frame.px, frame.w, r, &mut buf)?; + } else { + header(&mut buf, r, ENC_RAW); + encode::raw(&pf, &frame.px, frame.w, r, &mut buf); + } + } + stream.write_all(&buf).map_err(|e| e.to_string())?; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn frame(w: usize, h: usize, tile_gen: Vec) -> Frame { + Frame { + w, h, px: Arc::new(vec![0; w * h]), tiles_x: w.div_ceil(TILE), + tile_gen: Arc::new(tile_gen), gen: 5, size_gen: 0, + } + } + + #[test] + fn dirty_tiles_merge() { + // 3x2 tiles, 150x100: a 2-tile run on both rows (stacks), and one tile. + let f = frame(150, 100, vec![5, 5, 0, 5, 5, 4]); + let all = Rect { x: 0, y: 0, w: 150, h: 100 }; + let r = dirty_rects(&f, 4, all); + assert_eq!(r, vec![Rect { x: 0, y: 0, w: 128, h: 100 }]); + let r = dirty_rects(&f, 3, all); + assert_eq!(r, vec![Rect { x: 0, y: 0, w: 128, h: 64 }, Rect { x: 0, y: 64, w: 150, h: 36 }]); + // Clipped to the requested area. + let r = dirty_rects(&f, 4, Rect { x: 10, y: 10, w: 20, h: 20 }); + assert_eq!(r, vec![Rect { x: 10, y: 10, w: 20, h: 20 }]); + assert!(dirty_rects(&f, 5, all).is_empty()); + } +} diff --git a/src/workstation.rs b/src/workstation.rs index a2f4277ae..1c9b1ccd9 100644 --- a/src/workstation.rs +++ b/src/workstation.rs @@ -155,6 +155,9 @@ pub trait Workstation: Send { } /// The keyboard and mouse the host window types into. fn get_ps2(&self) -> Arc; + /// Add a host-side command set (the screen, pointer and key commands) + /// to this machine's monitor. + fn register_monitor_device(&self, device: Arc); fn get_timer_manager(&self) -> Arc; /// The CPU's debug interface, for the GDB stub. fn get_cpu_debug(&self) -> Arc; diff --git a/tools/vnc-check.py b/tools/vnc-check.py new file mode 100644 index 000000000..833597481 --- /dev/null +++ b/tools/vnc-check.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""Minimal RFB client for checking IRIS's VNC server from scripts. + + tools/vnc-check.py [--addr 127.0.0.1:5900] [--zrle] shot OUT.png + tools/vnc-check.py ... watch SECONDS # incremental updates: rects, pixels + # (--no-decode with --zrle times the server) + tools/vnc-check.py ... click X Y [BUTTON] # 1 left, 2 middle, 4 right + tools/vnc-check.py ... move X Y + tools/vnc-check.py ... key KEYSYM # e.g. 0xff0d (Return), 0x61 (a) + +Needs Pillow for `shot`. Decodes Raw and ZRLE (with --zrle). +""" +import argparse +import socket +import struct +import sys +import time +import zlib + + +def recvn(s, n): + b = bytearray() + while len(b) < n: + c = s.recv(n - len(b)) + if not c: + raise EOFError("server closed") + b += c + return bytes(b) + + +class Rfb: + def __init__(self, addr, zrle, decode=True): + self.decode = decode + host, port = addr.rsplit(":", 1) + self.s = socket.create_connection((host, int(port)), timeout=20) + ver = recvn(self.s, 12) + assert ver.startswith(b"RFB "), ver + self.s.sendall(b"RFB 003.008\n") + n = recvn(self.s, 1)[0] + types = recvn(self.s, n) + assert 1 in types, types + self.s.sendall(b"\x01") + assert struct.unpack(">I", recvn(self.s, 4))[0] == 0 + self.s.sendall(b"\x01") # shared + self.w, self.h = struct.unpack(">HH", recvn(self.s, 4)) + recvn(self.s, 16) + (nl,) = struct.unpack(">I", recvn(self.s, 4)) + self.name = recvn(self.s, nl).decode() + # 32 bpp, R at shift 16 (the usual viewer format) to exercise conversion. + pf = struct.pack(">BBBBHHHBBBxxx", 32, 24, 0, 1, 255, 255, 255, 16, 8, 0) + self.s.sendall(b"\x00\x00\x00\x00" + pf) + encs = ([16] if zrle else []) + [0, -223, -258] + self.s.sendall(struct.pack(">BxH", 2, len(encs)) + b"".join(struct.pack(">i", e) for e in encs)) + self.fb = bytearray(self.w * self.h * 4) + self.z = zlib.decompressobj() + + def request(self, incremental, x=0, y=0, w=None, h=None): + self.s.sendall(struct.pack(">BBHHHH", 3, incremental, x, y, w or self.w, h or self.h)) + + def pointer(self, x, y, mask): + self.s.sendall(struct.pack(">BBHH", 5, mask, x, y)) + + def key(self, sym, down): + self.s.sendall(struct.pack(">BBxxI", 4, down, sym)) + + def read_update(self): + """One server message; returns list of (x, y, w, h, enc) for an update.""" + t = recvn(self.s, 1)[0] + if t != 0: + raise RuntimeError(f"message type {t}") + (n,) = struct.unpack(">xH", recvn(self.s, 3)) + rects = [] + for _ in range(n): + x, y, w, h, enc = struct.unpack(">HHHHi", recvn(self.s, 12)) + rects.append((x, y, w, h, enc)) + if enc == 0: + data = recvn(self.s, w * h * 4) + for r in range(h): + o = ((y + r) * self.w + x) * 4 + self.fb[o:o + w * 4] = data[r * w * 4:(r + 1) * w * 4] + elif enc == 16: + (ln,) = struct.unpack(">I", recvn(self.s, 4)) + d = self.z.decompress(recvn(self.s, ln)) + if self.decode: + self.unzrle(d, x, y, w, h) + elif enc == -223: + self.w, self.h = w, h + self.fb = bytearray(w * h * 4) + elif enc == -258: + pass + else: + raise RuntimeError(f"encoding {enc}") + return rects + + def unzrle(self, d, x, y, w, h): + p = 0 + + def cpix(): + nonlocal p + v = d[p:p + 3] + b"\x00" + p += 3 + return v + + def rl(): + nonlocal p + n = 1 + while True: + b = d[p] + p += 1 + n += b + if b != 255: + return n + + for ty in range(y, y + h, 64): + th = min(64, y + h - ty) + for tx in range(x, x + w, 64): + tw = min(64, x + w - tx) + sub = d[p] + p += 1 + pal = [] + if 2 <= sub <= 16 or sub >= 130: + pal = [cpix() for _ in range(sub if sub <= 16 else sub - 128)] + tile = [] + if sub == 0: + tile = [cpix() for _ in range(tw * th)] + elif sub == 1: + tile = [cpix()] * (tw * th) + elif sub <= 16: + bits = 1 if sub == 2 else 2 if sub <= 4 else 4 + rowb = (tw * bits + 7) // 8 + for _ in range(th): + for i in range(tw): + b = d[p + (i * bits) // 8] + idx = (b >> (8 - bits - (i * bits) % 8)) & ((1 << bits) - 1) + tile.append(pal[idx]) + p += rowb + elif sub == 128: + while len(tile) < tw * th: + v = cpix() + tile += [v] * rl() + else: + while len(tile) < tw * th: + b = d[p] + p += 1 + tile += [pal[b & 127]] * (rl() if b & 128 else 1) + for r in range(th): + for c in range(tw): + o = ((ty + r) * self.w + tx + c) * 4 + self.fb[o:o + 4] = tile[r * tw + c] + assert p == len(d), (p, len(d)) + + def save(self, path): + from PIL import Image + # Pixels are little-endian with R at shift 16: bytes B, G, R, 0. + Image.frombytes("RGBX", (self.w, self.h), bytes(self.fb), "raw", "BGRX").convert("RGB").save(path) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--addr", default="127.0.0.1:5900") + ap.add_argument("--zrle", action="store_true") + ap.add_argument("--no-decode", action="store_true", help="watch: inflate ZRLE but do not decode tiles (times the server)") + ap.add_argument("cmd") + ap.add_argument("args", nargs="*") + a = ap.parse_args() + c = Rfb(a.addr, a.zrle, not a.no_decode) + print(f"connected: {c.name} {c.w}x{c.h}") + if a.cmd == "shot": + c.request(0) + t = time.time() + rects = c.read_update() + # The ext-key ack or a resize may come first; make sure we have pixels. + while not any(r[4] in (0, 16) for r in rects): + c.request(0) + rects = c.read_update() + c.save(a.args[0]) + print(f"saved {a.args[0]} {c.w}x{c.h} in {time.time() - t:.2f}s") + elif a.cmd == "watch": + end = time.time() + float(a.args[0]) + c.request(0) + c.read_update() + c.request(1) + n = 0 + c.s.settimeout(max(0.1, end - time.time())) + try: + while time.time() < end: + rects = c.read_update() + n += 1 + px = sum(r[2] * r[3] for r in rects) + print(f"update {n}: {len(rects)} rects, {px} px: {rects[:4]}") + c.request(1) + except socket.timeout: + pass + print(f"{n} updates") + elif a.cmd in ("click", "move"): + x, y = int(a.args[0]), int(a.args[1]) + b = int(a.args[2]) if len(a.args) > 2 else 1 + c.pointer(x, y, 0) + if a.cmd == "click": + time.sleep(0.05) + c.pointer(x, y, b) + time.sleep(0.05) + c.pointer(x, y, 0) + time.sleep(0.8) # let the server steer before we hang up + elif a.cmd == "key": + sym = int(a.args[0], 0) + c.key(sym, 1) + c.key(sym, 0) + time.sleep(0.2) + else: + sys.exit(f"unknown command {a.cmd}") + + +if __name__ == "__main__": + main()