From d03d3c6702e77ebd1d82dca32eb3ee69dd670006 Mon Sep 17 00:00:00 2001
From: Ravindu Somawansa <ravindu.somawansa@gmail.com>
Date: Thu, 29 Oct 2026 16:31:31 +0100
Subject: [PATCH] feat(cameras): live alignment score and lighting match
 against the reference frame

---
 .../recording/CameraConfiguration.test.tsx    |  18 ++
 .../recording/CameraConfiguration.tsx         |  11 +-
 .../recording/CameraReferenceControls.tsx     |  28 ++++
 frontend/src/hooks/useReferenceScores.ts      |  75 +++++++++
 frontend/src/lib/cameraReferenceScore.test.ts |  52 ++++++
 frontend/src/lib/cameraReferenceScore.ts      | 156 ++++++++++++++++++
 6 files changed, 339 insertions(+), 1 deletion(-)
 create mode 100644 frontend/src/hooks/useReferenceScores.ts
 create mode 100644 frontend/src/lib/cameraReferenceScore.test.ts
 create mode 100644 frontend/src/lib/cameraReferenceScore.ts

diff --git a/frontend/src/components/recording/CameraConfiguration.test.tsx b/frontend/src/components/recording/CameraConfiguration.test.tsx
index 571552e9c..60b35e99d 100644
--- a/frontend/src/components/recording/CameraConfiguration.test.tsx
+++ b/frontend/src/components/recording/CameraConfiguration.test.tsx
@@ -8,6 +8,8 @@ vi.mock("@/hooks/useAvailableCameras", () => ({ useAvailableCameras: () => ({ ca
 vi.mock("@/hooks/use-toast", () => ({ useToast: () => ({ toast: vi.fn() }) }));
 const reference = { url: null as string | null, error: null, save: vi.fn(), clear: vi.fn() };
 vi.mock("@/hooks/useCameraReference", () => ({ useCameraReference: () => reference }));
+const scores = { alignment: null as number | null, lighting: null as { match: number; brightnessDiff: number; colorDiff: number } | null };
+vi.mock("@/hooks/useReferenceScores", () => ({ useReferenceScores: () => scores }));
 vi.mock("@/hooks/useCameraStream", () => ({ useCameraStream: () => ({ videoRef: React.createRef<HTMLVideoElement>(), hasError: false }) }));
 let root: Root;
 let container: HTMLDivElement;
@@ -59,3 +61,19 @@ test("reference controls appear only for a named robot in read-only mode", () =>
   expect(slider).not.toBeNull();
   reference.url = null;
 });
+
+test("with a reference, the live scores show and the preview gets a halo by alignment", () => {
+  reference.url = "blob:ref";
+  scores.alignment = 95;
+  scores.lighting = { match: 72, brightnessDiff: 7.04, colorDiff: 0.3 };
+  act(() => root.render(<CameraConfiguration cameras={[camera]} onCamerasChange={vi.fn()} readOnly robotName="arm" />));
+  expect(container.textContent).toContain("Alignment 95% · Lighting 72% (brightness 7.0%, color 0.3%)");
+  const preview = () => container.querySelector("video")!.parentElement!.parentElement!;
+  expect(preview().className).toContain("border-green-500");
+  scores.alignment = 60;
+  act(() => root.render(<CameraConfiguration cameras={[camera]} onCamerasChange={vi.fn()} readOnly robotName="arm" />));
+  expect(preview().className).toContain("border-orange-500");
+  reference.url = null;
+  scores.alignment = null;
+  scores.lighting = null;
+});
diff --git a/frontend/src/components/recording/CameraConfiguration.tsx b/frontend/src/components/recording/CameraConfiguration.tsx
index 60c6b6907..45f7cf425 100644
--- a/frontend/src/components/recording/CameraConfiguration.tsx
+++ b/frontend/src/components/recording/CameraConfiguration.tsx
@@ -19,9 +19,11 @@ import {
 import { useToast } from "@/hooks/use-toast";
 import { useAvailableCameras } from "@/hooks/useAvailableCameras";
 import { useCameraReference } from "@/hooks/useCameraReference";
+import { useReferenceScores } from "@/hooks/useReferenceScores";
 import BrowserCameraPreview from "@/components/control/BrowserCameraPreview";
 import CameraRotationSelect from "@/components/control/CameraRotationSelect";
 import CameraReferenceControls from "@/components/recording/CameraReferenceControls";
+import { ALIGNED_SCORE } from "@/lib/cameraReferenceScore";
 import {
   CameraConfig,
   cameraRotation,
@@ -361,9 +363,15 @@ const CameraPreview: React.FC<CameraPreviewProps> = ({
   const rotationError = cameraRotationError(camera);
   const rotation = rotationError ? null : cameraRotation(camera.rotation);
   const output = rotation === null ? null : rotatedDimensions(camera.width, camera.height, rotation);
+  const scores = useReferenceScores(previewRef, reference.url, !paused && !rotationError);
+  // Halo around the preview: green once the framing lines up with the reference, orange while it's off.
+  const halo =
+    scores.alignment === null
+      ? ""
+      : `border-2 ${scores.alignment >= ALIGNED_SCORE ? "border-green-500" : "border-orange-500"}`;
   return (
     <div className="bg-gray-900 rounded-lg border border-gray-700 overflow-hidden">
-      <div ref={previewRef}>
+      <div ref={previewRef} className={halo}>
       <BrowserCameraPreview
         deviceId={deviceId}
         paused={paused}
@@ -404,6 +412,7 @@ const CameraPreview: React.FC<CameraPreviewProps> = ({
             onClear={() => void reference.clear()}
             disabled={paused || !!rotationError}
             error={reference.error}
+            scores={scores}
           />
         )}
         {rotationError && <p role="alert" className="text-xs text-red-400">{rotationError} Select a valid rotation.</p>}
diff --git a/frontend/src/components/recording/CameraReferenceControls.tsx b/frontend/src/components/recording/CameraReferenceControls.tsx
index c23e97acc..be959e09e 100644
--- a/frontend/src/components/recording/CameraReferenceControls.tsx
+++ b/frontend/src/components/recording/CameraReferenceControls.tsx
@@ -1,5 +1,7 @@
 import React from "react";
 import { Button } from "@/components/ui/button";
+import { ALIGNED_SCORE, LIGHTING_FAIR, LIGHTING_GOOD } from "@/lib/cameraReferenceScore";
+import type { ReferenceScores } from "@/hooks/useReferenceScores";
 
 interface Props {
   hasReference: boolean;
@@ -9,8 +11,32 @@ interface Props {
   onClear: () => void;
   disabled: boolean;
   error: string | null;
+  scores?: ReferenceScores;
 }
 
+const lightingTone = (match: number) =>
+  match >= LIGHTING_GOOD ? "text-green-400" : match >= LIGHTING_FAIR ? "text-amber-400" : "text-red-400";
+
+/** Live alignment and lighting match against the reference, with the lighting breakdown as text. */
+const ScoreLine: React.FC<{ scores: ReferenceScores }> = ({ scores: { alignment, lighting } }) => (
+  <p className="text-xs text-gray-400" aria-live="polite">
+    Alignment{" "}
+    {alignment === null ? (
+      <span>–</span>
+    ) : (
+      <span className={alignment >= ALIGNED_SCORE ? "text-green-400" : "text-orange-400"}>
+        {Math.round(alignment)}%
+      </span>
+    )}
+    {lighting && (
+      <>
+        {" · "}Lighting <span className={lightingTone(lighting.match)}>{Math.round(lighting.match)}%</span>
+        {` (brightness ${lighting.brightnessDiff.toFixed(1)}%, color ${lighting.colorDiff.toFixed(1)}%)`}
+      </>
+    )}
+  </p>
+);
+
 /** Save the current frame as the reference, then slide the ghost overlay over the live preview. */
 const CameraReferenceControls: React.FC<Props> = ({
   hasReference,
@@ -20,6 +46,7 @@ const CameraReferenceControls: React.FC<Props> = ({
   onClear,
   disabled,
   error,
+  scores,
 }) => (
   <div className="space-y-1.5">
     <div className="flex items-center gap-2">
@@ -60,6 +87,7 @@ const CameraReferenceControls: React.FC<Props> = ({
         />
       </label>
     )}
+    {hasReference && scores && <ScoreLine scores={scores} />}
     {error && <p role="alert" className="text-xs text-red-400">{error}</p>}
   </div>
 );
diff --git a/frontend/src/hooks/useReferenceScores.ts b/frontend/src/hooks/useReferenceScores.ts
new file mode 100644
index 000000000..7b37e1b7b
--- /dev/null
+++ b/frontend/src/hooks/useReferenceScores.ts
@@ -0,0 +1,75 @@
+import { RefObject, useEffect, useState } from "react";
+import {
+  LightingMatch,
+  Pixels,
+  alignmentInput,
+  alignmentScore,
+  lightingMatch,
+  lightingSignature,
+} from "@/lib/cameraReferenceScore";
+
+const POLL_MS = 500;
+
+export interface ReferenceScores {
+  /** 0-100, or null while there is no frame or the sizes differ. */
+  alignment: number | null;
+  lighting: LightingMatch | null;
+}
+
+const EMPTY: ReferenceScores = { alignment: null, lighting: null };
+
+function pixelsOf(source: CanvasImageSource, width: number, height: number): Pixels | null {
+  if (!width || !height) return null;
+  const canvas = document.createElement("canvas");
+  canvas.width = width;
+  canvas.height = height;
+  const ctx = canvas.getContext("2d", { willReadFrequently: true });
+  if (!ctx) return null;
+  ctx.drawImage(source, 0, 0);
+  return ctx.getImageData(0, 0, width, height);
+}
+
+/**
+ * Alignment and lighting scores of the `<video>` inside `previewRef` against the
+ * reference frame at `referenceUrl`, refreshed twice a second. Both frames are raw
+ * (unrotated) stream frames, so rotation doesn't enter the comparison.
+ */
+export function useReferenceScores(
+  previewRef: RefObject<HTMLElement>,
+  referenceUrl: string | null,
+  enabled: boolean,
+): ReferenceScores {
+  const [scores, setScores] = useState<ReferenceScores>(EMPTY);
+
+  useEffect(() => {
+    setScores(EMPTY);
+    if (!enabled || !referenceUrl) return;
+    let cancelled = false;
+    let timer: ReturnType<typeof setInterval> | undefined;
+    const img = new Image();
+    img.onload = () => {
+      if (cancelled) return;
+      const ref = pixelsOf(img, img.naturalWidth, img.naturalHeight);
+      if (!ref) return;
+      const refAlign = alignmentInput(ref);
+      const refLight = lightingSignature(ref);
+      timer = setInterval(() => {
+        const video = previewRef.current?.querySelector("video");
+        const live = video && video.readyState >= 2 ? pixelsOf(video, video.videoWidth, video.videoHeight) : null;
+        if (!live) return setScores(EMPTY);
+        const sameSize = live.width === ref.width && live.height === ref.height;
+        setScores({
+          alignment: sameSize ? alignmentScore(alignmentInput(live), refAlign) : null,
+          lighting: lightingMatch(lightingSignature(live), refLight),
+        });
+      }, POLL_MS);
+    };
+    img.src = referenceUrl;
+    return () => {
+      cancelled = true;
+      if (timer) clearInterval(timer);
+    };
+  }, [previewRef, referenceUrl, enabled]);
+
+  return scores;
+}
diff --git a/frontend/src/lib/cameraReferenceScore.test.ts b/frontend/src/lib/cameraReferenceScore.test.ts
new file mode 100644
index 000000000..8be550e17
--- /dev/null
+++ b/frontend/src/lib/cameraReferenceScore.test.ts
@@ -0,0 +1,52 @@
+import { expect, test } from "vitest";
+
+import { Pixels, alignmentInput, alignmentScore, lightingMatch, lightingSignature } from "./cameraReferenceScore";
+
+const W = 160;
+const H = 120;
+
+// A synthetic scene: a smooth pattern plus a bright block, optionally shifted, brightened or warmed.
+function scene({ dx = 0, gain = 1, warm = 0 } = {}): Pixels {
+  const data = new Uint8ClampedArray(W * H * 4);
+  for (let y = 0; y < H; y++) {
+    for (let x0 = 0; x0 < W; x0++) {
+      const x = x0 - dx;
+      const block = x >= 40 && x < 80 && y >= 30 && y < 70 ? 80 : 0;
+      const f = 128 + 60 * Math.sin(x / 9) * Math.cos(y / 7) + block;
+      const p = (y * W + x0) * 4;
+      data[p] = Math.round(f * gain * (1 + warm));
+      data[p + 1] = Math.round((0.8 * f + 20) * gain);
+      data[p + 2] = Math.round((0.6 * f + 40) * gain * (1 - warm));
+      data[p + 3] = 255;
+    }
+  }
+  return { data, width: W, height: H };
+}
+
+const align = (live: Pixels, ref: Pixels) => alignmentScore(alignmentInput(live), alignmentInput(ref));
+const light = (live: Pixels, ref: Pixels) => lightingMatch(lightingSignature(live), lightingSignature(ref));
+
+// Expected values come from teleop-setup-align's OpenCV implementation on the same scenes.
+test("alignment matches the OpenCV reference: falls with a shift, ignores a brightness change", () => {
+  const ref = scene();
+  expect(align(scene(), ref)).toBeCloseTo(100, 1);
+  expect(align(scene({ dx: 5 }), ref)).toBeCloseTo(86.87, 0);
+  expect(align(scene({ dx: 20 }), ref)).toBe(0);
+  expect(align(scene({ gain: 1.1 }), ref)).toBeGreaterThan(99.5);
+});
+
+test("frames of different sizes have no alignment score", () => {
+  const small = { data: new Uint8ClampedArray(4 * 4 * 4), width: 4, height: 4 };
+  expect(align(small, scene())).toBeNull();
+});
+
+test("lighting matches the OpenCV reference: a brighter or warmer scene lowers the match", () => {
+  const ref = scene();
+  expect(light(scene(), ref).match).toBe(100);
+  const brighter = light(scene({ gain: 1.1 }), ref);
+  expect(brighter.brightnessDiff).toBeCloseTo(9.92, 1);
+  expect(brighter.match).toBeCloseTo(60.3, 0);
+  const warmer = light(scene({ warm: 0.05 }), ref);
+  expect(warmer.colorDiff).toBeCloseTo(3.33, 1);
+  expect(warmer.match).toBeCloseTo(86.7, 0);
+});
diff --git a/frontend/src/lib/cameraReferenceScore.ts b/frontend/src/lib/cameraReferenceScore.ts
new file mode 100644
index 000000000..32e08af0d
--- /dev/null
+++ b/frontend/src/lib/cameraReferenceScore.ts
@@ -0,0 +1,156 @@
+/**
+ * How closely a live camera frame matches its saved reference frame, computed in
+ * the browser from the preview's own pixels (the backend never opens a camera).
+ * Ported from teleop-setup-align (https://github.com/ravediamond/teleop-setup-align),
+ * where both scores were tuned on a two-camera SO-101 rig.
+ */
+
+/** RGBA pixels, as returned by `CanvasRenderingContext2D.getImageData`. */
+export interface Pixels {
+  data: Uint8ClampedArray;
+  width: number;
+  height: number;
+}
+
+/** Alignment at or above this reads as lined up (green halo). */
+export const ALIGNED_SCORE = 90;
+/** Lighting match bands: at or above GOOD is green, at or above FAIR is amber, else red. */
+export const LIGHTING_GOOD = 85;
+export const LIGHTING_FAIR = 60;
+
+// OpenCV's default for a 9x9 Gaussian: sigma = 0.3 * ((ksize - 1) / 2 - 1) + 0.8.
+const KERNEL = (() => {
+  const sigma = 1.7;
+  const k = Array.from({ length: 9 }, (_, i) => Math.exp(-((i - 4) ** 2) / (2 * sigma * sigma)));
+  const sum = k.reduce((a, b) => a + b, 0);
+  return k.map((v) => v / sum);
+})();
+
+function gray({ data, width, height }: Pixels): Float32Array {
+  const out = new Float32Array(width * height);
+  for (let i = 0, p = 0; i < out.length; i++, p += 4) {
+    out[i] = 0.299 * data[p] + 0.587 * data[p + 1] + 0.114 * data[p + 2];
+  }
+  return out;
+}
+
+// Mirror without repeating the edge pixel, like OpenCV's BORDER_REFLECT_101.
+function reflect(i: number, n: number): number {
+  if (n === 1) return 0;
+  while (i < 0 || i >= n) i = i < 0 ? -i : 2 * n - 2 - i;
+  return i;
+}
+
+function blur(src: Float32Array, width: number, height: number): Float32Array {
+  const tmp = new Float32Array(src.length);
+  const out = new Float32Array(src.length);
+  for (let y = 0; y < height; y++) {
+    const row = y * width;
+    for (let x = 0; x < width; x++) {
+      let s = 0;
+      for (let k = 0; k < 9; k++) s += KERNEL[k] * src[row + reflect(x + k - 4, width)];
+      tmp[row + x] = s;
+    }
+  }
+  for (let y = 0; y < height; y++) {
+    for (let x = 0; x < width; x++) {
+      let s = 0;
+      for (let k = 0; k < 9; k++) s += KERNEL[k] * tmp[reflect(y + k - 4, height) * width + x];
+      out[y * width + x] = s;
+    }
+  }
+  return out;
+}
+
+/** Blurred grayscale of a frame, the input `alignmentScore` compares. Cache it for the reference. */
+export function alignmentInput(frame: Pixels): Float32Array {
+  return blur(gray(frame), frame.width, frame.height);
+}
+
+/**
+ * 0-100: how well the live framing and arm pose line up with the reference.
+ *
+ * Normalized cross-correlation subtracts each image's own mean, so an overall
+ * brightness change (the lighting score's job) barely moves it; only a shift in
+ * what's in frame does. The 9x9 blur first turns NCC's cliff at a 1-2 px shift into
+ * a graduated curve (on 640x480: ~99 with sensor noise only, ~93 at 10 px, ~73 at
+ * 40 px, ~46 at 80 px), which a person nudging a camera by hand can follow.
+ * Returns null when the two frames differ in size.
+ */
+export function alignmentScore(live: Float32Array, reference: Float32Array): number | null {
+  if (live.length !== reference.length || live.length === 0) return null;
+  let ma = 0;
+  let mb = 0;
+  for (let i = 0; i < live.length; i++) {
+    ma += live[i];
+    mb += reference[i];
+  }
+  ma /= live.length;
+  mb /= live.length;
+  let ab = 0;
+  let aa = 0;
+  let bb = 0;
+  for (let i = 0; i < live.length; i++) {
+    const a = live[i] - ma;
+    const b = reference[i] - mb;
+    ab += a * b;
+    aa += a * a;
+    bb += b * b;
+  }
+  const denom = Math.sqrt(aa * bb);
+  return denom ? Math.max(0, ab / denom) * 100 : 0;
+}
+
+export interface LightingSignature {
+  brightness: number;
+  /** Each channel's share of total brightness, as [r, g, b]. */
+  colorBalance: [number, number, number];
+}
+
+/**
+ * Mean brightness plus color balance. Auto-exposure renormalizes brightness, so two
+ * differently lit scenes (curtains open vs closed) can average out the same; it
+ * doesn't correct color temperature, which the per-channel shares still catch.
+ */
+export function lightingSignature({ data }: Pixels): LightingSignature {
+  let r = 0;
+  let g = 0;
+  let b = 0;
+  const n = data.length / 4;
+  for (let p = 0; p < data.length; p += 4) {
+    r += data[p];
+    g += data[p + 1];
+    b += data[p + 2];
+  }
+  r /= n;
+  g /= n;
+  b /= n;
+  const total = r + g + b || 1;
+  return { brightness: 0.299 * r + 0.587 * g + 0.114 * b, colorBalance: [r / total, g / total, b / total] };
+}
+
+export interface LightingMatch {
+  /** 0-100. */
+  match: number;
+  /** Percent change in mean brightness from the reference. */
+  brightnessDiff: number;
+  /** Color-balance shift, on the same percent scale. */
+  colorDiff: number;
+}
+
+// Real lighting changes on the rig only gave single-digit percent diffs even when
+// clearly visible, so the worst diff is amplified to move the match visibly.
+const LIGHTING_GAIN = 4;
+
+/**
+ * Which signal moves depends on the camera (closing curtains moved one camera's
+ * brightness 3% -> 6% and another's color 0.3% -> 6.1%), so the score uses the
+ * larger of the two rather than diluting it with an average.
+ */
+export function lightingMatch(live: LightingSignature, reference: LightingSignature): LightingMatch {
+  const pct = (a: number, b: number) => (b ? (Math.abs(a - b) / b) * 100 : a === b ? 0 : 100);
+  const brightnessDiff = pct(live.brightness, reference.brightness);
+  const colorDiff = live.colorBalance.reduce((s, v, i) => s + Math.abs(v - reference.colorBalance[i]), 0) * 100;
+  const worst = Math.max(brightnessDiff, colorDiff);
+  return { match: Math.max(0, 100 - LIGHTING_GAIN * worst), brightnessDiff, colorDiff };
+}