feat(cameras): live alignment score and lighting match against the reference frame

frontend/src/components/recording/CameraConfiguration.test.tsx

@@ -8,6 +8,8 @@ vi.mock("@/hooks/useAvailableCameras", () => ({ useAvailableCameras: () => ({ ca
 vi.mock("@/hooks/use-toast", () => ({ useToast: () => ({ toast: vi.fn() }) }));
 const reference = { url: null as string | null, error: null, save: vi.fn(), clear: vi.fn() };
 vi.mock("@/hooks/useCameraReference", () => ({ useCameraReference: () => reference }));
+const scores = { alignment: null as number | null, lighting: null as { match: number; brightnessDiff: number; colorDiff: number } | null };
+vi.mock("@/hooks/useReferenceScores", () => ({ useReferenceScores: () => scores }));
 vi.mock("@/hooks/useCameraStream", () => ({ useCameraStream: () => ({ videoRef: React.createRef<HTMLVideoElement>(), hasError: false }) }));
 let root: Root;
 let container: HTMLDivElement;
@@ -59,3 +61,19 @@ test("reference controls appear only for a named robot in read-only mode", () =>
   expect(slider).not.toBeNull();
   reference.url = null;
 });
+
+test("with a reference, the live scores show and the preview gets a halo by alignment", () => {
+  reference.url = "blob:ref";
+  scores.alignment = 95;
+  scores.lighting = { match: 72, brightnessDiff: 7.04, colorDiff: 0.3 };
+  act(() => root.render(<CameraConfiguration cameras={[camera]} onCamerasChange={vi.fn()} readOnly robotName="arm" />));
+  expect(container.textContent).toContain("Alignment 95% · Lighting 72% (brightness 7.0%, color 0.3%)");
+  const preview = () => container.querySelector("video")!.parentElement!.parentElement!;
+  expect(preview().className).toContain("border-green-500");
+  scores.alignment = 60;
+  act(() => root.render(<CameraConfiguration cameras={[camera]} onCamerasChange={vi.fn()} readOnly robotName="arm" />));
+  expect(preview().className).toContain("border-orange-500");
+  reference.url = null;
+  scores.alignment = null;
+  scores.lighting = null;
+});

frontend/src/components/recording/CameraConfiguration.tsx

@@ -19,9 +19,11 @@ import {
 import { useToast } from "@/hooks/use-toast";
 import { useAvailableCameras } from "@/hooks/useAvailableCameras";
 import { useCameraReference } from "@/hooks/useCameraReference";
+import { useReferenceScores } from "@/hooks/useReferenceScores";
 import BrowserCameraPreview from "@/components/control/BrowserCameraPreview";
 import CameraRotationSelect from "@/components/control/CameraRotationSelect";
 import CameraReferenceControls from "@/components/recording/CameraReferenceControls";
+import { ALIGNED_SCORE } from "@/lib/cameraReferenceScore";
 import {
   CameraConfig,
   cameraRotation,
@@ -361,9 +363,15 @@ const CameraPreview: React.FC<CameraPreviewProps> = ({
   const rotationError = cameraRotationError(camera);
   const rotation = rotationError ? null : cameraRotation(camera.rotation);
   const output = rotation === null ? null : rotatedDimensions(camera.width, camera.height, rotation);
+  const scores = useReferenceScores(previewRef, reference.url, !paused && !rotationError);
+  // Halo around the preview: green once the framing lines up with the reference, orange while it's off.
+  const halo =
+    scores.alignment === null
+      ? ""
+      : `border-2 ${scores.alignment >= ALIGNED_SCORE ? "border-green-500" : "border-orange-500"}`;
   return (
     <div className="bg-gray-900 rounded-lg border border-gray-700 overflow-hidden">
-      <div ref={previewRef}>
+      <div ref={previewRef} className={halo}>
       <BrowserCameraPreview
         deviceId={deviceId}
         paused={paused}
@@ -404,6 +412,7 @@ const CameraPreview: React.FC<CameraPreviewProps> = ({
             onClear={() => void reference.clear()}
             disabled={paused || !!rotationError}
             error={reference.error}
+            scores={scores}
           />
         )}
         {rotationError && <p role="alert" className="text-xs text-red-400">{rotationError} Select a valid rotation.</p>}

frontend/src/components/recording/CameraReferenceControls.tsx

@@ -1,5 +1,7 @@
 import React from "react";
 import { Button } from "@/components/ui/button";
+import { ALIGNED_SCORE, LIGHTING_FAIR, LIGHTING_GOOD } from "@/lib/cameraReferenceScore";
+import type { ReferenceScores } from "@/hooks/useReferenceScores";
 
 interface Props {
   hasReference: boolean;
@@ -9,8 +11,32 @@ interface Props {
   onClear: () => void;
   disabled: boolean;
   error: string | null;
+  scores?: ReferenceScores;
 }
 
+const lightingTone = (match: number) =>
+  match >= LIGHTING_GOOD ? "text-green-400" : match >= LIGHTING_FAIR ? "text-amber-400" : "text-red-400";
+
+/** Live alignment and lighting match against the reference, with the lighting breakdown as text. */
+const ScoreLine: React.FC<{ scores: ReferenceScores }> = ({ scores: { alignment, lighting } }) => (
+  <p className="text-xs text-gray-400" aria-live="polite">
+    Alignment{" "}
+    {alignment === null ? (
+      <span>–</span>
+    ) : (
+      <span className={alignment >= ALIGNED_SCORE ? "text-green-400" : "text-orange-400"}>
+        {Math.round(alignment)}%
+      </span>
+    )}
+    {lighting && (
+      <>
+        {" · "}Lighting <span className={lightingTone(lighting.match)}>{Math.round(lighting.match)}%</span>
+        {` (brightness ${lighting.brightnessDiff.toFixed(1)}%, color ${lighting.colorDiff.toFixed(1)}%)`}
+      </>
+    )}
+  </p>
+);
+
 /** Save the current frame as the reference, then slide the ghost overlay over the live preview. */
 const CameraReferenceControls: React.FC<Props> = ({
   hasReference,
@@ -20,6 +46,7 @@ const CameraReferenceControls: React.FC<Props> = ({
   onClear,
   disabled,
   error,
+  scores,
 }) => (
   <div className="space-y-1.5">
     <div className="flex items-center gap-2">
@@ -60,6 +87,7 @@ const CameraReferenceControls: React.FC<Props> = ({
         />
       </label>
     )}
+    {hasReference && scores && <ScoreLine scores={scores} />}
     {error && <p role="alert" className="text-xs text-red-400">{error}</p>}
   </div>
 );

frontend/src/hooks/useReferenceScores.ts

@@ -0,0 +1,75 @@
+import { RefObject, useEffect, useState } from "react";
+import {
+  LightingMatch,
+  Pixels,
+  alignmentInput,
+  alignmentScore,
+  lightingMatch,
+  lightingSignature,
+} from "@/lib/cameraReferenceScore";
+
+const POLL_MS = 500;
+
+export interface ReferenceScores {
+  /** 0-100, or null while there is no frame or the sizes differ. */
+  alignment: number | null;
+  lighting: LightingMatch | null;
+}
+
+const EMPTY: ReferenceScores = { alignment: null, lighting: null };
+
+function pixelsOf(source: CanvasImageSource, width: number, height: number): Pixels | null {
+  if (!width || !height) return null;
+  const canvas = document.createElement("canvas");
+  canvas.width = width;
+  canvas.height = height;
+  const ctx = canvas.getContext("2d", { willReadFrequently: true });
+  if (!ctx) return null;
+  ctx.drawImage(source, 0, 0);
+  return ctx.getImageData(0, 0, width, height);
+}
+
+/**
+ * Alignment and lighting scores of the `<video>` inside `previewRef` against the
+ * reference frame at `referenceUrl`, refreshed twice a second. Both frames are raw
+ * (unrotated) stream frames, so rotation doesn't enter the comparison.
+ */
+export function useReferenceScores(
+  previewRef: RefObject<HTMLElement>,
+  referenceUrl: string | null,
+  enabled: boolean,
+): ReferenceScores {
+  const [scores, setScores] = useState<ReferenceScores>(EMPTY);
+
+  useEffect(() => {
+    setScores(EMPTY);
+    if (!enabled || !referenceUrl) return;
+    let cancelled = false;
+    let timer: ReturnType<typeof setInterval> | undefined;
+    const img = new Image();
+    img.onload = () => {
+      if (cancelled) return;
+      const ref = pixelsOf(img, img.naturalWidth, img.naturalHeight);
+      if (!ref) return;
+      const refAlign = alignmentInput(ref);
+      const refLight = lightingSignature(ref);
+      timer = setInterval(() => {
+        const video = previewRef.current?.querySelector("video");
+        const live = video && video.readyState >= 2 ? pixelsOf(video, video.videoWidth, video.videoHeight) : null;
+        if (!live) return setScores(EMPTY);
+        const sameSize = live.width === ref.width && live.height === ref.height;
+        setScores({
+          alignment: sameSize ? alignmentScore(alignmentInput(live), refAlign) : null,
+          lighting: lightingMatch(lightingSignature(live), refLight),
+        });
+      }, POLL_MS);
+    };
+    img.src = referenceUrl;
+    return () => {
+      cancelled = true;
+      if (timer) clearInterval(timer);
+    };
+  }, [previewRef, referenceUrl, enabled]);
+
+  return scores;
+}

frontend/src/lib/cameraReferenceScore.test.ts

@@ -0,0 +1,52 @@
+import { expect, test } from "vitest";
+
+import { Pixels, alignmentInput, alignmentScore, lightingMatch, lightingSignature } from "./cameraReferenceScore";
+
+const W = 160;
+const H = 120;
+
+// A synthetic scene: a smooth pattern plus a bright block, optionally shifted, brightened or warmed.
+function scene({ dx = 0, gain = 1, warm = 0 } = {}): Pixels {
+  const data = new Uint8ClampedArray(W * H * 4);
+  for (let y = 0; y < H; y++) {
+    for (let x0 = 0; x0 < W; x0++) {
+      const x = x0 - dx;
+      const block = x >= 40 && x < 80 && y >= 30 && y < 70 ? 80 : 0;
+      const f = 128 + 60 * Math.sin(x / 9) * Math.cos(y / 7) + block;
+      const p = (y * W + x0) * 4;
+      data[p] = Math.round(f * gain * (1 + warm));
+      data[p + 1] = Math.round((0.8 * f + 20) * gain);
+      data[p + 2] = Math.round((0.6 * f + 40) * gain * (1 - warm));
+      data[p + 3] = 255;
+    }
+  }
+  return { data, width: W, height: H };
+}
+
+const align = (live: Pixels, ref: Pixels) => alignmentScore(alignmentInput(live), alignmentInput(ref));
+const light = (live: Pixels, ref: Pixels) => lightingMatch(lightingSignature(live), lightingSignature(ref));
+
+// Expected values come from teleop-setup-align's OpenCV implementation on the same scenes.
+test("alignment matches the OpenCV reference: falls with a shift, ignores a brightness change", () => {
+  const ref = scene();
+  expect(align(scene(), ref)).toBeCloseTo(100, 1);
+  expect(align(scene({ dx: 5 }), ref)).toBeCloseTo(86.87, 0);
+  expect(align(scene({ dx: 20 }), ref)).toBe(0);
+  expect(align(scene({ gain: 1.1 }), ref)).toBeGreaterThan(99.5);
+});
+
+test("frames of different sizes have no alignment score", () => {
+  const small = { data: new Uint8ClampedArray(4 * 4 * 4), width: 4, height: 4 };
+  expect(align(small, scene())).toBeNull();
+});
+
+test("lighting matches the OpenCV reference: a brighter or warmer scene lowers the match", () => {
+  const ref = scene();
+  expect(light(scene(), ref).match).toBe(100);
+  const brighter = light(scene({ gain: 1.1 }), ref);
+  expect(brighter.brightnessDiff).toBeCloseTo(9.92, 1);
+  expect(brighter.match).toBeCloseTo(60.3, 0);
+  const warmer = light(scene({ warm: 0.05 }), ref);
+  expect(warmer.colorDiff).toBeCloseTo(3.33, 1);
+  expect(warmer.match).toBeCloseTo(86.7, 0);
+});