Admin board: check and repair the R2 objects' custom metadata

apps/admin/src/audit.ts

@@ -5,7 +5,7 @@
  */
 import { assertNumber, assertUuid, HttpError } from "@openclapper/service-kit";
 
-export type AuditAction = "user.create" | "user.update" | "user.password" | "user.delete" | "settings.update";
+export type AuditAction = "user.create" | "user.update" | "user.password" | "user.delete" | "settings.update" | "storage.repair";
 
 export interface AuditEntry {
   id: string;

apps/admin/src/index.ts

@@ -9,6 +9,7 @@ import {
 } from "@openclapper/service-kit";
 import { auditEntry, listAudit, type AuditAction } from "./audit.ts";
 import { configured, MAX_DAYS, overlay, snapshotCloudflare } from "./cloudflare.ts";
+import { repairPage } from "./objects.ts";
 import { marksStatement, parseMarks, readMarks, storageReport } from "./storage.ts";
 
 export interface Env {
@@ -220,6 +221,23 @@ router.on("PUT", "/api/settings", async (ctx) => {
   await env.DB.batch([marksStatement(env.DB, marks, ctx.admin), audit(ctx, "settings.update", {}, { storage_marks: marks })]);
   return json({ storage_marks: marks });
 });
+/**
+ * The R2 objects' custom metadata (#255), one page per call: JSON { cursor?: string | null, check?: boolean }
+ * → { checked, repaired, unknown, examples, cursor }; the board calls again with the returned cursor until
+ * it is null. `check` only counts. A page that rewrote objects is one audit row.
+ */
+router.on("POST", "/api/storage/repair-metadata", async (ctx) => {
+  const { env, request } = ctx;
+  const b = await readJson<Record<string, unknown>>(request);
+  for (const k of Object.keys(b)) if (k !== "cursor" && k !== "check") throw new HttpError(400, `unknown field ${k}`);
+  const cursor = b.cursor === undefined || b.cursor === null ? null : assertString(b.cursor, "cursor", 1024);
+  if (b.check !== undefined && typeof b.check !== "boolean") throw new HttpError(400, "check must be true or false");
+  const check = b.check === true;
+  const page = await repairPage(env.DB, env.SHOTS_BUCKET, cursor, check);
+  if (!check && page.repaired) await audit(ctx, "storage.repair", {}, { checked: page.checked, repaired: page.repaired, unknown: page.unknown }).run();
+  return json(page);
+});
+
 /** The audit trail, newest first: `?limit=` (1..200, default 50), `before` + `before_id` from the previous page's `next`. */
 router.on("GET", "/api/audit", async ({ env, url }) => json(await listAudit(env.DB, url.searchParams)));
 

apps/admin/src/objects.ts

@@ -0,0 +1,82 @@
+/**
+ * The R2 objects' custom metadata (#255): what each one should carry, and a repair for objects that lost
+ * it. The app writes it on upload (apps/worker/src/shots.ts: shot_id, photo_id, workspace_id;
+ * compose.ts: kind, id), as a second record of ownership on the object itself. A bucket copied without
+ * metadata (the reference instance's rename, 2026-10-20; a clone with rclone, which adds mtime/btime)
+ * leaves objects bare; the repair derives the metadata again from D1 and the keys.
+ */
+
+export type ObjectMetadata = Record<string, string>;
+
+/** One page of a repair: what was looked at and what was (or, in a check, would be) rewritten. */
+export interface RepairPage {
+  checked: number;
+  /** Objects whose metadata was missing or different: rewritten, or only counted in a check. */
+  repaired: number;
+  /** Objects the app does not know: no photo row for the key, or a key outside photos/ and renders/. */
+  unknown: number;
+  /** A few repaired or unknown keys, for the operator to look at. */
+  examples: { key: string; reason: "missing" | "different" | "unknown" }[];
+  /** Where the next page starts; null after the last one. */
+  cursor: string | null;
+}
+
+/** Objects per call: each may take a get and a put, well inside a Worker request's limits. */
+export const PAGE = 100;
+
+interface PhotoRow { k: string; photo_id: string; shot_id: string; workspace_id: string | null }
+
+/**
+ * The metadata an object should carry: a photo's from its row (the workspace through its shot's
+ * project), a render's from its key (renders/<kind>/<id>.<ext>). Null for an object the app does not know.
+ */
+export function expectedMetadata(key: string, photo: PhotoRow | undefined): ObjectMetadata | null {
+  if (key.startsWith("photos/")) {
+    if (!photo || !photo.workspace_id) return null;
+    return { shot_id: photo.shot_id, photo_id: photo.photo_id, workspace_id: photo.workspace_id };
+  }
+  const render = key.match(/^renders\/(overlays|sketches)\/([^/]+)\.[a-z0-9]+$/);
+  return render ? { kind: render[1], id: render[2] } : null;
+}
+
+const same = (a: ObjectMetadata, b: ObjectMetadata) => {
+  const ka = Object.keys(a), kb = Object.keys(b);
+  return ka.length === kb.length && ka.every((k) => a[k] === b[k]);
+};
+
+/**
+ * One page of the bucket: every object whose metadata is not exactly what it should be is rewritten in
+ * place (the same body and content type, the derived metadata), unless `check` only counts them.
+ * Unknown objects are left alone.
+ */
+export async function repairPage(db: D1Database, bucket: R2Bucket, cursor: string | null, check: boolean): Promise<RepairPage> {
+  const listed = await bucket.list({ limit: PAGE, cursor: cursor ?? undefined, include: ["customMetadata"] });
+  const photoKeys = listed.objects.map((o) => o.key).filter((k) => k.startsWith("photos/"));
+  const rows = photoKeys.length
+    ? (await db.prepare(
+        `SELECT p.r2_object_key AS k, p.id AS photo_id, p.shot_id, pr.workspace_id
+           FROM photos p JOIN shots s ON s.id = p.shot_id JOIN projects pr ON pr.id = s.project_id
+          WHERE p.r2_object_key IN (SELECT value FROM json_each(?1))`,
+      ).bind(JSON.stringify(photoKeys)).all<PhotoRow>()).results
+    : [];
+  const byKey = new Map(rows.map((r) => [r.k, r]));
+  const page: RepairPage = { checked: 0, repaired: 0, unknown: 0, examples: [], cursor: listed.truncated ? listed.cursor : null };
+  for (const o of listed.objects) {
+    page.checked++;
+    const want = expectedMetadata(o.key, byKey.get(o.key));
+    if (!want) {
+      page.unknown++;
+      if (page.examples.length < 10) page.examples.push({ key: o.key, reason: "unknown" });
+      continue;
+    }
+    const has = o.customMetadata ?? {};
+    if (same(has, want)) continue;
+    page.repaired++;
+    if (page.examples.length < 10) page.examples.push({ key: o.key, reason: Object.keys(has).length ? "different" : "missing" });
+    if (check) continue;
+    const obj = await bucket.get(o.key);
+    if (!obj) continue; // deleted meanwhile
+    await bucket.put(o.key, obj.body, { httpMetadata: obj.httpMetadata, customMetadata: want });
+  }
+  return page;
+}

apps/admin/test/objects.test.ts

@@ -0,0 +1,89 @@
+/**
+ * The R2 objects' custom metadata (#255): derived from D1 and the keys, checked and repaired page by
+ * page, unknown objects left alone, bodies and content types kept.
+ */
+import { env } from "cloudflare:workers";
+import { describe, expect, test } from "vitest";
+import worker, { type Env } from "../src/index.ts";
+import { expectedMetadata, type RepairPage } from "../src/objects.ts";
+
+const bypass: Env = { ...env, ACCESS_TEAM_DOMAIN: "", ACCESS_AUD: "", ADMIN_DEV_BYPASS: "true" };
+function call(body: unknown): Promise<Response> {
+  return worker.fetch(new Request("http://localhost:8788/api/storage/repair-metadata", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }), bypass);
+}
+async function page(body: unknown, status = 200): Promise<RepairPage> {
+  const r = await call(body);
+  const text = await r.text();
+  expect(r.status, text).toBe(status);
+  return JSON.parse(text) as RepairPage;
+}
+/** Every page from the start, as the board does. */
+async function all(check: boolean): Promise<RepairPage> {
+  const sum: RepairPage = { checked: 0, repaired: 0, unknown: 0, examples: [], cursor: null };
+  let cursor: string | null = null;
+  do {
+    const p: RepairPage = await page({ cursor, check });
+    sum.checked += p.checked; sum.repaired += p.repaired; sum.unknown += p.unknown; sum.examples.push(...p.examples);
+    cursor = p.cursor;
+  } while (cursor);
+  return sum;
+}
+
+async function photo(ws: string): Promise<{ key: string; shot: string; id: string }> {
+  const project = crypto.randomUUID(), shot = crypto.randomUUID(), id = crypto.randomUUID(), key = `photos/${id}.jpg`;
+  await env.DB.batch([
+    env.DB.prepare("INSERT INTO projects (id, name, workspace_id) VALUES (?1, ?1, ?2)").bind(project, ws),
+    env.DB.prepare("INSERT INTO shots (id, project_id, captured_at) VALUES (?1, ?2, '2026-10-01T10:00:00Z')").bind(shot, project),
+    env.DB.prepare("INSERT INTO photos (id, shot_id, ordinal, timestamp, lat, lon, lens_mm, r2_object_key) VALUES (?1, ?2, 0, '2026-10-01T10:00:00Z', 0, 0, 35, ?3)").bind(id, shot, key),
+  ]);
+  return { key, shot, id };
+}
+
+describe("expected metadata", () => {
+  test("a photo's from its row, a render's from its key, nothing for the rest", () => {
+    expect(expectedMetadata("photos/p.jpg", { k: "photos/p.jpg", photo_id: "p", shot_id: "s", workspace_id: "w" })).toEqual({ shot_id: "s", photo_id: "p", workspace_id: "w" });
+    expect(expectedMetadata("photos/p.jpg", undefined)).toBeNull();
+    expect(expectedMetadata("renders/overlays/o1.png", undefined)).toEqual({ kind: "overlays", id: "o1" });
+    expect(expectedMetadata("renders/sketches/s1.webp", undefined)).toEqual({ kind: "sketches", id: "s1" });
+    expect(expectedMetadata("renders/other/x.png", undefined)).toBeNull();
+    expect(expectedMetadata("misc/readme.txt", undefined)).toBeNull();
+  });
+});
+
+describe("repair", () => {
+  test("a check counts, a repair rewrites bare and altered objects with the same body, and leaves the rest", async () => {
+    const ws = crypto.randomUUID();
+    await env.DB.prepare("INSERT INTO workspaces (id, name, owner_user_id, kind) VALUES (?1, 'Crew', NULL, 'shared')").bind(ws).run();
+    const bare = await photo(ws), cloned = await photo(ws), fine = await photo(ws);
+    const b = env.SHOTS_BUCKET;
+    await b.put(bare.key, "bare-bytes", { httpMetadata: { contentType: "image/jpeg" } });
+    await b.put(cloned.key, "cloned-bytes", { httpMetadata: { contentType: "image/jpeg" }, customMetadata: { mtime: "1792510933.004", btime: "2026-10-20T15:42:13.004Z" } });
+    await b.put(fine.key, "fine-bytes", { httpMetadata: { contentType: "image/jpeg" }, customMetadata: { shot_id: fine.shot, photo_id: fine.id, workspace_id: ws } });
+    await b.put("renders/sketches/sk1.png", "render-bytes", { httpMetadata: { contentType: "image/png" } });
+    await b.put("photos/orphan.jpg", "orphan-bytes");
+
+    const checked = await all(true);
+    expect(checked).toMatchObject({ checked: 5, repaired: 3, unknown: 1 });
+    expect((await b.head(bare.key))!.customMetadata).toEqual({});
+
+    const repaired = await all(false);
+    expect(repaired).toMatchObject({ checked: 5, repaired: 3, unknown: 1 });
+    expect(repaired.examples).toEqual(expect.arrayContaining([{ key: bare.key, reason: "missing" }, { key: cloned.key, reason: "different" }, { key: "photos/orphan.jpg", reason: "unknown" }]));
+    const got = await b.get(cloned.key);
+    expect(got!.customMetadata).toEqual({ shot_id: cloned.shot, photo_id: cloned.id, workspace_id: ws });
+    expect(got!.httpMetadata?.contentType).toBe("image/jpeg");
+    expect(await got!.text()).toBe("cloned-bytes");
+    expect((await b.head("renders/sketches/sk1.png"))!.customMetadata).toEqual({ kind: "sketches", id: "sk1" });
+    expect((await b.head("photos/orphan.jpg"))!.customMetadata).toEqual({});
+
+    expect(await all(false)).toMatchObject({ repaired: 0, unknown: 1 });
+    const audit = await env.DB.prepare("SELECT detail FROM admin_audit WHERE action = 'storage.repair'").all<{ detail: string }>();
+    expect(audit.results.map((r) => JSON.parse(r.detail))).toEqual([{ checked: 5, repaired: 3, unknown: 1 }]);
+  });
+
+  test("the body is strict", async () => {
+    await page({ cursor: 5 }, 400);
+    await page({ check: "yes" }, 400);
+    await page({ force: true }, 400);
+  });
+});

apps/admin/ui/src/Audit.tsx

@@ -10,6 +10,7 @@ const ACTION: Record<string, { label: string; icon: string }> = {
   "user.password": { label: "Password reset", icon: "key-variant" },
   "user.delete": { label: "Account deleted", icon: "delete-outline" },
   "settings.update": { label: "Settings changed", icon: "cog-outline" },
+  "storage.repair": { label: "Object metadata repaired", icon: "tag-check-outline" },
 };
 
 const when = (iso: string) => new Date(iso).toLocaleString([], { day: "numeric", month: "short", year: "numeric", hour: "2-digit", minute: "2-digit", second: "2-digit" });
@@ -21,6 +22,7 @@ function detail(e: AuditEntry): string {
     const marks = d.storage_marks as { bytes: number; label: string }[];
     return marks.length ? `Storage marks: ${marks.map((m) => `${m.label} (${bytes(m.bytes)})`).join(", ")}` : "Storage marks removed";
   }
+  if (e.action === "storage.repair") return `${String(d.repaired)} of ${String(d.checked)} objects rewritten${d.unknown ? `, ${String(d.unknown)} unknown left alone` : ""}`;
   const parts: string[] = [];
   if ("email" in d) parts.push(`email → ${String(d.email)}`);
   if ("name" in d) parts.push(d.name ? `name → ${String(d.name)}` : e.action === "user.create" ? "" : "name removed");

apps/admin/ui/src/Monitoring.tsx

@@ -4,7 +4,7 @@
  */
 import { useState } from "preact/hooks";
 import { Banner, Button, EmptyNote, IconButton, Input, Seg, Select, toast } from "@openclapper/ui";
-import { putMarks, type CfOverlay, type Mark, type StorageReport } from "./api";
+import { putMarks, type CfOverlay, type Mark, type StorageReport, repairMetadata, type RepairPage } from "./api";
 import { LineChart, Sparkline, type Point } from "./charts";
 import { bytes, day, num, Section, Tile } from "./shared";
 
@@ -52,6 +52,7 @@ export function Monitoring({ report, overlay, range, onRange, onSaved }: { repor
       </Section>
       <Marks marks={report.marks} onSaved={onSaved} />
       <Cloudflare overlay={overlay} marks={report.marks} />
+      <ObjectMetadata />
     </>
   );
 }
@@ -154,3 +155,51 @@ export function Cloudflare({ overlay, marks }: { overlay: CfOverlay | { configur
     </Section>
   );
 }
+
+/**
+ * The objects' custom metadata (#255): the app writes the owning workspace, shot and photo (renders: kind
+ * and id) on every object; a bucket copy can lose it. Check counts what is missing or different; Repair
+ * rewrites it from the database, page by page, keeping each object's body and type.
+ */
+function ObjectMetadata() {
+  const [busy, setBusy] = useState<"check" | "repair" | null>(null);
+  const [result, setResult] = useState<{ check: boolean; total: RepairPage } | null>(null);
+  const run = async (check: boolean) => {
+    setBusy(check ? "check" : "repair");
+    const total: RepairPage = { checked: 0, repaired: 0, unknown: 0, examples: [], cursor: null };
+    try {
+      let cursor: string | null = null;
+      do {
+        const p: RepairPage = await repairMetadata(cursor, check);
+        total.checked += p.checked; total.repaired += p.repaired; total.unknown += p.unknown;
+        total.examples.push(...p.examples.slice(0, Math.max(0, 10 - total.examples.length)));
+        cursor = p.cursor;
+        setResult({ check, total: { ...total } });
+      } while (cursor);
+      if (!check) toast(total.repaired ? `${num(total.repaired)} objects repaired` : "Every object already had its metadata");
+    } catch (e) {
+      toast((e as Error).message, "danger");
+    } finally {
+      setBusy(null);
+    }
+  };
+  const t = result?.total;
+  return (
+    <Section icon="tag-check-outline" title="Object metadata" action={<>
+      <Button size="sm" kind="ghost" icon="magnify" disabled={!!busy} onClick={() => void run(true)}>{busy === "check" ? "Checking…" : "Check"}</Button>
+      <Button size="sm" icon="wrench-outline" disabled={!!busy} onClick={() => void run(false)}>{busy === "repair" ? "Repairing…" : "Repair"}</Button>
+    </>}>
+      <div class="admin__pad">
+        <p class="meta">Every object in the bucket carries its owner (workspace, shot, photo; renders their kind and id). A bucket copy can lose it. Check counts the objects whose metadata is missing or different; Repair writes it again from the database, keeping each object as it is. Objects the app does not know are left alone.</p>
+        {t && (
+          <p>
+            {num(t.checked)} objects {busy ? "so far" : "checked"}: {t.repaired ? <strong>{num(t.repaired)} {result!.check ? "to repair" : "repaired"}</strong> : "all in order"}{t.unknown ? `, ${num(t.unknown)} unknown` : ""}.
+          </p>
+        )}
+        {t && t.examples.length > 0 && (
+          <ul class="meta">{t.examples.map((e) => <li key={e.key}><code>{e.key}</code> · {e.reason}</li>)}</ul>
+        )}
+      </div>
+    </Section>
+  );
+}

apps/admin/ui/src/api.ts

@@ -45,6 +45,9 @@ export interface StorageReport {
   crossed: Mark[];
 }
 export const fetchStorage = (days: number) => request<StorageReport>(`/api/storage?days=${days}`);
+/** One page of the object metadata check or repair (#255); call again with `cursor` until it is null. */
+export interface RepairPage { checked: number; repaired: number; unknown: number; examples: { key: string; reason: "missing" | "different" | "unknown" }[]; cursor: string | null }
+export const repairMetadata = (cursor: string | null, check: boolean) => send<RepairPage>("POST", "/api/storage/repair-metadata", { cursor, check });
 export const putMarks = (marks: Mark[]) => send<{ storage_marks: Mark[] }>("PUT", "/api/settings", { storage_marks: marks }).then((r) => r.storage_marks);
 
 // ---------- The optional Cloudflare overlay (#217) ----------

docs/features/admin.md

@@ -56,6 +56,7 @@ Every change an operator makes (account created, changed, blocked or unblocked,
 | GET | `/api/audit?limit=&before=&before_id=` | The audit trail, newest first: `{ entries: [{ id, at, operator_email, action, target_user_id, target_email, detail }], next: { before, before_id } \| null }`. `limit` 1..200 (default 50); pass `next` back for the following page |
 | GET | `/api/settings` | `{ storage_marks }` |
 | PUT | `/api/settings` | `{ storage_marks: [{ bytes, label }] }`, at most 10, strict |
+| POST | `/api/storage/repair-metadata` | `{ cursor?, check? }` → `{ checked, repaired, unknown, examples, cursor }`: one page (100 objects) of the R2 objects' custom metadata checked or rewritten from D1 ([monitoring](monitoring.md)); called again with `cursor` until null. A page that rewrote objects is audited as `storage.repair` |
 
 Without a valid Access JWT of the admin application every request is `401`. Until its AUD is set (`ADMIN_ACCESS_AUD`, [self-hosting](../self-hosting.md#4-the-admin-planes-sign-in)) that includes every request.
 

docs/features/monitoring.md

@@ -39,6 +39,7 @@ Tested in `apps/admin/test/storage.test.ts`.
 - **Storage**: the bucket total over 30 days, 90 days or a year as a line, the operator's marks as dashed lines with their label, a crosshair and a tooltip on hover. Before the first snapshot the chart says so.
 - **Workspaces**: photos, bytes, share of the photos' bytes and a sparkline of the window, largest first.
 - **Marks**: a size with a unit (MB, GB) and a label per row, at most 10; Save sends the whole list. The empty state explains what marks do.
+- **Object metadata** (#255): Check counts the bucket's objects whose custom metadata is missing or different from what the app writes (photos: `workspace_id`, `shot_id`, `photo_id` from D1; renders: `kind`, `id` from the key); Repair rewrites it page by page with the same body and content type (`apps/admin/src/objects.ts`). Objects the app does not know (no photo row, another prefix) are counted and left alone. Needed after a bucket copy that drops metadata (the reference instance's rename on 2026-10-20; a staging recreate, whose rclone copy also adds `mtime`/`btime`).
 - **Banner** on both tabs while the latest bucket total is past a mark, naming the highest crossed one (`role="alert"`); on the Board it links to Monitoring.
 
 The charts (`charts.tsx`) are hand-written SVG with no dependency, one hue (the accent) only: the design system has no categorical palette, so a comparison across workspaces is a table with sparklines, never stacked colours. Marks are drawn on the warning ink and always carry their text. Text is sized by the container's width, not by viewBox scaling. Both themes (Sun, Set) were checked with screenshots.