From 40efb47956a466cda6b619d5e6e2180664b0d65d Mon Sep 17 00:00:00 2001 From: Melvin Carvalho Date: Mon, 19 Oct 2026 15:11:09 +0200 Subject: [PATCH 1/2] forge/: mirror repos on their agent's kind-30618 state (opt-in config.mirror) gitmark 0.0.84 publishes a NIP-34 repository state after every mark. A forge told to follow (pubkey, name) pairs subscribes to those states on the configured relays and pulls when one changes: one REQ per relay for kinds 30617 and 30618 from the followed keys, one git fetch per real change, into refs/mirror/*. A branch moves only to a commit the fetch brought and, when the state carries a mark, whose blocktrails.json lists the marked commit. Mirrors are keyed by pubkey plus name, never name alone. Reconnect with backoff; every connect re-sends the REQ, which is the catch-up. Unset, nothing happens. The record is pluginDir/mirror//.json, served at api/repos///mirror. forge/lib/mirror.js is the pure part (config, event checks, the plan, the verdict); the sockets and the git are in plugin.js. Ten tests: six on the library, four against an in-process relay and a local upstream. --- forge/README.md | 53 ++++++++++ forge/lib/mirror.js | 141 ++++++++++++++++++++++++++ forge/plugin.js | 172 +++++++++++++++++++++++++++++++- forge/test.js | 238 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 603 insertions(+), 1 deletion(-) create mode 100644 forge/lib/mirror.js diff --git a/forge/README.md b/forge/README.md index dc0cc19..0eeaae5 100644 --- a/forge/README.md +++ b/forge/README.md @@ -727,6 +727,59 @@ Every announce emits **both** events (30617 then 30618) to the relays. 30618]`) — inspect the exact shapes, including the commit-precise refs, without a relay. +### Mirroring on kind-30618 (opt-in) + +The other direction. [gitmark](https://git-mark.com/) publishes a +kind-30618 repository state after every `git mark announce`, signed by the +agent's own key: `d` = repo name, `HEAD`, one `refs/heads/` tag per +branch with its commit, and the latest mark as `chain`, `txo`, `commit`. +It is replaceable, so the newest from the key says where the repo is now. +A forge told to follow that key pulls when the state changes: one small +event per announce instead of polling, one `git fetch` per real change. + +```js +mirror: { + relays: ['wss://relay.damus.io', 'wss://nos.lol'], + follow: [ + { pubkey: '<64-hex agent key>', name: 'proj', clone: 'https://github.com/o/proj.git' }, + { pubkey: '<64-hex agent key>', name: 'other' }, // clone URL from the key's newest 30617 + ], + minIntervalMs: 30000, // at most one fetch per repo per 30 s (default) +} +``` + +**Unset ⇒ nothing happens**: no sockets, no fetches. With it, activation +opens one `ws` socket per relay and sends one `REQ` for kinds 30617 and +30618 from the followed pubkeys; the relay filters, so the forge receives +only events about repos it mirrors. Sockets reconnect with backoff (1 s +doubling to 60 s) and every connect re-sends the `REQ`, which is the +catch-up: a replaceable event comes back as its newest version. + +A 30618 is acted on only when the signature verifies, `(pubkey, d)` is a +followed repo (never name alone: anyone can publish a 30618 with any `d`), +`created_at` is newer than the last state acted on, and a branch differs +from the local one. Then one fetch from the clone URL into `refs/mirror/*`, +and per changed branch a verdict: + +- the commit the state names was not fetched ⇒ the branch stays; +- no mark on the state ⇒ the branch moves; +- a mark ⇒ the branch moves only if `blocktrails.json` **at that commit** + lists the marked commit among its states. A state that claims a mark its + own trail does not carry moves nothing, and the reason is recorded. + +The mirror lives at `repos//.git` like a pushed repo, so it +has the usual pages and the Anchors tab reads its repo-held trail. The +record is `pluginDir/mirror//.json` and +`GET /api/repos///mirror` returns it: the last state seen +with its action and reason, the last fetch with per-branch verdicts, the +clone URL in use. Caps: 50 followed repos, 60 s per fetch. + +The stance holds. The plugin talks to relays and to git remotes, never to +a chain API; whether the mark is really on the chain is still the +browser's question on the Anchors tab. `forge/lib/mirror.js` is the pure +part (config, event checks, the plan, the verdict), tested on its own; +the sockets and the git are in plugin.js. + ### Who signs — forge instance key, not the maintainer (a Finding) The forge signs with its **own instance key**, not the maintainer's diff --git a/forge/lib/mirror.js b/forge/lib/mirror.js new file mode 100644 index 0000000..365c2b0 --- /dev/null +++ b/forge/lib/mirror.js @@ -0,0 +1,141 @@ +// forge/lib/mirror.js — mirror repos on their NIP-34 repository state. +// +// gitmark's `git mark announce` publishes a kind-30618 "repository state" +// after every mark: d = repo name, HEAD, one refs/heads/ tag per +// branch with its commit, and the latest mark as chain/txo/commit tags. It +// is replaceable, so the newest one from the agent's key says where the repo +// is now. A forge that follows that key pulls when the state changes and +// moves a branch only to a commit it fetched and, when the state carries a +// mark, whose trail file lists the marked commit. +// +// This module is the pure part: config parsing, event checks, the plan for +// one state, and the verdict for one branch. No I/O; plugin.js does the +// sockets and the git. +import crypto from 'node:crypto'; +import { schnorr } from '@noble/curves/secp256k1'; + +export const KIND_ANNOUNCE = 30617; +export const KIND_STATE = 30618; +export const MAX_FOLLOW = 50; +export const MIN_FETCH_INTERVAL_MS = 30_000; + +const HEX64 = /^[0-9a-f]{64}$/; +const SHA40 = /^[0-9a-f]{40}$/; +const NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; +const BRANCH = /^refs\/heads\/[A-Za-z0-9][A-Za-z0-9._/-]{0,200}$/; +const RELAY = /^wss?:\/\/[^\s;'"]+$/i; +const CLONE = /^(https?:\/\/|file:\/\/)[^\s;'"]+$/i; + +/** + * config.mirror -> { relays, follow, minIntervalMs } or null when the mirror + * is off. Off is the default: no sockets, no fetches. Every followed repo is + * (pubkey, name): the agent key that signs the state and the d tag it uses. + * A clone URL may be given; otherwise the newest 30617 from the same key + * supplies one. + */ +export function parseMirrorConfig(cfg) { + if (!cfg || typeof cfg !== 'object') return null; + const relays = (Array.isArray(cfg.relays) ? cfg.relays : []).filter((r) => typeof r === 'string' && RELAY.test(r)); + const follow = []; + for (const f of Array.isArray(cfg.follow) ? cfg.follow : []) { + if (!f || typeof f !== 'object') continue; + const pubkey = String(f.pubkey || '').toLowerCase(); + const name = String(f.name || ''); + if (!HEX64.test(pubkey) || !NAME.test(name)) continue; + const clone = typeof f.clone === 'string' && CLONE.test(f.clone) ? f.clone : null; + if (!follow.some((x) => x.pubkey === pubkey && x.name === name)) follow.push({ pubkey, name, clone }); + if (follow.length >= MAX_FOLLOW) break; + } + if (!relays.length || !follow.length) return null; + const minIntervalMs = Number.isInteger(cfg.minIntervalMs) && cfg.minIntervalMs >= 0 ? cfg.minIntervalMs : MIN_FETCH_INTERVAL_MS; + return { relays, follow, minIntervalMs }; +} + +/** The followed entry a (pubkey, d) pair names, or null. Never by name alone. */ +export function followedEntry(follow, pubkey, d) { + return follow.find((f) => f.pubkey === pubkey && f.name === d) || null; +} + +/** NIP-01: the id is the hash of the canonical form and the sig is over the id. */ +export function verifyEvent(ev) { + if (!ev || typeof ev !== 'object') return false; + if (!HEX64.test(String(ev.pubkey)) || !HEX64.test(String(ev.id)) || typeof ev.sig !== 'string') return false; + if (!Number.isInteger(ev.created_at) || !Number.isInteger(ev.kind) || !Array.isArray(ev.tags) || typeof ev.content !== 'string') return false; + if (!ev.tags.every((t) => Array.isArray(t) && t.every((x) => typeof x === 'string'))) return false; + const id = crypto.createHash('sha256') + .update(JSON.stringify([0, ev.pubkey, ev.created_at, ev.kind, ev.tags, ev.content]), 'utf8').digest('hex'); + if (id !== ev.id) return false; + try { return schnorr.verify(ev.sig, ev.id, ev.pubkey); } catch { return false; } +} + +const tagValue = (ev, name) => { const t = ev.tags.find((x) => x[0] === name); return t ? t[1] : null; }; + +/** + * What a 30618 says: its d, the symbolic HEAD, every well-formed branch ref + * with its full sha, and the mark when all three tags are present. + */ +export function readState(ev) { + const d = tagValue(ev, 'd'); + if (d === null) return null; + const refs = {}; + for (const t of ev.tags) if (BRANCH.test(t[0]) && SHA40.test(String(t[1] || '').toLowerCase())) refs[t[0]] = t[1].toLowerCase(); + const headTag = tagValue(ev, 'HEAD'); + const head = headTag && /^ref: (refs\/heads\/\S+)$/.test(headTag) ? headTag.slice(5) : null; + const chain = tagValue(ev, 'chain'); + const txo = tagValue(ev, 'txo'); + const commit = tagValue(ev, 'commit'); + const mark = chain && txo && SHA40.test(String(commit || '').toLowerCase()) ? { chain, txo, commit: commit.toLowerCase() } : null; + return { d, head, refs, mark, createdAt: ev.created_at, id: ev.id }; +} + +/** The clone URLs a 30617 offers, in order, https and http only. */ +export function readClones(ev) { + const out = []; + for (const t of ev.tags) if (t[0] === 'clone') for (const u of t.slice(1)) if (CLONE.test(u) && !u.startsWith('file:') && !out.includes(u)) out.push(u); + return out; +} + +/** + * Whether to fetch for a state, and which branches it would move. + * @param {object} o + * @param {object} o.state - from readState + * @param {Record} o.local - refs/heads/ -> sha the mirror has now + * @param {number} [o.lastSeenAt] - created_at of the last state acted on + * @param {number} [o.lastFetchAt] - ms clock of the last fetch + * @param {number} o.now - ms clock + * @param {number} o.minIntervalMs + * @returns {{action: 'fetch'|'skip', reason: string, changed: [string, string][]}} + */ +export function planMirror({ state, local, lastSeenAt = 0, lastFetchAt = 0, now, minIntervalMs }) { + if (!state) return { action: 'skip', reason: 'not a repository state', changed: [] }; + if (state.createdAt <= lastSeenAt) return { action: 'skip', reason: 'not newer than the last state acted on', changed: [] }; + const changed = Object.entries(state.refs).filter(([ref, sha]) => local[ref] !== sha); + if (!Object.keys(state.refs).length) return { action: 'skip', reason: 'the state names no branch', changed: [] }; + if (!changed.length) return { action: 'skip', reason: 'already at this state', changed: [] }; + if (now - lastFetchAt < minIntervalMs) return { action: 'skip', reason: `fetched ${Math.round((now - lastFetchAt) / 1000)} s ago; next in ${Math.ceil((minIntervalMs - (now - lastFetchAt)) / 1000)} s`, changed }; + return { action: 'fetch', reason: `${changed.length} branch${changed.length === 1 ? '' : 'es'} moved`, changed }; +} + +/** Does a gitmark trail document list this commit among its states? */ +export function trailLists(raw, commit) { + let doc; + try { doc = JSON.parse(raw); } catch { return false; } + if (!doc || doc.profile !== 'gitmark' || !Array.isArray(doc.states)) return false; + return doc.states.some((s) => String(s).toLowerCase() === commit); +} + +/** + * The verdict for one branch after the fetch. + * @param {object} o + * @param {boolean} o.fetched - the commit is in the mirror after the fetch + * @param {object|null} o.mark - the state's mark, if any + * @param {string|null} o.trail - the trail file at the commit, or null when absent + * @returns {{move: boolean, verdict: string}} + */ +export function branchVerdict({ fetched, mark, trail }) { + if (!fetched) return { move: false, verdict: 'the remote does not have the commit the state names' }; + if (!mark) return { move: true, verdict: 'fetched; the state carries no mark' }; + if (trail === null) return { move: false, verdict: 'the state carries a mark but the commit has no trail file' }; + if (!trailLists(trail, mark.commit)) return { move: false, verdict: `the trail at the commit does not list the marked commit ${mark.commit.slice(0, 8)}` }; + return { move: true, verdict: `fetched; the trail lists mark ${mark.commit.slice(0, 8)} on ${mark.chain}` }; +} diff --git a/forge/plugin.js b/forge/plugin.js index f9087ce..9946a1a 100644 --- a/forge/plugin.js +++ b/forge/plugin.js @@ -42,6 +42,7 @@ // GET ///blocktrails.json (CORS-readable trail doc) // nostr POST api/repos///announce (owner: publish NIP-34 30617 + 30618 to relays) // GET api/repos///nostr (the signed 30617 + 30618 that WOULD be published) +// mirror GET api/repos///mirror (a repo pulled on its agent's 30618: last state, last fetch, verdicts) // fork POST /api/repos///fork // web edit POST api/repos///edit {path,content,message?,branch?} // (owner-signed by default; config.openEdit => anon DEMO; CORS+preflight) @@ -143,6 +144,9 @@ import { ICON_PR_OPEN, ICON_PR_TAB, ICON_PR_MERGED, ICON_PR_CLOSED, } from './lib/helpers.js'; import { renderMarkdown } from './lib/markdown.js'; +import { + parseMirrorConfig, followedEntry, verifyEvent, readState, readClones, planMirror, branchVerdict, KIND_ANNOUNCE, KIND_STATE, +} from './lib/mirror.js'; const execFileP = promisify(execFile); @@ -1559,6 +1563,171 @@ export async function activate(api) { return Promise.all(relays.map((relay) => publishToRelay(WS, event, relay, timeoutMs))); } + // ------------------------------------------ mirror on 30618 (opt-in) + // The other direction. gitmark's `git mark announce` publishes a kind-30618 + // repository state after every mark, signed by the agent's key. A forge + // told to follow (pubkey, name) pairs subscribes to those states on the + // configured relays and pulls when one changes: one small event per + // announce instead of polling, one git fetch per real change. A branch + // moves only to a commit the fetch brought and, when the state carries a + // mark, whose trail file lists the marked commit. The stance holds: the + // plugin talks to relays and git remotes, never to a chain API; the mark + // itself is checked in the browser. Off unless config.mirror names relays + // and repos. pluginDir/mirror//.json is the record. + const mirror = parseMirrorConfig(api.config.mirror); + const mirrorDir = path.join(api.storage.pluginDir(), 'mirror'); + const mirrorSockets = []; + const mirrorTimers = new Set(); + let mirrorClosed = false; + const mirrorQueues = new Map(); // `${pubkey}/${name}` -> promise chain, one state at a time per repo + const MIRROR_FETCH_TIMEOUT_MS = 60_000; + + const mirrorRecordFile = (owner, name) => path.join(mirrorDir, owner, `${name}.json`); + function mirrorRecord(owner, name) { + try { return JSON.parse(fs.readFileSync(mirrorRecordFile(owner, name), 'utf8')); } catch { return null; } + } + function saveMirrorRecord(owner, name, rec) { + fs.mkdirSync(path.join(mirrorDir, owner), { recursive: true }); + fs.writeFileSync(mirrorRecordFile(owner, name), JSON.stringify(rec, null, 2)); + } + + async function localHeads(dir) { + const out = {}; + try { + const txt = await gitText(dir, ['for-each-ref', '--format=%(refname)%00%(objectname)', 'refs/heads']); + for (const l of txt.split('\n').filter(Boolean)) { const [r, sha] = l.split('\0'); out[r] = sha; } + } catch { /* no repo yet */ } + return out; + } + + /** One 30618 for one followed repo: plan, fetch into refs/mirror, verify each branch, move what passes. */ + async function mirrorApply(entry, ev) { + const owner = entry.pubkey; + const { name } = entry; + const state = readState(ev); + const rec = mirrorRecord(owner, name) || { pubkey: owner, name, clone: entry.clone, events: 0 }; + const dir = repoDirOf(owner, name); + const local = repoExists(owner, name) ? await localHeads(dir) : {}; + const plan = planMirror({ state, local, lastSeenAt: rec.lastEvent?.created_at, lastFetchAt: rec.lastFetchAt, now: Date.now(), minIntervalMs: mirror.minIntervalMs }); + rec.events += 1; + rec.lastSeen = { id: ev.id, created_at: ev.created_at, at: Date.now(), action: plan.action, reason: plan.reason }; + if (plan.action === 'skip') { saveMirrorRecord(owner, name, rec); return rec; } + const clone = entry.clone || rec.clone; + if (!clone) { + rec.lastSeen.action = 'skip'; rec.lastSeen.reason = 'no clone URL: none configured and no 30617 from this key seen yet'; + saveMirrorRecord(owner, name, rec); return rec; + } + if (!repoExists(owner, name)) await materialize(owner, name, `mirror of ${didShort(owner)}`); + rec.lastFetchAt = Date.now(); + try { + await execFileP('git', ['-C', dir, 'fetch', '--quiet', '--no-tags', clone, '+refs/heads/*:refs/mirror/*'], + { env: { ...gitEnv, GIT_TERMINAL_PROMPT: '0' }, timeout: MIRROR_FETCH_TIMEOUT_MS, maxBuffer: MAX_EXEC_BUFFER }); + } catch (err) { + rec.lastSeen.action = 'fetch-failed'; rec.lastSeen.reason = err.message.split('\n')[0]; + saveMirrorRecord(owner, name, rec); + api.log.warn(`forge: mirror ${didShort(owner)}/${name}: fetch from ${clone} failed: ${rec.lastSeen.reason}`); + return rec; + } + const branches = {}; + let moved = 0; + for (const [ref, sha] of plan.changed) { + let fetched = false; + try { await execFileP('git', ['-C', dir, 'cat-file', '-e', `${sha}^{commit}`], { env: gitEnv }); fetched = true; } catch { fetched = false; } + let trail = null; + if (fetched && state.mark) { try { trail = await gitText(dir, ['show', `${sha}:${TRAIL_FILE}`]); } catch { trail = null; } } + const v = branchVerdict({ fetched, mark: state.mark, trail }); + if (v.move) { await execFileP('git', ['-C', dir, 'update-ref', ref, sha], { env: gitEnv }); moved += 1; } + branches[ref] = { sha, moved: v.move, verdict: v.verdict }; + } + if (moved && state.head && branches[state.head]?.moved) { + await execFileP('git', ['-C', dir, 'symbolic-ref', 'HEAD', state.head], { env: gitEnv }).catch(() => {}); + } + rec.lastEvent = { id: ev.id, created_at: ev.created_at }; + rec.lastFetch = { at: rec.lastFetchAt, clone, branches, moved, mark: state.mark }; + rec.lastSeen.action = moved ? 'moved' : 'held'; rec.lastSeen.reason = moved ? `${moved} of ${plan.changed.length} moved` : 'nothing passed'; + saveMirrorRecord(owner, name, rec); + repoTrailSeen.delete(`${owner}/${name}`); + api.log.info(`forge: mirror ${didShort(owner)}/${name}: ${rec.lastSeen.reason}${state.mark ? ` (mark ${state.mark.commit.slice(0, 8)} on ${state.mark.chain})` : ''}`); + return rec; + } + + function mirrorEnqueue(entry, ev) { + const key = `${entry.pubkey}/${entry.name}`; + const prev = mirrorQueues.get(key) || Promise.resolve(); + const next = prev.then(() => mirrorApply(entry, ev)).catch((err) => api.log.warn(`forge: mirror ${key}: ${err.message}`)); + mirrorQueues.set(key, next); + return next; + } + + function mirrorOnEvent(ev) { + if (!verifyEvent(ev)) return; + if (ev.kind !== KIND_ANNOUNCE && ev.kind !== KIND_STATE) return; + const d = (ev.tags.find((t) => t[0] === 'd') || [])[1]; + const entry = followedEntry(mirror.follow, ev.pubkey, d); + if (!entry) return; + if (ev.kind === KIND_ANNOUNCE) { + // the newest announcement from the key supplies the clone URL when none is configured + const clones = readClones(ev); + if (!clones.length) return; + const rec = mirrorRecord(entry.pubkey, entry.name) || { pubkey: entry.pubkey, name: entry.name, events: 0 }; + if (rec.announceAt && rec.announceAt >= ev.created_at) return; + rec.clone = clones[0]; rec.announceAt = ev.created_at; rec.clones = clones; + saveMirrorRecord(entry.pubkey, entry.name, rec); + return; + } + mirrorEnqueue(entry, ev); + } + + /** One socket per relay, one REQ for both kinds from the followed keys; reconnect with backoff. */ + async function mirrorConnect(relay, attempt = 0) { + if (mirrorClosed) return; + const WS = await getWebSocket(); + let ws; + try { ws = new WS(relay); } catch (err) { return mirrorRetry(relay, attempt, err.message); } + mirrorSockets.push(ws); + const authors = [...new Set(mirror.follow.map((f) => f.pubkey))]; + ws.on('open', () => { + attempt = 0; + ws.send(JSON.stringify(['REQ', 'forge-mirror', { kinds: [KIND_ANNOUNCE, KIND_STATE], authors }])); + }); + ws.on('message', (data) => { + let msg; + try { msg = JSON.parse(data.toString()); } catch { return; } + if (Array.isArray(msg) && msg[0] === 'EVENT' && msg[1] === 'forge-mirror') mirrorOnEvent(msg[2]); + }); + ws.on('error', () => { /* close follows */ }); + ws.on('close', () => { + const i = mirrorSockets.indexOf(ws); + if (i >= 0) mirrorSockets.splice(i, 1); + mirrorRetry(relay, attempt, 'closed'); + }); + } + function mirrorRetry(relay, attempt, why) { + if (mirrorClosed) return; + const delay = Math.min(60_000, 1000 * 2 ** Math.min(attempt, 6)); + const t = setTimeout(() => { mirrorTimers.delete(t); mirrorConnect(relay, attempt + 1); }, delay); + mirrorTimers.add(t); + if (attempt === 0) api.log.warn(`forge: mirror relay ${relay} ${why}; reconnecting`); + } + function mirrorStop() { + mirrorClosed = true; + for (const t of mirrorTimers) clearTimeout(t); + mirrorTimers.clear(); + for (const ws of mirrorSockets.splice(0)) { try { ws.close(); } catch { /* already closed */ } } + } + if (mirror) { + fs.mkdirSync(mirrorDir, { recursive: true }); + for (const relay of mirror.relays) mirrorConnect(relay); + api.log.info(`forge: mirroring ${mirror.follow.length} repo${mirror.follow.length === 1 ? '' : 's'} on kind-30618 state from ${mirror.relays.length} relay${mirror.relays.length === 1 ? '' : 's'}`); + } + + /** GET api/repos///mirror: the mirror record, or 404 when the repo is not a mirror. */ + async function apiMirror(reply, owner, name) { + const rec = mirrorRecord(owner, name); + if (!rec) return apiErr(reply, 404, 'not a mirror'); + return sendJson(reply, 200, rec); + } + /** * Build + publish BOTH NIP-34 events for one repo: the 30617 announcement * (discovery: clone/web URLs, anchor) AND the 30618 repo-state (the refs a @@ -4736,6 +4905,7 @@ ${issuesScript({ api: `${prefix}/api/repos/${owner}/${name}`, owner, name, base, if (request.method !== 'POST') return apiErr(reply, 405, 'method not allowed'); return await apiRepoPreview(request, reply, owner, name); case 'nostr': return tail.length === 0 && isRead ? await apiNostr(reply, owner, name) : apiErr(reply, 404, 'not found'); + case 'mirror': return tail.length === 0 && isRead ? await apiMirror(reply, owner, name) : apiErr(reply, 404, 'not found'); case 'search': return tail.length === 0 ? await apiSearch(reply, owner, name, request.query) : apiErr(reply, 404, 'not found'); case 'releases': return tail.length === 0 ? await apiReleases(reply, owner, name) : apiErr(reply, 404, 'not found'); case 'fork': @@ -4932,5 +5102,5 @@ ${issuesScript({ api: `${prefix}/api/repos/${owner}/${name}`, owner, name, base, + `anchoring on ${chain} — derive-and-record only, no chain I/O)`, ); - return { deactivate() { /* nothing persistent to tear down */ } }; + return { deactivate() { mirrorStop(); } }; } diff --git a/forge/test.js b/forge/test.js index 2145cd3..6ea6079 100644 --- a/forge/test.js +++ b/forge/test.js @@ -2895,3 +2895,241 @@ describe('forge web-edit openEdit demo (anonymous edits)', () => { assert.strictEqual(res.status, 400); }); }); + +// --------------------------------------------------------------------------- +// Mirror on kind-30618 (issue #12). The pure library first, then a live run: +// an in-process relay that answers the forge's REQ with stored events and +// pushes new ones, a local upstream repo as the clone URL, and states signed +// by an agent key the forge is configured to follow. +import { + parseMirrorConfig, verifyEvent, readState, readClones, planMirror, branchVerdict, trailLists, MAX_FOLLOW, +} from './lib/mirror.js'; + +function signNostr(sk, { kind, tags, content = '', created_at }) { + const pubkey = Buffer.from(schnorr.getPublicKey(sk)).toString('hex'); + const ev = { pubkey, created_at, kind, tags, content }; + ev.id = crypto.createHash('sha256').update(JSON.stringify([0, pubkey, created_at, kind, tags, content]), 'utf8').digest('hex'); + ev.sig = Buffer.from(schnorr.sign(ev.id, sk)).toString('hex'); + return ev; +} + +describe('forge mirror library (pure)', () => { + const sk = Buffer.from(schnorr.utils.randomPrivateKey()).toString('hex'); + const pub = Buffer.from(schnorr.getPublicKey(sk)).toString('hex'); + const sha1 = 'a'.repeat(40); + const sha2 = 'b'.repeat(40); + + it('config: off unless relays and well-formed follow entries are given; capped; keyed by pubkey+name', () => { + assert.strictEqual(parseMirrorConfig(undefined), null); + assert.strictEqual(parseMirrorConfig({}), null); + assert.strictEqual(parseMirrorConfig({ relays: ['wss://r.example'], follow: [] }), null); + assert.strictEqual(parseMirrorConfig({ relays: [], follow: [{ pubkey: pub, name: 'x' }] }), null, 'no relays, no mirror'); + const m = parseMirrorConfig({ + relays: ['wss://r.example', 'http://not-a-relay', 'ws://127.0.0.1:1'], + follow: [ + { pubkey: pub.toUpperCase(), name: 'x', clone: 'https://example.org/x.git' }, + { pubkey: pub, name: 'x' }, // duplicate + { pubkey: 'nothex', name: 'y' }, // bad key + { pubkey: pub, name: '../etc' }, // bad name + { pubkey: pub, name: 'z', clone: 'ssh://nope' }, // clone scheme not allowed -> null clone, entry kept + ], + }); + assert.deepStrictEqual(m.relays, ['wss://r.example', 'ws://127.0.0.1:1']); + assert.deepStrictEqual(m.follow, [{ pubkey: pub, name: 'x', clone: 'https://example.org/x.git' }, { pubkey: pub, name: 'z', clone: null }]); + assert.strictEqual(m.minIntervalMs, 30_000); + const many = parseMirrorConfig({ relays: ['wss://r'], follow: Array.from({ length: 80 }, (_, i) => ({ pubkey: pub, name: `r${i}` })) }); + assert.strictEqual(many.follow.length, MAX_FOLLOW); + }); + + it('verifyEvent: id and schnorr signature must check out; a tampered tag fails', () => { + const ev = signNostr(sk, { kind: 30618, tags: [['d', 'x'], ['refs/heads/main', sha1]], created_at: 1760000000 }); + assert.strictEqual(verifyEvent(ev), true); + const tampered = { ...ev, tags: [['d', 'x'], ['refs/heads/main', sha2]] }; + assert.strictEqual(verifyEvent(tampered), false); + assert.strictEqual(verifyEvent({ ...ev, sig: 'f'.repeat(128) }), false); + assert.strictEqual(verifyEvent(null), false); + }); + + it('readState: d, HEAD, well-formed branch refs, and the mark only when all three tags are present', () => { + const ev = signNostr(sk, { + kind: 30618, + tags: [['d', 'x'], ['HEAD', 'ref: refs/heads/main'], ['refs/heads/main', sha1], ['refs/heads/dev', 'short'], ['refs/tags/v1', sha2], + ['chain', 'txbt4'], ['txo', 'txo:txbt4:' + 'c'.repeat(64) + ':0'], ['commit', sha1]], + created_at: 1760000000, + }); + const st = readState(ev); + assert.deepStrictEqual(st.refs, { 'refs/heads/main': sha1 }); + assert.strictEqual(st.head, 'refs/heads/main'); + assert.deepStrictEqual(st.mark, { chain: 'txbt4', txo: 'txo:txbt4:' + 'c'.repeat(64) + ':0', commit: sha1 }); + const noMark = readState(signNostr(sk, { kind: 30618, tags: [['d', 'x'], ['chain', 'txbt4']], created_at: 1 })); + assert.strictEqual(noMark.mark, null); + assert.strictEqual(readState(signNostr(sk, { kind: 30618, tags: [], created_at: 1 })), null, 'no d, no state'); + }); + + it('readClones: http(s) clone URLs from a 30617, in order, file: never', () => { + const ev = signNostr(sk, { kind: 30617, tags: [['d', 'x'], ['clone', 'https://a.example/x.git', 'git@a.example:x.git'], ['clone', 'file:///tmp/x'], ['clone', 'http://b.example/x.git']], created_at: 1 }); + assert.deepStrictEqual(readClones(ev), ['https://a.example/x.git', 'http://b.example/x.git']); + }); + + it('planMirror: skips older states, in-sync states and rate-limited fetches; otherwise names the moved branches', () => { + const state = { createdAt: 100, refs: { 'refs/heads/main': sha2, 'refs/heads/dev': sha1 } }; + const base = { state, local: { 'refs/heads/main': sha1, 'refs/heads/dev': sha1 }, now: 1_000_000, minIntervalMs: 30_000 }; + assert.strictEqual(planMirror({ ...base, lastSeenAt: 100 }).action, 'skip'); + assert.strictEqual(planMirror({ ...base, local: state.refs }).reason, 'already at this state'); + assert.match(planMirror({ ...base, lastFetchAt: 990_000 }).reason, /fetched 10 s ago; next in 20 s/); + const p = planMirror(base); + assert.strictEqual(p.action, 'fetch'); + assert.deepStrictEqual(p.changed, [['refs/heads/main', sha2]]); + assert.strictEqual(planMirror({ ...base, state: { createdAt: 100, refs: {} } }).reason, 'the state names no branch'); + }); + + it('branchVerdict: unfetched never moves; no mark moves; a mark needs the trail at the commit to list it', () => { + const trail = JSON.stringify({ '@type': 'Blocktrail', profile: 'gitmark', states: [sha1] }); + assert.strictEqual(branchVerdict({ fetched: false, mark: null, trail: null }).move, false); + assert.strictEqual(branchVerdict({ fetched: true, mark: null, trail: null }).move, true); + const mark = { chain: 'txbt4', txo: 'txo:…', commit: sha1 }; + assert.strictEqual(branchVerdict({ fetched: true, mark, trail: null }).move, false); + assert.strictEqual(branchVerdict({ fetched: true, mark, trail }).move, true); + assert.strictEqual(branchVerdict({ fetched: true, mark: { ...mark, commit: sha2 }, trail }).move, false); + assert.strictEqual(trailLists('not json', sha1), false); + }); +}); + +describe('forge mirrors a repo on its agent\'s kind-30618 state (in-process relay, local upstream)', () => { + const tmp3 = fs.mkdtempSync(path.join(os.tmpdir(), 'forge-mirror-')); + const up = path.join(tmp3, 'up'); + const sk = Buffer.from(schnorr.utils.randomPrivateKey()).toString('hex'); + const pub = Buffer.from(schnorr.getPublicKey(sk)).toString('hex'); + const other = Buffer.from(schnorr.utils.randomPrivateKey()).toString('hex'); + let relay; let relayUrl; let jss; let base; + const stored = []; // events the relay holds; served on REQ + const clients = new Set(); + let reqs = 0; + let t = 1760000000; + const g = (args, opts = {}) => git(args, { ...opts, env: { HOME: tmp3, ...(opts.env ?? {}) } }); + const head = async () => (await g(['rev-parse', 'HEAD'], { cwd: up })).stdout.trim(); + const until = async (fn, what, ms = 8000) => { + const t0 = Date.now(); + for (;;) { + const v = await fn(); + if (v) return v; + if (Date.now() - t0 > ms) throw new Error(`timed out waiting for ${what}`); + await new Promise((r) => { setTimeout(r, 50); }); + } + }; + const record = async () => { + const r = await fetch(`${base}/forge/api/repos/${pub}/up/mirror`); + return r.status === 200 ? r.json() : null; + }; + const mirrorHead = async () => { + const r = await g(['ls-remote', `${base}/forge/${pub}/up.git`, 'refs/heads/main']).catch(() => ({ stdout: '' })); + return r.stdout.trim().split('\t')[0] || null; + }; + const stateEvent = (key, refs, mark, extra = {}) => signNostr(key, { + kind: 30618, + tags: [['d', 'up'], ['HEAD', 'ref: refs/heads/main'], ...Object.entries(refs), ...(mark ? [['chain', mark.chain], ['txo', mark.txo], ['commit', mark.commit]] : [])], + created_at: extra.created_at ?? ++t, + }); + const push = (ev) => { stored.push(ev); for (const ws of clients) ws.send(JSON.stringify(['EVENT', 'forge-mirror', ev])); }; + + before(async () => { + fs.writeFileSync(path.join(tmp3, '.gitconfig'), '[user]\n\temail = agent@example.org\n\tname = Agent\n[init]\n\tdefaultBranch = main\n'); + fs.mkdirSync(up); + fs.writeFileSync(path.join(up, 'README.md'), '# up\n'); + await g(['init', '--quiet'], { cwd: up }); + await g(['add', '-A'], { cwd: up }); + await g(['commit', '--quiet', '-m', 'first'], { cwd: up }); + + const { WebSocketServer } = await import('ws'); + relay = new WebSocketServer({ port: 0 }); + await new Promise((r) => relay.on('listening', r)); + relayUrl = `ws://127.0.0.1:${relay.address().port}`; + relay.on('connection', (ws) => { + clients.add(ws); + ws.on('close', () => clients.delete(ws)); + ws.on('message', (data) => { + let msg; + try { msg = JSON.parse(data.toString()); } catch { return; } + if (Array.isArray(msg) && msg[0] === 'REQ') { + reqs += 1; + const [, sub, filter] = msg; + for (const ev of stored) { + if ((filter.kinds || []).includes(ev.kind) && (filter.authors || []).includes(ev.pubkey)) ws.send(JSON.stringify(['EVENT', sub, ev])); + } + ws.send(JSON.stringify(['EOSE', sub])); + } + }); + }); + + // a state already on the relay before the forge boots: the catch-up path + push(stateEvent(sk, { 'refs/heads/main': await head() }, null)); + + jss = await startJss({ + plugins: [{ id: 'forge', module: module_, prefix: '/forge', config: { + mirror: { relays: [relayUrl], follow: [{ pubkey: pub, name: 'up', clone: `file://${up}` }], minIntervalMs: 0 }, + } }], + }); + base = jss.base; + }); + + after(async () => { + if (jss) await jss.close(); + await new Promise((r) => relay.close(r)); + fs.rmSync(tmp3, { recursive: true, force: true }); + }); + + it('subscribes with one REQ for both kinds from the followed key, and pulls the state it finds on the relay', async () => { + const c1 = await head(); + const rec = await until(async () => { const r = await record(); return r && r.lastFetch ? r : null; }, 'the first fetch'); + assert.ok(reqs >= 1, 'the forge sent a REQ'); + assert.strictEqual(rec.lastFetch.moved, 1); + assert.strictEqual(rec.lastFetch.branches['refs/heads/main'].sha, c1); + assert.strictEqual(rec.lastSeen.action, 'moved'); + assert.strictEqual(await mirrorHead(), c1, 'the mirror serves the upstream commit'); + const page = await fetch(`${base}/forge/${pub}/up`); + assert.strictEqual(page.status, 200, 'the mirrored repo has a page like any pushed repo'); + }); + + it('a state with a mark moves the branch only when the trail at the commit lists the marked commit', async () => { + const c1 = await head(); + const txo = `txo:txbt4:${'c'.repeat(64)}:0`; + fs.writeFileSync(path.join(up, 'blocktrails.json'), JSON.stringify({ + '@type': 'Blocktrail', profile: 'gitmark', chain: 'txbt4', pubkeyBase: `02${pub}`, states: [c1], txo: [txo], + }, null, 2)); + await g(['add', '-A'], { cwd: up }); + await g(['commit', '--quiet', '-m', 'trail: first mark'], { cwd: up }); + const c2 = await head(); + push(stateEvent(sk, { 'refs/heads/main': c2 }, { chain: 'txbt4', txo, commit: c1 })); + const rec = await until(async () => { const r = await record(); return r && r.lastFetch && r.lastFetch.branches['refs/heads/main'].sha === c2 ? r : null; }, 'the marked state'); + assert.strictEqual(rec.lastFetch.moved, 1); + assert.match(rec.lastFetch.branches['refs/heads/main'].verdict, /trail lists mark/); + assert.strictEqual(await mirrorHead(), c2); + + // a state whose mark the trail does not list: fetched, held, nothing moves + fs.writeFileSync(path.join(up, 'NOTES.md'), 'more\n'); + await g(['add', '-A'], { cwd: up }); + await g(['commit', '--quiet', '-m', 'unmarked work'], { cwd: up }); + const c3 = await head(); + push(stateEvent(sk, { 'refs/heads/main': c3 }, { chain: 'txbt4', txo, commit: 'd'.repeat(40) })); + const held = await until(async () => { const r = await record(); return r && r.lastSeen.action === 'held' ? r : null; }, 'the held state'); + assert.strictEqual(held.lastFetch.moved, 0); + assert.match(held.lastFetch.branches['refs/heads/main'].verdict, /does not list the marked commit dddddddd/); + assert.strictEqual(await mirrorHead(), c2, 'the branch stayed where the last verified state put it'); + }); + + it('ignores a state from another key with the same d, and one older than the last acted on', async () => { + const before = (await record()).events; + push(stateEvent(other, { 'refs/heads/main': 'e'.repeat(40) }, null)); + push(stateEvent(sk, { 'refs/heads/main': 'f'.repeat(40) }, null, { created_at: 1760000000 })); + const rec = await until(async () => { const r = await record(); return r.events === before + 1 ? r : null; }, 'the old state to be counted'); + assert.strictEqual(rec.lastSeen.action, 'skip'); + assert.match(rec.lastSeen.reason, /not newer/); + await new Promise((r) => { setTimeout(r, 200); }); + assert.strictEqual((await record()).events, before + 1, 'the other key\'s state was never counted'); + }); + + it('a repo that is not a mirror has no mirror record', async () => { + const r = await fetch(`${base}/forge/api/repos/nobody/none/mirror`); + assert.strictEqual(r.status, 404); + }); +}); From bd55eda1ac035004fa7acf00931224dd0a0b2f4c Mon Sep 17 00:00:00 2001 From: Melvin Carvalho Date: Mon, 19 Oct 2026 15:22:02 +0200 Subject: [PATCH 2/2] forge/: the mirror record is served before the repo-exists guard, so an unfetchable mirror still shows why --- forge/plugin.js | 5 ++++- forge/test.js | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/forge/plugin.js b/forge/plugin.js index 9946a1a..65e941d 100644 --- a/forge/plugin.js +++ b/forge/plugin.js @@ -4877,6 +4877,10 @@ ${issuesScript({ api: `${prefix}/api/repos/${owner}/${name}`, owner, name, base, const name = rest[1]; if (!REPO_NAME.test(name) || name.includes('..') || name.endsWith('.git')) return apiErr(reply, 404, 'not found'); + // A followed repo has a mirror record before it has a bare repo (nothing fetched + // yet: no clone URL, a failing remote), and that record is the diagnosis the + // operator needs, so it is served ahead of the repo-exists guard. + if (rest.length === 3 && rest[2] === 'mirror' && isRead) return apiMirror(reply, owner, name); if (!repoExists(owner, name)) return apiErr(reply, 404, 'not found'); if (rest.length === 2) { if (request.method === 'PATCH') return apiRepoPatch(request, reply, owner, name); @@ -4905,7 +4909,6 @@ ${issuesScript({ api: `${prefix}/api/repos/${owner}/${name}`, owner, name, base, if (request.method !== 'POST') return apiErr(reply, 405, 'method not allowed'); return await apiRepoPreview(request, reply, owner, name); case 'nostr': return tail.length === 0 && isRead ? await apiNostr(reply, owner, name) : apiErr(reply, 404, 'not found'); - case 'mirror': return tail.length === 0 && isRead ? await apiMirror(reply, owner, name) : apiErr(reply, 404, 'not found'); case 'search': return tail.length === 0 ? await apiSearch(reply, owner, name, request.query) : apiErr(reply, 404, 'not found'); case 'releases': return tail.length === 0 ? await apiReleases(reply, owner, name) : apiErr(reply, 404, 'not found'); case 'fork': diff --git a/forge/test.js b/forge/test.js index 6ea6079..e5e1939 100644 --- a/forge/test.js +++ b/forge/test.js @@ -3132,4 +3132,28 @@ describe('forge mirrors a repo on its agent\'s kind-30618 state (in-process rela const r = await fetch(`${base}/forge/api/repos/nobody/none/mirror`); assert.strictEqual(r.status, 404); }); + + it('a followed repo that could not be fetched yet still serves its record, so the reason is visible', async () => { + // a second followed repo with no clone URL: its key never announced a clone + const sk2 = Buffer.from(schnorr.utils.randomPrivateKey()).toString('hex'); + const pub2 = Buffer.from(schnorr.getPublicKey(sk2)).toString('hex'); + const jss2 = await startJss({ + plugins: [{ id: 'forge', module: module_, prefix: '/forge', config: { + mirror: { relays: [relayUrl], follow: [{ pubkey: pub2, name: 'noclone' }], minIntervalMs: 0 }, + } }], + }); + try { + push(signNostr(sk2, { kind: 30618, tags: [['d', 'noclone'], ['refs/heads/main', 'a'.repeat(40)]], created_at: ++t })); + const rec = await until(async () => { + const r = await fetch(`${jss2.base}/forge/api/repos/${pub2}/noclone/mirror`); + return r.status === 200 ? r.json() : null; + }, 'the record of the unfetchable mirror'); + assert.strictEqual(rec.lastSeen.action, 'skip'); + assert.match(rec.lastSeen.reason, /no clone URL/); + const page = await fetch(`${jss2.base}/forge/api/repos/${pub2}/noclone`); + assert.strictEqual(page.status, 404, 'no bare repo exists, so the repo itself is still 404'); + } finally { + await jss2.close(); + } + }); });