Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ has a matching GitHub release whose body comes from its section below.

- Gateway: `POST /v1/berth/switch` and `POST /v1/berth/stop` switch the card to an engine or give it back without
sending a prompt. Stop never touches a pinned holder. The `/gui/` board's Launch now uses the same switch.
- Gateway: `GET /v1/berth/board` reads the node in one call: the card and who holds it, every engine's state and
actions, and each engine's models with which are loaded.
- Linux AMD support (experimental), contributed by @jenerestain. `berth daemon --gpu-backend amd`
governs a discrete RDNA3 or RDNA4 card, or a Phoenix APU (Radeon 780M), read from sysfs and DRM
`fdinfo` with no ROCm. `--gpu N` names DRM cardN. On an APU, reserved VRAM and GTT (shared host
Expand Down
5 changes: 5 additions & 0 deletions docs/gateway.md
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,11 @@ curl -s http://127.0.0.1:9799/v1/berth/stop -H "Authorization: Bearer $(berth
pinned holder (a game, a render, `berth hold`) is a `409`: Berth never stops it from the network. An engine that is
not running under a lease is a `409` too, and berthd not answering is a `503`.

`GET /v1/berth/board` is the node in one read, for a board or a script: the card (who holds it, and whether it is
free, held, draining, in a VM or failing its probe), then every engine in `berth.toml` with its state (up, offline,
waiting, starting, failed), whether it can be switched to (`launch`) or stopped (`stoppable`), and its models with a
`loaded` flag from the engine's own reader. `cards` is a list so a node with more cards needs no new shape.

## Open a stopped engine's web UI

A bookmark to ComfyUI or Strata's chat page is a dead link while that engine is stopped. Give the engine a
Expand Down
118 changes: 118 additions & 0 deletions internal/gateway/board.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
// Copyright 2026 Trevin Tindall (T-Crypt)
// SPDX-License-Identifier: Apache-2.0

package gateway

import (
"encoding/json"
"net/http"
"os"

"github.com/T-Crypt/berth/internal/proto"
)

// The switchboard's read side: one node as GET /v1/berth/board serves it.
// The gateway is the only process that knows a start in flight, so a node's
// board is built here; the harbor reads one per node. Cards is a list and
// every action names its engine, so more cards (v0.5) and more nodes (v0.6)
// add rows, not a new shape.

type boardNode struct {
Name string `json:"name"`
Reachable bool `json:"reachable"` // berthd answered; false = cards and holders unknown
Cards []boardCard `json:"cards"`
Engines []boardEngine `json:"engines"`
}

type boardCard struct {
Index int `json:"index"`
Name string `json:"name"`
// State is free, held, draining (a holder is giving it back),
// passthrough (in a VM) or probe-failed.
State string `json:"state"`
Total uint64 `json:"total"`
Used uint64 `json:"used"`
Holders []guiHold `json:"holders"`
}

type boardEngine struct {
guiRow
Stoppable bool `json:"stoppable"` // holds the card under an active lease that is not pinned
Models []boardModel `json:"models"`
}

type boardModel struct {
ID string `json:"id"`
Loaded bool `json:"loaded,omitempty"` // the engine's reader reports it loaded
}

func (g *Gateway) board(w http.ResponseWriter, r *http.Request) {
node := g.nodeStatus()
rows := g.guiRows(r.Context(), node)
listed := map[string][]string{}
for i := range g.Engines {
e := &g.Engines[i]
if rows[i].State == "up" {
listed[e.Name] = g.listed(r.Context(), e.URL)
}
if len(listed[e.Name]) == 0 {
listed[e.Name] = e.Serves
}
}
host, _ := os.Hostname()
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "no-store")
_ = json.NewEncoder(w).Encode(boardOf(host, node, rows, listed))
}

// boardOf assembles the board from berthd's status (nil = not answering),
// the engines' rows in config order and each engine's model list.
func boardOf(host string, node *proto.Status, rows []guiRow, listed map[string][]string) boardNode {
n := boardNode{Name: host, Reachable: node != nil, Cards: []boardCard{}, Engines: make([]boardEngine, 0, len(rows))}
loaded := map[string]map[string]bool{}
if node != nil {
for _, gp := range node.GPUs {
if !gp.Scheduled {
continue
}
c := boardCard{Index: gp.Index, Name: gp.Name, State: "free", Total: gp.Total, Used: gp.Used, Holders: []guiHold{}}
for _, l := range node.Leases {
c.Holders = append(c.Holders, guiHold{Name: holderName(l), State: l.State, Pinned: l.Pinned})
if c.State == "free" {
c.State = "held"
}
if l.State != "active" {
c.State = "draining"
}
}
switch {
case node.Passthrough != nil:
c.State = "passthrough"
case node.ProbeErr != "":
c.State = "probe-failed"
}
n.Cards = append(n.Cards, c)
}
for _, e := range node.Engines {
loaded[e.Name] = map[string]bool{}
for _, m := range e.Models {
loaded[e.Name][m.Name] = m.State == "loaded" || m.State == "ready"
}
}
}
for _, row := range rows {
be := boardEngine{guiRow: row, Models: []boardModel{}}
if node != nil {
for _, l := range node.Leases {
if (l.Engine == row.Name || l.Group == row.Name) && !l.Pinned && l.State == "active" {
be.Stoppable = true
}
}
}
for _, id := range listed[row.Name] {
be.Models = append(be.Models, boardModel{ID: id, Loaded: loaded[row.Name][id]})
}
n.Engines = append(n.Engines, be)
}
return n
}
133 changes: 133 additions & 0 deletions internal/gateway/board_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
// Copyright 2026 Trevin Tindall (T-Crypt)
// SPDX-License-Identifier: Apache-2.0

//go:build unix

package gateway

import (
"encoding/json"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"os"
"reflect"
"testing"
"time"

"github.com/T-Crypt/berth/internal/config"
"github.com/T-Crypt/berth/internal/proto"
)

func readBoard(t *testing.T, url string) boardNode {
t.Helper()
resp := call(t, "GET", url+"/v1/berth/board", "")
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("board = %d", resp.StatusCode)
}
var n boardNode
if err := json.NewDecoder(resp.Body).Decode(&n); err != nil {
t.Fatal(err)
}
return n
}

// The board is one node: its card with who holds it, and every engine with
// its state, what it can do, and its models.
func TestBoard(t *testing.T) {
sock, _ := daemon(t)
addr := freeAddr(t)
t.Setenv("BERTH_GW_ENGINE", addr+",strata-model,300ms")
g := &Gateway{Socket: sock, Ready: 10 * time.Second, Key: testKey, Log: slog.New(slog.NewTextHandler(io.Discard, nil)),
Engines: []config.Engine{
{Name: "strata", Type: "llamacpp", URL: "http://" + addr, Start: []string{os.Args[0]}, Serves: []string{"strata-model"}},
{Name: "comfyui", Type: "comfyui", URL: "http://127.0.0.1:1", Start: []string{"false"}, Serves: []string{"sdxl"}, GUI: "https://box/"},
}}
srv := httptest.NewServer(g.Handler())
defer srv.Close()

n := readBoard(t, srv.URL)
if !n.Reachable || len(n.Cards) != 1 || n.Cards[0].State != "free" || len(n.Cards[0].Holders) != 0 {
t.Fatalf("idle node = %+v", n)
}

resp := call(t, "POST", srv.URL+"/v1/berth/switch", `{"engine":"strata"}`)
resp.Body.Close()
until(t, "strata up on the board", func() bool {
for _, e := range readBoard(t, srv.URL).Engines {
if e.Name == "strata" && e.State == "up" {
return true
}
}
return false
})

n = readBoard(t, srv.URL)
if c := n.Cards[0]; c.State != "held" || !reflect.DeepEqual(c.Holders, []guiHold{{Name: "strata", State: "active"}}) {
t.Fatalf("held card = %+v", c)
}
want := []struct {
name, state string
launch, stop bool
models []boardModel
}{
{"strata", "up", true, true, []boardModel{{ID: "strata-model"}}},
{"comfyui", "offline", true, false, []boardModel{{ID: "sdxl"}}},
}
if len(n.Engines) != len(want) {
t.Fatalf("engines = %+v", n.Engines)
}
for i, w := range want {
e := n.Engines[i]
if e.Name != w.name || e.State != w.state || e.Launch != w.launch || e.Stoppable != w.stop || !reflect.DeepEqual(e.Models, w.models) {
t.Errorf("engine %d = %+v, want %+v", i, e, w)
}
}
}

// boardOf reads the card's state and each model's loaded flag from berthd's
// status alone; berthd not answering leaves the cards unknown.
func TestBoardOf(t *testing.T) {
gpu := []proto.GPU{{Index: 0, Name: "RTX 4090", Scheduled: true, Total: 24, Used: 23}}
rows := []guiRow{{Name: "llama-swap", State: "up"}, {Name: "game", State: "offline"}}
listed := map[string][]string{"llama-swap": {"qwen", "gemma"}}
for _, c := range []struct {
name string
node *proto.Status
card string
stoppable []bool
loaded []boardModel
}{
{"unreachable", nil, "", []bool{false, false}, []boardModel{{ID: "qwen"}, {ID: "gemma"}}},
{"held, one model ready", &proto.Status{GPUs: gpu,
Leases: []proto.Lease{{Engine: "llama-swap", Group: "qwen", State: "active"}},
Engines: []proto.Engine{{Name: "llama-swap", Models: []proto.Model{{Name: "qwen", State: "ready"}, {Name: "gemma", State: "starting"}}}}},
"held", []bool{true, false}, []boardModel{{ID: "qwen", Loaded: true}, {ID: "gemma"}}},
{"pinned game", &proto.Status{GPUs: gpu, Leases: []proto.Lease{{Engine: "game", State: "active", Pinned: true}}},
"held", []bool{false, false}, []boardModel{{ID: "qwen"}, {ID: "gemma"}}},
{"draining", &proto.Status{GPUs: gpu, Leases: []proto.Lease{{Engine: "llama-swap", State: "draining"}}},
"draining", []bool{false, false}, []boardModel{{ID: "qwen"}, {ID: "gemma"}}},
{"in a VM", &proto.Status{GPUs: gpu, Passthrough: &proto.Passthrough{State: proto.CardPassthrough}},
"passthrough", []bool{false, false}, []boardModel{{ID: "qwen"}, {ID: "gemma"}}},
} {
t.Run(c.name, func(t *testing.T) {
n := boardOf("box", c.node, rows, listed)
if n.Reachable != (c.node != nil) {
t.Fatalf("reachable = %v", n.Reachable)
}
if c.card == "" && len(n.Cards) != 0 || c.card != "" && (len(n.Cards) != 1 || n.Cards[0].State != c.card) {
t.Fatalf("cards = %+v, want state %q", n.Cards, c.card)
}
for i, s := range c.stoppable {
if n.Engines[i].Stoppable != s {
t.Errorf("%s stoppable = %v, want %v", n.Engines[i].Name, n.Engines[i].Stoppable, s)
}
}
if !reflect.DeepEqual(n.Engines[0].Models, c.loaded) {
t.Errorf("models = %+v, want %+v", n.Engines[0].Models, c.loaded)
}
})
}
}
1 change: 1 addition & 0 deletions internal/gateway/gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,7 @@ func (g *Gateway) Handler() http.Handler {
mux.HandleFunc("GET /v1/berth/wait", g.wait)
mux.HandleFunc("GET /v1/berth/cost", g.cost)
mux.HandleFunc("POST /v1/berth/plan", g.plan)
mux.HandleFunc("GET /v1/berth/board", g.board)
mux.HandleFunc("POST /v1/berth/switch", g.apiSwitch)
mux.HandleFunc("POST /v1/berth/stop", g.apiStop)
for _, p := range []string{"/v1/chat/completions", "/v1/completions", "/v1/embeddings"} {
Expand Down
7 changes: 3 additions & 4 deletions internal/gateway/gui.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ func (g *Gateway) GUIHandler() http.Handler {
// The static route beats GET /gui/{name} in Go's ServeMux, so an engine
// named "state" loses that one URL to the plane's state endpoint.
mux.HandleFunc("GET /gui/state", func(w http.ResponseWriter, r *http.Request) {
rows := g.guiStateAll(r.Context())
rows := g.guiRows(r.Context(), g.nodeStatus())
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "no-store")
_ = json.NewEncoder(w).Encode(rows)
Expand Down Expand Up @@ -172,11 +172,10 @@ func (g *Gateway) guiIndex(w http.ResponseWriter, _ *http.Request) {
_ = guiPage.Execute(w, v)
}

// guiStateAll is the plane's state: one row per engine in config order.
// guiRows is the plane's state: one row per engine in config order.
// Up rows come from the probe; everything else reuses cardView, exactly like
// the per-engine state handler.
func (g *Gateway) guiStateAll(ctx context.Context) []guiRow {
node := g.nodeStatus()
func (g *Gateway) guiRows(ctx context.Context, node *proto.Status) []guiRow {
rows := make([]guiRow, 0, len(g.Engines))
for i := range g.Engines {
e := &g.Engines[i]
Expand Down
Loading