diff --git a/Makefile b/Makefile index a3a6454..da6d44d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ .PHONY: test test: - cd apps/api && bun test + cd apps/api && bun test --timeout 15000 diff --git a/apps/api/src/api/__tests__/domains-status.test.ts b/apps/api/src/api/__tests__/domains-status.test.ts index fe83594..2cade48 100644 --- a/apps/api/src/api/__tests__/domains-status.test.ts +++ b/apps/api/src/api/__tests__/domains-status.test.ts @@ -8,7 +8,10 @@ const run = () => exec("bun", ["run", "src/api/__tests__/domains-status-runner.ts"], { cwd: `${import.meta.dir}/../../..`, timeout: 30000, - env: { ...process.env, TEST_DATABASE_URL: "postgresql://dequel:dequel@localhost:5433/dequel" }, + env: { + ...process.env, + TEST_DATABASE_URL: process.env.TEST_DATABASE_URL ?? "postgresql://dequel:dequel@localhost:5433/dequel", + }, }); const parse = (stdout: string) => { @@ -18,7 +21,7 @@ const parse = (stdout: string) => { describe("GET /projects/:id/domains/status", () => { it("returns array and 404 for missing project", async () => { - const { stdout, stderr } = await run(); + const { stdout } = await run(); const results = parse(stdout); expect(results.test1.status).toBe(200); expect(results.test1.isArray).toBe(true); diff --git a/apps/api/src/api/deployments/index.ts b/apps/api/src/api/deployments/index.ts index 9d2b44e..b7d32ec 100644 --- a/apps/api/src/api/deployments/index.ts +++ b/apps/api/src/api/deployments/index.ts @@ -23,7 +23,7 @@ import { terminateWithEscalation } from "../../utils/process-exec"; import { isPrivateGitUrl } from "../../utils/validate"; import { created, fail, ok } from "../response"; -const dispatchDeployment = async ( +export const dispatchDeployment = async ( deployment: Awaited>, project: Awaited>, server: Awaited>, @@ -217,9 +217,9 @@ export const deploymentsRoutes = new Elysia() set.status = 404; return fail("Deployment not found"); } - if (original.status !== "running") { + if (original.status === "pending" || original.status === "building" || original.status === "deploying") { set.status = 400; - return fail("Can only redeploy the currently running deployment"); + return fail("Cannot redeploy an in-progress deployment"); } if (original.sourceType === "image") { set.status = 400; @@ -233,11 +233,16 @@ export const deploymentsRoutes = new Elysia() set.status = 400; return fail("Selected deployment server does not exist"); } + let sourceRef = original.sourceRef; + if (original.sourceType === "upload" && project?.buildType === "compose") { + const { ensureValidComposeArchive } = await import("../projects/compose"); + sourceRef = await ensureValidComposeArchive(project, original.sourceRef); + } const deployment = await createDeployment({ projectId: original.projectId || undefined, serverId: original.serverId, sourceType: original.sourceType, - sourceRef: original.sourceRef, + sourceRef, branch: original.branch || project?.repoBranch || undefined, environment: original.environment || undefined, }); diff --git a/apps/api/src/api/index.ts b/apps/api/src/api/index.ts index 7e052a5..6be2a8b 100644 --- a/apps/api/src/api/index.ts +++ b/apps/api/src/api/index.ts @@ -1,6 +1,5 @@ import { Elysia } from "elysia"; import { fixdiagRoutes } from "../fixdiag/routes"; -import { rateLimitMiddleware } from "./rate-limit"; import { agentRoutes } from "./agents"; import { alertsRoutes } from "./alerts"; import { apiKeysRoutes } from "./api-keys"; @@ -15,8 +14,10 @@ import { envVarsRoutes } from "./env-vars"; import { githubRoutes } from "./github"; import { healthRoutes } from "./health"; import { projectsRoutes } from "./projects"; +import { projectComposeRoutes } from "./projects/compose"; import { projectStatusRoutes } from "./projects/status"; import { prometheusRoutes } from "./prometheus"; +import { rateLimitMiddleware } from "./rate-limit"; import { fail } from "./response"; import { routesRoutes } from "./routes"; import { scalingRoutes } from "./scaling"; @@ -26,6 +27,7 @@ import { settingsRoutes } from "./settings"; import { llmSettingsRoutes } from "./settings/llm"; import { sharedEnvLinksRoutes, sharedEnvVarsRoutes } from "./shared-env-vars"; import { sshKeysRoutes } from "./ssh-keys"; +import { templateRoutes } from "./templates"; import { volumesRoutes } from "./volumes"; const INTERNAL_ERROR = @@ -62,7 +64,9 @@ export const apiRoutes = new Elysia({ .use(agentRoutes) .use(healthRoutes) .use(projectsRoutes) + .use(projectComposeRoutes) .use(projectStatusRoutes) + .use(templateRoutes) .use(deploymentsRoutes) .use(envVarsRoutes) .use(sharedEnvVarsRoutes) diff --git a/apps/api/src/api/projects/__tests__/compose.test.ts b/apps/api/src/api/projects/__tests__/compose.test.ts new file mode 100644 index 0000000..5ce3cb8 --- /dev/null +++ b/apps/api/src/api/projects/__tests__/compose.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { parse as parseYaml } from "yaml"; +import { createZipStore } from "../../../utils/zip-store"; + +describe("project compose management", () => { + it("creates a zip store containing docker-compose.yml and unzips correctly", async () => { + const tempDir = mkdtempSync(join(process.cwd(), "scratch-compose-test-")); + try { + const composeContent = "services:\n app:\n image: nginx:alpine\n ports:\n - 80:80\n"; + const zipBytes = createZipStore([{ name: "docker-compose.yml", content: composeContent }]); + const zipPath = join(tempDir, "compose.zip"); + writeFileSync(zipPath, zipBytes); + + const { safeSpawn } = await import("../../../utils/process-exec"); + const res = await safeSpawn("unzip", ["-p", zipPath, "docker-compose.yml"]); + expect(res.code).toBe(0); + expect(res.stdout).toBe(composeContent.trim()); + + const parsed = parseYaml(res.stdout) as any; + expect(parsed.services.app.image).toBe("nginx:alpine"); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("validates compose yaml structure", () => { + const invalidYaml = "services: [bad yaml"; + expect(() => parseYaml(invalidYaml)).toThrow(); + + const validYaml = "services:\n web:\n image: node:20-alpine\n ports:\n - 3000:3000\n"; + const parsed = parseYaml(validYaml) as any; + expect(parsed.services).toBeDefined(); + expect(parsed.services.web).toBeDefined(); + }); + + it("finds matching template for a project and recovers full compose yaml with version 3.8", async () => { + const { findTemplateForProject, getOrRecoverProjectCompose } = await import("../compose"); + const wpProject = { + id: "proj-wp-test", + name: "my-wordpress", + description: "Content management system (template: wordpress)", + composeService: "wordpress", + composePort: 80, + }; + const template = findTemplateForProject(wpProject); + expect(template).toBeDefined(); + expect(template?.key).toBe("wordpress"); + + const recovered = await getOrRecoverProjectCompose(wpProject); + expect(recovered).toContain('version: "3.8"'); + expect(recovered).toContain("services:"); + expect(recovered).toContain("wordpress:"); + expect(recovered).toContain("db:"); + expect(recovered).toContain("wp-content:"); + expect(recovered).toContain("wp-db:"); + }); +}); diff --git a/apps/api/src/api/projects/compose.ts b/apps/api/src/api/projects/compose.ts new file mode 100644 index 0000000..9fc8863 --- /dev/null +++ b/apps/api/src/api/projects/compose.ts @@ -0,0 +1,296 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { basename, join } from "node:path"; +import { Elysia } from "elysia"; +import { parse as parseYaml } from "yaml"; +import { createDeployment, listDeployments } from "../../db/repo/deployments"; +import { getProjectById, updateProject } from "../../db/repo/projects"; +import { getServerById } from "../../db/repo/servers"; +import { parseContainerTargetPort } from "../../orchestrator/compose"; +import { config } from "../../utils/config"; +import { safeSpawn } from "../../utils/process-exec"; +import { validateComposeServices } from "../../utils/validate"; +import { createZipStore } from "../../utils/zip-store"; +import { created, fail, ok } from "../response"; +import { getTemplate, listTemplates } from "../templates/registry"; +import type { DeployTemplate } from "../templates/types"; + +const getWorkspaceRoot = (): string => config.workspaceRoot || process.env.WORKSPACE_ROOT || "./workspace"; + +const readComposeFromArchive = async (archivePath: string): Promise => { + if (!existsSync(archivePath)) return null; + const candidates = ["docker-compose.yml", "docker-compose.yaml", "compose.yml", "compose.yaml"]; + for (const candidate of candidates) { + try { + const res = await safeSpawn("unzip", ["-p", archivePath, candidate], { timeoutMs: 10_000 }); + if (res.code === 0 && res.stdout.trim()) { + return res.stdout; + } + } catch {} + } + return null; +}; + +const readComposeFromDir = (dirPath: string): string | null => { + const candidates = ["docker-compose.yml", "docker-compose.yaml", "compose.yml", "compose.yaml"]; + for (const candidate of candidates) { + const full = join(dirPath, candidate); + if (existsSync(full)) { + try { + return readFileSync(full, "utf8"); + } catch {} + } + } + return null; +}; + +export const findTemplateForProject = (project: { + name?: string | null; + description?: string | null; + composeService?: string | null; +}): DeployTemplate | undefined => { + const desc = project.description || ""; + const match = desc.match(/template:\s*([a-zA-Z0-9_-]+)/i); + if (match?.[1]) { + const t = getTemplate(match[1]); + if (t) return t; + } + if (project.name) { + const byName = getTemplate(project.name.toLowerCase()); + if (byName) return byName; + const stripped = project.name.toLowerCase().replace(/[^a-z0-9]/g, ""); + const found = listTemplates().find( + (t) => + t.key.toLowerCase().replace(/[^a-z0-9]/g, "") === stripped || + t.name.toLowerCase().replace(/[^a-z0-9]/g, "") === stripped, + ); + if (found) return found; + } + if (project.composeService) { + const byService = listTemplates().find((t) => t.service.toLowerCase() === project.composeService?.toLowerCase()); + if (byService) return byService; + } + return undefined; +}; + +const fallbackComposeFor = (project: { composeService?: string | null; composePort?: number | null }) => { + const primaryService = project.composeService || "app"; + const primaryPort = project.composePort || 80; + return `version: "3.8"\n\nservices:\n ${primaryService}:\n image: nginx:alpine\n ports:\n - "${primaryPort}:80"\n`; +}; + +const isFallbackCompose = (text: string | null | undefined): boolean => { + if (!text) return false; + try { + const parsed = parseYaml(text) as any; + const services = parsed?.services; + if (!services || typeof services !== "object") return false; + const entries = Object.entries(services); + if (entries.length !== 1) return false; + const svc = entries[0][1] as any; + if (svc?.image !== "nginx:alpine") return false; + const extraKeys = Object.keys(svc).filter((k) => !["image", "ports", "restart"].includes(k)); + if (extraKeys.length > 0) return false; + const ports = Array.isArray(svc.ports) ? svc.ports : []; + return ports.length === 1 && String(ports[0]).trim().endsWith(":80"); + } catch { + return false; + } +}; + +export const getOrRecoverProjectCompose = async (project: any): Promise => { + let recovered: string | null = null; + let recoveredStub: string | null = null; + const accept = (candidate: string | null): boolean => { + if (!candidate) return false; + if (isFallbackCompose(candidate)) { + if (!recoveredStub) recoveredStub = candidate; + return false; + } + recovered = candidate; + return true; + }; + + const deps = await listDeployments(project.id, 0, 10).catch(() => []); + for (const d of deps) { + if (d.sourceRef?.endsWith(".zip")) { + let archiveFile = d.sourceRef; + if (!existsSync(archiveFile)) { + const inUploads = join(getWorkspaceRoot(), "uploads", basename(archiveFile)); + if (existsSync(inUploads)) archiveFile = inUploads; + } + const candidate = await readComposeFromArchive(archiveFile); + if (accept(candidate)) break; + } + } + + if (!recovered && deps.length > 0) { + for (const d of deps) { + const wsDir = join(getWorkspaceRoot(), d.id); + if (accept(readComposeFromDir(wsDir))) break; + } + } + + if (!recovered) { + const projectWs = join(getWorkspaceRoot(), project.id); + accept(readComposeFromDir(projectWs)); + } + + if (!recovered) { + const template = findTemplateForProject(project); + if (template) { + const { renderCompose } = await import("../templates/registry"); + const { listEnvironmentVariablesForDeploy } = await import("../../db/repo/env-vars"); + const envs = await listEnvironmentVariablesForDeploy(project.id, "production").catch(() => []); + const envMap: Record = {}; + for (const e of envs) envMap[e.key] = e.value; + const rendered = renderCompose(template, envMap); + recovered = rendered.compose; + } + } + + if (recovered) { + if (!recovered.trim().startsWith("version:")) { + recovered = `version: "3.8"\n\n${recovered.trim()}`; + } + return recovered; + } + + return recoveredStub ?? fallbackComposeFor(project); +}; + +export const ensureValidComposeArchive = async (project: any, preferredRef?: string | null): Promise => { + if (preferredRef && existsSync(preferredRef)) { + const existing = await readComposeFromArchive(preferredRef); + if (existing && !isFallbackCompose(existing)) return preferredRef; + } + if (preferredRef) { + const inUploads = join(getWorkspaceRoot(), "uploads", basename(preferredRef)); + if (existsSync(inUploads)) { + const existing = await readComposeFromArchive(inUploads); + if (existing && !isFallbackCompose(existing)) return inUploads; + } + } + + const uploadsDir = join(getWorkspaceRoot(), "uploads"); + mkdirSync(uploadsDir, { recursive: true }); + + const compose = await getOrRecoverProjectCompose(project); + const targetPath = preferredRef ? join(uploadsDir, basename(preferredRef)) : null; + const targetArchive = + targetPath && !existsSync(targetPath) ? targetPath : join(uploadsDir, `${Date.now()}-compose-${project.id}.zip`); + + writeFileSync(targetArchive, createZipStore([{ name: "docker-compose.yml", content: compose }])); + return targetArchive; +}; + +export const projectComposeRoutes = new Elysia() + .get("/projects/:id/compose", async ({ params: { id }, set }) => { + const project = await getProjectById(id); + if (!project) { + set.status = 404; + return fail("Project not found"); + } + + const compose = await getOrRecoverProjectCompose(project); + + return ok({ + compose, + composeService: project.composeService, + composePort: project.composePort, + composeServices: project.composeServices, + }); + }) + .put("/projects/:id/compose", async ({ params: { id }, body, set }: any) => { + const project = await getProjectById(id); + if (!project) { + set.status = 404; + return fail("Project not found"); + } + + const compose = typeof body?.compose === "string" ? body.compose.trim() : ""; + if (!compose) { + set.status = 400; + return fail("Compose YAML content is required"); + } + + let parsed: any; + try { + parsed = parseYaml(compose); + } catch (e: any) { + set.status = 400; + return fail(`Invalid YAML syntax: ${e.message}`); + } + + if (!parsed || typeof parsed !== "object" || !parsed.services || typeof parsed.services !== "object") { + set.status = 400; + return fail("Compose file must contain a 'services' mapping"); + } + + const serviceEntries = Object.entries(parsed.services); + if (serviceEntries.length === 0) { + set.status = 400; + return fail("At least one service must be defined in the compose file"); + } + + const requestedService = typeof body.composeService === "string" ? body.composeService.trim() : ""; + const targetService = + requestedService && (parsed.services as any)[requestedService] ? requestedService : serviceEntries[0][0]; + + let targetPort = body.composePort ? Number(body.composePort) : null; + if (!targetPort || Number.isNaN(targetPort)) { + const svcCfg = (parsed.services as any)[targetService]; + if (svcCfg?.ports && Array.isArray(svcCfg.ports)) { + for (const p of svcCfg.ports) { + const parsedPort = parseContainerTargetPort(p); + if (parsedPort) { + targetPort = parsedPort; + break; + } + } + } + } + if (!targetPort) targetPort = 3000; + + let composeServicesPayload: string | null = null; + if (body.composeServices) { + const validation = validateComposeServices(body.composeServices); + if (!validation.ok) { + set.status = 400; + return fail(validation.error); + } + composeServicesPayload = + typeof body.composeServices === "string" ? body.composeServices : JSON.stringify(body.composeServices); + } + + const uploadsDir = join(getWorkspaceRoot(), "uploads"); + mkdirSync(uploadsDir, { recursive: true }); + const archivePath = join(uploadsDir, `${Date.now()}-compose-${project.id}.zip`); + writeFileSync(archivePath, createZipStore([{ name: "docker-compose.yml", content: compose }])); + + const serverId = project.serverId || "local"; + const server = await getServerById(serverId); + if (!server) { + set.status = 400; + return fail("Project deployment server not found"); + } + + const updatedProject = await updateProject(project.id, { + buildType: "compose", + sourceType: "upload", + composeService: targetService, + composePort: targetPort, + composeServices: (composeServicesPayload ?? project.composeServices) as any, + }); + + const deployment = await createDeployment({ + projectId: project.id, + serverId, + sourceType: "upload", + sourceRef: archivePath, + environment: "production", + }); + + const { dispatchDeployment } = await import("../deployments"); + await dispatchDeployment(deployment, updatedProject, server); + return created({ project: updatedProject, deployment }); + }); diff --git a/apps/api/src/api/projects/index.ts b/apps/api/src/api/projects/index.ts index 56127a3..a8e2b9d 100644 --- a/apps/api/src/api/projects/index.ts +++ b/apps/api/src/api/projects/index.ts @@ -284,19 +284,22 @@ export const projectsRoutes = new Elysia() const data = (await response.json()) as any; if (data.status === "success" && data.data?.result?.[0]?.values) { return ok({ - resultType: "matrix", - result: [ - { - metric: {}, - values: data.data.result[0].values, - }, - ], + status: "success", + data: { + resultType: "matrix", + result: [ + { + metric: {}, + values: data.data.result[0].values, + }, + ], + }, }); } - return ok({ resultType: "matrix", result: [] }); + return ok({ status: "success", data: { resultType: "matrix", result: [] } }); } catch (e) { console.error("Loki query failed", e); - return ok({ resultType: "matrix", result: [] }); + return ok({ status: "success", data: { resultType: "matrix", result: [] } }); } }) .get("/projects/:id/request-logs/stream", async ({ params: { id }, request, set }) => { diff --git a/apps/api/src/api/templates/__tests__/templates-routes.test.ts b/apps/api/src/api/templates/__tests__/templates-routes.test.ts new file mode 100644 index 0000000..affc317 --- /dev/null +++ b/apps/api/src/api/templates/__tests__/templates-routes.test.ts @@ -0,0 +1,194 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "bun:test"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { drizzle } from "drizzle-orm/node-postgres"; +import type { Pool } from "pg"; +import { setDbProvider } from "../../../db/db-provider"; +import { createDeployment } from "../../../db/repo/deployments"; +import { + createEnvironmentVariable, + getEnvironmentVariablePlaintext, + listEnvironmentVariables, +} from "../../../db/repo/env-vars"; +import { createProject, listProjects } from "../../../db/repo/projects"; +import { createServer, ensureLocalServer, getServerById } from "../../../db/repo/servers"; +import * as schema from "../../../db/schema"; +import { createTestPool, truncateAllTables } from "../../../db/test-helper"; +import { createTemplateRoutes } from "../index"; + +let pool: Pool; +let workspaceRoot: string; +let routes: ReturnType; + +const repoOverrides = { + createProject, + createDeployment, + createEnvironmentVariable, + getServerById, + listProjects, +}; + +beforeAll(async () => { + pool = createTestPool(); + const db = drizzle(pool, { schema }); + setDbProvider(async () => db); + workspaceRoot = mkdtempSync(join(tmpdir(), "dequel-template-routes-")); + routes = createTemplateRoutes({ + repo: repoOverrides, + resolveServerId: async () => "local", + enqueue: () => {}, + workspaceRoot, + }); +}); + +beforeEach(async () => { + await truncateAllTables(pool); + await ensureLocalServer(); +}); + +afterAll(async () => { + await truncateAllTables(pool); + await pool.end(); + rmSync(workspaceRoot, { recursive: true, force: true }); +}); + +const postDeploy = (body: Record) => + routes.handle( + new Request("http://localhost/templates/n8n/deployments", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }), + ); + +describe("template routes", () => { + it("returns template metadata without raw compose", async () => { + const res = await routes.handle(new Request("http://localhost/templates")); + expect(res.status).toBe(200); + const json = (await res.json()) as { status: string; data: { key: string; service: string; port: number }[] }; + expect(json.status).toBe("success"); + expect(json.data.some((t) => t.key === "n8n")).toBe(true); + const entry = json.data.find((t) => t.key === "wordpress"); + expect(entry?.service).toBe("wordpress"); + expect(entry?.port).toBe(80); + }); + + it("creates a project, archive and deployment from a template", async () => { + const res = await postDeploy({ + name: "n8n-check", + values: { GENERIC_TIMEZONE: "Europe/Berlin" }, + }); + expect(res.status).toBe(200); + const json = (await res.json()) as { + status: string; + data: { project: Record; deployment: Record }; + }; + expect(json.status).toBe("success"); + expect(json.data.project.buildType).toBe("compose"); + expect(json.data.project.composeService).toBe("n8n"); + expect(json.data.project.composePort).toBe(5678); + expect(json.data.project.sourceType).toBe("upload"); + expect(json.data.deployment.sourceType).toBe("upload"); + + const archivePath = json.data.deployment.sourceRef as string; + expect(existsSync(archivePath)).toBe(true); + + const out = join(workspaceRoot, "verify-extract"); + mkdirSync(out, { recursive: true }); + const proc = Bun.spawnSync(["unzip", "-q", archivePath, "-d", out]); + expect(proc.exitCode).toBe(0); + const compose = readFileSync(join(out, "docker-compose.yml"), "utf8"); + expect(compose).toContain(`GENERIC_TIMEZONE=\${GENERIC_TIMEZONE}`); + expect(compose).toContain(`N8N_ENCRYPTION_KEY=\${N8N_ENCRYPTION_KEY}`); + expect(compose).not.toMatch(/(? { + const res = await postDeploy({ name: "n8n-persist" }); + expect(res.status).toBe(200); + const json = (await res.json()) as { data: { project: { id: string }; deployment: { id: string } } }; + const projects = await listProjects(); + expect(projects.some((p) => p.id === json.data.project.id)).toBe(true); + }); + + it("stores the rendered variables as project env vars", async () => { + const res = await postDeploy({ name: "n8n-env", values: { GENERIC_TIMEZONE: "Europe/Berlin" } }); + expect(res.status).toBe(200); + const json = (await res.json()) as { data: { project: { id: string } } }; + const envs = await listEnvironmentVariables(json.data.project.id); + const keys = envs.map((v) => v.key); + expect(keys).toContain("GENERIC_TIMEZONE"); + expect(keys).toContain("N8N_ENCRYPTION_KEY"); + const secret = envs.find((v) => v.key === "N8N_ENCRYPTION_KEY"); + expect(secret).toBeDefined(); + const plain = await getEnvironmentVariablePlaintext(secret!.id); + expect(plain?.length).toBeGreaterThan(0); + }); + + it("rejects duplicate project names", async () => { + const first = await postDeploy({ name: "n8n-dup" }); + expect(first.status).toBe(200); + const second = await postDeploy({ name: "n8n-dup" }); + expect(second.status).toBe(400); + const json = (await second.json()) as { status: string; message: string }; + expect(json.status).toBe("error"); + }); + + it("rejects unknown template keys", async () => { + const res = await routes.handle( + new Request("http://localhost/templates/nope/deployments", { + method: "POST", + headers: { "content-type": "application/json" }, + body: "{}", + }), + ); + expect(res.status).toBe(404); + }); + + it("rejects non-local servers", async () => { + const remoteRoutes = createTemplateRoutes({ + repo: { + createProject, + createDeployment, + getServerById: async () => ({ id: "srv", name: "srv", mode: "ssh" }) as never, + listProjects, + }, + resolveServerId: async () => "srv", + enqueue: () => {}, + workspaceRoot, + }); + const res = await remoteRoutes.handle( + new Request("http://localhost/templates/n8n/deployments", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ name: "n8n-remote" }), + }), + ); + expect(res.status).toBe(400); + const json = (await res.json()) as { message: string }; + expect(json.message).toContain("remote servers support Git sources only"); + }); + + it("deploys to an explicitly selected local server", async () => { + const res = await postDeploy({ name: "n8n-pick", serverId: "local" }); + expect(res.status).toBe(200); + const json = (await res.json()) as { data: { project: { serverId: string | null } } }; + expect(json.data.project.serverId).toBe("local"); + }); + + it("rejects an explicitly selected non-local server", async () => { + const server = await createServer({ name: "edge-box", host: "203.0.113.9", mode: "ssh", sshUser: "deploy" }); + const res = await postDeploy({ name: "n8n-edge", serverId: server.id }); + expect(res.status).toBe(400); + const json = (await res.json()) as { message: string }; + expect(json.message).toContain("remote servers support Git sources only"); + }); + + it("rejects an unknown server id", async () => { + const res = await postDeploy({ name: "n8n-ghost", serverId: "no-such-server" }); + expect(res.status).toBe(400); + const json = (await res.json()) as { message: string }; + expect(json.message).toContain("does not exist"); + }); +}); diff --git a/apps/api/src/api/templates/__tests__/templates.test.ts b/apps/api/src/api/templates/__tests__/templates.test.ts new file mode 100644 index 0000000..f1a189c --- /dev/null +++ b/apps/api/src/api/templates/__tests__/templates.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from "bun:test"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { parse as parseYaml } from "yaml"; +import { createZipStore } from "../../../utils/zip-store"; +import { getTemplate, listTemplates, renderCompose } from "../registry"; +import type { DeployTemplate } from "../types"; + +describe("deploy template registry", () => { + it("lists templates with unique keys, services and ports", () => { + const templates = listTemplates(); + expect(templates.length).toBeGreaterThanOrEqual(11); + expect(new Set(templates.map((t) => t.key)).size).toBe(templates.length); + for (const t of templates) { + expect(t.service.length).toBeGreaterThan(0); + expect(t.port).toBeGreaterThan(0); + expect(t.compose).toContain(t.service); + expect(t.vars.every((v) => /^[A-Z][A-Z0-9_]*$/.test(v.key))).toBe(true); + } + }); + + it("renders env references into the compose file", () => { + const template = getTemplate("wordpress"); + expect(template).toBeDefined(); + const { compose, vars } = renderCompose(template!, { + MYSQL_DATABASE: "wpdata", + MYSQL_USER: "wp-user", + MYSQL_ROOT_PASSWORD: "root-secret", + MYSQL_PASSWORD: "app-secret", + }); + expect(compose).toContain(`MYSQL_DATABASE=\${MYSQL_DATABASE}`); + expect(compose).toContain(`MYSQL_USER=\${MYSQL_USER}`); + expect(compose).toContain(`MYSQL_ROOT_PASSWORD=\${MYSQL_ROOT_PASSWORD}`); + expect(compose).toContain(`MYSQL_PASSWORD=\${MYSQL_PASSWORD}`); + expect(compose).not.toContain("wpdata"); + expect(vars.map((v) => [v.key, v.value])).toContainEqual(["MYSQL_DATABASE", "wpdata"]); + expect(compose).not.toMatch(/(? { + const template = getTemplate("n8n"); + expect(template).toBeDefined(); + const first = renderCompose(template!, {}); + expect(first.vars.find((v) => v.key === "N8N_ENCRYPTION_KEY")?.value.length).toBeGreaterThan(0); + const second = renderCompose(template!, {}); + expect(first.vars.find((v) => v.key === "N8N_ENCRYPTION_KEY")?.value).not.toEqual( + second.vars.find((v) => v.key === "N8N_ENCRYPTION_KEY")?.value, + ); + expect(first.compose).toContain(`N8N_ENCRYPTION_KEY=\${N8N_ENCRYPTION_KEY}`); + }); + + it("renders with defaults into parseable compose with the primary service", () => { + for (const template of listTemplates()) { + const { compose } = renderCompose(template, {}); + const doc = parseYaml(compose) as { services?: Record }; + expect(doc?.services).toBeDefined(); + expect(Object.keys(doc.services)).toContain(template.service); + } + }); + + it("uses provided values over defaults", () => { + const template = getTemplate("n8n")!; + const { compose, vars } = renderCompose(template, { GENERIC_TIMEZONE: "Europe/Berlin" }); + expect(vars.find((v) => v.key === "GENERIC_TIMEZONE")?.value).toBe("Europe/Berlin"); + expect(compose).toContain(`GENERIC_TIMEZONE=\${GENERIC_TIMEZONE}`); + }); + + it("rejects unresolved references and line breaks in values", () => { + const fake: DeployTemplate = { + key: "fake", + name: "fake", + description: "", + category: "", + service: "svc", + port: 80, + compose: "services:\n svc:\n image: alpine\n port: {MYSTERY_VAR}\n", + vars: [], + }; + expect(() => renderCompose(fake, {})).toThrow(/MYSTERY_VAR/); + + const oneVar: DeployTemplate = { ...fake, compose: "value: {ONLY_VAR}\n", vars: [{ key: "ONLY_VAR", label: "v" }] }; + expect(() => renderCompose(oneVar, { ONLY_VAR: "line\nbreak" })).toThrow(/line breaks/i); + }); +}); + +describe("zip store writer", () => { + it("produces an archive the platform extractor can read", () => { + const dir = mkdtempSync(join(tmpdir(), "dequel-zip-")); + try { + const compose = "services:\n svc:\n image: alpine\n"; + const zip = createZipStore([ + { name: "docker-compose.yml", content: compose }, + { name: "README.md", content: "hello world" }, + ]); + const zipPath = join(dir, "template.zip"); + writeFileSync(zipPath, zip); + const out = join(dir, "out"); + mkdirSync(out, { recursive: true }); + + const proc = Bun.spawnSync(["unzip", "-q", zipPath, "-d", out]); + expect(proc.exitCode).toBe(0); + + expect(readFileSync(join(out, "docker-compose.yml"), "utf8")).toBe(compose); + expect(readFileSync(join(out, "README.md"), "utf8")).toBe("hello world"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/apps/api/src/api/templates/index.ts b/apps/api/src/api/templates/index.ts new file mode 100644 index 0000000..cea6a0a --- /dev/null +++ b/apps/api/src/api/templates/index.ts @@ -0,0 +1,209 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { Elysia } from "elysia"; +import type { + CreateDeploymentInput, + CreateEnvironmentVariableInput, + CreateProjectInput, + Deployment, + Project, + Server, +} from "../../types"; +import { created, fail, ok } from "../response"; +import { getTemplate, listTemplates, type RenderedTemplate, renderCompose } from "./registry"; +import type { DeployTemplate } from "./types"; + +interface TemplateRepo { + createProject: (input: CreateProjectInput) => Promise; + createDeployment: (input: CreateDeploymentInput) => Promise; + createEnvironmentVariable: (input: CreateEnvironmentVariableInput) => Promise; + getServerById: (id: string) => Promise; + listProjects: () => Promise; + deleteProjectCascade?: (id: string) => Promise; +} + +export interface TemplateRouteOverrides { + repo?: TemplateRepo; + resolveServerId?: (preferred?: string | null, exclude?: string | null, modes?: string[]) => Promise; + enqueue?: (deploymentId: string) => void; + workspaceRoot?: string; +} + +type DeployBody = { + name?: unknown; + description?: unknown; + serverId?: unknown; + values?: unknown; +}; + +const isObjectType = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +export const templateMeta = (t: DeployTemplate) => ({ + key: t.key, + name: t.name, + description: t.description, + category: t.category, + docsUrl: t.docsUrl, + service: t.service, + port: t.port, + vars: t.vars.map((v) => ({ + key: v.key, + label: v.label, + description: v.description, + default: v.default, + secret: v.secret ?? false, + required: !v.default, + })), +}); + +export const createTemplateRoutes = (overrides?: TemplateRouteOverrides): Elysia => { + const loadRepo = async (): Promise => + overrides?.repo ?? ((await import("../../db/repo")) as any as TemplateRepo); + + const resolveServer = async ( + preferred?: string | null, + exclude?: string | null, + modes?: string[], + ): Promise => { + if (overrides?.resolveServerId) return overrides.resolveServerId(preferred, exclude, modes); + const { resolveDefaultServerId } = await import("../../utils/server-default"); + return resolveDefaultServerId(preferred, exclude, modes); + }; + + const enqueueDeployment = async (deploymentId: string): Promise => { + if (overrides?.enqueue) return overrides.enqueue(deploymentId); + const { orchestrator } = await import("../../orchestrator"); + orchestrator.enqueue(deploymentId); + }; + + const uploadsDir = async (): Promise => { + if (overrides?.workspaceRoot) return join(overrides.workspaceRoot, "uploads"); + const { config } = await import("../../utils/config"); + return join(config.workspaceRoot, "uploads"); + }; + + return new Elysia() + .get("/templates", () => ok(listTemplates().map(templateMeta))) + .get("/templates/:key", ({ params: { key }, set }) => { + const template = getTemplate(key); + if (!template) { + set.status = 404; + return fail(`Unknown template "${key}"`); + } + return ok(templateMeta(template)); + }) + .post("/templates/:key/deployments", async ({ params: { key }, body, set }: any) => { + const template = getTemplate(key); + if (!template) { + set.status = 404; + return fail(`Unknown template "${key}"`); + } + const input: DeployBody = isObjectType(body) ? (body as DeployBody) : {}; + const values: Record = isObjectType(input.values) ? (input.values as Record) : {}; + + let rendered: RenderedTemplate; + try { + rendered = renderCompose(template, values); + } catch (error) { + set.status = 400; + return fail(error instanceof Error ? error.message : "Failed to render template"); + } + + const name = String(input.name ?? template.name).trim(); + if (!name) { + set.status = 400; + return fail("Project name is required"); + } + + const repo = await loadRepo(); + const projects = await repo.listProjects(); + if (projects.some((p) => p.name === name)) { + set.status = 400; + return fail(`A project named "${name}" already exists`); + } + + const requestedServerId = + typeof input.serverId === "string" && input.serverId.trim() ? input.serverId.trim() : undefined; + let serverId: string | undefined; + if (requestedServerId) { + serverId = requestedServerId; + } else { + serverId = await resolveServer(undefined, undefined, ["local"]); + if (!serverId) { + set.status = 400; + return fail("No server available for template deployment"); + } + } + const server = await repo.getServerById(serverId); + if (!server) { + set.status = 400; + return fail("Selected deployment server does not exist"); + } + if (server.mode !== "local") { + set.status = 400; + return fail("Templates deploy Docker Compose stacks - remote servers support Git sources only right now"); + } + + const dir = await uploadsDir(); + mkdirSync(dir, { recursive: true }); + const archivePath = join(dir, `${Date.now()}-template-${template.key}.zip`); + const { createZipStore } = await import("../../utils/zip-store"); + writeFileSync(archivePath, createZipStore([{ name: "docker-compose.yml", content: rendered.compose }])); + + const project = await repo.createProject({ + name, + serverId, + description: + typeof input.description === "string" + ? input.description + : `${template.description} (template: ${template.key})`, + sourceType: "upload", + projectType: "web", + buildType: "compose", + composeService: template.service, + composePort: template.port, + }); + + try { + for (const v of rendered.vars) { + await repo.createEnvironmentVariable({ + projectId: project.id, + key: v.key, + value: v.value, + environment: "production", + }); + } + } catch (error) { + const cleanupFn = repo.deleteProjectCascade ?? (await import("../../db/repo")).deleteProjectCascade; + if (cleanupFn) { + await cleanupFn(project.id).catch(() => {}); + } + set.status = 500; + return fail(error instanceof Error ? error.message : "Failed to store template variables"); + } + + let deployment: Deployment; + try { + deployment = await repo.createDeployment({ + projectId: project.id, + serverId, + sourceType: "upload", + sourceRef: archivePath, + environment: "production", + }); + } catch (error) { + const cleanupFn = repo.deleteProjectCascade ?? (await import("../../db/repo")).deleteProjectCascade; + if (cleanupFn) { + await cleanupFn(project.id).catch(() => {}); + } + set.status = 500; + return fail(error instanceof Error ? error.message : "Failed to create deployment"); + } + + await enqueueDeployment(deployment.id); + return created({ project, deployment }); + }); +}; + +export const templateRoutes = createTemplateRoutes(); diff --git a/apps/api/src/api/templates/registry.ts b/apps/api/src/api/templates/registry.ts new file mode 100644 index 0000000..574bff8 --- /dev/null +++ b/apps/api/src/api/templates/registry.ts @@ -0,0 +1,34 @@ +import { giteaTemplate } from "./templates/gitea"; +import { itToolsTemplate } from "./templates/it-tools"; +import { memosTemplate } from "./templates/memos"; +import { n8nTemplate } from "./templates/n8n"; +import { nextcloudTemplate } from "./templates/nextcloud"; +import { paperlessNgxTemplate } from "./templates/paperless-ngx"; +import { plausibleTemplate } from "./templates/plausible"; +import { umamiTemplate } from "./templates/umami"; +import { uptimeKumaTemplate } from "./templates/uptime-kuma"; +import { vaultwardenTemplate } from "./templates/vaultwarden"; +import { wordPressTemplate } from "./templates/wordpress"; +import type { DeployTemplate } from "./types"; + +export type { DeployTemplate, RenderedTemplate, TemplateVar } from "./types"; +export { renderCompose } from "./types"; + +const templates: DeployTemplate[] = [ + n8nTemplate, + wordPressTemplate, + uptimeKumaTemplate, + giteaTemplate, + nextcloudTemplate, + paperlessNgxTemplate, + plausibleTemplate, + umamiTemplate, + vaultwardenTemplate, + memosTemplate, + itToolsTemplate, +]; + +export const listTemplates = (): DeployTemplate[] => templates; + +export const getTemplate = (key: string): DeployTemplate | undefined => + templates.find((t) => t.key.toLowerCase() === key.toLowerCase()); diff --git a/apps/api/src/api/templates/templates/gitea.ts b/apps/api/src/api/templates/templates/gitea.ts new file mode 100644 index 0000000..f97cc86 --- /dev/null +++ b/apps/api/src/api/templates/templates/gitea.ts @@ -0,0 +1,37 @@ +import type { DeployTemplate } from "../types"; + +export const giteaTemplate: DeployTemplate = { + key: "gitea", + name: "Gitea", + description: "Lightweight self-hosted Git service with issues, pulls and CI.", + category: "developer", + docsUrl: "https://docs.gitea.com/next/installation/install-with-docker", + service: "gitea", + port: 3000, + compose: `services: + gitea: + image: gitea/gitea:1 + pull_policy: always + restart: unless-stopped + environment: + - USER_UID=1000 + - USER_GID=1000 + - GITEA__server__DISABLE_SSH=true + - GITEA__server__HTTP_PORT=3000 + - GITEA__server__ROOT_URL={ROOT_URL} + - GITEA__database__DB_TYPE=sqlite3 + - GITEA__database__PATH=/data/gitea/gitea.db + volumes: + - gitea-data:/data +volumes: + gitea-data: +`, + vars: [ + { + key: "ROOT_URL", + label: "Instance URL", + default: "http://localhost", + description: "Public URL of this Gitea instance, e.g. https://git.example.com. Used for links and SSH/web URLs.", + }, + ], +}; diff --git a/apps/api/src/api/templates/templates/it-tools.ts b/apps/api/src/api/templates/templates/it-tools.ts new file mode 100644 index 0000000..a755ee8 --- /dev/null +++ b/apps/api/src/api/templates/templates/it-tools.ts @@ -0,0 +1,18 @@ +import type { DeployTemplate } from "../types"; + +export const itToolsTemplate: DeployTemplate = { + key: "it-tools", + name: "IT Tools", + description: "Collection of handy online tools for developers, all in one place.", + category: "developer", + docsUrl: "https://github.com/CorentinTh/it-tools", + service: "it-tools", + port: 80, + compose: `services: + it-tools: + image: corentinth/it-tools:latest + pull_policy: always + restart: unless-stopped +`, + vars: [], +}; diff --git a/apps/api/src/api/templates/templates/memos.ts b/apps/api/src/api/templates/templates/memos.ts new file mode 100644 index 0000000..84c22b1 --- /dev/null +++ b/apps/api/src/api/templates/templates/memos.ts @@ -0,0 +1,22 @@ +import type { DeployTemplate } from "../types"; + +export const memosTemplate: DeployTemplate = { + key: "memos", + name: "Memos", + description: "Lightweight note-taking service, like a private stream of your thoughts.", + category: "productivity", + docsUrl: "https://github.com/usememos/memos", + service: "memos", + port: 5230, + compose: `services: + memos: + image: neosmemo/memos:stable + pull_policy: always + restart: unless-stopped + volumes: + - memos-data:/var/opt/memos +volumes: + memos-data: +`, + vars: [], +}; diff --git a/apps/api/src/api/templates/templates/n8n.ts b/apps/api/src/api/templates/templates/n8n.ts new file mode 100644 index 0000000..51c01f1 --- /dev/null +++ b/apps/api/src/api/templates/templates/n8n.ts @@ -0,0 +1,35 @@ +import type { DeployTemplate } from "../types"; + +export const n8nTemplate: DeployTemplate = { + key: "n8n", + name: "n8n", + description: "Workflow automation platform. Self-hosted alternative to Zapier.", + category: "automation", + docsUrl: "https://docs.n8n.io/hosting/", + service: "n8n", + port: 5678, + compose: `services: + n8n: + image: docker.n8n.io/n8nio/n8n:latest + pull_policy: always + restart: unless-stopped + environment: + - N8N_PORT=5678 + - N8N_PROTOCOL=http + - N8N_ENCRYPTION_KEY={N8N_ENCRYPTION_KEY} + - GENERIC_TIMEZONE={GENERIC_TIMEZONE} + volumes: + - n8n-data:/home/node/.n8n +volumes: + n8n-data: +`, + vars: [ + { + key: "N8N_ENCRYPTION_KEY", + label: "Encryption key", + description: "Used to encrypt credentials stored in n8n. Changing it after first run breaks stored credentials.", + secret: true, + }, + { key: "GENERIC_TIMEZONE", label: "Timezone", default: "UTC" }, + ], +}; diff --git a/apps/api/src/api/templates/templates/nextcloud.ts b/apps/api/src/api/templates/templates/nextcloud.ts new file mode 100644 index 0000000..0ef4ea6 --- /dev/null +++ b/apps/api/src/api/templates/templates/nextcloud.ts @@ -0,0 +1,54 @@ +import type { DeployTemplate } from "../types"; + +export const nextcloudTemplate: DeployTemplate = { + key: "nextcloud", + name: "Nextcloud", + description: "Files, calendar, contacts and collaboration, self-hosted in one place.", + category: "storage", + docsUrl: "https://docs.nextcloud.com/server/latest/admin_manual/installation/docker_install_config.html", + service: "nextcloud", + port: 80, + compose: `services: + db: + image: postgres:17-alpine + pull_policy: always + restart: unless-stopped + environment: + - POSTGRES_DB=nextcloud + - POSTGRES_USER=nextcloud + - POSTGRES_PASSWORD={NEXTCLOUD_DB_PASSWORD} + volumes: + - nextcloud-db:/var/lib/postgresql/data + nextcloud: + image: nextcloud:35-apache + pull_policy: always + restart: unless-stopped + depends_on: + - db + environment: + - POSTGRES_HOST=db + - POSTGRES_DB=nextcloud + - POSTGRES_USER=nextcloud + - POSTGRES_PASSWORD={NEXTCLOUD_DB_PASSWORD} + - NEXTCLOUD_TRUSTED_DOMAINS={TRUSTED_DOMAINS} + volumes: + - nextcloud-data:/var/www/html +volumes: + nextcloud-db: + nextcloud-data: +`, + vars: [ + { + key: "NEXTCLOUD_DB_PASSWORD", + label: "Database password", + secret: true, + }, + { + key: "TRUSTED_DOMAINS", + label: "Trusted domains", + default: "localhost", + description: + "Space-separated list of domains allowed to use this Nextcloud, e.g. mycloud.example.com. Include your Dequel domain here.", + }, + ], +}; diff --git a/apps/api/src/api/templates/templates/paperless-ngx.ts b/apps/api/src/api/templates/templates/paperless-ngx.ts new file mode 100644 index 0000000..8f7e851 --- /dev/null +++ b/apps/api/src/api/templates/templates/paperless-ngx.ts @@ -0,0 +1,102 @@ +import type { DeployTemplate } from "../types"; + +export const paperlessNgxTemplate: DeployTemplate = { + key: "paperless-ngx", + name: "Paperless-ngx", + description: "Document management with OCR, full-text search and automatic tagging.", + category: "productivity", + docsUrl: "https://docs.paperless-ngx.com/", + service: "webserver", + port: 8000, + compose: `services: + broker: + image: redis:7 + pull_policy: always + restart: unless-stopped + gotenberg: + image: gotenberg/gotenberg:8 + pull_policy: always + restart: unless-stopped + command: + - gotenberg + - --chromium-disable-routes=1 + tika: + image: apache/tika:latest + pull_policy: always + restart: unless-stopped + db: + image: postgres:16 + pull_policy: always + restart: unless-stopped + environment: + - POSTGRES_DB=paperless + - POSTGRES_USER=paperless + - POSTGRES_PASSWORD={PAPERLESS_DB_PASSWORD} + volumes: + - paperless-db:/var/lib/postgresql/data + webserver: + image: ghcr.io/paperless-ngx/paperless-ngx:latest + pull_policy: always + restart: unless-stopped + depends_on: + - broker + - gotenberg + - tika + - db + environment: + - PAPERLESS_REDIS=redis://broker:6379 + - PAPERLESS_DBHOST=db + - PAPERLESS_DBNAME=paperless + - PAPERLESS_DBUSER=paperless + - PAPERLESS_DBPASS={PAPERLESS_DB_PASSWORD} + - PAPERLESS_SECRET_KEY={PAPERLESS_SECRET_KEY} + - PAPERLESS_TIME_ZONE={TIME_ZONE} + - PAPERLESS_TIKA_ENABLED=1 + - PAPERLESS_TIKA_ENDPOINT=http://tika:9998 + - PAPERLESS_TIKA_GOTENBERG_ENDPOINT=http://gotenberg:3000 + - PAPERLESS_ADMIN_USER={ADMIN_USER} + - PAPERLESS_ADMIN_PASSWORD={ADMIN_PASSWORD} + - PAPERLESS_CONSUMER_POLLING=30 + volumes: + - paperless-data:/usr/src/paperless/data + - paperless-media:/usr/src/paperless/media + - paperless-export:/usr/src/paperless/export + - paperless-consume:/usr/src/paperless/consume +volumes: + paperless-db: + paperless-data: + paperless-media: + paperless-export: + paperless-consume: +`, + vars: [ + { + key: "PAPERLESS_DB_PASSWORD", + label: "Database password", + secret: true, + }, + { + key: "PAPERLESS_SECRET_KEY", + label: "Secret key", + description: "Django secret used for session signing and password resets.", + secret: true, + }, + { + key: "ADMIN_USER", + label: "Admin username", + default: "admin", + }, + { + key: "ADMIN_PASSWORD", + label: "Admin password", + description: "Used to auto-create the first Paperless account on startup.", + secret: true, + }, + { + key: "TIME_ZONE", + label: "Timezone", + default: "UTC", + description: "IANA timezone, e.g. Europe/Berlin. Used for date display and document dating.", + }, + ], +}; diff --git a/apps/api/src/api/templates/templates/plausible.ts b/apps/api/src/api/templates/templates/plausible.ts new file mode 100644 index 0000000..cea3b88 --- /dev/null +++ b/apps/api/src/api/templates/templates/plausible.ts @@ -0,0 +1,80 @@ +import type { DeployTemplate } from "../types"; + +export const plausibleTemplate: DeployTemplate = { + key: "plausible", + name: "Plausible", + description: "Privacy-friendly web analytics, no cookies and GDPR compliant.", + category: "analytics", + docsUrl: "https://plausible.io/docs/self-hosting", + service: "plausible", + port: 8000, + compose: `services: + db: + image: postgres:16-alpine + pull_policy: always + restart: unless-stopped + environment: + - POSTGRES_USER=plausible + - POSTGRES_DB=plausible + - POSTGRES_PASSWORD={POSTGRES_PASSWORD} + volumes: + - plausible-db:/var/lib/postgresql/data + events-db: + image: clickhouse/clickhouse-server:24.12-alpine + pull_policy: always + restart: unless-stopped + environment: + - CLICKHOUSE_USER=plausible + - CLICKHOUSE_PASSWORD={CLICKHOUSE_PASSWORD} + - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 + volumes: + - plausible-events:/var/lib/clickhouse + - plausible-events-user:/etc/clickhouse-users + - plausible-events-logs:/etc/clickhouse-server/server_logs + plausible: + image: ghcr.io/plausible/community-edition:latest + pull_policy: always + restart: unless-stopped + depends_on: + - db + - events-db + command: sh -c "sleep 10 && /entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh run" + environment: + - BASE_URL={BASE_URL} + - SECRET_KEY_BASE={SECRET_KEY_BASE} + - DATABASE_URL=postgres://plausible:{POSTGRES_PASSWORD}@db:5432/plausible + - CLICKHOUSE_DATABASE_URL=http://events-db:8123/plausible_events_db + - CLICKHOUSE_DATABASE_USER=plausible + - CLICKHOUSE_DATABASE_PASSWORD={CLICKHOUSE_PASSWORD} +volumes: + plausible-db: + plausible-events: + plausible-events-user: + plausible-events-logs: +`, + vars: [ + { + key: "POSTGRES_PASSWORD", + label: "Database password", + secret: true, + }, + { + key: "CLICKHOUSE_PASSWORD", + label: "Events database password", + secret: true, + }, + { + key: "SECRET_KEY_BASE", + label: "Secret key", + description: "Server-side secret used to sign sessions and tokens.", + secret: true, + }, + { + key: "BASE_URL", + label: "Base URL", + default: "http://localhost", + description: + "Public URL of the Plausible dashboard, e.g. https://plausible.example.com. Update after deploy if your domain differs.", + }, + ], +}; diff --git a/apps/api/src/api/templates/templates/umami.ts b/apps/api/src/api/templates/templates/umami.ts new file mode 100644 index 0000000..802dcb2 --- /dev/null +++ b/apps/api/src/api/templates/templates/umami.ts @@ -0,0 +1,48 @@ +import type { DeployTemplate } from "../types"; + +export const umamiTemplate: DeployTemplate = { + key: "umami", + name: "Umami", + description: "Simple, privacy-focused web analytics with a clean dashboard.", + category: "analytics", + docsUrl: "https://umami.is/docs", + service: "umami", + port: 3000, + compose: `services: + db: + image: postgres:15-alpine + pull_policy: always + restart: unless-stopped + environment: + - POSTGRES_DB=umami + - POSTGRES_USER=umami + - POSTGRES_PASSWORD={UMAMI_DB_PASSWORD} + volumes: + - umami-db:/var/lib/postgresql/data + umami: + image: ghcr.io/umami-software/umami:latest + pull_policy: always + restart: unless-stopped + depends_on: + - db + environment: + - DATABASE_TYPE=postgresql + - DATABASE_URL=postgresql://umami:{UMAMI_DB_PASSWORD}@db:5432/umami + - APP_SECRET={APP_SECRET} +volumes: + umami-db: +`, + vars: [ + { + key: "UMAMI_DB_PASSWORD", + label: "Database password", + secret: true, + }, + { + key: "APP_SECRET", + label: "Session secret", + description: "Used by Umami to sign session cookies and tokens.", + secret: true, + }, + ], +}; diff --git a/apps/api/src/api/templates/templates/uptime-kuma.ts b/apps/api/src/api/templates/templates/uptime-kuma.ts new file mode 100644 index 0000000..730c4ac --- /dev/null +++ b/apps/api/src/api/templates/templates/uptime-kuma.ts @@ -0,0 +1,22 @@ +import type { DeployTemplate } from "../types"; + +export const uptimeKumaTemplate: DeployTemplate = { + key: "uptime-kuma", + name: "Uptime Kuma", + description: "Self-hosted monitoring tool with status pages and alerts.", + category: "monitoring", + docsUrl: "https://github.com/louislam/uptime-kuma", + service: "uptime-kuma", + port: 3001, + compose: `services: + uptime-kuma: + image: louislam/uptime-kuma:1 + pull_policy: always + restart: unless-stopped + volumes: + - uptime-kuma-data:/app/data +volumes: + uptime-kuma-data: +`, + vars: [], +}; diff --git a/apps/api/src/api/templates/templates/vaultwarden.ts b/apps/api/src/api/templates/templates/vaultwarden.ts new file mode 100644 index 0000000..a28020c --- /dev/null +++ b/apps/api/src/api/templates/templates/vaultwarden.ts @@ -0,0 +1,39 @@ +import type { DeployTemplate } from "../types"; + +export const vaultwardenTemplate: DeployTemplate = { + key: "vaultwarden", + name: "Vaultwarden", + description: "Lightweight password manager compatible with Bitwarden clients.", + category: "security", + docsUrl: "https://github.com/dani-garcia/vaultwarden", + service: "vaultwarden", + port: 80, + compose: `services: + vaultwarden: + image: vaultwarden/server:latest + pull_policy: always + restart: unless-stopped + environment: + - ADMIN_TOKEN={ADMIN_TOKEN} + - SIGNUPS_ALLOWED={SIGNUPS_ALLOWED} + volumes: + - vaultwarden-data:/data +volumes: + vaultwarden-data: +`, + vars: [ + { + key: "ADMIN_TOKEN", + label: "Admin panel token", + description: + "Token used to access the /admin panel. Keep it secret or flip SIGNUPS_ALLOWED down once accounts exist.", + secret: true, + }, + { + key: "SIGNUPS_ALLOWED", + label: "Allow public signups", + default: "true", + description: "Set to false to stop new account registrations once your accounts exist.", + }, + ], +}; diff --git a/apps/api/src/api/templates/templates/wordpress.ts b/apps/api/src/api/templates/templates/wordpress.ts new file mode 100644 index 0000000..f9055f3 --- /dev/null +++ b/apps/api/src/api/templates/templates/wordpress.ts @@ -0,0 +1,46 @@ +import type { DeployTemplate } from "../types"; + +export const wordPressTemplate: DeployTemplate = { + key: "wordpress", + name: "WordPress", + description: "Content management system with PHP and MySQL backend, bundled with MariaDB.", + category: "cms", + docsUrl: "https://hub.docker.com/_/wordpress", + service: "wordpress", + port: 80, + compose: `services: + db: + image: mariadb:11 + pull_policy: always + restart: unless-stopped + environment: + - MYSQL_ROOT_PASSWORD={MYSQL_ROOT_PASSWORD} + - MYSQL_DATABASE={MYSQL_DATABASE} + - MYSQL_USER={MYSQL_USER} + - MYSQL_PASSWORD={MYSQL_PASSWORD} + volumes: + - wp-db:/var/lib/mysql + wordpress: + image: wordpress:latest + pull_policy: always + restart: unless-stopped + depends_on: + - db + environment: + - WORDPRESS_DB_HOST=db + - WORDPRESS_DB_NAME={MYSQL_DATABASE} + - WORDPRESS_DB_USER={MYSQL_USER} + - WORDPRESS_DB_PASSWORD={MYSQL_PASSWORD} + volumes: + - wp-content:/var/www/html +volumes: + wp-db: + wp-content: +`, + vars: [ + { key: "MYSQL_DATABASE", label: "Database name", default: "wordpress" }, + { key: "MYSQL_USER", label: "Database user", default: "wordpress" }, + { key: "MYSQL_ROOT_PASSWORD", label: "MariaDB root password", secret: true }, + { key: "MYSQL_PASSWORD", label: "WordPress database password", secret: true }, + ], +}; diff --git a/apps/api/src/api/templates/types.ts b/apps/api/src/api/templates/types.ts new file mode 100644 index 0000000..4f29fb2 --- /dev/null +++ b/apps/api/src/api/templates/types.ts @@ -0,0 +1,56 @@ +import { randomBytes } from "node:crypto"; + +export interface TemplateVar { + key: string; + label: string; + description?: string; + default?: string; + secret?: boolean; +} + +export interface DeployTemplate { + key: string; + name: string; + description: string; + category: string; + docsUrl?: string; + service: string; + port: number; + compose: string; + vars: TemplateVar[]; +} + +export interface RenderedTemplate { + compose: string; + vars: { key: string; value: string; secret: boolean }[]; +} + +const generateSecret = (): string => randomBytes(24).toString("base64url"); + +export const renderCompose = (template: DeployTemplate, values: Record = {}): RenderedTemplate => { + const resolved: { key: string; value: string; secret: boolean }[] = []; + let result = template.compose.trim(); + if (!result.startsWith("version:")) { + result = `version: "3.8"\n\n${result}`; + } + + for (const v of template.vars) { + const provided = values[v.key]?.trim(); + const value = provided ? provided : v.default ? v.default : v.secret ? generateSecret() : undefined; + if (value === undefined) { + throw new Error(`Missing value for "${v.key}" (${v.label})`); + } + if (value.includes("\n") || value.includes("\r")) { + throw new Error(`Value for "${v.key}" must not contain line breaks`); + } + result = result.split(`{${v.key}}`).join(`\${${v.key}}`); + resolved.push({ key: v.key, value, secret: v.secret ?? false }); + } + + const leftover = result.match(/(? ok(await listVolumes(params.id))) + .get("/projects/:id/volumes", async ({ params }) => + ok( + (await listVolumes(params.id)).map((v) => ({ + ...v, + managed: isComposeStackVolume(v.dockerVolumeName, params.id), + })), + ), + ) .post("/projects/:id/volumes", async ({ params, body }: any) => created( await createVolume({ @@ -21,6 +29,15 @@ export const volumesRoutes = new Elysia() return ok(volume); }) .delete("/volumes/:id", async ({ params: { id }, set }) => { + const volume = await getVolumeById(id); + if (!volume) { + set.status = 404; + return fail("Volume not found"); + } + if (isComposeStackVolume(volume.dockerVolumeName, volume.projectId)) { + set.status = 400; + return fail("This volume belongs to the project's Docker Compose stack - edit the compose file to change it"); + } const deleted = await deleteVolume(id); if (!deleted) { set.status = 404; diff --git a/apps/api/src/db/__tests__/databases.test.ts b/apps/api/src/db/__tests__/databases.test.ts index 64c49d8..13ed47d 100644 --- a/apps/api/src/db/__tests__/databases.test.ts +++ b/apps/api/src/db/__tests__/databases.test.ts @@ -8,8 +8,8 @@ const runRepoScenarios = (): any => { const result = spawnSync("bun", [runnerPath], { env: { ...process.env, - DATABASE_URL: "postgresql://dequel:dequel@localhost:5433/dequel", - TEST_DATABASE_URL: "postgresql://dequel:dequel@localhost:5433/dequel", + DATABASE_URL: process.env.TEST_DATABASE_URL ?? "postgresql://dequel:dequel@localhost:5433/dequel", + TEST_DATABASE_URL: process.env.TEST_DATABASE_URL ?? "postgresql://dequel:dequel@localhost:5433/dequel", }, encoding: "utf8", }); diff --git a/apps/api/src/db/__tests__/shared-env-links.test.ts b/apps/api/src/db/__tests__/shared-env-links.test.ts index 7598ef1..94b2de9 100644 --- a/apps/api/src/db/__tests__/shared-env-links.test.ts +++ b/apps/api/src/db/__tests__/shared-env-links.test.ts @@ -8,8 +8,8 @@ const runScenarios = (): any => { const result = spawnSync("bun", [runnerPath], { env: { ...process.env, - DATABASE_URL: "postgresql://dequel:dequel@localhost:5433/dequel", - TEST_DATABASE_URL: "postgresql://dequel:dequel@localhost:5433/dequel", + DATABASE_URL: process.env.TEST_DATABASE_URL ?? "postgresql://dequel:dequel@localhost:5433/dequel", + TEST_DATABASE_URL: process.env.TEST_DATABASE_URL ?? "postgresql://dequel:dequel@localhost:5433/dequel", }, encoding: "utf8", }); diff --git a/apps/api/src/db/__tests__/volumes-sync.test.ts b/apps/api/src/db/__tests__/volumes-sync.test.ts new file mode 100644 index 0000000..17b35ca --- /dev/null +++ b/apps/api/src/db/__tests__/volumes-sync.test.ts @@ -0,0 +1,75 @@ +import { afterAll, beforeAll, describe, expect, it } from "bun:test"; +import { drizzle } from "drizzle-orm/node-postgres"; +import type { Pool } from "pg"; +import { setDbProvider } from "../db-provider"; +import { isComposeStackVolume, listVolumes, syncComposeStackVolumes } from "../repo/volumes"; +import * as schema from "../schema"; +import { createTestPool, truncateAllTables } from "../test-helper"; + +let pool: Pool; +const PROJECT_ID = "vol-sync-proj-1"; + +const seed = async () => { + await pool.query( + `INSERT INTO projects (id, name, source_type, created_at, updated_at) + VALUES ($1, 'Volumes Sync Project', 'upload', NOW(), NOW()) ON CONFLICT (id) DO NOTHING`, + [PROJECT_ID], + ); +}; + +beforeAll(async () => { + pool = createTestPool(); + const db = drizzle(pool, { schema }); + setDbProvider(async () => db); + await truncateAllTables(pool); + await seed(); +}); + +afterAll(async () => { + if (pool) await pool.end(); +}); + +describe("compose stack volume sync", () => { + it("registers compose volumes with joined mount paths", async () => { + await syncComposeStackVolumes(PROJECT_ID, [ + { volumeKey: "dq-volsyncproj1-wp-db", containerPath: "/var/lib/mysql" }, + { volumeKey: "dq-volsyncproj1-wp-db", containerPath: "/var/lib/mysql2" }, + { volumeKey: "dq-volsyncproj1-wp-content", containerPath: "/var/www/html" }, + ]); + const rows = await listVolumes(PROJECT_ID); + expect(rows).toHaveLength(2); + const dbVolume = rows.find((v) => v.dockerVolumeName === "dq-volsyncproj1-wp-db"); + expect(dbVolume?.mountPath).toBe("/var/lib/mysql, /var/lib/mysql2"); + expect(rows.find((v) => v.dockerVolumeName === "dq-volsyncproj1-wp-content")?.mountPath).toBe("/var/www/html"); + }); + + it("is idempotent and prunes dropped compose volumes while keeping platform volumes", async () => { + await syncComposeStackVolumes(PROJECT_ID, [ + { volumeKey: "dq-volsyncproj1-wp-db", containerPath: "/var/lib/mysql" }, + ]); + const rows = await listVolumes(PROJECT_ID); + expect(rows).toHaveLength(1); + expect(rows.find((v) => v.dockerVolumeName === "dq-volsyncproj1-wp-db")?.mountPath).toBe("/var/lib/mysql"); + expect(rows.find((v) => v.dockerVolumeName === "dq-volsyncproj1-wp-content")).toBeUndefined(); + + await pool.query( + `INSERT INTO volumes (id, project_id, mount_path, docker_volume_name, created_at) + VALUES ('vol-platform-keep', $1, '/app/data', 'vol-ab12cd34', NOW()) ON CONFLICT (id) DO NOTHING`, + [PROJECT_ID], + ); + await syncComposeStackVolumes(PROJECT_ID, [ + { volumeKey: "dq-volsyncproj1-wp-db", containerPath: "/var/lib/mysql" }, + ]); + const after = await listVolumes(PROJECT_ID); + expect(after).toHaveLength(2); + expect(after.find((v) => v.dockerVolumeName === "vol-ab12cd34")).toBeDefined(); + expect(after.find((v) => v.dockerVolumeName === "dq-volsyncproj1-wp-db")).toBeDefined(); + }); + + it("identifies compose volumes by stable name prefix", () => { + expect(isComposeStackVolume("dq-volsyncproj1-wp-db", "vol-sync-proj-1")).toBe(true); + expect(isComposeStackVolume("dq-a08b4c0da2084fb58617-umami-db", "a08b4c0d-a208-4fb5-8617-6b43db10c74c")).toBe(true); + expect(isComposeStackVolume("vol-ab12cd34", "vol-sync-proj-1")).toBe(false); + expect(isComposeStackVolume(null, "vol-sync-proj-1")).toBe(false); + }); +}); diff --git a/apps/api/src/db/repo/env-vars.ts b/apps/api/src/db/repo/env-vars.ts index 4867e1b..27283ce 100644 --- a/apps/api/src/db/repo/env-vars.ts +++ b/apps/api/src/db/repo/env-vars.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { and, asc, eq } from "drizzle-orm"; +import { and, asc, eq, or } from "drizzle-orm"; import type { CreateEnvironmentVariableInput, EnvironmentVariable } from "../../types"; import { config } from "../../utils/config"; import { decryptValue, encryptValue } from "../../utils/crypto"; @@ -111,7 +111,14 @@ export const listEnvironmentVariablesForDeploy = async ( ): Promise<{ key: string; value: string }[]> => { const db = await getDb(); const cond = environment - ? and(eq(environmentVariables.projectId, projectId), eq(environmentVariables.environment, environment)) + ? and( + eq(environmentVariables.projectId, projectId), + or( + eq(environmentVariables.environment, environment), + eq(environmentVariables.environment, ""), + eq(environmentVariables.environment, "all"), + ), + ) : eq(environmentVariables.projectId, projectId); const rows = await db .select({ diff --git a/apps/api/src/db/repo/volumes.ts b/apps/api/src/db/repo/volumes.ts index 19bfbd6..0750a64 100644 --- a/apps/api/src/db/repo/volumes.ts +++ b/apps/api/src/db/repo/volumes.ts @@ -14,6 +14,78 @@ const mapVolume = (row: typeof volumes.$inferSelect): Volume => ({ createdAt: row.createdAt, }); +export const composeVolumeNamePrefix = (projectId: string): string => { + const idPart = projectId.replace(/[^a-zA-Z0-9]/g, "").slice(0, 20); + return `dq-${idPart}-`; +}; + +export const isComposeStackVolume = (dockerVolumeName: string | null | undefined, projectId: string): boolean => + Boolean(dockerVolumeName?.startsWith(composeVolumeNamePrefix(projectId))); + +const safeMountPath = (raw: string): string => { + const path = raw.split(":")[0]?.trim() ?? ""; + return path.startsWith("/") ? path : ""; +}; + +export interface ComposeVolumeMount { + volumeKey: string; + containerPath: string; +} + +export const joinComposeMounts = (mounts: ComposeVolumeMount[]): ComposeVolumeEntry[] => { + const byVolume = new Map(); + for (const m of mounts) { + const path = safeMountPath(m.containerPath); + if (!path) continue; + const list = byVolume.get(m.volumeKey) ?? []; + if (!list.includes(path)) list.push(path); + byVolume.set(m.volumeKey, list); + } + return [...byVolume.entries()].map(([volumeKey, paths]) => ({ + dockerVolumeName: volumeKey, + mountPath: paths.join(", "), + })); +}; + +export interface ComposeVolumeEntry { + dockerVolumeName: string; + mountPath: string; +} + +export const syncComposeStackVolumes = async (projectId: string, mounts: ComposeVolumeMount[]): Promise => { + const entries = joinComposeMounts(mounts); + const db = await getDb(); + const existing = await db.select().from(volumes).where(eq(volumes.projectId, projectId)).execute(); + const prefix = composeVolumeNamePrefix(projectId); + const desired = new Set(entries.map((e) => e.dockerVolumeName)); + + for (const entry of entries) { + const matched = existing.find((row) => row.dockerVolumeName === entry.dockerVolumeName); + if (matched) { + if (matched.mountPath !== entry.mountPath) { + await db.update(volumes).set({ mountPath: entry.mountPath }).where(eq(volumes.id, matched.id)).execute(); + } + continue; + } + await db + .insert(volumes) + .values({ + id: randomUUID(), + projectId, + mountPath: entry.mountPath, + dockerVolumeName: entry.dockerVolumeName, + createdAt: now(), + }) + .execute(); + } + + for (const row of existing) { + if (row.dockerVolumeName?.startsWith(prefix) && !desired.has(row.dockerVolumeName)) { + await db.delete(volumes).where(eq(volumes.id, row.id)).execute(); + } + } +}; + export const createVolume = async (input: CreateVolumeInput): Promise => { const id = randomUUID(); const timestamp = now(); diff --git a/apps/api/src/db/test-helper.ts b/apps/api/src/db/test-helper.ts index 1e5b9b7..317999d 100644 --- a/apps/api/src/db/test-helper.ts +++ b/apps/api/src/db/test-helper.ts @@ -3,7 +3,7 @@ import { Pool } from "pg"; import { setDbProvider } from "./db-provider"; import * as schema from "./schema"; -const TEST_DATABASE_URL = process.env.TEST_DATABASE_URL ?? "postgresql://dequel:dequel@localhost:5433/dequel"; +const TEST_DATABASE_URL = process.env.TEST_DATABASE_URL ?? "postgresql://dequel:dequel@localhost:5434/dequel"; const TABLE_NAMES = [ "agent_credentials", diff --git a/apps/api/src/executors/__tests__/dispatch.test.ts b/apps/api/src/executors/__tests__/dispatch.test.ts index 3ed6ebb..d30187a 100644 --- a/apps/api/src/executors/__tests__/dispatch.test.ts +++ b/apps/api/src/executors/__tests__/dispatch.test.ts @@ -20,6 +20,7 @@ describe("remote SSH build script", () => { it("embeds inputs with single-quote escaping", () => { const script = buildRemoteDeployScript(input); expect(script).toContain("set -euo pipefail"); + expect(script).toContain("trap 'rm -rf \"$PROJECT_DIR\"' EXIT"); expect(script).toContain("git clone --depth 1 'https://github.com/example/api.git' ."); expect(script).toContain("--env 'NODE_ENV=production'"); expect(script).toContain("--name 'example-api-deploym:latest'"); @@ -63,11 +64,11 @@ describe("remote SSH build script", () => { it("installs bun before generating the config", () => { const script = buildRemoteDeployScript(input); - expect(script).toContain("bun-linux-${BUN_ARCH}.zip"); + expect(script).toContain("bun-linux-$BUN_ARCH.zip"); expect(script).toContain("python3 -m zipfile -e"); expect(script).toContain("bun.sh/install"); expect(script).toContain('export PATH="$HOME/.bun/bin:$PATH"'); - expect(script.indexOf("bun-linux-${BUN_ARCH}.zip")).toBeLessThan(script.indexOf(".dequel-railpack-gen.ts")); + expect(script.indexOf("bun-linux-$BUN_ARCH.zip")).toBeLessThan(script.indexOf(".dequel-railpack-gen.ts")); }); it("passes source dir and project type to the generator", () => { diff --git a/apps/api/src/executors/ssh-build-script.ts b/apps/api/src/executors/ssh-build-script.ts index 2c897c1..d6798c2 100644 --- a/apps/api/src/executors/ssh-build-script.ts +++ b/apps/api/src/executors/ssh-build-script.ts @@ -70,6 +70,7 @@ export const buildRemoteDeployScript = (input: SshBuildScriptInput): string => { 'echo "[build] Ensuring workspace"', 'rm -rf "$PROJECT_DIR"', 'mkdir -p "$PROJECT_DIR"', + "trap 'rm -rf \"$PROJECT_DIR\"' EXIT", 'cd "$PROJECT_DIR"', "", "if command -v railpack >/dev/null 2>&1; then", @@ -103,9 +104,9 @@ export const buildRemoteDeployScript = (input: SshBuildScriptInput): string => { " if command -v python3 >/dev/null 2>&1; then", ' mkdir -p "$HOME/.bun/bin"', ' BUN_TMP="$(mktemp -d)"', - ' curl -fsSL -o "$BUN_TMP/bun.zip" "https://github.com/oven-sh/bun/releases/latest/download/bun-linux-${BUN_ARCH}.zip"', + ' curl -fsSL -o "$BUN_TMP/bun.zip" "https://github.com/oven-sh/bun/releases/latest/download/bun-linux-$BUN_ARCH.zip"', ' python3 -m zipfile -e "$BUN_TMP/bun.zip" "$BUN_TMP"', - ' mv "$BUN_TMP/bun-linux-${BUN_ARCH}/bun" "$HOME/.bun/bin/bun"', + ' mv "$BUN_TMP/bun-linux-$BUN_ARCH/bun" "$HOME/.bun/bin/bun"', ' chmod +x "$HOME/.bun/bin/bun"', ' rm -rf "$BUN_TMP"', " elif command -v unzip >/dev/null 2>&1; then", @@ -164,3 +165,15 @@ export const parseRemoteBuildResult = (stdout: string): RemoteBuildResult | null return null; } }; + +export const cleanupRemoteWorkspace = async ( + server: any, + deploymentId: string, + run?: (server: any, script: string) => Promise, +): Promise => { + const { config } = await import("../utils/config"); + const { runRemoteScript } = await import("../utils/ssh"); + const remoteWorkspace = config.workspaceRoot.startsWith("/app") ? "$HOME/.dequel/workspace" : config.workspaceRoot; + const runFn = run ?? runRemoteScript; + await runFn(server, `rm -rf "${remoteWorkspace}/${deploymentId}"`).catch(() => {}); +}; diff --git a/apps/api/src/executors/ssh.ts b/apps/api/src/executors/ssh.ts index 000ea69..bf68d17 100644 --- a/apps/api/src/executors/ssh.ts +++ b/apps/api/src/executors/ssh.ts @@ -6,7 +6,7 @@ import { config } from "../utils/config"; import { CANCELLED_FAILURE_REASON } from "../utils/failure-outcome"; import { RemoteScriptAbortedError, removeRemoteCaddyRoute, runRemoteScript, syncRemoteCaddyRoute } from "../utils/ssh"; import { emitLog } from "./logging"; -import { buildRemoteDeployScript, parseRemoteBuildResult } from "./ssh-build-script"; +import { buildRemoteDeployScript, cleanupRemoteWorkspace, parseRemoteBuildResult } from "./ssh-build-script"; import { buildRemoteComposeDestroyScript, buildRemoteComposeScript, @@ -157,7 +157,7 @@ const deployComposeRemote = async (deployment: Deployment, project: Project, ser } const { buildCaddySnippet } = await import("../utils/domain-verifier"); - const { shouldRouteViaIngress, syncIngressRoute, upsertIngressRoute } = await import("../utils/ingress"); + const { shouldRouteViaIngress } = await import("../utils/ingress"); const { baseDomainFor } = await import("../utils/routes"); const { getIngressServer } = await import("../utils/ingress"); const { upsertRoute } = await import("../db/repo"); @@ -174,7 +174,7 @@ const deployComposeRemote = async (deployment: Deployment, project: Project, ser if (!snippet.trim() || snippet.trim().startsWith(":")) { const fallbackDomain = `${slug}.${rawBaseDomain === "localhost" ? "localhost" : rawBaseDomain}`; const domain = rawBaseDomain === "localhost" ? `${fallbackDomain}:80` : fallbackDomain; - snippet = `${domain} {\n log {\n output stdout\n format json\n }\n reverse_proxy ${primary.container}:${primary.port} {\n header_up Host {upstream_hostport}\n }\n}\n`; + snippet = `${domain} {\n log {\n output stdout\n format json\n }\n reverse_proxy ${primary.container}:${primary.port}\n}\n`; } const { isDbServiceName } = await import("../utils/compose-ingress"); for (const svc of webServices) { @@ -190,7 +190,7 @@ const deployComposeRemote = async (deployment: Deployment, project: Project, ser domains.push(`api.${slug}.${baseDomainForCaddy}`); } } - snippet += `\n${domains.join(", ")} {\n log {\n output stdout\n format json\n }\n reverse_proxy ${svc.container}:${svc.port} {\n header_up Host {upstream_hostport}\n }\n}\n`; + snippet += `\n${domains.join(", ")} {\n log {\n output stdout\n format json\n }\n reverse_proxy ${svc.container}:${svc.port}\n}\n`; } const hostname = `${slug}.${baseDomainFor()}`; @@ -380,6 +380,7 @@ export const sshExecutor: DeploymentExecutor = { } } finally { remoteBuilds.delete(deployment.id); + await cleanupRemoteWorkspace(server, deployment.id); } }, diff --git a/apps/api/src/orchestrator/__tests__/compose.test.ts b/apps/api/src/orchestrator/__tests__/compose.test.ts index 8de6cc2..8ccc1c2 100644 --- a/apps/api/src/orchestrator/__tests__/compose.test.ts +++ b/apps/api/src/orchestrator/__tests__/compose.test.ts @@ -1,8 +1,56 @@ import { afterEach, describe, expect, mock, test } from "bun:test"; -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { parseAllComposeServices, parseComposeTarget, parseContainerTargetPort } from "../compose"; +import { + evaluateComposeContainerState, + parseAllComposeServices, + parseComposeTarget, + parseContainerTargetPort, +} from "../compose"; + +describe("evaluateComposeContainerState", () => { + test("running after stability window is ok", () => { + const v = evaluateComposeContainerState("c", "running", { primary: true, everSawRunning: true }); + expect(v.kind).toBe("ok"); + }); + + test("first running observation still requires a stability re-check", () => { + const v = evaluateComposeContainerState("c", "running", { primary: true, everSawRunning: false }); + expect(v.kind).toBe("retry"); + }); + + test("created containers retry", () => { + const v = evaluateComposeContainerState("c", "created", { primary: true }); + expect(v.kind).toBe("retry"); + }); + + test("missing containers retry", () => { + const v = evaluateComposeContainerState("c", null, { primary: true }); + expect(v.kind).toBe("retry"); + }); + + test("primary crash loops are fatal", () => { + const v = evaluateComposeContainerState("c", "restarting", { primary: true }); + expect(v.kind).toBe("fatal"); + }); + + test("primary exits are fatal", () => { + const v = evaluateComposeContainerState("c", "exited", { primary: true }); + expect(v.kind).toBe("fatal"); + }); + + test("secondary exits retry instead of failing the deployment", () => { + const v = evaluateComposeContainerState("c", "exited", { primary: false }); + expect(v.kind).toBe("retry"); + }); + + test("unknown states retry with the state surfaced", () => { + const v = evaluateComposeContainerState("c", "paused", { primary: false }); + expect(v.kind).toBe("retry"); + expect(v.reason).toContain("paused"); + }); +}); describe("parseContainerTargetPort", () => { test("parses number ports", () => { @@ -213,3 +261,42 @@ describe("spawnComposeCommand env merge with project env vars", () => { } }); }); + +describe("writeComposeEnvFile", () => { + const runFailingBuild = async (dir: string, envVars: Record) => { + const { buildWithCompose } = await import("../compose"); + try { + await buildWithCompose(dir, "envfile-proj", async () => {}, null, envVars); + } catch { + // compose build fails fast on the bad build context; the .env write happens before spawn + } + }; + + const withComposeDir = async (fn: (dir: string) => Promise) => { + const dir = mkdtempSync(join(tmpdir(), "dequel-compose-envfile-")); + writeFileSync(join(dir, "docker-compose.yml"), `services:\n app:\n build: {NEVER_EXISTS}\n`); + try { + await fn(dir); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }; + + test("writes project env vars as an escaped .env beside the compose file", async () => { + await withComposeDir(async (dir) => { + await runFailingBuild(dir, { SECRET_ONE: "pa55$$word", MULTI: 'a\nb"c', "1BAD-KEY": "skip" }); + const content = readFileSync(join(dir, ".env"), "utf8"); + expect(content).toContain('SECRET_ONE="pa55$$$$word"'); + expect(content).toContain('MULTI="a\\nb\\"c"'); + expect(content).not.toContain("1BAD-KEY"); + }); + }); + + test("does not overwrite an existing .env from the archive", async () => { + await withComposeDir(async (dir) => { + writeFileSync(join(dir, ".env"), "FROM_ARCHIVE=1\n"); + await runFailingBuild(dir, { SECRET_ONE: "fresh" }); + expect(readFileSync(join(dir, ".env"), "utf8")).toBe("FROM_ARCHIVE=1\n"); + }); + }); +}); diff --git a/apps/api/src/orchestrator/__tests__/failover-stale-routes.test.ts b/apps/api/src/orchestrator/__tests__/failover-stale-routes.test.ts index 7517e94..5002a42 100644 --- a/apps/api/src/orchestrator/__tests__/failover-stale-routes.test.ts +++ b/apps/api/src/orchestrator/__tests__/failover-stale-routes.test.ts @@ -8,7 +8,10 @@ const run = () => exec("bun", ["run", "src/orchestrator/__tests__/failover-stale-routes-runner.ts"], { cwd: `${import.meta.dir}/../../..`, timeout: 30000, - env: { ...process.env, TEST_DATABASE_URL: "postgresql://dequel:dequel@localhost:5433/dequel" }, + env: { + ...process.env, + TEST_DATABASE_URL: process.env.TEST_DATABASE_URL ?? "postgresql://dequel:dequel@localhost:5433/dequel", + }, }); const parse = (stdout: string) => { diff --git a/apps/api/src/orchestrator/__tests__/ingress-sync.test.ts b/apps/api/src/orchestrator/__tests__/ingress-sync.test.ts index 3f4a289..f3324f0 100644 --- a/apps/api/src/orchestrator/__tests__/ingress-sync.test.ts +++ b/apps/api/src/orchestrator/__tests__/ingress-sync.test.ts @@ -8,7 +8,10 @@ const run = () => exec("bun", ["run", "src/orchestrator/__tests__/ingress-sync-runner.ts"], { cwd: `${import.meta.dir}/../../..`, timeout: 30000, - env: { ...process.env, TEST_DATABASE_URL: "postgresql://dequel:dequel@localhost:5433/dequel" }, + env: { + ...process.env, + TEST_DATABASE_URL: process.env.TEST_DATABASE_URL ?? "postgresql://dequel:dequel@localhost:5433/dequel", + }, }); const parse = (stdout: string) => { diff --git a/apps/api/src/orchestrator/__tests__/pipeline-cleanup-runner.ts b/apps/api/src/orchestrator/__tests__/pipeline-cleanup-runner.ts new file mode 100644 index 0000000..c4493b5 --- /dev/null +++ b/apps/api/src/orchestrator/__tests__/pipeline-cleanup-runner.ts @@ -0,0 +1,153 @@ +import { beforeEach, describe, expect, it, mock } from "bun:test"; + +const fileUrl = (relPath: string) => new URL(relPath, import.meta.url).toString(); + +let cleanupFailedDeploymentCalled = false; +let buildShouldFail = false; + +beforeEach(() => { + cleanupFailedDeploymentCalled = false; + buildShouldFail = false; +}); + +const mockDb = { + getDeploymentById: mock(() => + Promise.resolve({ + id: "dep-1", + projectId: "proj-1", + sourceType: "git", + sourceRef: "https://github.com/test/repo.git", + branch: "main", + status: "pending", + commitSha: null, + clearCache: false, + environment: null, + imageTag: null, + containerName: null, + }), + ), + getProjectById: mock(() => + Promise.resolve({ + id: "proj-1", + name: "Test Project", + sourceDir: null, + port: 3000, + cpuLimit: null, + memoryLimitMb: null, + repoUrl: "https://github.com/test/repo.git", + }), + ), + updateDeploymentStatus: mock(() => Promise.resolve()), + updateDeploymentCommitSha: mock(() => Promise.resolve()), + appendLog: mock(async (_depId: string, stage: string, message: string) => { + if (stage === "system" && message === "Deployment is running") { + throw new Error("Simulated DB write failure"); + } + return { sequence: 1 }; + }), + getProjectGithubToken: mock(() => Promise.resolve(null)), + listEnvironmentVariablesForDeploy: mock(() => Promise.resolve([])), + listVolumes: mock(() => Promise.resolve([])), + listDeployments: mock(() => Promise.resolve([])), + listProjectEvents: mock(() => Promise.resolve([])), + listRoutesByDeployment: mock(() => Promise.resolve([])), + listAllDatabases: mock(() => Promise.resolve([])), + deleteDeploymentAndLogs: mock(() => Promise.resolve()), + getScalingPolicy: mock(() => Promise.resolve(null)), + getServerById: mock(() => Promise.resolve(null)), + createAgentJob: mock(() => Promise.resolve("job-1")), + upsertRoute: mock(() => Promise.resolve({})), + updateServerStatus: mock(() => Promise.resolve()), + createAgentRegistrationToken: mock(() => Promise.resolve({ token: "dqr_test", expiresAt: new Date().toISOString() })), + getPlatformSettings: mock(() => Promise.resolve({ ingressServerId: null })), + listServers: mock(() => Promise.resolve([])), + listProjects: mock(() => Promise.resolve([])), + ensureLocalServer: mock(() => Promise.resolve()), + deleteRoutesByDeployment: mock(() => Promise.resolve()), + updateRouteStatus: mock(() => Promise.resolve()), + updateDomainValidation: mock(() => Promise.resolve()), + listDomains: mock(() => Promise.resolve([])), + createDeploymentEvent: mock(() => Promise.resolve()), + recordDeploymentFailure: mock(() => Promise.resolve({ outcome: "recorded" })), + recordDeploymentCancellation: mock(() => Promise.resolve({ outcome: "recorded" })), + getLogs: mock(() => Promise.resolve([])), +}; + +mock.module(fileUrl("../../db/repo"), () => mockDb); + +mock.module(fileUrl("../runtime"), () => ({ + run: mock(() => Promise.resolve("")), + deployContainer: mock(() => + Promise.resolve({ + containerName: "test-project-abc12345", + liveUrl: "http://test-project.localhost", + }), + ), + cleanupFailedDeployment: mock(async () => { + cleanupFailedDeploymentCalled = true; + }), + ensureContainerRunning: mock(() => Promise.resolve()), + reloadCaddy: mock(() => Promise.resolve()), + tryRun: mock(() => Promise.resolve("")), +})); + +mock.module(fileUrl("../railpack"), () => ({ + buildWithRailpack: mock( + async (_workspace: string, _imageTag: string, _onLog: (line: string) => Promise, _opts?: unknown) => { + if (buildShouldFail) throw new Error("Build failed"); + }, + ), + CancelledError: class CancelledError extends Error { + constructor() { + super("Cancelled"); + } + }, +})); + +mock.module(fileUrl("../source"), () => ({ + prepareSourceWorkspace: mock(() => Promise.resolve("/tmp/workspace/dep-1")), + prepareUploadWorkspace: mock(() => Promise.resolve("/tmp/workspace/dep-1")), + cleanupWorkspace: mock(() => Promise.resolve()), + getHeadSha: mock(() => Promise.resolve("abc123def456")), +})); + +mock.module(fileUrl("../../utils/grafana"), () => ({ + ensureProjectDashboard: mock(() => Promise.resolve()), +})); + +mock.module("ioredis", () => ({ + default: class FakeRedis { + queue: string[] = []; + rpush = mock(async (_key: string, ...values: string[]) => { + this.queue.push(...values); + }); + blpop = mock(async () => { + const v = this.queue.shift(); + return v ? ["queue", v] : null; + }); + lrem = mock(async () => {}); + zadd = mock(async () => {}); + zrem = mock(async () => {}); + zrangebyscore = mock(async () => []); + quit = mock(async () => {}); + }, +})); + +const { PipelineOrchestrator } = await import("../pipeline"); + +describe("runDeployment cleanup behavior", () => { + it("does NOT call cleanupFailedDeployment when failure occurs after deployContainer succeeds", async () => { + const orchestrator = new PipelineOrchestrator(); + await (orchestrator as any).runDeployment("dep-1"); + + expect(cleanupFailedDeploymentCalled).toBe(false); + }); + + it("calls cleanupFailedDeployment when failure occurs during build (before deployContainer)", async () => { + buildShouldFail = true; + const orchestrator = new PipelineOrchestrator(); + await (orchestrator as any).runDeployment("dep-1"); + + expect(cleanupFailedDeploymentCalled).toBe(true); + }); +}); diff --git a/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts b/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts index f3beac6..979911a 100644 --- a/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts +++ b/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts @@ -1,152 +1,25 @@ -import { beforeEach, describe, expect, it, mock } from "bun:test"; - -const fileUrl = (relPath: string) => new URL(relPath, import.meta.url).toString(); - -let cleanupFailedDeploymentCalled = false; -let buildShouldFail = false; - -beforeEach(() => { - cleanupFailedDeploymentCalled = false; - buildShouldFail = false; -}); - -const mockDb = { - getDeploymentById: mock(() => - Promise.resolve({ - id: "dep-1", - projectId: "proj-1", - sourceType: "git", - sourceRef: "https://github.com/test/repo.git", - branch: "main", - status: "pending", - commitSha: null, - clearCache: false, - environment: null, - imageTag: null, - containerName: null, - }), - ), - getProjectById: mock(() => - Promise.resolve({ - id: "proj-1", - name: "Test Project", - sourceDir: null, - port: 3000, - cpuLimit: null, - memoryLimitMb: null, - repoUrl: "https://github.com/test/repo.git", - }), - ), - updateDeploymentStatus: mock(() => Promise.resolve()), - updateDeploymentCommitSha: mock(() => Promise.resolve()), - appendLog: mock(async (_depId: string, stage: string, message: string) => { - if (stage === "system" && message === "Deployment is running") { - throw new Error("Simulated DB write failure"); - } - return { sequence: 1 }; - }), - getProjectGithubToken: mock(() => Promise.resolve(null)), - listEnvironmentVariablesForDeploy: mock(() => Promise.resolve([])), - listVolumes: mock(() => Promise.resolve([])), - listDeployments: mock(() => Promise.resolve([])), - listProjectEvents: mock(() => Promise.resolve([])), - listRoutesByDeployment: mock(() => Promise.resolve([])), - listAllDatabases: mock(() => Promise.resolve([])), - deleteDeploymentAndLogs: mock(() => Promise.resolve()), - getScalingPolicy: mock(() => Promise.resolve(null)), - getServerById: mock(() => Promise.resolve(null)), - createAgentJob: mock(() => Promise.resolve("job-1")), - upsertRoute: mock(() => Promise.resolve({})), - updateServerStatus: mock(() => Promise.resolve()), - createAgentRegistrationToken: mock(() => Promise.resolve({ token: "dqr_test", expiresAt: new Date().toISOString() })), - getPlatformSettings: mock(() => Promise.resolve({ ingressServerId: null })), - listServers: mock(() => Promise.resolve([])), - listProjects: mock(() => Promise.resolve([])), - ensureLocalServer: mock(() => Promise.resolve()), - deleteRoutesByDeployment: mock(() => Promise.resolve()), - updateRouteStatus: mock(() => Promise.resolve()), - updateDomainValidation: mock(() => Promise.resolve()), - listDomains: mock(() => Promise.resolve([])), - createDeploymentEvent: mock(() => Promise.resolve()), - recordDeploymentFailure: mock(() => Promise.resolve({ outcome: "recorded" })), - recordDeploymentCancellation: mock(() => Promise.resolve({ outcome: "recorded" })), -}; - -mock.module(fileUrl("../../db/repo"), () => mockDb); - -mock.module(fileUrl("../runtime"), () => ({ - run: mock(() => Promise.resolve("")), - deployContainer: mock(() => - Promise.resolve({ - containerName: "test-project-abc12345", - liveUrl: "http://test-project.localhost", - }), - ), - cleanupFailedDeployment: mock(async () => { - cleanupFailedDeploymentCalled = true; - }), - ensureContainerRunning: mock(() => Promise.resolve()), - reloadCaddy: mock(() => Promise.resolve()), - tryRun: mock(() => Promise.resolve("")), -})); - -mock.module(fileUrl("../railpack"), () => ({ - buildWithRailpack: mock( - async (_workspace: string, _imageTag: string, _onLog: (line: string) => Promise, _opts?: unknown) => { - if (buildShouldFail) throw new Error("Build failed"); +import { describe, expect, it } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { join } from "node:path"; + +const runCleanupSuite = () => { + const result = spawnSync("bun", ["test", join(import.meta.dir, "pipeline-cleanup-runner.ts")], { + cwd: join(import.meta.dir, "../.."), + env: { + ...process.env, + TEST_DATABASE_URL: process.env.TEST_DATABASE_URL ?? "postgresql://dequel:dequel@localhost:5434/dequel", }, - ), - CancelledError: class CancelledError extends Error { - constructor() { - super("Cancelled"); - } - }, -})); - -mock.module(fileUrl("../source"), () => ({ - prepareSourceWorkspace: mock(() => Promise.resolve("/tmp/workspace/dep-1")), - prepareUploadWorkspace: mock(() => Promise.resolve("/tmp/workspace/dep-1")), - cleanupWorkspace: mock(() => Promise.resolve()), - getHeadSha: mock(() => Promise.resolve("abc123def456")), -})); - -mock.module(fileUrl("../../utils/grafana"), () => ({ - ensureProjectDashboard: mock(() => Promise.resolve()), -})); - -mock.module("ioredis", () => ({ - default: class FakeRedis { - queue: string[] = []; - rpush = mock(async (_key: string, ...values: string[]) => { - this.queue.push(...values); - }); - blpop = mock(async () => { - const v = this.queue.shift(); - return v ? ["queue", v] : null; - }); - lrem = mock(async () => {}); - zadd = mock(async () => {}); - zrem = mock(async () => {}); - zrangebyscore = mock(async () => []); - quit = mock(async () => {}); - }, -})); - -const { PipelineOrchestrator } = await import("../pipeline"); - -describe("runDeployment cleanup behavior", () => { - it("does NOT call cleanupFailedDeployment when failure occurs after deployContainer succeeds", async () => { - const orchestrator = new PipelineOrchestrator(); - await (orchestrator as any).runDeployment("dep-1"); - - expect(cleanupFailedDeploymentCalled).toBe(false); + encoding: "utf8", + timeout: 60_000, }); + if (result.status !== 0) { + throw new Error(`Pipeline cleanup suite failed:\n${result.stdout}\n${result.stderr}`); + } + return result.status; +}; - it("calls cleanupFailedDeployment when failure occurs during build (before deployContainer)", async () => { - buildShouldFail = true; - const orchestrator = new PipelineOrchestrator(); - await (orchestrator as any).runDeployment("dep-1"); - - expect(cleanupFailedDeploymentCalled).toBe(true); - }); +describe("pipeline cleanup", () => { + it("passes every cleanup scenario", () => { + expect(runCleanupSuite()).toBe(0); + }, 60_000); }); diff --git a/apps/api/src/orchestrator/__tests__/stable-volumes.test.ts b/apps/api/src/orchestrator/__tests__/stable-volumes.test.ts new file mode 100644 index 0000000..0cf82c8 --- /dev/null +++ b/apps/api/src/orchestrator/__tests__/stable-volumes.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "bun:test"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { parse as parseYaml } from "yaml"; +import { rewriteVolumesForProject, stableVolumeName } from "../../orchestrator/compose-deploy"; + +describe("compose stable volume rewrite", () => { + it("declares stable per-project volume names and keeps references aligned", async () => { + const compose = [ + "services:", + " db:", + " image: mariadb:11", + " volumes:", + " - wp-db:/var/lib/mysql", + " - ./local-folder:/opt/data", + " - /run.sock:/var/run.sock", + " wordpress:", + " image: wordpress:7.1-php8.4-apache", + " volumes:", + " - wp-content:/var/www/html", + " - wp-db:/var/lib/mysql2", + "volumes:", + " wp-db:", + " wp-content:", + ].join("\n"); + const dir = mkdtempSync(join("/tmp", "stablerewrite-")); + const file = join(dir, "docker-compose.yml"); + writeFileSync(file, compose); + writeFileSync(file, compose); + const mounts = await rewriteVolumesForProject(file, "proj-1234-abcd"); + const parsed = parseYaml(readFileSync(file, "utf8")) as any; + expect(parsed.volumes["dq-proj1234abcd-wp-db"]).toBeDefined(); + expect(parsed.volumes["dq-proj1234abcd-wp-db"].name).toBe("dq-proj1234abcd-wp-db"); + expect(parsed.volumes["dq-proj1234abcd-wp-content"]).toBeDefined(); + expect(parsed.volumes["wp-db"]).toBeUndefined(); + expect(mounts).toContainEqual({ volumeKey: "dq-proj1234abcd-wp-db", containerPath: "/var/lib/mysql" }); + expect(mounts).toContainEqual({ volumeKey: "dq-proj1234abcd-wp-db", containerPath: "/var/lib/mysql2" }); + expect(mounts).toContainEqual({ volumeKey: "dq-proj1234abcd-wp-content", containerPath: "/var/www/html" }); + expect(mounts.find((m) => m.containerPath.startsWith("./local-folder"))).toBeUndefined(); + expect(mounts.find((m) => m.containerPath.startsWith("/run.sock"))).toBeUndefined(); + expect(parsed.services.db.volumes).toContain("dq-proj1234abcd-wp-db:/var/lib/mysql"); + expect(parsed.services.wordpress.volumes).toContain("dq-proj1234abcd-wp-db:/var/lib/mysql2"); + expect(parsed.services.wordpress.volumes).toContain("dq-proj1234abcd-wp-content:/var/www/html"); + expect(parsed.services.db.volumes.join("\n")).toContain("./local-folder"); + expect(parsed.services.db.volumes.join("\n")).toContain("/run.sock"); + rmSync(dir, { recursive: true, force: true }); + }); + + it("names volumes deterministically per project", async () => { + expect(stableVolumeName("c424652c-9d1e-4d5f-bc9a-1234567890ab", "wp-db")).toBe("dq-c424652c9d1e4d5fbc9a-wp-db"); + expect(stableVolumeName("a08b4c0d-a208-4fb5-8617-6b43db10c74c", "umami-db")).toBe( + "dq-a08b4c0da2084fb58617-umami-db", + ); + expect(stableVolumeName("ddc6a4f2-9a8c-4286-bda1-aa4b3ae989f8", "memos-data")).toBe( + "dq-ddc6a4f29a8c4286bda1-memos-data", + ); + expect(stableVolumeName("x", "weird/../key")).toBe("dq-x-weird..key"); + }); +}); diff --git a/apps/api/src/orchestrator/compose-deploy.ts b/apps/api/src/orchestrator/compose-deploy.ts index fbbd773..e9e379a 100644 --- a/apps/api/src/orchestrator/compose-deploy.ts +++ b/apps/api/src/orchestrator/compose-deploy.ts @@ -1,9 +1,20 @@ -import { writeFile } from "node:fs/promises"; +import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { isMap, isScalar, isSeq, parseDocument } from "yaml"; +import { type ComposeVolumeMount, syncComposeStackVolumes } from "../db/repo/volumes"; import { config } from "../utils/config"; import { dockerBin } from "../utils/docker-bin"; import { buildCaddySnippet } from "../utils/domain-verifier"; -import { deployWithCompose, destroyComposeStack, getComposeContainerNames, parseAllComposeServices } from "./compose"; +import { + deployWithCompose, + destroyComposeStack, + evaluateComposeContainerState, + fetchContainerExitLogs, + findComposeFilePath, + getComposeContainerNames, + inspectComposeContainerStates, + parseAllComposeServices, +} from "./compose"; import { buildComposeEnvOverrideYaml } from "./compose-env"; import { reloadCaddy, tryRun } from "./runtime"; @@ -12,6 +23,154 @@ export interface ComposeDeployResult { liveUrl: string; } +const VERIFY_ATTEMPTS = 6; +const VERIFY_INTERVAL_MS = 2000; + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +const verifyComposeStackAfterUp = async ( + serviceContainers: Map, + primaryService: string, + onLog: (line: string) => Promise, + signal?: AbortSignal, +): Promise => { + if (signal?.throwIfAborted?.()) throw new Error("Deploy aborted"); + const names = [...serviceContainers.values()]; + const sawRunning = new Set(); + + await onLog(`Waiting for ${names.length} compose service(s) to reach running state...`); + + for (let attempt = 1; attempt <= VERIFY_ATTEMPTS; attempt++) { + signal?.throwIfAborted?.(); + const states = await inspectComposeContainerStates(names); + let okCount = 0; + for (const [service, container] of serviceContainers) { + const status = states.get(container) ?? null; + const verdict = evaluateComposeContainerState(container, status, { + primary: service === primaryService, + everSawRunning: sawRunning.has(service), + }); + if (status === "running") sawRunning.add(service); + if (verdict.kind === "ok") { + okCount++; + continue; + } + if (verdict.kind === "fatal") { + const logs = await fetchContainerExitLogs(container); + await onLog(`Service verification failed: ${verdict.reason}. Last logs:\n${logs}`); + throw new Error(`Compose service ${verdict.reason}`); + } + await onLog(`Service ${service} ${verdict.reason} (attempt ${attempt}/${VERIFY_ATTEMPTS})`); + } + if (okCount === serviceContainers.size) { + await onLog("Compose services verified: all containers running."); + return; + } + await sleep(VERIFY_INTERVAL_MS); + } + + const states = await inspectComposeContainerStates(names); + const primaryContainer = serviceContainers.get(primaryService) ?? primaryService; + const primaryStatus = states.get(primaryContainer) ?? null; + if (primaryStatus !== "running") { + const logs = await fetchContainerExitLogs(primaryContainer); + await onLog(`Primary service did not reach running state. Last logs:\n${logs}`); + throw new Error( + `Primary compose service ${primaryContainer} is ${primaryStatus ?? "missing"} after ${VERIFY_ATTEMPTS} attempts`, + ); + } + for (const [service, container] of serviceContainers) { + const st = states.get(container) ?? null; + if (st !== "running" && service !== primaryService) { + await onLog( + `Warning: service ${service} (${container}) is "${st}" after ${VERIFY_ATTEMPTS} attempts; continuing`, + ); + } + } + await onLog("Compose services verified: primary container running."); +}; + +export const stableVolumeName = (projectId: string, volumeKey: string): string => { + const idPart = projectId.replace(/[^a-zA-Z0-9]/g, "").slice(0, 20); + const keyPart = volumeKey.replace(/[^a-zA-Z0-9_.-]/g, "").slice(0, 60); + return `dq-${idPart}-${keyPart}`; +}; + +const VOLUME_KEY_RE = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/; + +export const rewriteVolumesForProject = async ( + composeFile: string, + projectId: string, +): Promise => { + const content = await readFile(composeFile, "utf8"); + const doc = parseDocument(content); + if (doc.errors.length > 0) return []; + + const volumes = doc.get("volumes"); + if (!isMap(volumes)) return []; + + const renames = new Map(); + for (const item of volumes.items) { + const key = isScalar(item.key) && typeof item.key.value === "string" ? item.key.value : ""; + if (!VOLUME_KEY_RE.test(key)) continue; + if (isMap(item.value)) { + const external = item.value.get("external"); + if (external === true || (typeof external === "string" && external.length > 0)) continue; + if (item.value.get("name") === undefined || item.value.get("name") === null) { + item.value.set("name", stableVolumeName(projectId, key)); + } + } else if ( + item.value === null || + item.value === undefined || + (isScalar(item.value) && (item.value.value === null || item.value.value === undefined)) + ) { + volumes.set(key, doc.createNode({ name: stableVolumeName(projectId, key) })); + } + renames.set(key, stableVolumeName(projectId, key)); + } + if (renames.size === 0) return []; + + const services = doc.get("services"); + const mounts: ComposeVolumeMount[] = []; + if (isMap(services)) { + for (const serviceItem of services.items) { + if (!isMap(serviceItem.value)) continue; + const serviceVolumes = serviceItem.value.get("volumes"); + if (!isSeq(serviceVolumes)) continue; + for (const vol of serviceVolumes.items) { + if (isScalar(vol) && typeof vol.value === "string") { + const m = vol.value.match(/^([A-Za-z0-9][A-Za-z0-9_.-]*):(.*)$/); + if (m && renames.has(m[1])) { + const stable = renames.get(m[1])!; + vol.value = `${stable}:${m[2]}`; + mounts.push({ volumeKey: stable, containerPath: m[2] }); + } + } else if (isMap(vol)) { + const source = vol.get("source"); + if (isScalar(source) && typeof source.value === "string" && renames.has(source.value)) { + const stable = renames.get(source.value)!; + vol.set("source", stable); + const target = vol.get("target"); + mounts.push({ + volumeKey: stable, + containerPath: isScalar(target) && typeof target.value === "string" ? target.value : "", + }); + } + } + } + } + } + + for (const item of volumes.items) { + if (isScalar(item.key) && typeof item.key.value === "string" && renames.has(item.key.value)) { + item.key.value = renames.get(item.key.value)!; + } + } + + await writeFile(composeFile, doc.toString(), "utf8"); + return mounts; +}; + export interface ComposeDeployOptions { workspacePath: string; deploymentId: string; @@ -44,8 +203,15 @@ export const deployComposeStack = async (options: ComposeDeployOptions): Promise } = options; if (oldDeploymentId) { - await onLog(`Stopping previous compose stack deploy-${oldDeploymentId}`); - await destroyComposeStack(`deploy-${oldDeploymentId}`); + await onLog( + `Blue-green deploy: keeping previous stack deploy-${oldDeploymentId} running until the new stack passes verification`, + ); + } + + const composeFile = findComposeFilePath(workspacePath, sourceDir); + if (composeFile) { + const mounts = await rewriteVolumesForProject(composeFile, projectId); + await syncComposeStackVolumes(projectId, mounts); } const allServices = parseAllComposeServices(workspacePath, sourceDir, composeService, composePort); @@ -71,12 +237,15 @@ export const deployComposeStack = async (options: ComposeDeployOptions): Promise const containerFor = (serviceName: string) => composeContainers.get(serviceName) || `deploy-${deploymentId}-${serviceName}-1`; + const target = allServices.find((s) => s.isPrimary) || allServices[0]; + const verifyTargets = new Map(allServices.map((s) => [s.serviceName, containerFor(s.serviceName)])); + await verifyComposeStackAfterUp(verifyTargets, target.serviceName, onLog, signal); + for (const s of allServices) { const cName = containerFor(s.serviceName); await tryRun(dockerBin, ["network", "connect", config.dockerNetwork, cName]); } - const target = allServices.find((s) => s.isPrimary) || allServices[0]; const containerName = containerFor(target.serviceName); const slug = projectName @@ -112,7 +281,7 @@ export const deployComposeStack = async (options: ComposeDeployOptions): Promise } } const targetPort = customMatch?.port ? Number(customMatch.port) : s.port; - snippet += `\n${domains.join(", ")} {\n log {\n output stdout\n format json\n }\n reverse_proxy ${sContainer}:${targetPort} {\n header_up Host {upstream_hostport}\n }\n}\n`; + snippet += `\n${domains.join(", ")} {\n log {\n output stdout\n format json\n }\n reverse_proxy ${sContainer}:${targetPort}\n}\n`; } const caddyRouteFile = join(config.caddyRoutesDir, `${slug}.caddy`); @@ -123,6 +292,13 @@ export const deployComposeStack = async (options: ComposeDeployOptions): Promise console.warn("[Pipeline] Caddy not ready for reload after compose deploy"); } + if (oldDeploymentId) { + await onLog(`New stack verified and routed; tearing down previous stack deploy-${oldDeploymentId}`); + await destroyComposeStack(`deploy-${oldDeploymentId}`).catch((e) => + onLog(`Warning: failed to tear down previous stack deploy-${oldDeploymentId}: ${e?.message ?? e}`), + ); + } + const liveUrl = rawBaseDomain === "localhost" ? `http://${slug}.localhost` : `https://${slug}.${rawBaseDomain}`; return { containerName, liveUrl }; diff --git a/apps/api/src/orchestrator/compose.ts b/apps/api/src/orchestrator/compose.ts index d79d40e..12f3ccd 100644 --- a/apps/api/src/orchestrator/compose.ts +++ b/apps/api/src/orchestrator/compose.ts @@ -1,5 +1,6 @@ import { spawn } from "node:child_process"; import { existsSync, readFileSync, realpathSync } from "node:fs"; +import { writeFile } from "node:fs/promises"; import { dirname, join, resolve, sep } from "node:path"; import type { Server } from "../types"; import { dockerBin } from "../utils/docker-bin"; @@ -203,7 +204,6 @@ const spawnComposeCommand = ( signal?: AbortSignal, ): Promise<{ code: number; stdout: string; stderr: string }> => { return new Promise((resolve, reject) => { - const isPosix = process.platform !== "win32"; const child = spawn(dockerBin, ["compose", ...args], { cwd, env: { ...process.env, ...(envVars || {}) }, @@ -279,6 +279,21 @@ const spawnComposeCommand = ( }); }; +const dotenvValue = (value: string): string => + `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\$/g, "$$$$").replace(/\r?\n/g, "\\n")}"`; + +const writeComposeEnvFile = async ( + cwd: string, + envVars: Record | undefined, + onLog: (line: string) => Promise, +): Promise => { + const entries = Object.entries(envVars ?? {}).filter(([key]) => /^[A-Za-z_][A-Za-z0-9_]*$/.test(key)); + if (entries.length === 0 || existsSync(join(cwd, ".env"))) return; + const content = entries.map(([key, value]) => `${key}=${dotenvValue(value)}`).join("\n"); + await writeFile(join(cwd, ".env"), `${content}\n`); + await onLog(`Wrote ${entries.length} environment variables to .env`); +}; + export const buildWithCompose = async ( workspacePath: string, projectName: string, @@ -293,6 +308,7 @@ export const buildWithCompose = async ( } const cwd = dirname(composeFile); + await writeComposeEnvFile(cwd, envVars, onLog); await onLog(`Starting Docker Compose build (project: ${projectName})...`); const res = await spawnComposeCommand(["-f", composeFile, "-p", projectName, "build"], cwd, envVars, onLog, signal); @@ -318,6 +334,7 @@ export const deployWithCompose = async ( } const cwd = dirname(composeFile); + await writeComposeEnvFile(cwd, envVars, onLog); await onLog(`Starting Docker Compose services (project: ${projectName})...`); const args = ["-f", composeFile]; @@ -362,3 +379,64 @@ export const getComposeContainerNames = async (projectName: string): Promise => { return safeSpawn(dockerBin, args, { timeoutMs: 60_000 }); }; + +export interface ComposeContainerState { + container: string; + status: string | null; +} + +export type ComposeContainerVerdict = + | { kind: "ok" } + | { kind: "retry"; reason: string } + | { kind: "fatal"; reason: string }; + +export const evaluateComposeContainerState = ( + container: string, + status: string | null, + { primary, everSawRunning = false }: { primary: boolean; everSawRunning?: boolean } = { primary: false }, +): ComposeContainerVerdict => { + if (!status) return { kind: "retry", reason: `${container} has not been created yet` }; + if (status === "running" && everSawRunning) return { kind: "ok" }; + if (status === "running") return { kind: "retry", reason: "reinforcing stability window" }; + if (status === "created") return { kind: "retry", reason: `${container} is still initializing` }; + if (status === "restarting") { + const reason = `${container} is restarting continuously`; + return primary ? { kind: "fatal", reason } : { kind: "retry", reason }; + } + return { + kind: primary ? "fatal" : "retry", + reason: `${container} ${status === "exited" ? "exited immediately" : `is in unknown state "${status}"`}`, + }; +}; + +export const inspectComposeContainerStates = async (containerNames: string[]): Promise> => { + const states = new Map(); + if (containerNames.length === 0) return states; + const res = await runDocker(["inspect", "--format", "{{.Name}}|{{.State.Status}}", ...containerNames]).catch(() => ({ + code: 1, + stdout: "", + stderr: "", + })); + for (const line of res.stdout.split("\n")) { + const [rawName, status] = line.split("|"); + if (!rawName) continue; + states.set(rawName.replace(/^\//, ""), status?.trim() || null); + } + for (const name of containerNames) { + if (!states.has(name)) states.set(name, null); + } + return states; +}; + +export const fetchContainerExitLogs = async (container: string): Promise => { + const res = await runDocker(["logs", "--tail", "30", container]).catch(() => ({ + code: 1, + stdout: "", + stderr: "", + })); + const logs = [res.stdout, res.stderr] + .filter((s) => s?.trim()) + .join("\n") + .trim(); + return logs || "No logs produced"; +}; diff --git a/apps/api/src/orchestrator/pipeline.ts b/apps/api/src/orchestrator/pipeline.ts index a481b44..428be42 100644 --- a/apps/api/src/orchestrator/pipeline.ts +++ b/apps/api/src/orchestrator/pipeline.ts @@ -215,7 +215,6 @@ export class PipelineOrchestrator { this.abortControllers.set(deploymentId, controller); let workspacePath = ""; - let uploadedArchivePath: string | null = null; let imageTag: string | undefined; let projectName: string | undefined; let project: import("../../types").Project | null = null; @@ -259,8 +258,7 @@ export class PipelineOrchestrator { } } else { await emitLog(deploymentId, "build", `Extracting archive: ${deployment.sourceRef}`); - uploadedArchivePath = deployment.sourceRef; - workspacePath = await prepareUploadWorkspace(deploymentId, deployment.sourceRef); + workspacePath = await prepareUploadWorkspace(deploymentId, deployment.sourceRef, deployment.projectId); } await this.checkCancelled(deploymentId); @@ -509,14 +507,6 @@ export class PipelineOrchestrator { } catch { // cleanup failures must never mask the deployment result } - try { - if (uploadedArchivePath) - await rm(uploadedArchivePath, { - force: true, - }); - } catch { - // cleanup failures must never mask the deployment result - } } } diff --git a/apps/api/src/orchestrator/source.ts b/apps/api/src/orchestrator/source.ts index f9ea9a2..a08680c 100644 --- a/apps/api/src/orchestrator/source.ts +++ b/apps/api/src/orchestrator/source.ts @@ -100,18 +100,39 @@ export const prepareSourceWorkspace = async ( return root; }; -export const prepareUploadWorkspace = async (deploymentId: string, archivePath: string) => { +export const prepareUploadWorkspace = async (deploymentId: string, archivePath: string, projectId?: string | null) => { const root = join(config.workspaceRoot, deploymentId); await rm(root, { recursive: true, force: true }); await mkdir(root, { recursive: true }); - if (archivePath.endsWith(".zip")) { - await run("unzip", ["-q", archivePath, "-d", root]); + let resolvedArchive = archivePath; + const { existsSync } = await import("node:fs"); + const { basename } = await import("node:path"); + if (!existsSync(resolvedArchive)) { + const inUploads = join(config.workspaceRoot, "uploads", basename(archivePath)); + if (existsSync(inUploads)) { + resolvedArchive = inUploads; + } else if (projectId) { + try { + const { getProjectById } = await import("../db/repo"); + const proj = await getProjectById(projectId); + if (proj) { + const { ensureValidComposeArchive } = await import("../api/projects/compose"); + resolvedArchive = await ensureValidComposeArchive(proj, archivePath); + } + } catch (error) { + console.warn(`[Source] Could not recover compose archive for project ${projectId}:`, error); + } + } + } + + if (resolvedArchive.endsWith(".zip")) { + await run("unzip", ["-q", resolvedArchive, "-d", root]); return ensureSingleRoot(root); } - if (archivePath.endsWith(".tar") || archivePath.endsWith(".tar.gz") || archivePath.endsWith(".tgz")) { - await run("tar", ["-xf", archivePath, "-C", root]); + if (resolvedArchive.endsWith(".tar") || resolvedArchive.endsWith(".tar.gz") || resolvedArchive.endsWith(".tgz")) { + await run("tar", ["-xf", resolvedArchive, "-C", root]); return ensureSingleRoot(root); } diff --git a/apps/api/src/scaling/__tests__/engine-runner.ts b/apps/api/src/scaling/__tests__/engine-runner.ts new file mode 100644 index 0000000..6f09535 --- /dev/null +++ b/apps/api/src/scaling/__tests__/engine-runner.ts @@ -0,0 +1,478 @@ +import { afterAll, beforeEach, describe, expect, it, mock } from "bun:test"; + +const TEST_DEPLOYMENT = { id: "dep-1", projectId: "proj-1", containerName: "deploy-dep-1" }; + +const TEST_POLICY = { + id: "pol-1", + projectId: "proj-1", + enabled: true, + minReplicas: 1, + maxReplicas: 5, + cpuThresholdPercent: 70, + memoryThresholdPercent: 85, + cooldownSeconds: 120, + createdAt: "", + updatedAt: "", +}; + +// Mock state variables +let mockRunImpl: ((cmd: string, args: string[]) => Promise) | null = null; +let mockTryRunImpl: ((cmd: string, args: string[]) => Promise) | null = null; +let mockReadFileContent: string | null = null; +let mockWriteFilePath: string | null = null; +let mockWriteContent: string | null = null; +let mockRedisData: Record = {}; +let mockGetScalingPolicyResult: ((id: string) => any) | null = null; +let mockListEnvVarsResult: any[] = []; + +beforeEach(() => { + mockRunImpl = null; + mockTryRunImpl = null; + mockReadFileContent = null; + mockWriteFilePath = null; + mockWriteContent = null; + mockRedisData = {}; + mockGetScalingPolicyResult = null; + mockListEnvVarsResult = []; +}); + +afterAll(() => { + mockRunImpl = null; + mockTryRunImpl = null; + mockReadFileContent = null; + mockWriteFilePath = null; + mockWriteContent = null; + mockRedisData = {}; + mockGetScalingPolicyResult = null; + mockListEnvVarsResult = []; +}); + +// Mock docker-utils +const fileUrl = (path: string) => new URL(path, import.meta.url).toString(); +mock.module(fileUrl("../docker-utils"), () => ({ + run: mock((cmd: string, args: string[]) => { + if (mockRunImpl) return mockRunImpl(cmd, args); + return Promise.resolve(""); + }), + tryRun: mock((cmd: string, args: string[]) => { + if (mockTryRunImpl) return mockTryRunImpl(cmd, args); + return Promise.resolve(""); + }), +})); + +// Mock fs +mock.module("node:fs/promises", () => { + const fs = require("node:fs"); + const pathMod = require("node:path"); + const base = { ...fs.promises }; + return { + ...base, + rm: mock(async (path: string, opts?: any) => { + if (fs.existsSync(path)) fs.rmSync(path, opts || {}); + }), + readFile: mock(async (path: string, options?: any) => { + if (mockReadFileContent !== null) { + return mockReadFileContent; + } + if (fs.existsSync(path)) { + return fs.readFileSync(path, options || "utf8"); + } + throw new Error(`ENOENT: no such file or directory, open '${path}'`); + }), + writeFile: mock(async (path: string, content: string, options?: any) => { + mockWriteFilePath = path; + mockWriteContent = content; + const dir = pathMod.dirname(path); + if (fs.existsSync(dir)) { + fs.writeFileSync(path, content, options || "utf8"); + } + }), + }; +}); + +// Mock redis +mock.module("ioredis", () => ({ + default: class FakeRedis { + hgetall = mock(async () => mockRedisData); + hset = mock(async (_key: string, data: Record) => { + mockRedisData = { ...mockRedisData, ...data }; + }); + get = mock(async (key: string) => mockRedisData[key] ?? null); + set = mock(async (key: string, value: string) => { + mockRedisData[key] = value; + }); + quit = mock(async () => {}); + }, +})); + +// Mock db/repo +let mockScaleJobs: any[] = []; +mock.module(fileUrl("../../db/repo"), () => ({ + getProjectById: mock(() => Promise.resolve({ id: "proj-1", name: "proj-1", cpuLimit: 0.5, memoryLimitMb: 512 })), + listDeployments: mock(() => Promise.resolve([])), + getScalingPolicy: mock((id: string) => + mockGetScalingPolicyResult ? Promise.resolve(mockGetScalingPolicyResult(id)) : Promise.resolve(null), + ), + getServerById: mock(async (id: string) => { + if (id === "server-1") return { id, mode: "agent", lastHeartbeat: new Date().toISOString() }; + if (id === "server-2") return { id, mode: "agent", lastHeartbeat: new Date(Date.now() - 600_000).toISOString() }; + return null; + }), + createAgentJob: mock(async (input: any) => { + mockScaleJobs.push(input); + return "job-1"; + }), + upsertRoute: mock(async () => ({})), + updateServerStatus: mock(() => Promise.resolve()), + createAgentRegistrationToken: mock(() => Promise.resolve({ token: "dqr_test", expiresAt: new Date().toISOString() })), + getPlatformSettings: mock(() => Promise.resolve({ ingressServerId: null })), + listServers: mock(() => Promise.resolve([])), + listProjects: mock(() => Promise.resolve([])), + ensureLocalServer: mock(() => Promise.resolve()), + updateDeploymentStatus: mock(() => Promise.resolve()), + listEnvironmentVariablesForDeploy: mock(() => Promise.resolve(mockListEnvVarsResult)), + listDomains: mock(() => Promise.resolve([])), + updateDomainValidation: mock(() => Promise.resolve()), + getLogs: mock(() => Promise.resolve([])), +})); + +mock.module(fileUrl("../../utils/config"), () => ({ + config: { + redisUrl: "redis://localhost:6379", + caddyRoutesDir: "/tmp/dequel-test-routes", + caddyBaseDomain: "localhost", + appInternalPort: 3000, + dockerNetwork: "dequel_net", + envEncryptionKey: "dev-env-key-change-me", + workspaceRoot: "./workspace", + }, +})); + +mock.module(fileUrl("../../utils/docker-bin"), () => ({ dockerBin: "/usr/bin/docker" })); + +const { scalingEngine } = await import("../engine"); + +// --- Tests --- + +describe("parseMemToMb", () => { + it("converts GiB to MB", () => { + expect((scalingEngine as any).parseMemToMb("2GiB")).toBe(2048); + }); + + it("converts MiB directly", () => { + expect((scalingEngine as any).parseMemToMb("512MiB")).toBe(512); + }); + + it("converts KiB to MB", () => { + expect((scalingEngine as any).parseMemToMb("1024KiB")).toBe(1); + }); + + it("returns 0 for unrecognized format", () => { + expect((scalingEngine as any).parseMemToMb("")).toBe(0); + expect((scalingEngine as any).parseMemToMb("abc")).toBe(0); + }); + + it("handles MB format", () => { + expect((scalingEngine as any).parseMemToMb("256MB")).toBe(256); + }); +}); + +describe("getCurrentReplicas", () => { + it("returns 1 when no route file exists", async () => { + mockReadFileContent = null; + const result = await (scalingEngine as any).getCurrentReplicas("proj-1"); + expect(result).toBe(1); + }); + + it("returns 1 when no reverse_proxy in file", async () => { + mockReadFileContent = "proj-1.localhost:80 {\n log\n}\n"; + const result = await (scalingEngine as any).getCurrentReplicas("proj-1"); + expect(result).toBe(1); + }); + + it("counts unique containers in reverse_proxy targets", async () => { + mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n"; + const result = await (scalingEngine as any).getCurrentReplicas("proj-1"); + expect(result).toBe(2); + }); + + it("ignores port suffixes when counting containers", async () => { + mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy a:80 b:80 c:80\n}\n"; + const result = await (scalingEngine as any).getCurrentReplicas("proj-1"); + expect(result).toBe(3); + }); + + it("counts only the primary site block when custom-domain blocks exist", async () => { + mockReadFileContent = [ + "verify-app.custom-domain.com {", + " reverse_proxy some-other-container:3000", + "}", + "proj-1.localhost:80 {", + " reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000", + "}", + ].join("\n"); + const result = await (scalingEngine as any).getCurrentReplicas("proj-1"); + expect(result).toBe(2); + }); + + it("falls back to whole-file counting when no primary block matches", async () => { + mockReadFileContent = "other.localhost:80 {\n reverse_proxy a:3000\n}\n"; + const result = await (scalingEngine as any).getCurrentReplicas("proj-1"); + expect(result).toBe(1); + }); +}); + +describe("evaluate", () => { + beforeEach(() => { + mockGetScalingPolicyResult = () => TEST_POLICY; + mockListEnvVarsResult = []; + mockReadFileContent = null; + mockRunImpl = null; + mockTryRunImpl = null; + }); + + it("returns early when deployment has no projectId", async () => { + await (scalingEngine as any).evaluate({ id: "dep-1", projectId: null, containerName: null }); + }); + + it("returns early when no scaling policy", async () => { + mockGetScalingPolicyResult = null; + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + }); + + it("returns early when policy is disabled", async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, enabled: false }); + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + }); + + it("returns early when container stats fail", async () => { + mockRunImpl = async () => "not-json"; + const result = await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(result).toBeUndefined(); + }); + + it("sets highCpuSince on first high CPU reading", async () => { + mockRunImpl = async () => '{"CPUPerc":"85%","MemUsage":"128MiB / 512MiB"}'; + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockRedisData.highCpuSince).toBeTruthy(); + }); + + it("does not scale up if highCpuSince is not old enough", async () => { + mockRedisData = { highCpuSince: String(Date.now() - 5000), lastScaleUp: "0" }; + mockRunImpl = async () => '{"CPUPerc":"85%","MemUsage":"128MiB / 512MiB"}'; + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockWriteFilePath).toBeNull(); + }); + + it("scales up when CPU exceeds threshold past cooldown", async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cooldownSeconds: 0 }); + mockRedisData = { highCpuSince: String(Date.now() - 10000), lastScaleUp: "0" }; + mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000\n}\n"; + + mockRunImpl = async (_cmd: string, args: string[]) => { + if (args.some((a) => a.includes("stats"))) return '{"CPUPerc":"85%","MemUsage":"128MiB / 512MiB"}'; + if (args.some((a) => a.includes("Image"))) return "my-app:latest"; + if (args.some((a) => a.includes("Env"))) return '["PORT=3000"]'; + if (args.some((a) => a.includes("Mounts"))) return "[]"; + if (args[0] === "run") return "new-container-id"; + if (args[0] === "ps") return "caddy-abc123\n"; + if (args.some((a) => a.includes("reload"))) return ""; + return ""; + }; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + + expect(mockWriteFilePath).toBeTruthy(); + expect(mockWriteContent).toContain("deploy-dep-1-replica-2"); + }); + + it("does not scale up when already at max replicas", async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, maxReplicas: 2, cooldownSeconds: 0 }); + mockRedisData = { highCpuSince: String(Date.now() - 10000), lastScaleUp: "0" }; + mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n"; + mockRunImpl = async () => '{"CPUPerc":"85%","MemUsage":"128MiB / 512MiB"}'; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockWriteFilePath).toBeNull(); + }); + + it("does not advance lastScaleUp when already at max replicas", async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, maxReplicas: 2, cooldownSeconds: 0 }); + const lastScaleUpBefore = String(Date.now() - 60_000); + mockRedisData = { highCpuSince: String(Date.now() - 10000), lastScaleUp: lastScaleUpBefore }; + mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n"; + mockRunImpl = async () => '{"CPUPerc":"85%","MemUsage":"128MiB / 512MiB"}'; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockRedisData.lastScaleUp).toBe(lastScaleUpBefore); + expect(mockRedisData.highCpuSince).toBeFalsy(); + }); + + it("keeps accruing the low-CPU timer between threshold and 30%", async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cpuThresholdPercent: 1, cooldownSeconds: 0 }); + const lowCpuSinceBefore = String(Date.now() - 240_000); + mockRedisData = { lowCpuSince: lowCpuSinceBefore, highCpuSince: "0", lastScaleUp: "0" }; + mockRunImpl = async () => '{"CPUPerc":"22%","MemUsage":"64MiB / 512MiB"}'; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockRedisData.lowCpuSince).toBe(lowCpuSinceBefore); + }); + + it("scales down in the overlap zone (threshold < cpu < 30) once idle for 5 min", async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cpuThresholdPercent: 1, cooldownSeconds: 0, minReplicas: 1 }); + mockRedisData = { lowCpuSince: String(Date.now() - 310_000), lastScaleDown: "0" }; + mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n"; + mockTryRunImpl = async () => ""; + mockRunImpl = async (_cmd: string, args: string[]) => { + if (args.includes("stats")) return '{"CPUPerc":"22%","MemUsage":"64MiB / 512MiB"}'; + if (args[0] === "stop") return ""; + if (args[0] === "rm") return ""; + if (args[0] === "ps") return "caddy-abc123\n"; + if (args.includes("reload")) return ""; + return ""; + }; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockWriteFilePath).toBeTruthy(); + expect(mockWriteContent).not.toContain("replica-2"); + }); + + it("does not scale down when CPU is moderate", async () => { + mockRunImpl = async () => '{"CPUPerc":"50%","MemUsage":"128MiB / 512MiB"}'; + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockWriteFilePath).toBeNull(); + }); + + it("scales down when CPU stays low for 5 min", async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, minReplicas: 1, cooldownSeconds: 0 }); + mockRedisData = { lowCpuSince: String(Date.now() - 310_000), lastScaleDown: "0" }; + mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n"; + mockTryRunImpl = async () => ""; + + let _callCount = 0; + mockRunImpl = async (_cmd: string, args: string[]) => { + _callCount++; + if (args.includes("stats")) return '{"CPUPerc":"5%","MemUsage":"64MiB / 512MiB"}'; + if (args[0] === "stop") return ""; + if (args[0] === "rm") return ""; + if (args[0] === "ps") return "caddy-abc123\n"; + if (args.includes("reload")) return ""; + return ""; + }; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + + expect(mockWriteContent).toContain("deploy-dep-1"); + expect(mockWriteContent).not.toContain("replica-2"); + }); + + it("does not scale down below minReplicas", async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, minReplicas: 2, cooldownSeconds: 0 }); + mockRedisData = { lowCpuSince: String(Date.now() - 310_000), lastScaleDown: "0" }; + mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n"; + mockRunImpl = async () => '{"CPUPerc":"5%","MemUsage":"64MiB / 512MiB"}'; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockWriteFilePath).toBeNull(); + }); + + it("scales down at idle even when cpuThresholdPercent is 0", async () => { + mockGetScalingPolicyResult = () => ({ + ...TEST_POLICY, + cpuThresholdPercent: 0, + minReplicas: 1, + cooldownSeconds: 0, + }); + mockRedisData = { lowCpuSince: String(Date.now() - 310_000), lastScaleDown: "0" }; + mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n"; + mockTryRunImpl = async () => ""; + mockRunImpl = async (_cmd: string, args: string[]) => { + if (args.includes("stats")) return '{"CPUPerc":"0%","MemUsage":"64MiB / 512MiB"}'; + if (args[0] === "stop") return ""; + if (args[0] === "rm") return ""; + if (args[0] === "ps") return "caddy-abc123\n"; + if (args.includes("reload")) return ""; + return ""; + }; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockWriteFilePath).toBeTruthy(); + expect(mockWriteContent).toContain("deploy-dep-1"); + expect(mockWriteContent).not.toContain("replica-2"); + }); + + it("scales up on any observed CPU when cpuThresholdPercent is 0", async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cpuThresholdPercent: 0, cooldownSeconds: 0 }); + mockRedisData = { highCpuSince: String(Date.now() - 10_000), lastScaleUp: "0" }; + mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000\n}\n"; + + mockRunImpl = async (_cmd: string, args: string[]) => { + if (args.some((a) => a.includes("stats"))) return '{"CPUPerc":"1%","MemUsage":"128MiB / 512MiB"}'; + if (args.some((a) => a.includes("Image"))) return "my-app:latest"; + if (args.some((a) => a.includes("Env"))) return '["PORT=3000"]'; + if (args.some((a) => a.includes("Mounts"))) return "[]"; + if (args[0] === "run") return "new-container-id"; + if (args[0] === "ps") return "caddy-abc123\n"; + if (args.some((a) => a.includes("reload"))) return ""; + return ""; + }; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockWriteFilePath).toBeTruthy(); + expect(mockWriteContent).toContain("deploy-dep-1-replica-2"); + }); + + it("skips evaluation when the agent server is offline", async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cooldownSeconds: 0 }); + mockRedisData = { highCpuSince: String(Date.now() - 10_000), lastScaleUp: "0" }; + mockScaleJobs = []; + await (scalingEngine as any).evaluate({ ...TEST_DEPLOYMENT, serverId: "server-2" }); + expect(mockScaleJobs.length).toBe(0); + }); + + it("scales up through a scale job on an online agent server", async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cooldownSeconds: 0 }); + mockRedisData = { + highCpuSince: String(Date.now() - 10_000), + lastScaleUp: "0", + "dequel:agent-stats:server-1": JSON.stringify({ + updatedAt: new Date().toISOString(), + containers: [{ containerName: "deploy-dep-1", cpuPercent: 90, memoryMb: 200 }], + }), + }; + mockScaleJobs = []; + await (scalingEngine as any).evaluate({ ...TEST_DEPLOYMENT, serverId: "server-1" }); + expect(mockScaleJobs.length).toBe(2); + expect(mockScaleJobs[0].type).toBe("scale"); + expect(mockScaleJobs[0].payload.action).toBe("up"); + expect(mockScaleJobs[0].payload.replicas).toBe(2); + expect(mockScaleJobs[0].serverId).toBe("server-1"); + expect(mockScaleJobs[1].type).toBe("reload_routes"); + expect(mockScaleJobs[1].payload.action).toBe("add"); + }); + + it("counts agent replicas from cached stats", async () => { + mockRedisData = { + "dequel:agent-stats:server-1": JSON.stringify({ + updatedAt: new Date().toISOString(), + containers: [ + { containerName: "deploy-dep-1", cpuPercent: 40, memoryMb: 100, deploymentId: "dep-1", replica: false }, + { + containerName: "deploy-dep-1-replica-2", + cpuPercent: 41, + memoryMb: 110, + deploymentId: "dep-1", + replica: true, + }, + { containerName: "other-app", cpuPercent: 10, memoryMb: 10, deploymentId: "other", replica: false }, + ], + }), + }; + const count = await (scalingEngine as any).getCurrentReplicas( + "proj-1", + { server: { id: "server-1" }, mode: "agent" }, + TEST_DEPLOYMENT, + ); + expect(count).toBe(2); + }); +}); diff --git a/apps/api/src/scaling/__tests__/engine.test.ts b/apps/api/src/scaling/__tests__/engine.test.ts index e6daf3b..d874857 100644 --- a/apps/api/src/scaling/__tests__/engine.test.ts +++ b/apps/api/src/scaling/__tests__/engine.test.ts @@ -1,474 +1,25 @@ -import { afterAll, beforeEach, describe, expect, it, mock } from "bun:test"; - -const TEST_DEPLOYMENT = { id: "dep-1", projectId: "proj-1", containerName: "deploy-dep-1" }; - -const TEST_POLICY = { - id: "pol-1", - projectId: "proj-1", - enabled: true, - minReplicas: 1, - maxReplicas: 5, - cpuThresholdPercent: 70, - memoryThresholdPercent: 85, - cooldownSeconds: 120, - createdAt: "", - updatedAt: "", +import { describe, expect, it } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { join } from "node:path"; + +const runEngineSuite = () => { + const result = spawnSync("bun", ["test", join(import.meta.dir, "engine-runner.ts")], { + cwd: join(import.meta.dir, "../.."), + env: { + ...process.env, + TEST_DATABASE_URL: process.env.TEST_DATABASE_URL ?? "postgresql://dequel:dequel@localhost:5434/dequel", + }, + encoding: "utf8", + timeout: 120_000, + }); + if (result.status !== 0) { + throw new Error(`Scaling engine suite failed:\n${result.stdout}\n${result.stderr}`); + } + return result.status; }; -// Mock state variables -let mockRunImpl: ((cmd: string, args: string[]) => Promise) | null = null; -let mockTryRunImpl: ((cmd: string, args: string[]) => Promise) | null = null; -let mockReadFileContent: string | null = null; -let mockWriteFilePath: string | null = null; -let mockWriteContent: string | null = null; -let mockRedisData: Record = {}; -let mockGetScalingPolicyResult: ((id: string) => any) | null = null; -let mockListEnvVarsResult: any[] = []; - -beforeEach(() => { - mockRunImpl = null; - mockTryRunImpl = null; - mockReadFileContent = null; - mockWriteFilePath = null; - mockWriteContent = null; - mockRedisData = {}; - mockGetScalingPolicyResult = null; - mockListEnvVarsResult = []; -}); - -afterAll(() => { - mockRunImpl = null; - mockTryRunImpl = null; - mockReadFileContent = null; - mockWriteFilePath = null; - mockWriteContent = null; - mockRedisData = {}; - mockGetScalingPolicyResult = null; - mockListEnvVarsResult = []; -}); - -// Mock docker-utils -const fileUrl = (path: string) => new URL(path, import.meta.url).toString(); -mock.module(fileUrl("../docker-utils"), () => ({ - run: mock((cmd: string, args: string[]) => { - if (mockRunImpl) return mockRunImpl(cmd, args); - return Promise.resolve(""); - }), - tryRun: mock((cmd: string, args: string[]) => { - if (mockTryRunImpl) return mockTryRunImpl(cmd, args); - return Promise.resolve(""); - }), -})); - -// Mock fs -mock.module("node:fs/promises", () => { - const fs = require("node:fs"); - const pathMod = require("node:path"); - return { - rm: mock(async (path: string, opts?: any) => { - if (fs.existsSync(path)) fs.rmSync(path, opts || {}); - }), - readFile: mock(async (path: string, options?: any) => { - if (mockReadFileContent !== null) { - return mockReadFileContent; - } - if (fs.existsSync(path)) { - return fs.readFileSync(path, options || "utf8"); - } - throw new Error(`ENOENT: no such file or directory, open '${path}'`); - }), - writeFile: mock(async (path: string, content: string, options?: any) => { - mockWriteFilePath = path; - mockWriteContent = content; - const dir = pathMod.dirname(path); - if (fs.existsSync(dir)) { - fs.writeFileSync(path, content, options || "utf8"); - } - }), - }; -}); - -// Mock redis -mock.module("ioredis", () => ({ - default: class FakeRedis { - hgetall = mock(async () => mockRedisData); - hset = mock(async (_key: string, data: Record) => { - mockRedisData = { ...mockRedisData, ...data }; - }); - get = mock(async (key: string) => mockRedisData[key] ?? null); - set = mock(async (key: string, value: string) => { - mockRedisData[key] = value; - }); - quit = mock(async () => {}); - }, -})); - -// Mock db/repo -let mockScaleJobs: any[] = []; -mock.module(fileUrl("../../db/repo"), () => ({ - getProjectById: mock(() => Promise.resolve({ id: "proj-1", name: "proj-1", cpuLimit: 0.5, memoryLimitMb: 512 })), - listDeployments: mock(() => Promise.resolve([])), - getScalingPolicy: mock((id: string) => - mockGetScalingPolicyResult ? Promise.resolve(mockGetScalingPolicyResult(id)) : Promise.resolve(null), - ), - getServerById: mock(async (id: string) => { - if (id === "server-1") return { id, mode: "agent", lastHeartbeat: new Date().toISOString() }; - if (id === "server-2") return { id, mode: "agent", lastHeartbeat: new Date(Date.now() - 600_000).toISOString() }; - return null; - }), - createAgentJob: mock(async (input: any) => { - mockScaleJobs.push(input); - return "job-1"; - }), - upsertRoute: mock(async () => ({})), - updateServerStatus: mock(() => Promise.resolve()), - createAgentRegistrationToken: mock(() => Promise.resolve({ token: "dqr_test", expiresAt: new Date().toISOString() })), - getPlatformSettings: mock(() => Promise.resolve({ ingressServerId: null })), - listServers: mock(() => Promise.resolve([])), - listProjects: mock(() => Promise.resolve([])), - ensureLocalServer: mock(() => Promise.resolve()), - updateDeploymentStatus: mock(() => Promise.resolve()), - listEnvironmentVariablesForDeploy: mock(() => Promise.resolve(mockListEnvVarsResult)), - listDomains: mock(() => Promise.resolve([])), - updateDomainValidation: mock(() => Promise.resolve()), -})); - -mock.module(fileUrl("../../utils/config"), () => ({ - config: { - redisUrl: "redis://localhost:6379", - caddyRoutesDir: "/tmp/dequel-test-routes", - caddyBaseDomain: "localhost", - appInternalPort: 3000, - dockerNetwork: "dequel_net", - envEncryptionKey: "dev-env-key-change-me", - }, -})); - -mock.module(fileUrl("../../utils/docker-bin"), () => ({ dockerBin: "/usr/bin/docker" })); - -const { scalingEngine } = await import("../engine"); - -// --- Tests --- - -describe("parseMemToMb", () => { - it("converts GiB to MB", () => { - expect((scalingEngine as any).parseMemToMb("2GiB")).toBe(2048); - }); - - it("converts MiB directly", () => { - expect((scalingEngine as any).parseMemToMb("512MiB")).toBe(512); - }); - - it("converts KiB to MB", () => { - expect((scalingEngine as any).parseMemToMb("1024KiB")).toBe(1); - }); - - it("returns 0 for unrecognized format", () => { - expect((scalingEngine as any).parseMemToMb("")).toBe(0); - expect((scalingEngine as any).parseMemToMb("abc")).toBe(0); - }); - - it("handles MB format", () => { - expect((scalingEngine as any).parseMemToMb("256MB")).toBe(256); - }); -}); - -describe("getCurrentReplicas", () => { - it("returns 1 when no route file exists", async () => { - mockReadFileContent = null; - const result = await (scalingEngine as any).getCurrentReplicas("proj-1"); - expect(result).toBe(1); - }); - - it("returns 1 when no reverse_proxy in file", async () => { - mockReadFileContent = "proj-1.localhost:80 {\n log\n}\n"; - const result = await (scalingEngine as any).getCurrentReplicas("proj-1"); - expect(result).toBe(1); - }); - - it("counts unique containers in reverse_proxy targets", async () => { - mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n"; - const result = await (scalingEngine as any).getCurrentReplicas("proj-1"); - expect(result).toBe(2); - }); - - it("ignores port suffixes when counting containers", async () => { - mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy a:80 b:80 c:80\n}\n"; - const result = await (scalingEngine as any).getCurrentReplicas("proj-1"); - expect(result).toBe(3); - }); - - it("counts only the primary site block when custom-domain blocks exist", async () => { - mockReadFileContent = [ - "verify-app.custom-domain.com {", - " reverse_proxy some-other-container:3000", - "}", - "proj-1.localhost:80 {", - " reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000", - "}", - ].join("\n"); - const result = await (scalingEngine as any).getCurrentReplicas("proj-1"); - expect(result).toBe(2); - }); - - it("falls back to whole-file counting when no primary block matches", async () => { - mockReadFileContent = "other.localhost:80 {\n reverse_proxy a:3000\n}\n"; - const result = await (scalingEngine as any).getCurrentReplicas("proj-1"); - expect(result).toBe(1); - }); -}); - -describe("evaluate", () => { - beforeEach(() => { - mockGetScalingPolicyResult = () => TEST_POLICY; - mockListEnvVarsResult = []; - mockReadFileContent = null; - mockRunImpl = null; - mockTryRunImpl = null; - }); - - it("returns early when deployment has no projectId", async () => { - await (scalingEngine as any).evaluate({ id: "dep-1", projectId: null, containerName: null }); - }); - - it("returns early when no scaling policy", async () => { - mockGetScalingPolicyResult = null; - await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); - }); - - it("returns early when policy is disabled", async () => { - mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, enabled: false }); - await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); - }); - - it("returns early when container stats fail", async () => { - mockRunImpl = async () => "not-json"; - const result = await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); - expect(result).toBeUndefined(); - }); - - it("sets highCpuSince on first high CPU reading", async () => { - mockRunImpl = async () => '{"CPUPerc":"85%","MemUsage":"128MiB / 512MiB"}'; - await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); - expect(mockRedisData.highCpuSince).toBeTruthy(); - }); - - it("does not scale up if highCpuSince is not old enough", async () => { - mockRedisData = { highCpuSince: String(Date.now() - 5000), lastScaleUp: "0" }; - mockRunImpl = async () => '{"CPUPerc":"85%","MemUsage":"128MiB / 512MiB"}'; - await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); - expect(mockWriteFilePath).toBeNull(); - }); - - it("scales up when CPU exceeds threshold past cooldown", async () => { - mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cooldownSeconds: 0 }); - mockRedisData = { highCpuSince: String(Date.now() - 10000), lastScaleUp: "0" }; - mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000\n}\n"; - - mockRunImpl = async (_cmd: string, args: string[]) => { - if (args.some((a) => a.includes("stats"))) return '{"CPUPerc":"85%","MemUsage":"128MiB / 512MiB"}'; - if (args.some((a) => a.includes("Image"))) return "my-app:latest"; - if (args.some((a) => a.includes("Env"))) return '["PORT=3000"]'; - if (args.some((a) => a.includes("Mounts"))) return "[]"; - if (args[0] === "run") return "new-container-id"; - if (args[0] === "ps") return "caddy-abc123\n"; - if (args.some((a) => a.includes("reload"))) return ""; - return ""; - }; - - await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); - - expect(mockWriteFilePath).toBeTruthy(); - expect(mockWriteContent).toContain("deploy-dep-1-replica-2"); - }); - - it("does not scale up when already at max replicas", async () => { - mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, maxReplicas: 2, cooldownSeconds: 0 }); - mockRedisData = { highCpuSince: String(Date.now() - 10000), lastScaleUp: "0" }; - mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n"; - mockRunImpl = async () => '{"CPUPerc":"85%","MemUsage":"128MiB / 512MiB"}'; - - await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); - expect(mockWriteFilePath).toBeNull(); - }); - - it("does not advance lastScaleUp when already at max replicas", async () => { - mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, maxReplicas: 2, cooldownSeconds: 0 }); - const lastScaleUpBefore = String(Date.now() - 60_000); - mockRedisData = { highCpuSince: String(Date.now() - 10000), lastScaleUp: lastScaleUpBefore }; - mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n"; - mockRunImpl = async () => '{"CPUPerc":"85%","MemUsage":"128MiB / 512MiB"}'; - - await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); - expect(mockRedisData.lastScaleUp).toBe(lastScaleUpBefore); - expect(mockRedisData.highCpuSince).toBeFalsy(); - }); - - it("keeps accruing the low-CPU timer between threshold and 30%", async () => { - mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cpuThresholdPercent: 1, cooldownSeconds: 0 }); - const lowCpuSinceBefore = String(Date.now() - 240_000); - mockRedisData = { lowCpuSince: lowCpuSinceBefore, highCpuSince: "0", lastScaleUp: "0" }; - mockRunImpl = async () => '{"CPUPerc":"22%","MemUsage":"64MiB / 512MiB"}'; - - await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); - expect(mockRedisData.lowCpuSince).toBe(lowCpuSinceBefore); - }); - - it("scales down in the overlap zone (threshold < cpu < 30) once idle for 5 min", async () => { - mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cpuThresholdPercent: 1, cooldownSeconds: 0, minReplicas: 1 }); - mockRedisData = { lowCpuSince: String(Date.now() - 310_000), lastScaleDown: "0" }; - mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n"; - mockTryRunImpl = async () => ""; - mockRunImpl = async (_cmd: string, args: string[]) => { - if (args.includes("stats")) return '{"CPUPerc":"22%","MemUsage":"64MiB / 512MiB"}'; - if (args[0] === "stop") return ""; - if (args[0] === "rm") return ""; - if (args[0] === "ps") return "caddy-abc123\n"; - if (args.includes("reload")) return ""; - return ""; - }; - - await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); - expect(mockWriteFilePath).toBeTruthy(); - expect(mockWriteContent).not.toContain("replica-2"); - }); - - it("does not scale down when CPU is moderate", async () => { - mockRunImpl = async () => '{"CPUPerc":"50%","MemUsage":"128MiB / 512MiB"}'; - await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); - expect(mockWriteFilePath).toBeNull(); - }); - - it("scales down when CPU stays low for 5 min", async () => { - mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, minReplicas: 1, cooldownSeconds: 0 }); - mockRedisData = { lowCpuSince: String(Date.now() - 310_000), lastScaleDown: "0" }; - mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n"; - mockTryRunImpl = async () => ""; - - let _callCount = 0; - mockRunImpl = async (_cmd: string, args: string[]) => { - _callCount++; - if (args.includes("stats")) return '{"CPUPerc":"5%","MemUsage":"64MiB / 512MiB"}'; - if (args[0] === "stop") return ""; - if (args[0] === "rm") return ""; - if (args[0] === "ps") return "caddy-abc123\n"; - if (args.includes("reload")) return ""; - return ""; - }; - - await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); - - expect(mockWriteContent).toContain("deploy-dep-1"); - expect(mockWriteContent).not.toContain("replica-2"); - }); - - it("does not scale down below minReplicas", async () => { - mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, minReplicas: 2, cooldownSeconds: 0 }); - mockRedisData = { lowCpuSince: String(Date.now() - 310_000), lastScaleDown: "0" }; - mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n"; - mockRunImpl = async () => '{"CPUPerc":"5%","MemUsage":"64MiB / 512MiB"}'; - - await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); - expect(mockWriteFilePath).toBeNull(); - }); - - it("scales down at idle even when cpuThresholdPercent is 0", async () => { - mockGetScalingPolicyResult = () => ({ - ...TEST_POLICY, - cpuThresholdPercent: 0, - minReplicas: 1, - cooldownSeconds: 0, - }); - mockRedisData = { lowCpuSince: String(Date.now() - 310_000), lastScaleDown: "0" }; - mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n"; - mockTryRunImpl = async () => ""; - mockRunImpl = async (_cmd: string, args: string[]) => { - if (args.includes("stats")) return '{"CPUPerc":"0%","MemUsage":"64MiB / 512MiB"}'; - if (args[0] === "stop") return ""; - if (args[0] === "rm") return ""; - if (args[0] === "ps") return "caddy-abc123\n"; - if (args.includes("reload")) return ""; - return ""; - }; - - await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); - expect(mockWriteFilePath).toBeTruthy(); - expect(mockWriteContent).toContain("deploy-dep-1"); - expect(mockWriteContent).not.toContain("replica-2"); - }); - - it("scales up on any observed CPU when cpuThresholdPercent is 0", async () => { - mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cpuThresholdPercent: 0, cooldownSeconds: 0 }); - mockRedisData = { highCpuSince: String(Date.now() - 10_000), lastScaleUp: "0" }; - mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000\n}\n"; - - mockRunImpl = async (_cmd: string, args: string[]) => { - if (args.some((a) => a.includes("stats"))) return '{"CPUPerc":"1%","MemUsage":"128MiB / 512MiB"}'; - if (args.some((a) => a.includes("Image"))) return "my-app:latest"; - if (args.some((a) => a.includes("Env"))) return '["PORT=3000"]'; - if (args.some((a) => a.includes("Mounts"))) return "[]"; - if (args[0] === "run") return "new-container-id"; - if (args[0] === "ps") return "caddy-abc123\n"; - if (args.some((a) => a.includes("reload"))) return ""; - return ""; - }; - - await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); - expect(mockWriteFilePath).toBeTruthy(); - expect(mockWriteContent).toContain("deploy-dep-1-replica-2"); - }); - - it("skips evaluation when the agent server is offline", async () => { - mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cooldownSeconds: 0 }); - mockRedisData = { highCpuSince: String(Date.now() - 10_000), lastScaleUp: "0" }; - mockScaleJobs = []; - await (scalingEngine as any).evaluate({ ...TEST_DEPLOYMENT, serverId: "server-2" }); - expect(mockScaleJobs.length).toBe(0); - }); - - it("scales up through a scale job on an online agent server", async () => { - mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cooldownSeconds: 0 }); - mockRedisData = { - highCpuSince: String(Date.now() - 10_000), - lastScaleUp: "0", - "dequel:agent-stats:server-1": JSON.stringify({ - updatedAt: new Date().toISOString(), - containers: [{ containerName: "deploy-dep-1", cpuPercent: 90, memoryMb: 200 }], - }), - }; - mockScaleJobs = []; - await (scalingEngine as any).evaluate({ ...TEST_DEPLOYMENT, serverId: "server-1" }); - expect(mockScaleJobs.length).toBe(2); - expect(mockScaleJobs[0].type).toBe("scale"); - expect(mockScaleJobs[0].payload.action).toBe("up"); - expect(mockScaleJobs[0].payload.replicas).toBe(2); - expect(mockScaleJobs[0].serverId).toBe("server-1"); - expect(mockScaleJobs[1].type).toBe("reload_routes"); - expect(mockScaleJobs[1].payload.action).toBe("add"); - }); - - it("counts agent replicas from cached stats", async () => { - mockRedisData = { - "dequel:agent-stats:server-1": JSON.stringify({ - updatedAt: new Date().toISOString(), - containers: [ - { containerName: "deploy-dep-1", cpuPercent: 40, memoryMb: 100, deploymentId: "dep-1", replica: false }, - { - containerName: "deploy-dep-1-replica-2", - cpuPercent: 41, - memoryMb: 110, - deploymentId: "dep-1", - replica: true, - }, - { containerName: "other-app", cpuPercent: 10, memoryMb: 10, deploymentId: "other", replica: false }, - ], - }), - }; - const count = await (scalingEngine as any).getCurrentReplicas( - "proj-1", - { server: { id: "server-1" }, mode: "agent" }, - TEST_DEPLOYMENT, - ); - expect(count).toBe(2); - }); +describe("scaling engine", () => { + it("passes every scaling engine scenario", () => { + expect(runEngineSuite()).toBe(0); + }, 120_000); }); diff --git a/apps/api/src/utils/__tests__/domain-verifier.test.ts b/apps/api/src/utils/__tests__/domain-verifier.test.ts index c34f787..f3cf9fa 100644 --- a/apps/api/src/utils/__tests__/domain-verifier.test.ts +++ b/apps/api/src/utils/__tests__/domain-verifier.test.ts @@ -154,7 +154,7 @@ describe("buildCaddySnippet", () => { it("creates snippet with just localhost domain when no verified domains", async () => { const snippet = await buildCaddySnippet("elysia", "deploy-abc", "proj-1", noDomains, APP_PORT, snippetOpts); expect(snippet).toBe( - "elysia.localhost:80 {\n log {\n output stdout\n format json\n }\n reverse_proxy deploy-abc:3000 {\n header_up Host {upstream_hostport}\n }\n}\n", + "elysia.localhost:80 {\n log {\n output stdout\n format json\n }\n reverse_proxy deploy-abc:3000\n}\n", ); }); @@ -162,7 +162,7 @@ describe("buildCaddySnippet", () => { const listFn = mockDomains(["app.example.com"]); const snippet = await buildCaddySnippet("elysia", "deploy-abc", "proj-1", listFn, APP_PORT, snippetOpts); expect(snippet).toBe( - "elysia.localhost:80, app.example.com {\n log {\n output stdout\n format json\n }\n reverse_proxy deploy-abc:3000 {\n header_up Host {upstream_hostport}\n }\n}\n", + "elysia.localhost:80, app.example.com {\n log {\n output stdout\n format json\n }\n reverse_proxy deploy-abc:3000\n}\n", ); }); @@ -258,7 +258,7 @@ describe("buildCaddySnippet", () => { ]); const snippet = await buildCaddySnippet("my-app", "deploy-abc", "proj-1", listFn, APP_PORT, snippetOpts); expect(snippet).toContain( - "custom.example.com {\n log {\n output stdout\n format json\n }\n reverse_proxy deploy-abc:3000 {\n header_up Host {upstream_hostport}\n trusted_proxies private_ranges 173.245.48.0/20", + "custom.example.com {\n log {\n output stdout\n format json\n }\n reverse_proxy deploy-abc:3000 {\n trusted_proxies private_ranges 173.245.48.0/20", ); }); }); diff --git a/apps/api/src/utils/__tests__/loki.test.ts b/apps/api/src/utils/__tests__/loki.test.ts new file mode 100644 index 0000000..8a34b69 --- /dev/null +++ b/apps/api/src/utils/__tests__/loki.test.ts @@ -0,0 +1,11 @@ +import { describe, expect, test } from "bun:test"; +import { caddyRequestLogSelector } from "../loki"; + +describe("caddyRequestLogSelector", () => { + test("extracts nested request.host and filters by host regex", () => { + const sel = caddyRequestLogSelector("app\\.localhost"); + expect(sel.startsWith('{container="')).toBe(true); + expect(sel).toContain("| json |"); + expect(sel.endsWith('request_host =~ "^(app\\.localhost)$"')).toBe(true); + }); +}); diff --git a/apps/api/src/utils/__tests__/server-default.test.ts b/apps/api/src/utils/__tests__/server-default.test.ts index e0622d9..c4e6af6 100644 --- a/apps/api/src/utils/__tests__/server-default.test.ts +++ b/apps/api/src/utils/__tests__/server-default.test.ts @@ -81,6 +81,22 @@ describe("pickBestServer", () => { expect(await pickBestServer(null, "a")).toBe("b"); }); + it("weights free memory over cpu at equal project counts", async () => { + servers = [ + { id: "a", mode: "ssh", status: "connected", cpuUsedPercent: 5, memoryTotalMb: 8192, memoryUsedMb: 7500 }, + { id: "b", mode: "ssh", status: "connected", cpuUsedPercent: 95, memoryTotalMb: 8192, memoryUsedMb: 500 }, + ]; + expect(await pickBestServer()).toBe("b"); + }); + + it("avoids memory-saturated servers even at low cpu", async () => { + servers = [ + { id: "a", mode: "ssh", status: "connected", cpuUsedPercent: 5, memoryTotalMb: 8192, memoryUsedMb: 7500 }, + { id: "b", mode: "ssh", status: "connected", cpuUsedPercent: 80 }, + ]; + expect(await pickBestServer()).toBe("b"); + }); + it("filters candidates by mode", async () => { servers = [ { id: "a", mode: "ssh", status: "connected", cpuUsedPercent: 10 }, diff --git a/apps/api/src/utils/caddy-site.ts b/apps/api/src/utils/caddy-site.ts index 2dfd3d0..88faacc 100644 --- a/apps/api/src/utils/caddy-site.ts +++ b/apps/api/src/utils/caddy-site.ts @@ -34,9 +34,7 @@ export const caddySite = (hosts: string, reverseProxy: string, options?: CaddySi }; export const caddyReverseProxy = (targets: string, options?: CaddyReverseProxyOptions): string => { - const trustedProxies = options?.cloudflareProxied - ? `\n trusted_proxies private_ranges ${CADDY_CLOUDFLARE_TRUSTED_PROXIES}\n header_up X-Real-IP {client_ip}` - : ""; + if (!options?.cloudflareProxied) return ` reverse_proxy ${targets}`; - return ` reverse_proxy ${targets} {\n header_up Host {upstream_hostport}${trustedProxies}\n }`; + return ` reverse_proxy ${targets} {\n trusted_proxies private_ranges ${CADDY_CLOUDFLARE_TRUSTED_PROXIES}\n header_up X-Real-IP {client_ip}\n }`; }; diff --git a/apps/api/src/utils/config.ts b/apps/api/src/utils/config.ts index 5d94a19..98e2f1b 100644 --- a/apps/api/src/utils/config.ts +++ b/apps/api/src/utils/config.ts @@ -52,6 +52,7 @@ export const config = { ...SYSTEM, databaseUrl, caddyBaseDomain: withFile("CADDY_BASE_DOMAIN", "localhost"), + caddyContainer: withFile("CADDY_CONTAINER", "dequel-caddy-1"), controlPlaneUrl: withFile("CONTROL_PLANE_URL", ""), agentTunnelUrl: withFile("AGENT_TUNNEL_URL", ""), envEncryptionKey, diff --git a/apps/api/src/utils/crypto.ts b/apps/api/src/utils/crypto.ts index 0cafe4b..61ae2ab 100644 --- a/apps/api/src/utils/crypto.ts +++ b/apps/api/src/utils/crypto.ts @@ -22,9 +22,15 @@ export const encryptValue = (value: string, secret: string) => { }; export const decryptValue = (encrypted: string, iv: string, tag: string, secret: string) => { - const key = deriveKey(secret); - const decipher = createDecipheriv(ALGO, key, Buffer.from(iv, "base64")); - decipher.setAuthTag(Buffer.from(tag, "base64")); - const decrypted = Buffer.concat([decipher.update(Buffer.from(encrypted, "base64")), decipher.final()]); - return decrypted.toString("utf8"); + try { + const key = deriveKey(secret); + const decipher = createDecipheriv(ALGO, key, Buffer.from(iv, "base64")); + decipher.setAuthTag(Buffer.from(tag, "base64")); + const decrypted = Buffer.concat([decipher.update(Buffer.from(encrypted, "base64")), decipher.final()]); + return decrypted.toString("utf8"); + } catch { + throw new Error( + "Stored secret could not be decrypted - the encryption key no longer matches. Open the Env tab, re-save the affected values, and deploy again.", + ); + } }; diff --git a/apps/api/src/utils/grafana.ts b/apps/api/src/utils/grafana.ts index 2e9fd86..af5d78d 100644 --- a/apps/api/src/utils/grafana.ts +++ b/apps/api/src/utils/grafana.ts @@ -1,5 +1,5 @@ import { config } from "./config"; -import { buildProjectRequestHostRegex } from "./loki"; +import { buildProjectRequestHostRegex, caddyRequestLogSelector } from "./loki"; import { slugify } from "./routes"; interface GrafanaDashboard { @@ -16,41 +16,15 @@ interface GrafanaDashboard { overwrite: boolean; } -let sessionCookie: string | null = null; -let sessionExpires = 0; - -async function grafanaLogin(): Promise { - if (sessionCookie && Date.now() < sessionExpires) { - return sessionCookie; - } - try { - const res = await fetch(`${config.grafanaUrl}/login`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - user: config.grafanaUser, - password: config.grafanaPass, - }), - }); - const setCookie = res.headers.get("set-cookie"); - if (!setCookie) return null; - sessionCookie = setCookie.split(";")[0]; - sessionExpires = Date.now() + 60 * 60 * 1000; - return sessionCookie; - } catch { - return null; - } -} +const grafanaAuth = (): string => `Basic ${btoa(`${config.grafanaUser}:${config.grafanaPass}`)}`; async function grafanaPost(path: string, body: unknown): Promise { - const cookie = await grafanaLogin(); - if (!cookie) return null; try { const res = await fetch(`${config.grafanaUrl}/api${path}`, { method: "POST", headers: { "Content-Type": "application/json", - Cookie: cookie, + Authorization: grafanaAuth(), }, body: JSON.stringify(body), }); @@ -104,7 +78,7 @@ export async function ensureProjectDashboard( targets: [ { datasource: { type: "loki", uid: "loki" }, - expr: `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [$__range]))`, + expr: `sum(count_over_time(${caddyRequestLogSelector(regexEscaped)} [$__range]))`, refId: "A", }, ], @@ -136,7 +110,7 @@ export async function ensureProjectDashboard( targets: [ { datasource: { type: "loki", uid: "loki" }, - expr: `(sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | status >= 100 and status < 400 [$__range])) / sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [$__range])) * 100) or 0`, + expr: `(sum(count_over_time(${caddyRequestLogSelector(regexEscaped)} | status >= 100 and status < 400 [$__range])) / sum(count_over_time(${caddyRequestLogSelector(regexEscaped)} [$__range])) * 100) or 0`, refId: "A", }, ], @@ -168,7 +142,7 @@ export async function ensureProjectDashboard( targets: [ { datasource: { type: "loki", uid: "loki" }, - expr: `avg_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | unwrap duration [$__range])`, + expr: `avg_over_time(${caddyRequestLogSelector(regexEscaped)} | unwrap duration [$__range])`, refId: "A", }, ], @@ -193,7 +167,7 @@ export async function ensureProjectDashboard( targets: [ { datasource: { type: "loki", uid: "loki" }, - expr: `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [2m]))`, + expr: `sum(count_over_time(${caddyRequestLogSelector(regexEscaped)} [2m]))`, refId: "A", }, ], @@ -225,7 +199,7 @@ export async function ensureProjectDashboard( targets: [ { datasource: { type: "loki", uid: "loki" }, - expr: `(sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | status >= 100 and status < 400 [2m])) / sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [2m])) * 100) or 0`, + expr: `(sum(count_over_time(${caddyRequestLogSelector(regexEscaped)} | status >= 100 and status < 400 [2m])) / sum(count_over_time(${caddyRequestLogSelector(regexEscaped)} [2m])) * 100) or 0`, refId: "A", }, ], @@ -250,7 +224,7 @@ export async function ensureProjectDashboard( targets: [ { datasource: { type: "loki", uid: "loki" }, - expr: `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | status >= 100 and status < 300 [2m]))`, + expr: `sum(count_over_time(${caddyRequestLogSelector(regexEscaped)} | status >= 100 and status < 300 [2m]))`, refId: "A", }, ], @@ -275,7 +249,7 @@ export async function ensureProjectDashboard( targets: [ { datasource: { type: "loki", uid: "loki" }, - expr: `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | status >= 300 and status < 400 [2m]))`, + expr: `sum(count_over_time(${caddyRequestLogSelector(regexEscaped)} | status >= 300 and status < 400 [2m]))`, refId: "A", }, ], @@ -300,7 +274,7 @@ export async function ensureProjectDashboard( targets: [ { datasource: { type: "loki", uid: "loki" }, - expr: `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | status >= 400 and status < 500 [2m]))`, + expr: `sum(count_over_time(${caddyRequestLogSelector(regexEscaped)} | status >= 400 and status < 500 [2m]))`, refId: "A", }, ], @@ -325,7 +299,7 @@ export async function ensureProjectDashboard( targets: [ { datasource: { type: "loki", uid: "loki" }, - expr: `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | status >= 500 [2m]))`, + expr: `sum(count_over_time(${caddyRequestLogSelector(regexEscaped)} | status >= 500 [2m]))`, refId: "A", }, ], @@ -351,7 +325,7 @@ export async function ensureProjectDashboard( targets: [ { datasource: { type: "loki", uid: "loki" }, - expr: `sum(rate({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [$__interval])) by (request_host)`, + expr: `sum(rate(${caddyRequestLogSelector(regexEscaped)} [$__interval])) by (request_host)`, legendFormat: "{{request_host}}", refId: "A", }, @@ -372,7 +346,7 @@ export async function ensureProjectDashboard( targets: [ { datasource: { type: "loki", uid: "loki" }, - expr: `sum(rate({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [$__interval])) by (status)`, + expr: `sum(rate(${caddyRequestLogSelector(regexEscaped)} [$__interval])) by (status)`, legendFormat: "HTTP {{status}}", refId: "A", }, @@ -393,7 +367,7 @@ export async function ensureProjectDashboard( targets: [ { datasource: { type: "loki", uid: "loki" }, - expr: `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [$__interval]))`, + expr: `sum(count_over_time(${caddyRequestLogSelector(regexEscaped)} [$__interval]))`, legendFormat: "Requests", refId: "A", }, @@ -420,25 +394,25 @@ export async function ensureProjectDashboard( targets: [ { datasource: { type: "loki", uid: "loki" }, - expr: `quantile_over_time(0.99, {container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | unwrap duration [$__interval])`, + expr: `quantile_over_time(0.99, ${caddyRequestLogSelector(regexEscaped)} | unwrap duration [$__interval])`, legendFormat: "p99", refId: "A", }, { datasource: { type: "loki", uid: "loki" }, - expr: `quantile_over_time(0.95, {container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | unwrap duration [$__interval])`, + expr: `quantile_over_time(0.95, ${caddyRequestLogSelector(regexEscaped)} | unwrap duration [$__interval])`, legendFormat: "p95", refId: "B", }, { datasource: { type: "loki", uid: "loki" }, - expr: `quantile_over_time(0.50, {container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | unwrap duration [$__interval])`, + expr: `quantile_over_time(0.50, ${caddyRequestLogSelector(regexEscaped)} | unwrap duration [$__interval])`, legendFormat: "p50 (median)", refId: "C", }, { datasource: { type: "loki", uid: "loki" }, - expr: `avg_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | unwrap duration [$__interval])`, + expr: `avg_over_time(${caddyRequestLogSelector(regexEscaped)} | unwrap duration [$__interval])`, legendFormat: "Average", refId: "D", }, @@ -453,7 +427,7 @@ export async function ensureProjectDashboard( targets: [ { datasource: { type: "loki", uid: "loki" }, - expr: `log_histogram({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | unwrap duration [$__interval])`, + expr: `log_histogram(${caddyRequestLogSelector(regexEscaped)} | unwrap duration [$__interval])`, refId: "A", }, ], @@ -554,7 +528,7 @@ export async function ensureProjectDashboard( targets: [ { datasource: { type: "loki", uid: "loki" }, - expr: `{container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" | status >= 400`, + expr: `${caddyRequestLogSelector(regexEscaped)} | status >= 400`, refId: "A", }, ], diff --git a/apps/api/src/utils/loki.ts b/apps/api/src/utils/loki.ts index 253835f..353d054 100644 --- a/apps/api/src/utils/loki.ts +++ b/apps/api/src/utils/loki.ts @@ -3,7 +3,7 @@ import { config } from "./config"; import { slugify } from "./routes"; const LOKI_URL = "http://loki:3100"; -const CADDY_LOG_STREAM = '{container="dequel-caddy-1"}'; +const caddyLogStream = (): string => `{container="${config.caddyContainer}"}`; export const buildProjectRequestHostRegex = async (projectId: string): Promise => { const project = await getProjectById(projectId); @@ -22,7 +22,7 @@ export const buildProjectRequestHostRegex = async (projectId: string): Promise - `${CADDY_LOG_STREAM} | json | request_host =~ "^(${hostRegex})$"`; + `${caddyLogStream()} | json | request_host =~ "^(${hostRegex})$"`; export const lokiInstantQuery = async (query: string, timeoutMs = 5000): Promise => { const url = `${LOKI_URL}/loki/api/v1/query?query=${encodeURIComponent(query)}`; diff --git a/apps/api/src/utils/server-default.ts b/apps/api/src/utils/server-default.ts index 9ab23ba..48351c4 100644 --- a/apps/api/src/utils/server-default.ts +++ b/apps/api/src/utils/server-default.ts @@ -14,6 +14,17 @@ const isServerHealthy = (s: Awaited>[number]): bo return false; }; +const memoryLoadScore = (s: { memoryTotalMb: number | null; memoryUsedMb: number | null }): number => { + if (!s.memoryTotalMb || s.memoryUsedMb === null || s.memoryUsedMb === undefined) return 0; + return Math.min(1, Math.max(0, s.memoryUsedMb / s.memoryTotalMb)); +}; + +const resourceScore = (s: Awaited>[number]): number => { + const memoryLoad = memoryLoadScore(s); + const cpuLoad = Math.min(1, Math.max(0, (s.cpuUsedPercent ?? 0) / 100)); + return memoryLoad * 0.7 + cpuLoad * 0.3; +}; + export const pickBestServer = async ( preferredServerId?: string | null, excludeServerId?: string | null, @@ -45,6 +56,8 @@ export const pickBestServer = async ( .sort((a, b) => { const countDiff = (projectCount.get(a.id) ?? 0) - (projectCount.get(b.id) ?? 0); if (countDiff !== 0) return countDiff; + const resourceDiff = resourceScore(a) - resourceScore(b); + if (resourceDiff !== 0) return resourceDiff; return (a.cpuUsedPercent ?? 0) - (b.cpuUsedPercent ?? 0); }); if (candidates.length > 0) return candidates[0].id; diff --git a/apps/api/src/utils/zip-store.ts b/apps/api/src/utils/zip-store.ts new file mode 100644 index 0000000..170b03e --- /dev/null +++ b/apps/api/src/utils/zip-store.ts @@ -0,0 +1,93 @@ +const crcTable = (() => { + const table = new Uint32Array(256); + for (let n = 0; n < 256; n++) { + let c = n; + for (let k = 0; k < 8; k++) { + c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; + } + table[n] = c >>> 0; + } + return table; +})(); + +const crc32 = (data: Uint8Array): number => { + let c = 0xffffffff; + for (let i = 0; i < data.length; i++) { + c = crcTable[(c ^ data[i]) & 0xff] ^ (c >>> 8); + } + return (c ^ 0xffffffff) >>> 0; +}; + +export interface ZipEntry { + name: string; + content: string; +} + +export const createZipStore = (entries: ZipEntry[]): Uint8Array => { + const encoder = new TextEncoder(); + const chunks: Uint8Array[] = []; + const central: { name: Uint8Array; crc: number; size: number; offset: number }[] = []; + let offset = 0; + + for (const entry of entries) { + const name = encoder.encode(entry.name); + const data = encoder.encode(entry.content); + const crc = crc32(data); + const header = new Uint8Array(30 + name.length); + const view = new DataView(header.buffer); + view.setUint32(0, 0x04034b50, true); + view.setUint16(4, 20, true); + view.setUint16(6, 0, true); + view.setUint16(8, 0, true); + view.setUint16(10, 0, true); + view.setUint16(12, 0x21, true); + view.setUint32(14, crc, true); + view.setUint32(18, data.length, true); + view.setUint32(22, data.length, true); + view.setUint16(26, name.length, true); + view.setUint16(28, 0, true); + header.set(name, 30); + chunks.push(header, data); + central.push({ name, crc, size: data.length, offset }); + offset += header.length + data.length; + } + + const centralStart = offset; + for (const entry of central) { + const dir = new Uint8Array(46 + entry.name.length); + const view = new DataView(dir.buffer); + view.setUint32(0, 0x02014b50, true); + view.setUint16(4, 20, true); + view.setUint16(6, 20, true); + view.setUint16(8, 0, true); + view.setUint16(10, 0, true); + view.setUint16(12, 0, true); + view.setUint16(14, 0x21, true); + view.setUint32(16, entry.crc, true); + view.setUint32(20, entry.size, true); + view.setUint32(24, entry.size, true); + view.setUint16(28, entry.name.length, true); + view.setUint32(42, entry.offset, true); + dir.set(entry.name, 46); + chunks.push(dir); + offset += dir.length; + } + + const eocd = new Uint8Array(22); + const view = new DataView(eocd.buffer); + view.setUint32(0, 0x06054b50, true); + view.setUint16(8, central.length, true); + view.setUint16(10, central.length, true); + view.setUint32(12, offset - centralStart, true); + view.setUint32(16, centralStart, true); + chunks.push(eocd); + + const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0); + const output = new Uint8Array(total); + let cursor = 0; + for (const chunk of chunks) { + output.set(chunk, cursor); + cursor += chunk.length; + } + return output; +}; diff --git a/apps/docs/blume.config.ts b/apps/docs/blume.config.ts index 17204d0..4dac358 100644 --- a/apps/docs/blume.config.ts +++ b/apps/docs/blume.config.ts @@ -40,6 +40,7 @@ export default defineConfig({ "/docs/configuration", "/docs/env-vars", "/docs/shared-env-vars", + "/docs/templates", "/docs/volumes", "/docs/domains", "/docs/scaling", diff --git a/apps/docs/docs/templates.mdx b/apps/docs/docs/templates.mdx new file mode 100644 index 0000000..cf8ff19 --- /dev/null +++ b/apps/docs/docs/templates.mdx @@ -0,0 +1,50 @@ +--- +title: One-Click Templates +category: Projects +description: The registry of one-click app templates Dequel can deploy with routing, SSL and secrets wired automatically. +slug: templates +--- + +One-click templates deploy popular self-hosted apps as complete Docker +Compose stacks - app plus its database, secrets, reverse-proxy route and +(via your `CADDY_BASE_DOMAIN`) HTTPS - in a single click. Open the +dashboard's **Create project** page and switch to **From a template**, or +go straight to `/templates` in the web app. + +Every template declares which settings it needs. Secrets you leave blank +are generated for you at deploy time; all values land in the project's +**Environment Variables** tab (encrypted at rest) and are injected into +the stack on deploy through compose interpolation. + +## Registry + +| Template | Category | What it deploys | +| --- | --- | --- | +| Gitea | developer | Lightweight self-hosted Git service with issues, pulls and CI | +| IT Tools | developer | Collection of handy developer tools, all in one place | +| Memos | productivity | Lightweight note-taking, like a private stream of your thoughts | +| n8n | automation | Workflow automation platform, self-hosted Zapier alternative | +| Nextcloud | storage | Files, calendar, contacts and collaboration | +| Paperless-ngx | productivity | Document management with OCR, full-text search and tagging | +| Plausible | analytics | Privacy-friendly, cookieless web analytics | +| Umami | analytics | Simple, privacy-focused analytics with a clean dashboard | +| Uptime Kuma | monitoring | Monitoring with status pages and alerts | +| Vaultwarden | security | Lightweight password manager compatible with Bitwarden clients | +| WordPress | cms | Content management system with PHP and a MariaDB backend | + +Each deployment gets a subdomain named after the project +(`.`), a status page under the project's +deployments tab, and its services joined to the private `dequel` network. + +## Managing template settings + +Template variables become regular environment variables of the project: + +1. Secrets left empty are generated once at deploy and stored encrypted. +2. Open the project's **Env** tab to view (masked) or edit values; use + the reveal action for secrets. +3. Compose files reference values as `${VAR}` placeholders, so a changed + value is applied on the next deploy/redeploy of the project. + +The stack's own compose file is rendered from the template at deploy +time - no secrets are baked into stored archives. diff --git a/apps/web/src/api/client.ts b/apps/web/src/api/client.ts index 828d6c7..6c220ba 100644 --- a/apps/web/src/api/client.ts +++ b/apps/web/src/api/client.ts @@ -4,6 +4,7 @@ import type { ApiKey, CreateProjectInput, Deployment, + DeployTemplateMeta, DiagRun, DiagStage, Domain, @@ -16,6 +17,7 @@ import type { ScalingPolicy, Server, SmtpSettingsStatus, + TemplateDeployResult, Volume, } from "../types"; import { apiFetch, BASE } from "./http"; @@ -54,6 +56,28 @@ export const deleteProject = (id: string) => method: "DELETE", }); +export const getProjectCompose = (id: string) => + apiFetch<{ + compose: string; + composeService?: string | null; + composePort?: number | null; + composeServices?: any; + }>(`/projects/${id}/compose`); + +export const updateProjectCompose = ( + id: string, + data: { + compose: string; + composeService?: string; + composePort?: number; + composeServices?: string; + }, +) => + apiFetch<{ project: Project; deployment: Deployment }>(`/projects/${id}/compose`, { + method: "PUT", + body: JSON.stringify(data), + }); + // Deployments export const listDeployments = (projectId?: string, offset = 0, limit = 50) => { const params = new URLSearchParams(); @@ -73,6 +97,18 @@ export const createDeployment = (form: FormData) => method: "POST", body: form, }); + +// Templates +export const listTemplates = () => apiFetch("/templates"); +export const getTemplate = (key: string) => apiFetch(`/templates/${key}`); +export const deployTemplate = ( + key: string, + data: { name?: string; values?: Record; serverId?: string }, +) => + apiFetch(`/templates/${key}/deployments`, { + method: "POST", + body: JSON.stringify(data), + }); export const rollbackDeployment = (id: string) => apiFetch(`/deployments/${id}/rollback`, { method: "POST" }); export const redeployDeployment = (id: string) => diff --git a/apps/web/src/components/layout/Header.tsx b/apps/web/src/components/layout/Header.tsx index 49ea2cc..2abe8b2 100644 --- a/apps/web/src/components/layout/Header.tsx +++ b/apps/web/src/components/layout/Header.tsx @@ -29,7 +29,7 @@ export function Header({ currentProject, currentProjectId, location, setSidebarO {currentProject.name} diff --git a/apps/web/src/components/project/create/DeploymentTargetSection.tsx b/apps/web/src/components/project/create/DeploymentTargetSection.tsx index 564fbfa..192c64b 100644 --- a/apps/web/src/components/project/create/DeploymentTargetSection.tsx +++ b/apps/web/src/components/project/create/DeploymentTargetSection.tsx @@ -1,4 +1,4 @@ -import { Laptop, Server } from "lucide-react"; +import { Laptop, Layers, Server } from "lucide-react"; import { cn } from "../../../lib/utils"; import type { Server as DequelServer } from "../../../types"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "../../ui/select"; @@ -32,6 +32,13 @@ export function DeploymentTargetSelect({ serverId, setServerId, servers }: Deplo + + + + Auto + - picks the server with the most free resources + + {targets.map((s) => ( diff --git a/apps/web/src/components/project/deployments/DeploymentsTab.tsx b/apps/web/src/components/project/deployments/DeploymentsTab.tsx index b1f0fb7..ed64883 100644 --- a/apps/web/src/components/project/deployments/DeploymentsTab.tsx +++ b/apps/web/src/components/project/deployments/DeploymentsTab.tsx @@ -1,4 +1,5 @@ -import { Play, Rocket, Webhook } from "lucide-react"; +import { useNavigate } from "@tanstack/react-router"; +import { Layers, Play, Rocket, Webhook } from "lucide-react"; import type React from "react"; import { useEffect, useRef, useState } from "react"; import { getRepoHooks, registerRepoHook, removeRepoHook } from "../../../api/client"; @@ -15,9 +16,9 @@ import { Button } from "../../ui/button"; import { Card, CardContent, CardHeader, CardTitle } from "../../ui/card"; import { Input } from "../../ui/input"; import { ClearCacheToggle } from "./clear-cache-toggle"; -import { SwitchToGitCard } from "./SwitchToGitCard"; import { DeploymentHistory } from "./deployment-history"; import { ManualDeployDialog } from "./manual-deploy-dialog"; +import { SwitchToGitCard } from "./SwitchToGitCard"; const PAGE_SIZE = 5; @@ -26,6 +27,7 @@ interface DeploymentsTabProps { } export function DeploymentsTab({ projectId }: DeploymentsTabProps) { + const navigate = useNavigate(); const { data: project } = useProject(projectId); const [page, setPage] = useState(0); const { data, isLoading } = useDeployments(projectId, page, PAGE_SIZE); @@ -204,7 +206,40 @@ export function DeploymentsTab({ projectId }: DeploymentsTabProps) { )} - {sourceType === "git" ? ( + {project?.buildType === "compose" && !project?.repoUrl ? ( +
+
+
+
+
+ Docker Compose Stack +
+
+ Primary Service:{" "} + + {(project as any).composeService || "auto-detect"} + + {(project as any).composePort && ( + + (:{(project as any).composePort}) + + )} +
+
+ Update and manage services, ports, and docker-compose.yml configuration in Project Settings. +
+
+ +
+
+
+ ) : sourceType === "git" ? (
diff --git a/apps/web/src/components/project/deployments/deployment-history.tsx b/apps/web/src/components/project/deployments/deployment-history.tsx index 4c5e548..3fda6f1 100644 --- a/apps/web/src/components/project/deployments/deployment-history.tsx +++ b/apps/web/src/components/project/deployments/deployment-history.tsx @@ -1,4 +1,5 @@ -import { ChevronLeft, ChevronRight, History, RefreshCw } from "lucide-react"; +import { useNavigate } from "@tanstack/react-router"; +import { ChevronLeft, ChevronRight, ExternalLink, History, RefreshCw } from "lucide-react"; import { useState } from "react"; import { cn } from "../../../lib/utils"; import { StatusBadge } from "../../StatusBadge"; @@ -37,6 +38,7 @@ export function DeploymentHistory({ onCancel, onDelete, }: DeploymentHistoryProps) { + const navigate = useNavigate(); const [deleteConfirmId, setDeleteConfirmId] = useState(null); const [cancelConfirmId, setCancelConfirmId] = useState(null); @@ -47,6 +49,8 @@ export function DeploymentHistory({
{deployments.map((dep) => ( + // biome-ignore lint/a11y/noStaticElementInteractions: selection card with nested interactive buttons + // biome-ignore lint/a11y/useKeyWithClickEvents: selection card with nested interactive buttons
onSelect(selectedId === dep.id ? null : dep.id)} @@ -64,13 +68,16 @@ export function DeploymentHistory({ )}
-
e.stopPropagation()}> +
{dep.status === "running" && dep.imageTag && dep.sourceType !== "image" && ( - )} - {dep.imageTag && dep.status !== "running" && dep.status !== "pending" && dep.status !== "building" && dep.status !== "deploying" && ( - )} + {dep.imageTag && + dep.status !== "running" && + dep.status !== "pending" && + dep.status !== "building" && + dep.status !== "deploying" && ( + + )} {dep.status !== "running" && ( )} +
@@ -116,15 +170,15 @@ export function DeploymentHistory({ {depDisplayName(projectName, dep.id)} - - {formatTimeAgo(dep.createdAt)} - + {formatTimeAgo(dep.createdAt)}
{dep.branch ? ( - {dep.branch} + + {dep.branch} + ) : null} {dep.sourceType}
@@ -217,6 +271,7 @@ export function DeploymentHistory({ strokeLinecap="round" strokeLinejoin="round" > + Cancel deployment @@ -260,11 +315,29 @@ export function DeploymentHistory({ strokeLinecap="round" strokeLinejoin="round" > + Delete deployment )} +
diff --git a/apps/web/src/components/project/envtab/EnvVarsTab.tsx b/apps/web/src/components/project/envtab/EnvVarsTab.tsx index d8a641a..831e53f 100644 --- a/apps/web/src/components/project/envtab/EnvVarsTab.tsx +++ b/apps/web/src/components/project/envtab/EnvVarsTab.tsx @@ -21,10 +21,13 @@ export function EnvVarsTab({ projectId }: EnvVarsTabProps) { queryFn: () => api.listEnvVars(projectId), }); - const { data: deploymentsData } = useDeployments(projectId, 0, 5); + const { data: deploymentsData } = useDeployments(projectId, 0, 50); const redeploy = useRedeployDeployment(); const navigate = useNavigate(); - const runningDeployment = deploymentsData?.items?.find((d) => d.status === "running"); + const inProgressStatuses = new Set(["pending", "building", "deploying"]); + const targetDeployment = + deploymentsData?.items?.find((d) => d.status === "running") || + deploymentsData?.items?.find((d) => d.status && !inProgressStatuses.has(d.status)); const [isAddOpen, setIsAddOpen] = useState(false); const [isImportOpen, setIsImportOpen] = useState(false); @@ -129,8 +132,9 @@ export function EnvVarsTab({ projectId }: EnvVarsTabProps) { }; const handleRedeploy = async () => { + if (!targetDeployment) return; try { - await redeploy.mutateAsync(runningDeployment!.id); + await redeploy.mutateAsync(targetDeployment.id); setShowRedeployPrompt(false); navigate({ search: { tab: "deployments" } as any }); } catch (err) { @@ -144,7 +148,8 @@ export function EnvVarsTab({ projectId }: EnvVarsTabProps) { setShowRedeployPrompt(false)} onRedeploy={handleRedeploy} @@ -177,7 +182,13 @@ export function EnvVarsTab({ projectId }: EnvVarsTabProps) { /> )} - setShowRedeployPrompt(true)} /> + setShowRedeployPrompt(true)} + /> diff --git a/apps/web/src/components/project/envtab/RedeployBanner.tsx b/apps/web/src/components/project/envtab/RedeployBanner.tsx index bfee30d..bedaac1 100644 --- a/apps/web/src/components/project/envtab/RedeployBanner.tsx +++ b/apps/web/src/components/project/envtab/RedeployBanner.tsx @@ -3,14 +3,23 @@ import { Button } from "../../ui/button"; interface RedeployBannerProps { show: boolean; - runningDeploymentId: string | undefined; + runningDeploymentId?: string | undefined; + targetDeploymentId?: string | undefined; isPending: boolean; onDismiss: () => void; onRedeploy: () => Promise; } -export function RedeployBanner({ show, runningDeploymentId, isPending, onDismiss, onRedeploy }: RedeployBannerProps) { - if (!show || !runningDeploymentId) return null; +export function RedeployBanner({ + show, + runningDeploymentId, + targetDeploymentId, + isPending, + onDismiss, + onRedeploy, +}: RedeployBannerProps) { + const effectiveId = targetDeploymentId || runningDeploymentId; + if (!show || !effectiveId) return null; return (
diff --git a/apps/web/src/components/project/settings/BuildSettingsSection.tsx b/apps/web/src/components/project/settings/BuildSettingsSection.tsx new file mode 100644 index 0000000..cd8510a --- /dev/null +++ b/apps/web/src/components/project/settings/BuildSettingsSection.tsx @@ -0,0 +1,193 @@ +import { Box, CheckCircle, Globe, Save } from "lucide-react"; +import type React from "react"; +import { useEffect, useState } from "react"; +import { useUpdateProject } from "../../../hooks/useProjects"; +import { cn } from "../../../lib/utils"; +import type { Project } from "../../../types"; +import { Button } from "../../ui/button"; +import { Card, CardContent } from "../../ui/card"; +import { Input } from "../../ui/input"; + +interface BuildSettingsSectionProps { + project: Project; +} + +export function BuildSettingsSection({ project }: BuildSettingsSectionProps) { + const updateProjectMutation = useUpdateProject(); + + const [buildType, setBuildType] = useState("railpack"); + const [buildCommand, setBuildCommand] = useState(""); + const [installCommand, setInstallCommand] = useState(""); + const [outputDir, setOutputDir] = useState(""); + const [startCommand, setStartCommand] = useState(""); + const [saveSuccess, setSaveSuccess] = useState(false); + + useEffect(() => { + if (project) { + setBuildType((project as any).buildType || "railpack"); + setBuildCommand(project.buildCommand || ""); + setInstallCommand((project as any).installCommand || ""); + setOutputDir((project as any).outputDir || ""); + setStartCommand(project.startCommand || ""); + } + }, [project]); + + const handleSave = async (e: React.FormEvent) => { + e.preventDefault(); + try { + await updateProjectMutation.mutateAsync({ + id: project.id, + buildType, + buildCommand: buildCommand.trim() || null, + installCommand: installCommand.trim() || null, + outputDir: outputDir.trim() || null, + startCommand: startCommand.trim() || null, + }); + setSaveSuccess(true); + setTimeout(() => setSaveSuccess(false), 3000); + } catch (err) { + console.error("Failed to update build settings:", err); + } + }; + + return ( + + +
+
+
+

Build & Execution Strategy

+

Choose how Dequel compiles and launches your application.

+
+
+ {saveSuccess && ( +
+ + Strategy saved +
+ )} + +
+
+ +
+ Build Engine +
+ + +
+
+ + {buildType === "railpack" && ( +
+
+ + setInstallCommand(e.target.value)} + /> +

+ Override dependency installation. Leave blank for auto-detection. +

+
+ +
+ + setBuildCommand(e.target.value)} + /> +

+ Override build command. Leave blank for auto-detection. +

+
+ +
+ + setOutputDir(e.target.value)} + /> +

+ Directory containing compiled static assets (for Static Site projects). +

+
+ +
+ + setStartCommand(e.target.value)} + /> +

+ Override application launch command. Leave blank for auto-detection. +

+
+
+ )} +
+ + + ); +} diff --git a/apps/web/src/components/project/settings/ComposeSettingsSection.tsx b/apps/web/src/components/project/settings/ComposeSettingsSection.tsx new file mode 100644 index 0000000..edf455b --- /dev/null +++ b/apps/web/src/components/project/settings/ComposeSettingsSection.tsx @@ -0,0 +1,378 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { useNavigate } from "@tanstack/react-router"; +import { AlertCircle, CheckCircle2, Layers, Play, Plus, RefreshCw, Trash2 } from "lucide-react"; +import { useEffect, useRef, useState } from "react"; +import * as api from "../../../api/client"; +import type { Project } from "../../../types"; +import { Alert, AlertDescription } from "../../ui/alert"; +import { Badge } from "../../ui/badge"; +import { Button } from "../../ui/button"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "../../ui/card"; +import { Input } from "../../ui/input"; + +interface ComposeServiceRow { + id: string; + serviceName: string; + port: string; + subdomain: string; +} + +interface ComposeSettingsSectionProps { + project: Project; +} + +export function ComposeSettingsSection({ project }: ComposeSettingsSectionProps) { + const queryClient = useQueryClient(); + const navigate = useNavigate(); + + const { data, isLoading, error } = useQuery({ + queryKey: ["project-compose", project.id], + queryFn: () => api.getProjectCompose(project.id), + }); + + const [composeCode, setComposeCode] = useState(""); + const [primaryService, setPrimaryService] = useState(""); + const [primaryPort, setPrimaryPort] = useState(""); + const [servicesList, setServicesList] = useState([]); + const [parseError, setParseError] = useState(null); + const [detectedServices, setDetectedServices] = useState([]); + const [saveSuccess, setSaveSuccess] = useState(false); + + const initializedFor = useRef(null); + useEffect(() => { + if (data && initializedFor.current !== project.id) { + initializedFor.current = project.id; + setComposeCode(data.compose || ""); + setPrimaryService(data.composeService || (project as any).composeService || ""); + setPrimaryPort( + data.composePort + ? String(data.composePort) + : (project as any).composePort + ? String((project as any).composePort) + : "", + ); + + const rawServices = data.composeServices ?? (project as any).composeServices; + if (rawServices) { + try { + const parsed = typeof rawServices === "string" ? JSON.parse(rawServices) : rawServices; + if (Array.isArray(parsed) && parsed.length > 0) { + setServicesList(parsed); + return; + } + } catch {} + } + setServicesList([ + { + id: "1", + serviceName: data.composeService || (project as any).composeService || "", + port: data.composePort ? String(data.composePort) : "", + subdomain: "", + }, + ]); + } + }, [data, project]); + + useEffect(() => { + if (!composeCode) return; + try { + const lines = composeCode.split("\n"); + let inServices = false; + const found: string[] = []; + for (const line of lines) { + const trimmed = line.trimEnd(); + if (/^services:\s*$/.test(trimmed)) { + inServices = true; + continue; + } + if (inServices) { + if (/^[a-zA-Z0-9_-]+:\s*$/.test(trimmed)) { + inServices = false; + continue; + } + const match = trimmed.match(/^ {2}([a-zA-Z0-9_-]+):\s*$/); + if (match?.[1]) { + found.push(match[1]); + } + } + } + setDetectedServices(found); + setParseError(null); + } catch (err) { + setParseError(err instanceof Error ? err.message : "Error parsing compose"); + } + }, [composeCode]); + + const updateMutation = useMutation({ + mutationFn: (payload: { + compose: string; + composeService?: string; + composePort?: number; + composeServices?: any[]; + }) => api.updateProjectCompose(project.id, payload), + onSuccess: (res) => { + setSaveSuccess(true); + queryClient.invalidateQueries({ queryKey: ["project-compose", project.id] }); + queryClient.invalidateQueries({ queryKey: ["project", project.id] }); + queryClient.invalidateQueries({ queryKey: ["deployments", project.id] }); + setTimeout(() => setSaveSuccess(false), 3000); + + if (res?.deployment?.id) { + navigate({ + to: "/project/$projectId/deployments/$deploymentId", + params: { projectId: project.id, deploymentId: res.deployment.id }, + }); + } + }, + }); + + const handleSave = () => { + const targetService = primaryService.trim() || detectedServices[0] || undefined; + const targetPort = primaryPort.trim() ? Number(primaryPort) : undefined; + updateMutation.mutate({ + compose: composeCode, + composeService: targetService, + composePort: targetPort, + composeServices: servicesList, + }); + }; + + const addRow = () => { + setServicesList((prev) => [...prev, { id: String(Date.now()), serviceName: "", port: "", subdomain: "" }]); + }; + + const removeRow = (id: string) => { + if (servicesList.length <= 1) return; + setServicesList((prev) => prev.filter((r) => r.id !== id)); + }; + + const updateRow = (id: string, field: keyof ComposeServiceRow, val: string) => { + setServicesList((prev) => prev.map((r) => (r.id === id ? { ...r, [field]: val } : r))); + }; + + if (isLoading) { + return ( + + Loading compose stack configuration... + + ); + } + + if (error) { + return ( + + + Failed to load Docker Compose configuration + + ); + } + + return ( +
+ + +
+
+ + Docker Compose Stack + + + View and modify your stack definition. Saving will bundle the compose file and trigger a redeploy. + +
+
+ + +
+
+ + {updateMutation.error && ( + + + + {updateMutation.error instanceof Error + ? updateMutation.error.message + : "Failed to save and redeploy compose"} + + + )} + + {detectedServices.length > 0 && ( +
+ + Detected Services: + + {detectedServices.map((svc) => ( + + {svc} + + ))} +
+ )} + + + + {saveSuccess && ( + + + Compose configuration updated and deployment queued! + + )} + + {parseError && ( + + + {parseError} + + )} + +
+
+ + YAML syntax +
+