diff --git a/apps/api/src/api/__tests__/rate-limit.test.ts b/apps/api/src/api/__tests__/rate-limit.test.ts new file mode 100644 index 0000000..198c3a4 --- /dev/null +++ b/apps/api/src/api/__tests__/rate-limit.test.ts @@ -0,0 +1,49 @@ +import { beforeEach, describe, expect, it } from "bun:test"; +import { Elysia } from "elysia"; +import { RATE_LIMIT_RULES, getClientIp, rateLimitMiddleware, rateLimiter } from "../rate-limit"; + +describe("Rate Limiting Middleware", () => { + beforeEach(() => { + rateLimiter.reset(); + process.env.TEST_RATE_LIMIT = "true"; + }); + + it("extracts client IP from headers with priority", () => { + const req1 = new Request("http://localhost/test", { + headers: { "x-real-ip": "1.1.1.1", "cf-connecting-ip": "2.2.2.2" }, + }); + expect(getClientIp(req1)).toBe("1.1.1.1"); + + const req2 = new Request("http://localhost/test", { + headers: { "x-forwarded-for": "2.2.2.2, 3.3.3.3" }, + }); + expect(getClientIp(req2)).toBe("2.2.2.2"); + + const req3 = new Request("http://localhost/test", { + headers: { "cf-connecting-ip": "4.4.4.4" }, + }); + expect(getClientIp(req3)).toBe("4.4.4.4"); + + const req4 = new Request("http://localhost/test"); + expect(getClientIp(req4)).toBe("127.0.0.1"); + }); + + it("enforces rate limits and sets 429 Retry-After headers when exceeded", async () => { + const app = new Elysia().use(rateLimitMiddleware).post("/api/auth/login", () => ({ success: true })); + + const headers = { "x-real-ip": "198.51.100.1" }; + + for (let i = 0; i < RATE_LIMIT_RULES["/api/auth/login"].max; i++) { + const res = await app.handle(new Request("http://localhost/api/auth/login", { method: "POST", headers })); + expect(res.status).toBe(200); + expect(res.headers.get("RateLimit-Limit")).toBe(String(RATE_LIMIT_RULES["/api/auth/login"].max)); + } + + const blocked = await app.handle(new Request("http://localhost/api/auth/login", { method: "POST", headers })); + expect(blocked.status).toBe(429); + expect(blocked.headers.get("RateLimit-Remaining")).toBe("0"); + expect(blocked.headers.get("Retry-After")).toBeDefined(); + const body = await blocked.json(); + expect(body.status).toBe("error"); + }); +}); diff --git a/apps/api/src/api/__tests__/server-validate.test.ts b/apps/api/src/api/__tests__/server-validate.test.ts new file mode 100644 index 0000000..9c4a0a3 --- /dev/null +++ b/apps/api/src/api/__tests__/server-validate.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, test } from "bun:test"; +import { validateServerCreate } from "../servers/validate"; + +describe("validateServerCreate", () => { + test("rejects inline ssh keys", () => { + expect( + validateServerCreate({ + name: "node-1", + host: "10.0.0.1", + sshKey: "-----BEGIN OPENSSH PRIVATE KEY-----", + sshKeyId: "key-1", + }), + ).toContain("SSH key pool"); + }); + + test("rejects ssh mode without pool key or password", () => { + expect(validateServerCreate({ name: "node-1", host: "10.0.0.1" })).toContain("sshKeyId"); + }); + + test("accepts ssh mode with pool key", () => { + expect(validateServerCreate({ name: "node-1", host: "10.0.0.1", sshKeyId: "key-1" })).toBeNull(); + expect(validateServerCreate({ name: "node-1", host: "10.0.0.1", sshPassword: "secret" })).toBeNull(); + }); + + test("rejects non-string sshPassword values", () => { + expect(validateServerCreate({ name: "node-1", host: "10.0.0.1", sshPassword: { pw: true } })).toContain( + "sshPassword", + ); + }); + + test("agent mode needs no ssh credentials", () => { + expect(validateServerCreate({ name: "homelab", host: "10.0.0.1", mode: "agent" })).toBeNull(); + }); + + test("keeps host/mode/port validation", () => { + expect(validateServerCreate({ name: "node-1", host: "bad_host", sshKeyId: "k" })).toContain("host"); + expect(validateServerCreate({ name: "node-1", host: "10.0.0.1", mode: "docker", sshKeyId: "k" })).toContain("mode"); + expect(validateServerCreate({ name: "node-1", host: "10.0.0.1", port: 99999, sshKeyId: "k" })).toContain("port"); + expect(validateServerCreate({ name: "", host: "10.0.0.1", sshKeyId: "k" })).toContain("name and host"); + }); +}); diff --git a/apps/api/src/api/audit-logs/index.ts b/apps/api/src/api/audit-logs/index.ts new file mode 100644 index 0000000..0b05b47 --- /dev/null +++ b/apps/api/src/api/audit-logs/index.ts @@ -0,0 +1,21 @@ +import { Elysia } from "elysia"; +import { countAuditLogs, listAuditLogs } from "../../db/repo"; +import { ok } from "../response"; + +export const auditLogsRoutes = new Elysia().get("/audit-logs", async ({ query }: any) => { + const rawLimit = Number(query?.limit); + const limit = Number.isFinite(rawLimit) && rawLimit > 0 ? Math.min(Math.floor(rawLimit), 100) : 50; + const rawOffset = Number(query?.offset); + const offset = Number.isFinite(rawOffset) && rawOffset >= 0 ? Math.floor(rawOffset) : 0; + const filters = { + actorType: query?.actorType, + actorId: query?.actorId, + action: query?.action, + resourceType: query?.resourceType, + resourceId: query?.resourceId, + limit, + offset, + }; + const [items, total] = await Promise.all([listAuditLogs(filters), countAuditLogs(filters)]); + return ok({ items, total, limit, offset }); +}); diff --git a/apps/api/src/api/backups/index.ts b/apps/api/src/api/backups/index.ts index b052cb0..5cdb0ce 100644 --- a/apps/api/src/api/backups/index.ts +++ b/apps/api/src/api/backups/index.ts @@ -1,9 +1,12 @@ +import { randomUUID } from "node:crypto"; import { Elysia } from "elysia"; import { BackupOrchestrator } from "../../backup/orchestrator"; +import { runRestoreTest } from "../../backup/restore-test"; import type { BackupTarget, StorageConfig } from "../../backup/types"; import { S3_BACKUP_PREFIX } from "../../backup/types"; import { deleteBackupRecord, getBackupRecord, listBackupRecords } from "../../db/repo/backups"; import { getDatabaseById } from "../../db/repo/databases"; +import { createRestoreTestRecord, listRestoreTests, updateRestoreTestRecord } from "../../db/repo/restore-tests"; import { getBackupStorageSettings } from "../../db/repo/settings"; import { created, fail, ok } from "../response"; @@ -117,13 +120,84 @@ export const backupRoutes = new Elysia({ prefix: "/backups" }) return ok({ deleted: true }); }) .post("/:id/restore", async ({ params, set }) => { + const record = await getBackupRecord(params.id); + if (!record) { + set.status = 404; + return fail("Backup not found"); + } + let serverId: string | null = null; + if (record.targetId !== "internal") { + const db = await getDatabaseById(record.targetId); + serverId = db?.serverId ?? null; + } + const event = await createRestoreTestRecord({ + id: randomUUID(), + backupId: record.id, + targetId: record.targetId, + serverId, + kind: "restore", + }); + let restoreError: unknown = null; try { const storage = await loadStorageConfig(); const orchestrator = new BackupOrchestrator(storage); await orchestrator.restore(params.id, resolveTarget); - return ok({ restored: true }); + } catch (error) { + restoreError = error; + } + try { + await updateRestoreTestRecord(event.id, { + status: restoreError ? "failed" : "passed", + error: restoreError ? String(restoreError) : null, + completedAt: new Date(), + }); + } catch (eventError) { + console.error(`[backups] failed to record restore event ${event.id}:`, eventError); + } + if (restoreError) { + set.status = 500; + return fail(String(restoreError)); + } + return ok({ restored: true }); + }) + .post("/:id/restore-test", async ({ params, body, set }: any) => { + try { + const record = await getBackupRecord(params.id); + if (!record) { + set.status = 404; + return fail("Backup not found"); + } + const test = await runRestoreTest({ backupId: params.id, serverId: body?.serverId ?? null }); + if (test?.status !== "passed") { + set.status = 500; + return fail(test?.error || "Restore test failed"); + } + return created(test); } catch (error) { set.status = 500; return fail(String(error)); } + }) + .get("/restore-tests", async ({ query }: any) => { + const limit = query?.limit ? Number(query.limit) : undefined; + return ok( + await listRestoreTests({ + backupId: query?.backupId, + targetId: query?.targetId, + status: query?.status, + kind: query?.kind, + limit: Number.isInteger(limit) && (limit as number) > 0 ? limit : undefined, + }), + ); + }) + .get("/:id/restore-tests", async ({ params, query }: any) => { + const limit = query?.limit ? Number(query.limit) : undefined; + return ok( + await listRestoreTests({ + backupId: params.id, + status: query?.status, + kind: query?.kind, + limit: Number.isInteger(limit) && (limit as number) > 0 ? limit : undefined, + }), + ); }); diff --git a/apps/api/src/api/deployments/index.ts b/apps/api/src/api/deployments/index.ts index 69c9d17..9d2b44e 100644 --- a/apps/api/src/api/deployments/index.ts +++ b/apps/api/src/api/deployments/index.ts @@ -3,6 +3,7 @@ import { basename, join } from "node:path"; import { Elysia } from "elysia"; import { validateRemoteDeployment } from "../../agents/deployments"; import { + captureProjectGithubToken, countDeployments, createDeployment, getDeploymentById, @@ -131,6 +132,9 @@ export const deploymentsRoutes = new Elysia() commitSha, clearCache, }); + if (projectId) { + await captureProjectGithubToken(projectId, gitUrl, request.headers.get("cookie")); + } await dispatchDeployment(deployment, project, server); return created(deployment); } diff --git a/apps/api/src/api/domains/index.ts b/apps/api/src/api/domains/index.ts index 12c7d2a..e19ca9c 100644 --- a/apps/api/src/api/domains/index.ts +++ b/apps/api/src/api/domains/index.ts @@ -91,6 +91,7 @@ export const domainsRoutes = new Elysia() type: body.type ?? "custom", targetService: body.targetService || null, targetPort, + cloudflareProxied: Boolean(body.cloudflareProxied), }); const { validateDomain, resolveServerIp } = await import("../../utils/dns"); const project = await getProjectById(params.id); diff --git a/apps/api/src/api/index.ts b/apps/api/src/api/index.ts index 42708fc..7e052a5 100644 --- a/apps/api/src/api/index.ts +++ b/apps/api/src/api/index.ts @@ -1,8 +1,10 @@ import { Elysia } from "elysia"; import { fixdiagRoutes } from "../fixdiag/routes"; +import { rateLimitMiddleware } from "./rate-limit"; import { agentRoutes } from "./agents"; import { alertsRoutes } from "./alerts"; import { apiKeysRoutes } from "./api-keys"; +import { auditLogsRoutes } from "./audit-logs"; import { authRoutes } from "./auth"; import { authMiddleware } from "./auth-middleware"; import { backupRoutes } from "./backups"; @@ -53,8 +55,10 @@ export const apiRoutes = new Elysia({ } return fail(message); }) + .use(rateLimitMiddleware) .use(authRoutes) .use(authMiddleware) + .use(auditLogsRoutes) .use(agentRoutes) .use(healthRoutes) .use(projectsRoutes) diff --git a/apps/api/src/api/projects/index.ts b/apps/api/src/api/projects/index.ts index f1f6c9b..56127a3 100644 --- a/apps/api/src/api/projects/index.ts +++ b/apps/api/src/api/projects/index.ts @@ -2,6 +2,7 @@ import { unlink } from "node:fs/promises"; import { join } from "node:path"; import { Elysia } from "elysia"; import { + captureProjectGithubToken, createProject, deleteProjectCascade, getProjectById, @@ -58,7 +59,7 @@ export const projectsRoutes = new Elysia() } return ok(project); }) - .post("/projects", async ({ body, set }: any) => { + .post("/projects", async ({ body, request, set }: any) => { if (!body?.name) { set.status = 400; return fail("name is required"); @@ -117,6 +118,11 @@ export const projectsRoutes = new Elysia() project_type: project.projectType, source_type: project.sourceType, }).catch(() => {}); + await captureProjectGithubToken(project.id, body.repoUrl, request?.headers?.get("cookie") ?? null).then( + (captured) => { + if (captured) project.hasGithubToken = true; + }, + ); return created(project); }) .patch("/projects/:id", async ({ params: { id }, body, set }: any) => { diff --git a/apps/api/src/api/rate-limit.ts b/apps/api/src/api/rate-limit.ts new file mode 100644 index 0000000..d33d213 --- /dev/null +++ b/apps/api/src/api/rate-limit.ts @@ -0,0 +1,117 @@ +import { Elysia } from "elysia"; +import { fail } from "./response"; + +export interface RateLimitRule { + windowMs: number; + max: number; +} + +export const RATE_LIMIT_RULES: Record = { + "/api/auth/login": { windowMs: 60_000, max: 5 }, + "/api/auth/refresh": { windowMs: 60_000, max: 30 }, + default: { windowMs: 60_000, max: 300 }, +}; + +export const getClientIp = (request: Request): string => { + const realIp = request.headers.get("x-real-ip"); + if (realIp) return realIp.trim(); + const forwarded = request.headers.get("x-forwarded-for"); + if (forwarded) return forwarded.split(",")[0].trim(); + const cfIp = request.headers.get("cf-connecting-ip"); + if (cfIp) return cfIp.trim(); + return "127.0.0.1"; +}; + +class RateLimiter { + private stores = new Map(); + private cleanupInterval: NodeJS.Timeout | null = null; + + constructor() { + this.cleanupInterval = setInterval(() => this.cleanup(), 60_000); + if (this.cleanupInterval?.unref) { + this.cleanupInterval.unref(); + } + } + + public check( + key: string, + rule: RateLimitRule, + ): { + allowed: boolean; + limit: number; + remaining: number; + reset: number; + retryAfter: number; + } { + const now = Date.now(); + const windowStart = now - rule.windowMs; + const timestamps = (this.stores.get(key) ?? []).filter((ts) => ts > windowStart); + + if (timestamps.length >= rule.max) { + const earliest = timestamps[0] ?? now; + const reset = Math.ceil((earliest + rule.windowMs) / 1000); + const retryAfter = Math.max(1, Math.ceil((earliest + rule.windowMs - now) / 1000)); + this.stores.set(key, timestamps); + return { + allowed: false, + limit: rule.max, + remaining: 0, + reset, + retryAfter, + }; + } + + timestamps.push(now); + this.stores.set(key, timestamps); + const earliest = timestamps[0]; + const reset = Math.ceil((earliest + rule.windowMs) / 1000); + return { + allowed: true, + limit: rule.max, + remaining: rule.max - timestamps.length, + reset, + retryAfter: 0, + }; + } + + public reset(): void { + this.stores.clear(); + } + + private cleanup(): void { + const now = Date.now(); + for (const [key, timestamps] of this.stores.entries()) { + const valid = timestamps.filter((ts) => ts > now - 60_000); + if (valid.length === 0) { + this.stores.delete(key); + } else { + this.stores.set(key, valid); + } + } + } +} + +export const rateLimiter = new RateLimiter(); + +export const rateLimitMiddleware = (app: Elysia) => + app.onBeforeHandle(async ({ request, set, path }) => { + if (process.env.NODE_ENV === "test" && process.env.TEST_RATE_LIMIT !== "true") { + return; + } + + const ip = getClientIp(request); + const rule = RATE_LIMIT_RULES[path] ?? RATE_LIMIT_RULES.default; + const key = `${path}:${ip}`; + + const result = rateLimiter.check(key, rule); + + set.headers["RateLimit-Limit"] = String(result.limit); + set.headers["RateLimit-Remaining"] = String(result.remaining); + set.headers["RateLimit-Reset"] = String(result.reset); + + if (!result.allowed) { + set.status = 429; + set.headers["Retry-After"] = String(result.retryAfter); + return fail("Too many requests, please try again later"); + } + }); diff --git a/apps/api/src/api/servers/index.ts b/apps/api/src/api/servers/index.ts index 8910a07..00b9fd1 100644 --- a/apps/api/src/api/servers/index.ts +++ b/apps/api/src/api/servers/index.ts @@ -3,8 +3,8 @@ import { createServer, deleteServer, getServerById, listServers } from "../../db import { serverEventBus } from "../../servers/event-bus"; import { isServerPreparing, prepareServer } from "../../servers/prepare"; import { testSshConnection } from "../../utils/ssh"; -import { isPort, SERVER_HOST_RE } from "../../utils/validate"; import { created, fail, ok } from "../response"; +import { validateServerCreate } from "./validate"; const redactServer = (server: T): T => { if (server.sshKey) server.sshKey = "[redacted]"; @@ -15,21 +15,10 @@ const redactServer = ok((await listServers()).map((server) => redactServer(server)))) .post("/servers", async ({ body, set }: any) => { - if (!body?.name || !body?.host) { + const validationError = validateServerCreate(body); + if (validationError) { set.status = 400; - return fail("name and host are required"); - } - if (!SERVER_HOST_RE.test(body.host)) { - set.status = 400; - return fail("host must be a valid hostname or IP address"); - } - if (body.mode && body.mode !== "ssh" && body.mode !== "agent") { - set.status = 400; - return fail("mode must be 'ssh' or 'agent'"); - } - if (body.port !== undefined && body.port !== null && !isPort(Number(body.port))) { - set.status = 400; - return fail("port must be between 1 and 65535"); + return fail(validationError); } return created( redactServer( @@ -39,7 +28,6 @@ export const serversRoutes = new Elysia() port: body.port ? Number(body.port) : body.mode === "ssh" ? 22 : 2375, mode: body.mode ?? "ssh", sshUser: body.sshUser ?? "root", - sshKey: body.sshKey, sshKeyId: body.sshKeyId, sshPassword: body.sshPassword, authToken: body.authToken, diff --git a/apps/api/src/api/servers/validate.ts b/apps/api/src/api/servers/validate.ts new file mode 100644 index 0000000..0a5eaca --- /dev/null +++ b/apps/api/src/api/servers/validate.ts @@ -0,0 +1,24 @@ +import { isPort, SERVER_HOST_RE } from "../../utils/validate"; + +export const validateServerCreate = (body: any): string | null => { + if (!body?.name || !body?.host) return "name and host are required"; + if (typeof body.host !== "string" || !SERVER_HOST_RE.test(body.host)) { + return "host must be a valid hostname or IP address"; + } + const mode = body.mode ?? "ssh"; + if (mode !== "ssh" && mode !== "agent") return "mode must be 'ssh' or 'agent'"; + if (body.port !== undefined && body.port !== null && !isPort(Number(body.port))) { + return "port must be between 1 and 65535"; + } + if (body.sshKey) { + return "SSH keys cannot be attached to a server directly. Add the key to the SSH key pool, then pass its sshKeyId."; + } + if (body.sshKeyId && typeof body.sshKeyId !== "string") return "sshKeyId must be a string"; + if (body.sshPassword && typeof body.sshPassword !== "string") { + return "sshPassword must be a non-empty string"; + } + if (mode === "ssh" && !body.sshKeyId && !(typeof body.sshPassword === "string" && body.sshPassword)) { + return "SSH servers need a pool key (sshKeyId) or a password (sshPassword)"; + } + return null; +}; diff --git a/apps/api/src/backup/__tests__/crypto.test.ts b/apps/api/src/backup/__tests__/crypto.test.ts new file mode 100644 index 0000000..f70f33c --- /dev/null +++ b/apps/api/src/backup/__tests__/crypto.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from "bun:test"; +import { Readable } from "node:stream"; +import { createBackupDecryptStream, createBackupEncryptStream } from "../crypto"; + +describe("Backup Crypto Streams", () => { + it("encrypts and decrypts stream data round-trip", async () => { + const plainText = "CREATE TABLE users (id serial primary key, email text);"; + const source = Readable.from([Buffer.from(plainText, "utf-8")]); + + const { input: encInput, output: encOutput } = createBackupEncryptStream(); + source.pipe(encInput); + + const chunks: Buffer[] = []; + for await (const chunk of encOutput) { + chunks.push(chunk as Buffer); + } + const encryptedBytes = Buffer.concat(chunks); + + expect(encryptedBytes.toString("utf-8")).not.toContain(plainText); + expect(encryptedBytes.length).toBeGreaterThan(16); + + const encSource = Readable.from([encryptedBytes]); + const { input: decInput, output: decOutput } = createBackupDecryptStream(); + encSource.pipe(decInput); + + const decryptedChunks: Buffer[] = []; + for await (const chunk of decOutput) { + decryptedChunks.push(chunk as Buffer); + } + const decryptedText = Buffer.concat(decryptedChunks).toString("utf-8"); + + expect(decryptedText).toBe(plainText); + }); + + it("fails authentication when encrypted data is tampered with", async () => { + const plainText = "SELECT * FROM secrets;"; + const source = Readable.from([Buffer.from(plainText, "utf-8")]); + + const { input: encInput, output: encOutput } = createBackupEncryptStream(); + source.pipe(encInput); + + const chunks: Buffer[] = []; + for await (const chunk of encOutput) { + chunks.push(chunk as Buffer); + } + const encryptedBytes = Buffer.concat(chunks); + + // Tamper with one byte in the ciphertext + encryptedBytes[15] ^= 0xff; + + const encSource = Readable.from([encryptedBytes]); + const { input: decInput, output: decOutput } = createBackupDecryptStream(); + encSource.pipe(decInput); + + let errored = false; + try { + for await (const _ of decOutput) { + // drain + } + } catch { + errored = true; + } + + expect(errored).toBe(true); + }); +}); diff --git a/apps/api/src/backup/crypto.ts b/apps/api/src/backup/crypto.ts new file mode 100644 index 0000000..923e0bf --- /dev/null +++ b/apps/api/src/backup/crypto.ts @@ -0,0 +1,102 @@ +import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto"; +import { Transform } from "node:stream"; +import { config } from "../utils/config"; +import { deriveKey } from "../utils/crypto"; + +const ALGO = "aes-256-gcm"; +const IV_LEN = 12; +const TAG_LEN = 16; + +const getEncryptionKey = (): string => { + const key = config.envEncryptionKey; + if (!key || (process.env.NODE_ENV === "production" && key === "dev-env-key-change-me")) { + throw new Error("ENV_ENCRYPTION_KEY must be configured in production"); + } + return key; +}; + +export const createBackupEncryptStream = () => { + const key = deriveKey(getEncryptionKey()); + const iv = randomBytes(IV_LEN); + const cipher = createCipheriv(ALGO, key, iv); + let ivSent = false; + + const transform = new Transform({ + transform(chunk: Buffer, _encoding, callback) { + try { + if (!ivSent) { + this.push(iv); + ivSent = true; + } + const data = cipher.update(chunk); + if (data.length > 0) this.push(data); + callback(); + } catch (err) { + callback(err as Error); + } + }, + flush(callback) { + try { + if (!ivSent) { + this.push(iv); + ivSent = true; + } + const finalData = cipher.final(); + if (finalData.length > 0) this.push(finalData); + const tag = cipher.getAuthTag(); + this.push(tag); + callback(); + } catch (err) { + callback(err as Error); + } + }, + }); + + return { input: transform, output: transform }; +}; + +export const createBackupDecryptStream = () => { + const key = deriveKey(getEncryptionKey()); + let buffer = Buffer.alloc(0); + let decipher: ReturnType | null = null; + + const transform = new Transform({ + transform(chunk: Buffer, _encoding, callback) { + try { + buffer = Buffer.concat([buffer, chunk]); + if (!decipher) { + if (buffer.length < IV_LEN + TAG_LEN) { + return callback(); + } + const iv = buffer.subarray(0, IV_LEN); + decipher = createDecipheriv(ALGO, key, iv); + buffer = buffer.subarray(IV_LEN); + } + if (buffer.length > TAG_LEN) { + const cipherText = buffer.subarray(0, buffer.length - TAG_LEN); + buffer = buffer.subarray(buffer.length - TAG_LEN); + const plain = decipher.update(cipherText); + if (plain.length > 0) this.push(plain); + } + callback(); + } catch (err) { + callback(err as Error); + } + }, + flush(callback) { + try { + if (!decipher || buffer.length !== TAG_LEN) { + return callback(new Error("Invalid backup payload: missing or truncated authentication tag")); + } + decipher.setAuthTag(buffer); + const finalPlain = decipher.final(); + if (finalPlain.length > 0) this.push(finalPlain); + callback(); + } catch (err) { + callback(err as Error); + } + }, + }); + + return { input: transform, output: transform }; +}; diff --git a/apps/api/src/backup/orchestrator.ts b/apps/api/src/backup/orchestrator.ts index 3d2b1e3..54246e4 100644 --- a/apps/api/src/backup/orchestrator.ts +++ b/apps/api/src/backup/orchestrator.ts @@ -12,6 +12,7 @@ import type { BackupContext } from "./adapter"; import { getAdapter } from "./adapters"; import type { BackupStorage } from "./storage"; import { createStorage } from "./storage/index"; +import { createBackupDecryptStream, createBackupEncryptStream } from "./crypto"; import type { BackupJob, BackupTarget, StorageConfig } from "./types"; async function buildContext(target: BackupTarget): Promise { @@ -43,17 +44,29 @@ export class BackupOrchestrator { const dump = await adapter.dump(ctx); await this.updateJob(job.id, { status: "compressing" }); - const compressed = dump.pipe(createGzip()); + const gzip = createGzip(); + const { input: encInput, output: encOutput } = createBackupEncryptStream(); + + dump.on("error", (err) => { + gzip.destroy(err); + encInput.destroy(err); + }); + gzip.on("error", (err) => { + encInput.destroy(err); + }); + + dump.pipe(gzip).pipe(encInput); await this.updateJob(job.id, { status: "uploading" }); - const filename = `${target.id}-${new Date().toISOString().replace(/[:.]/g, "-")}.sql.gz`; - const { path: storagePath, size: sizeBytes } = await this.storage.upload(filename, compressed); + const filename = `${target.id}-${new Date().toISOString().replace(/[:.]/g, "-")}.sql.gz.enc`; + const { path: storagePath, size: sizeBytes } = await this.storage.upload(filename, encOutput); const completed = await this.updateJob(job.id, { status: "completed", filename, storagePath, sizeBytes, + isEncrypted: true, completedAt: new Date(), }); @@ -72,9 +85,29 @@ export class BackupOrchestrator { const ctx = await buildContext(target); const adapter = getAdapter(target.engine); - const compressed = await this.storage.download(job.storagePath!); + const downloaded = await this.storage.download(job.storagePath!); const { createGunzip } = await import("node:zlib"); - const dump = compressed.pipe(createGunzip()); + const gunzip = createGunzip(); + + let streamToGunzip: any = downloaded; + if (job.isEncrypted) { + const { input: decInput, output: decOutput } = createBackupDecryptStream(); + downloaded.on("error", (err) => { + decInput.destroy(err); + gunzip.destroy(err); + }); + decOutput.on("error", (err) => { + gunzip.destroy(err); + }); + downloaded.pipe(decInput); + streamToGunzip = decOutput; + } else { + downloaded.on("error", (err) => { + gunzip.destroy(err); + }); + } + + const dump = streamToGunzip.pipe(gunzip); await adapter.restore(ctx, dump); } @@ -100,6 +133,7 @@ export class BackupOrchestrator { storageType: this.storageType, storagePath: null, sizeBytes: null, + isEncrypted: true, status: "pending", error: null, }); diff --git a/apps/api/src/backup/restore-test.ts b/apps/api/src/backup/restore-test.ts new file mode 100644 index 0000000..378e209 --- /dev/null +++ b/apps/api/src/backup/restore-test.ts @@ -0,0 +1,150 @@ +import { randomUUID } from "node:crypto"; +import { createGunzip } from "node:zlib"; +import { resolveDbImage } from "../databases/manager"; +import { getDatabaseById, getServerById } from "../db/repo"; +import { getBackupRecord } from "../db/repo/backups"; +import { createRestoreTestRecord, updateRestoreTestRecord } from "../db/repo/restore-tests"; +import { getBackupStorageSettings } from "../db/repo/settings"; +import type { Server } from "../types"; +import { dockerRun, ensureContainerRemoved } from "../utils/docker-run"; +import type { BackupContext } from "./adapter"; +import { getAdapter } from "./adapters"; +import { createStorage } from "./storage/index"; +import type { BackupTarget, DatabaseType, StorageConfig } from "./types"; +import { S3_BACKUP_PREFIX } from "./types"; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function loadStorageConfig(): Promise { + const settings = await getBackupStorageSettings(); + if (settings.type === "s3") { + return { + type: "s3", + endpoint: settings.s3Endpoint, + accessKeyId: settings.s3AccessKeyId, + secretAccessKey: settings.s3SecretAccessKey, + bucket: settings.s3Bucket, + region: settings.s3Region, + prefix: S3_BACKUP_PREFIX, + }; + } + return { type: "local", path: settings.path }; +} + +const scratchEnv = (engine: DatabaseType, username: string, password: string, databaseName: string): string[] => { + if (engine === "postgresql") { + return [`POSTGRES_USER=${username}`, `POSTGRES_PASSWORD=${password}`, `POSTGRES_DB=${databaseName}`]; + } + if (engine === "mysql") { + return [ + `MYSQL_ROOT_PASSWORD=${password}`, + `MYSQL_DATABASE=${databaseName}`, + `MYSQL_USER=${username}`, + `MYSQL_PASSWORD=${password}`, + ]; + } + if (engine === "mongodb") { + return [ + `MONGO_INITDB_ROOT_USERNAME=${username}`, + `MONGO_INITDB_ROOT_PASSWORD=${password}`, + `MONGO_INITDB_DATABASE=${databaseName}`, + ]; + } + return []; +}; + +const scratchCommand = (engine: DatabaseType, password: string): string[] => { + if (engine === "redis") return ["redis-server", "--requirepass", password]; + return []; +}; + +export interface RestoreTestOptions { + backupId: string; + serverId?: string | null; +} + +export const runRestoreTest = async (options: RestoreTestOptions) => { + const job = await getBackupRecord(options.backupId); + if (!job) throw new Error("Backup not found"); + if (job.status !== "completed" || !job.storagePath) { + throw new Error("Backup has not completed successfully"); + } + + const engine = job.engine as DatabaseType; + const adapter = getAdapter(engine); + + let version: string | null = null; + let databaseName = "dequel"; + if (job.targetId !== "internal") { + const db = await getDatabaseById(job.targetId); + if (!db) throw new Error(`Database ${job.targetId} not found`); + version = db.version; + databaseName = db.databaseName; + } + + let server: Server | null = null; + let serverId: string | null = null; + if (options.serverId) { + server = await getServerById(options.serverId); + if (!server) throw new Error("Restore-test server not found"); + serverId = server.id; + } + + const test = await createRestoreTestRecord({ + id: randomUUID(), + backupId: job.id, + targetId: job.targetId, + serverId, + }); + + const scratchName = `dequel-restore-test-${test.id.slice(0, 8)}`; + const username = "dequel"; + const password = randomUUID().replace(/-/g, ""); + const target: BackupTarget = { + id: `restore-test-${test.id}`, + type: "managed", + engine, + containerName: scratchName, + databaseName, + serverId, + credentials: { username, password }, + }; + const ctx: BackupContext = { target, server }; + + try { + const image = resolveDbImage(engine, version); + const envArgs = scratchEnv(engine, username, password, databaseName).flatMap((e) => ["-e", e]); + await dockerRun( + "docker", + ["run", "-d", "--name", scratchName, ...envArgs, image, ...scratchCommand(engine, password)], + server, + ); + + const deadline = Date.now() + 90_000; + let ready = false; + while (Date.now() < deadline) { + if (await adapter.ping(ctx)) { + ready = true; + break; + } + await sleep(2_000); + } + if (!ready) throw new Error(`Scratch ${engine} container did not become ready`); + + const storage = createStorage(await loadStorageConfig()); + const compressed = await storage.download(job.storagePath); + await adapter.restore(ctx, compressed.pipe(createGunzip())); + + if (!(await adapter.ping(ctx))) throw new Error("Restored container failed health check"); + + return updateRestoreTestRecord(test.id, { status: "passed", completedAt: new Date() }); + } catch (error) { + return updateRestoreTestRecord(test.id, { + status: "failed", + error: String(error), + completedAt: new Date(), + }); + } finally { + await ensureContainerRemoved(scratchName, server).catch(() => {}); + } +}; diff --git a/apps/api/src/backup/types.ts b/apps/api/src/backup/types.ts index 65d824e..f688fb4 100644 --- a/apps/api/src/backup/types.ts +++ b/apps/api/src/backup/types.ts @@ -26,6 +26,7 @@ export interface BackupJob { storageType: "local" | "s3"; storagePath: string | null; sizeBytes: number | null; + isEncrypted?: boolean; status: BackupStatus; error: string | null; createdAt: Date; diff --git a/apps/api/src/db/__tests__/audit-logs.test.ts b/apps/api/src/db/__tests__/audit-logs.test.ts new file mode 100644 index 0000000..1f6a445 --- /dev/null +++ b/apps/api/src/db/__tests__/audit-logs.test.ts @@ -0,0 +1,58 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "bun:test"; +import { countAuditLogs, createAuditLog, listAuditLogs } from "../repo/audit-logs"; +import { setupTestDb, teardownTestDb, truncateAllTables } from "../test-helper"; + +describe("Audit Logs Repository", () => { + let pool: any; + + beforeAll(async () => { + const res = await setupTestDb(); + pool = res.pool; + }); + + beforeEach(async () => { + await truncateAllTables(pool); + }); + + afterAll(async () => { + await teardownTestDb(pool); + }); + + it("creates and lists audit logs with filtering", async () => { + const log1 = await createAuditLog({ + actorType: "user", + actorId: "admin", + action: "project.create", + resourceType: "project", + resourceId: "proj-1", + details: { name: "test-app" }, + ipAddress: "192.168.1.1", + userAgent: "Mozilla/5.0", + }); + + expect(log1.id).toBeDefined(); + expect(log1.action).toBe("project.create"); + expect(log1.actorId).toBe("admin"); + expect(log1.details).toEqual({ name: "test-app" }); + + const log2 = await createAuditLog({ + actorType: "api_key", + actorId: "key-123", + action: "deployment.trigger", + resourceType: "deployment", + resourceId: "dep-1", + }); + + const allLogs = await listAuditLogs(); + expect(allLogs).toHaveLength(2); + expect(await countAuditLogs()).toBe(2); + + const userLogs = await listAuditLogs({ actorType: "user" }); + expect(userLogs).toHaveLength(1); + expect(userLogs[0].id).toBe(log1.id); + + const depLogs = await listAuditLogs({ resourceType: "deployment" }); + expect(depLogs).toHaveLength(1); + expect(depLogs[0].id).toBe(log2.id); + }); +}); diff --git a/apps/api/src/db/__tests__/databases-repo-runner.ts b/apps/api/src/db/__tests__/databases-repo-runner.ts index 1c2491c..c4fb1c5 100644 --- a/apps/api/src/db/__tests__/databases-repo-runner.ts +++ b/apps/api/src/db/__tests__/databases-repo-runner.ts @@ -32,9 +32,8 @@ const db = drizzle(pool, { schema }); setDbProvider(async () => db); const cleanup = async () => { - for (const name of TABLE_NAMES) { - await pool.query(`TRUNCATE TABLE "${name}" CASCADE`); - } + const tables = TABLE_NAMES.map((name) => `"${name}"`).join(", "); + await pool.query(`TRUNCATE TABLE ${tables} CASCADE`); }; try { diff --git a/apps/api/src/db/__tests__/databases.test.ts b/apps/api/src/db/__tests__/databases.test.ts index 2c13f14..64c49d8 100644 --- a/apps/api/src/db/__tests__/databases.test.ts +++ b/apps/api/src/db/__tests__/databases.test.ts @@ -35,7 +35,7 @@ describe("Database Repository Tests", () => { expect(result.standalone.volumeName).toMatch(/^db-[a-f0-9-]{12}$/); expect(result.standalone.connectionPrefix).toBe("postgresql:"); expect(result.standalone.hasPassword).toBe(true); - }); + }, 30000); it("persists project attachment, allowlist, and lifecycle updates", () => { const result = runRepoScenarios(); @@ -50,5 +50,5 @@ describe("Database Repository Tests", () => { expect(result.updated.storageUsedMb).toBe(256); expect(result.deleted).toBe(true); expect(result.total).toBe(1); - }); + }, 30000); }); diff --git a/apps/api/src/db/__tests__/github-integration-encryption.test.ts b/apps/api/src/db/__tests__/github-integration-encryption.test.ts new file mode 100644 index 0000000..be654a8 --- /dev/null +++ b/apps/api/src/db/__tests__/github-integration-encryption.test.ts @@ -0,0 +1,86 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "bun:test"; +import { drizzle } from "drizzle-orm/node-postgres"; +import type { Pool } from "pg"; +import { setDbProvider } from "../db-provider"; +import * as schema from "../schema"; +import { createTestPool, truncateAllTables } from "../test-helper"; + +const TEST_SECRET = "test-jwt-secret-for-testing-purposes-only"; +let pool: Pool; + +beforeAll(async () => { + pool = createTestPool(); + const db = drizzle(pool, { schema }); + setDbProvider(async () => db); + const { initAuth } = await import("../../utils/auth"); + initAuth(TEST_SECRET); +}); + +beforeEach(async () => { + await truncateAllTables(pool); +}); + +afterAll(async () => { + await truncateAllTables(pool); +}); + +describe("github integrations store secrets encrypted", () => { + it("writes ciphertext for client and webhook secrets", async () => { + const gh = await import("../repo/github"); + await gh.setGithubIntegration({ + clientId: "Iv1.testapp", + clientSecret: "gh-canary-secret", + webhookSecret: "hook-canary-secret", + }); + + const [rawRow] = await pool + .query( + "SELECT client_secret, client_secret_iv, client_secret_tag, webhook_secret, webhook_secret_iv FROM github_integrations", + ) + .then((r: any) => r.rows); + expect(rawRow.client_secret).not.toBe("gh-canary-secret"); + expect(rawRow.client_secret).toMatch(/^[A-Za-z0-9+/]+=*$/); + expect(rawRow.client_secret_iv).toBeTruthy(); + expect(rawRow.client_secret_tag).toBeTruthy(); + expect(rawRow.webhook_secret).not.toBe("hook-canary-secret"); + expect(rawRow.webhook_secret_iv).toBeTruthy(); + + const read = await gh.getGithubIntegration(); + expect(read?.clientSecret).toBe("gh-canary-secret"); + expect(read?.webhookSecret).toBe("hook-canary-secret"); + }); + + it("keeps an existing webhook secret when the update omits one", async () => { + const gh = await import("../repo/github"); + await gh.setGithubIntegration({ + clientId: "Iv1.testapp", + clientSecret: "secret-a", + webhookSecret: "hook-keep-me", + }); + const second = await gh.setGithubIntegration({ + clientId: "Iv1.testapp2", + clientSecret: "secret-b", + }); + expect(second.clientSecret).toBe("secret-b"); + expect(second.webhookSecret).toBe("hook-keep-me"); + + const read = await gh.getGithubIntegration(); + expect(read?.webhookSecret).toBe("hook-keep-me"); + }); + + it("decrypts rows migrated from legacy plaintext", async () => { + const gh = await import("../repo/github"); + const db = drizzle(pool, { schema }); + + await db.insert(schema.githubIntegrations).values({ + id: "legacy-row", + clientId: "Iv1.legacy", + clientSecret: "gh-plan-secret", + appName: "Dequel", + createdAt: new Date(), + }); + + const read = await gh.getGithubIntegration(); + expect(read?.clientSecret).toBe("gh-plan-secret"); + }); +}); diff --git a/apps/api/src/db/encryption-migration.ts b/apps/api/src/db/encryption-migration.ts index b9ae84c..8245418 100644 --- a/apps/api/src/db/encryption-migration.ts +++ b/apps/api/src/db/encryption-migration.ts @@ -1,10 +1,11 @@ -import { and, eq, isNotNull } from "drizzle-orm"; +import { and, eq, isNotNull, isNull, or } from "drizzle-orm"; import { config } from "../utils/config"; import { DEFAULT_ENV_ENCRYPTION_KEY, decryptValue, encryptValue } from "../utils/crypto"; import { getDb } from "./db-provider"; import { backupStorageSettings, environmentVariables, + githubIntegrations, githubSessions, llmProviderKeys, projects, @@ -270,6 +271,86 @@ const STORES: Store[] = [ ); }, }, + { + name: "github_integrations", + run: async (activeKey) => { + const db = await getDb(); + const legacy = await db + .select() + .from(githubIntegrations) + .where( + or( + isNull(githubIntegrations.clientSecretIv), + and(isNotNull(githubIntegrations.webhookSecret), isNull(githubIntegrations.webhookSecretIv)), + ), + ); + let migrated = 0; + for (const row of legacy) { + const client = row.clientSecret && !row.clientSecretIv ? encryptValue(row.clientSecret, activeKey) : null; + const webhook = row.webhookSecret && !row.webhookSecretIv ? encryptValue(row.webhookSecret, activeKey) : null; + if (!client && !webhook) continue; + await db + .update(githubIntegrations) + .set({ + ...(client + ? { clientSecret: client.encrypted, clientSecretIv: client.iv, clientSecretTag: client.tag } + : {}), + ...(webhook + ? { webhookSecret: webhook.encrypted, webhookSecretIv: webhook.iv, webhookSecretTag: webhook.tag } + : {}), + }) + .where(eq(githubIntegrations.id, row.id)) + .execute(); + migrated += 1; + } + const encrypted = await db + .select() + .from(githubIntegrations) + .where(and(isNotNull(githubIntegrations.clientSecretIv), isNotNull(githubIntegrations.clientSecretTag))); + let unknown = 0; + for (const row of encrypted) { + const triples: { read: StoredTriple; write: (next: Triple) => Promise }[] = []; + if (row.clientSecretIv) { + triples.push({ + read: { enc: row.clientSecret, iv: row.clientSecretIv, tag: row.clientSecretTag }, + write: (next) => + db + .update(githubIntegrations) + .set({ clientSecret: next.encrypted, clientSecretIv: next.iv, clientSecretTag: next.tag }) + .where(eq(githubIntegrations.id, row.id)) + .execute(), + }); + } + if (row.webhookSecretIv) { + triples.push({ + read: { enc: row.webhookSecret, iv: row.webhookSecretIv, tag: row.webhookSecretTag }, + write: (next) => + db + .update(githubIntegrations) + .set({ webhookSecret: next.encrypted, webhookSecretIv: next.iv, webhookSecretTag: next.tag }) + .where(eq(githubIntegrations.id, row.id)) + .execute(), + }); + } + for (const triple of triples) { + const { enc, iv, tag } = triple.read; + if (!enc || !iv || !tag) continue; + try { + decryptValue(enc, iv, tag, activeKey); + continue; + } catch {} + try { + const plain = decryptValue(enc, iv, tag, DEFAULT_ENV_ENCRYPTION_KEY); + await triple.write(encryptValue(plain, activeKey)); + migrated += 1; + } catch { + unknown += 1; + } + } + } + return { migrated, unknown }; + }, + }, { name: "projects", run: async (activeKey) => { diff --git a/apps/api/src/db/migrations/0037_security_hardening.sql b/apps/api/src/db/migrations/0037_security_hardening.sql new file mode 100644 index 0000000..1f155af --- /dev/null +++ b/apps/api/src/db/migrations/0037_security_hardening.sql @@ -0,0 +1,25 @@ +CREATE TABLE IF NOT EXISTS "audit_logs" ( + "id" text PRIMARY KEY NOT NULL, + "actor_type" text NOT NULL, + "actor_id" text NOT NULL, + "action" text NOT NULL, + "resource_type" text NOT NULL, + "resource_id" text, + "details" jsonb DEFAULT '{}'::jsonb NOT NULL, + "ip_address" text, + "user_agent" text, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); + +CREATE INDEX IF NOT EXISTS "idx_audit_logs_actor" ON "audit_logs" ("actor_type", "actor_id"); +CREATE INDEX IF NOT EXISTS "idx_audit_logs_resource" ON "audit_logs" ("resource_type", "resource_id"); +CREATE INDEX IF NOT EXISTS "idx_audit_logs_created_at" ON "audit_logs" ("created_at"); + +ALTER TABLE "databases" ADD COLUMN IF NOT EXISTS "password_encrypted" text; +ALTER TABLE "databases" ADD COLUMN IF NOT EXISTS "password_iv" text; +ALTER TABLE "databases" ADD COLUMN IF NOT EXISTS "password_tag" text; +ALTER TABLE "databases" ALTER COLUMN "password" DROP NOT NULL; + +ALTER TABLE "backups" ADD COLUMN IF NOT EXISTS "is_encrypted" boolean DEFAULT false NOT NULL; + +ALTER TABLE "domains" ADD COLUMN IF NOT EXISTS "cloudflare_proxied" boolean DEFAULT false NOT NULL; diff --git a/apps/api/src/db/migrations/0038_restore_tests.sql b/apps/api/src/db/migrations/0038_restore_tests.sql new file mode 100644 index 0000000..f8d8166 --- /dev/null +++ b/apps/api/src/db/migrations/0038_restore_tests.sql @@ -0,0 +1,17 @@ +CREATE TABLE "restore_tests" ( + "id" text PRIMARY KEY NOT NULL, + "backup_id" text NOT NULL, + "target_id" text NOT NULL, + "server_id" text, + "kind" text DEFAULT 'test' NOT NULL, + "status" text DEFAULT 'running' NOT NULL, + "error" text, + "started_at" timestamp with time zone DEFAULT now() NOT NULL, + "completed_at" timestamp with time zone +); +--> statement-breakpoint +CREATE INDEX "idx_restore_tests_backup" ON "restore_tests" USING btree ("backup_id"); +--> statement-breakpoint +CREATE INDEX "idx_restore_tests_target" ON "restore_tests" USING btree ("target_id"); +--> statement-breakpoint +CREATE INDEX "idx_restore_tests_status" ON "restore_tests" USING btree ("status"); diff --git a/apps/api/src/db/migrations/0039_github_integration_encryption.sql b/apps/api/src/db/migrations/0039_github_integration_encryption.sql new file mode 100644 index 0000000..3c544dd --- /dev/null +++ b/apps/api/src/db/migrations/0039_github_integration_encryption.sql @@ -0,0 +1,4 @@ +ALTER TABLE "github_integrations" ADD COLUMN "client_secret_iv" text;--> statement-breakpoint +ALTER TABLE "github_integrations" ADD COLUMN "client_secret_tag" text;--> statement-breakpoint +ALTER TABLE "github_integrations" ADD COLUMN "webhook_secret_iv" text;--> statement-breakpoint +ALTER TABLE "github_integrations" ADD COLUMN "webhook_secret_tag" text; \ No newline at end of file diff --git a/apps/api/src/db/migrations/meta/_journal.json b/apps/api/src/db/migrations/meta/_journal.json index 850b822..c261cf1 100644 --- a/apps/api/src/db/migrations/meta/_journal.json +++ b/apps/api/src/db/migrations/meta/_journal.json @@ -92,6 +92,27 @@ "when": 1790700000000, "tag": "0036_fixdiag", "breakpoints": true + }, + { + "idx": 13, + "version": "7", + "when": 1791432207765, + "tag": "0037_security_hardening", + "breakpoints": true + }, + { + "idx": 14, + "version": "7", + "when": 1791861049522, + "tag": "0038_restore_tests", + "breakpoints": true + }, + { + "idx": 15, + "version": "7", + "when": 1792300000000, + "tag": "0039_github_integration_encryption", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/api/src/db/repo/audit-logs.ts b/apps/api/src/db/repo/audit-logs.ts new file mode 100644 index 0000000..3b22d21 --- /dev/null +++ b/apps/api/src/db/repo/audit-logs.ts @@ -0,0 +1,79 @@ +import { randomUUID } from "node:crypto"; +import { and, count, desc, eq } from "drizzle-orm"; +import type { AuditLog, CreateAuditLogInput, ListAuditLogsFilters } from "../../types"; +import { getDb } from "../db-provider"; +import { auditLogs } from "../schema"; + +const mapAuditLog = (row: typeof auditLogs.$inferSelect): AuditLog => ({ + id: row.id, + actorType: row.actorType as AuditLog["actorType"], + actorId: row.actorId, + action: row.action, + resourceType: row.resourceType, + resourceId: row.resourceId, + details: (row.details as Record) ?? {}, + ipAddress: row.ipAddress, + userAgent: row.userAgent, + createdAt: row.createdAt.toISOString(), +}); + +export const createAuditLog = async (input: CreateAuditLogInput): Promise => { + const db = await getDb(); + const id = randomUUID(); + const [row] = await db + .insert(auditLogs) + .values({ + id, + actorType: input.actorType, + actorId: input.actorId, + action: input.action, + resourceType: input.resourceType, + resourceId: input.resourceId ?? null, + details: input.details ?? {}, + ipAddress: input.ipAddress ?? null, + userAgent: input.userAgent ?? null, + }) + .returning(); + + return mapAuditLog(row); +}; + +export const listAuditLogs = async (filters: ListAuditLogsFilters = {}): Promise => { + const db = await getDb(); + const conditions = []; + + if (filters.actorType) conditions.push(eq(auditLogs.actorType, filters.actorType)); + if (filters.actorId) conditions.push(eq(auditLogs.actorId, filters.actorId)); + if (filters.action) conditions.push(eq(auditLogs.action, filters.action)); + if (filters.resourceType) conditions.push(eq(auditLogs.resourceType, filters.resourceType)); + if (filters.resourceId) conditions.push(eq(auditLogs.resourceId, filters.resourceId)); + + const query = db + .select() + .from(auditLogs) + .where(conditions.length > 0 ? and(...conditions) : undefined) + .orderBy(desc(auditLogs.createdAt)) + .limit(filters.limit ?? 50) + .offset(filters.offset ?? 0); + + const rows = await query.execute(); + return rows.map(mapAuditLog); +}; + +export const countAuditLogs = async (filters: ListAuditLogsFilters = {}): Promise => { + const db = await getDb(); + const conditions = []; + + if (filters.actorType) conditions.push(eq(auditLogs.actorType, filters.actorType)); + if (filters.actorId) conditions.push(eq(auditLogs.actorId, filters.actorId)); + if (filters.action) conditions.push(eq(auditLogs.action, filters.action)); + if (filters.resourceType) conditions.push(eq(auditLogs.resourceType, filters.resourceType)); + if (filters.resourceId) conditions.push(eq(auditLogs.resourceId, filters.resourceId)); + + const [res] = await db + .select({ count: count() }) + .from(auditLogs) + .where(conditions.length > 0 ? and(...conditions) : undefined); + + return Number(res?.count ?? 0); +}; diff --git a/apps/api/src/db/repo/backups.ts b/apps/api/src/db/repo/backups.ts index c12d0d8..083c2d2 100644 --- a/apps/api/src/db/repo/backups.ts +++ b/apps/api/src/db/repo/backups.ts @@ -11,6 +11,7 @@ export interface BackupRecord { storageType: string; storagePath: string | null; sizeBytes: number | null; + isEncrypted: boolean; error: string | null; status: string; createdAt: Date; @@ -25,6 +26,7 @@ export async function createBackupRecord( .insert(backups) .values({ ...record, + isEncrypted: record.isEncrypted ?? false, createdAt: new Date(), }) .returning(); diff --git a/apps/api/src/db/repo/databases.ts b/apps/api/src/db/repo/databases.ts index 019e617..b3e5355 100644 --- a/apps/api/src/db/repo/databases.ts +++ b/apps/api/src/db/repo/databases.ts @@ -1,49 +1,96 @@ import { randomUUID } from "node:crypto"; import { desc, eq } from "drizzle-orm"; import type { CreateDatabaseInput, Database, DatabaseStatus } from "../../types"; +import { config } from "../../utils/config"; +import { decryptValue, encryptValue } from "../../utils/crypto"; import { getDb } from "../db-provider"; import { databases } from "../schema"; import { getRowsAffected, now } from "./helpers"; -const mapDatabase = (row: typeof databases.$inferSelect): Database => ({ - id: row.id, - projectId: row.projectId ?? null, - serverId: row.serverId ?? null, - name: row.name, - type: row.type as Database["type"], - version: row.version, - databaseName: row.databaseName, - username: row.username, - password: row.password, - internalHost: row.internalHost, - internalPort: row.internalPort, - cpuLimit: row.cpuLimit, - memoryLimitMb: row.memoryLimitMb, - storageLimitMb: row.storageLimitMb, - storageUsedMb: row.storageUsedMb, - publicAccess: Boolean(row.publicAccess), - allowPublicAccessFromAnywhere: Boolean(row.allowPublicAccessFromAnywhere), - allowedCidrs: Array.isArray(row.allowedCidrs) - ? row.allowedCidrs - : (() => { - try { - return JSON.parse(row.allowedCidrs || "[]"); - } catch { - return []; - } - })(), - externalPort: row.externalPort, - proxyContainerName: row.proxyContainerName, - volumeName: row.volumeName, - connectionString: row.connectionString, - status: row.status as DatabaseStatus, - containerName: row.containerName, - backupEnabled: Boolean(row.backupEnabled), - backupSchedule: row.backupSchedule ?? "0 */6 * * *", - backupRetention: row.backupRetention ?? 7, - createdAt: row.createdAt, - updatedAt: row.updatedAt, -}); +export const buildConnectionString = ( + type: Database["type"], + username: string, + password: string, + host: string, + port: number, + databaseName: string, +): string => { + if (type === "redis") { + return password ? `redis://:${password}@${host}:${port}` : `redis://${host}:${port}`; + } + if (type === "mongodb") { + const auth = password ? `${username}:${password}@` : ""; + return `mongodb://${auth}${host}:${port}/${databaseName}?authSource=admin`; + } + if (type === "mysql") { + const auth = password ? `${username}:${password}@` : `${username}@`; + return `mysql://${auth}${host}:${port}/${databaseName}`; + } + const auth = password ? `${username}:${password}@` : `${username}@`; + return `postgresql://${auth}${host}:${port}/${databaseName}`; +}; + +const mapDatabase = (row: typeof databases.$inferSelect): Database => { + const password = (() => { + if (row.passwordEncrypted && row.passwordIv && row.passwordTag) { + try { + return decryptValue(row.passwordEncrypted, row.passwordIv, row.passwordTag, config.envEncryptionKey); + } catch { + return row.password ?? ""; + } + } + return row.password ?? ""; + })(); + + return { + id: row.id, + projectId: row.projectId ?? null, + serverId: row.serverId ?? null, + name: row.name, + type: row.type as Database["type"], + version: row.version, + databaseName: row.databaseName, + username: row.username, + password, + internalHost: row.internalHost, + internalPort: row.internalPort, + cpuLimit: row.cpuLimit, + memoryLimitMb: row.memoryLimitMb, + storageLimitMb: row.storageLimitMb, + storageUsedMb: row.storageUsedMb, + publicAccess: Boolean(row.publicAccess), + allowPublicAccessFromAnywhere: Boolean(row.allowPublicAccessFromAnywhere), + allowedCidrs: Array.isArray(row.allowedCidrs) + ? row.allowedCidrs + : (() => { + try { + return JSON.parse(row.allowedCidrs || "[]"); + } catch { + return []; + } + })(), + externalPort: row.externalPort, + proxyContainerName: row.proxyContainerName, + volumeName: row.volumeName, + connectionString: password + ? buildConnectionString( + row.type as Database["type"], + row.username, + password, + row.internalHost, + row.internalPort, + row.databaseName, + ) + : row.connectionString, + status: row.status as DatabaseStatus, + containerName: row.containerName, + backupEnabled: Boolean(row.backupEnabled), + backupSchedule: row.backupSchedule ?? "0 */6 * * *", + backupRetention: row.backupRetention ?? 7, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }; +}; export const createDatabase = async (input: CreateDatabaseInput): Promise => { const id = randomUUID(); @@ -52,20 +99,12 @@ export const createDatabase = async (input: CreateDatabaseInput): Promise => { const db = await getDb(); - const rows = await db.select().from(diagStages).where(eq(diagStages.runId, runId)).execute(); + const rows = await db + .select() + .from(diagStages) + .where(eq(diagStages.runId, runId)) + .orderBy(asc(diagStages.createdAt)) + .execute(); return rows.map((r) => ({ stage: r.stage as DiagStageName, payload: r.payload ?? null })); }; diff --git a/apps/api/src/db/repo/domains.ts b/apps/api/src/db/repo/domains.ts index c12a4d0..8ac6cbe 100644 --- a/apps/api/src/db/repo/domains.ts +++ b/apps/api/src/db/repo/domains.ts @@ -14,6 +14,7 @@ const mapDomain = (row: typeof domains.$inferSelect): Domain => ({ sslStatus: row.sslStatus as SslStatus, targetService: row.targetService ?? null, targetPort: row.targetPort ?? null, + cloudflareProxied: Boolean(row.cloudflareProxied), createdAt: row.createdAt, updatedAt: row.updatedAt, }); @@ -33,6 +34,7 @@ export const createDomain = async (input: CreateDomainInput): Promise => sslStatus: "pending", targetService: input.targetService ?? null, targetPort: input.targetPort ?? null, + cloudflareProxied: Boolean(input.cloudflareProxied), createdAt: timestamp, updatedAt: timestamp, }) diff --git a/apps/api/src/db/repo/github.ts b/apps/api/src/db/repo/github.ts index 1b5c308..11db546 100644 --- a/apps/api/src/db/repo/github.ts +++ b/apps/api/src/db/repo/github.ts @@ -1,16 +1,30 @@ import { randomUUID } from "node:crypto"; import { desc, eq } from "drizzle-orm"; import type { GithubIntegration } from "../../types"; +import { config } from "../../utils/config"; +import { decryptValue, encryptValue } from "../../utils/crypto"; import { getDb } from "../db-provider"; import { githubIntegrations } from "../schema"; import { now } from "./helpers"; +const decryptOrRaw = (value: string, iv: string | null, tag: string | null, field: string): string => { + if (!iv || !tag) return value; + try { + return decryptValue(value, iv, tag, config.envEncryptionKey); + } catch { + console.warn(`[GithubIntegration] could not decrypt ${field} (check ENV_ENCRYPTION_KEY)`); + return ""; + } +}; + const mapGithubIntegration = (row: typeof githubIntegrations.$inferSelect): GithubIntegration => ({ id: row.id, clientId: row.clientId, - clientSecret: row.clientSecret, + clientSecret: decryptOrRaw(row.clientSecret, row.clientSecretIv, row.clientSecretTag, "clientSecret"), appName: row.appName, - webhookSecret: row.webhookSecret ?? null, + webhookSecret: row.webhookSecret + ? decryptOrRaw(row.webhookSecret, row.webhookSecretIv, row.webhookSecretTag, "webhookSecret") + : null, createdAt: row.createdAt, }); @@ -32,6 +46,8 @@ export const setGithubIntegration = async (input: { webhookSecret?: string; }): Promise => { const db = await getDb(); + const client = encryptValue(input.clientSecret, config.envEncryptionKey); + const webhook = input.webhookSecret ? encryptValue(input.webhookSecret, config.envEncryptionKey) : null; return db.transaction(async (tx) => { const [existing] = await tx @@ -47,9 +63,13 @@ export const setGithubIntegration = async (input: { .update(githubIntegrations) .set({ clientId: input.clientId, - clientSecret: input.clientSecret, + clientSecret: client.encrypted, + clientSecretIv: client.iv, + clientSecretTag: client.tag, appName: input.appName ?? "Dequel", - webhookSecret: input.webhookSecret ?? existing.webhookSecret, + ...(webhook + ? { webhookSecret: webhook.encrypted, webhookSecretIv: webhook.iv, webhookSecretTag: webhook.tag } + : {}), }) .where(eq(githubIntegrations.id, existing.id)) .execute(); @@ -67,9 +87,13 @@ export const setGithubIntegration = async (input: { .values({ id, clientId: input.clientId, - clientSecret: input.clientSecret, + clientSecret: client.encrypted, + clientSecretIv: client.iv, + clientSecretTag: client.tag, appName: input.appName ?? "Dequel", - webhookSecret: input.webhookSecret ?? null, + webhookSecret: webhook?.encrypted ?? null, + webhookSecretIv: webhook?.iv ?? null, + webhookSecretTag: webhook?.tag ?? null, createdAt: timestamp, }) .execute(); diff --git a/apps/api/src/db/repo/index.ts b/apps/api/src/db/repo/index.ts index 7add6c0..2a1e8b3 100644 --- a/apps/api/src/db/repo/index.ts +++ b/apps/api/src/db/repo/index.ts @@ -18,6 +18,7 @@ export { } from "./agents"; export { createAlert, deleteAlert, getAlertById, listAlerts, updateAlertEnabled } from "./alerts"; export { createApiKey, deleteApiKey, listApiKeys, validateApiKey } from "./api-keys"; +export { countAuditLogs, createAuditLog, listAuditLogs } from "./audit-logs"; export { createDatabase, @@ -51,6 +52,14 @@ export { updateDeploymentCommitSha, updateDeploymentStatus, } from "./deployments"; +export type { DiagAction, DiagActionKind, DiagActionStatus } from "./diag-actions"; +export { + claimDiagAction, + completeDiagAction, + failDiagAction, + findCompletedAction, + getDiagAction, +} from "./diag-actions"; export { createDiagRun, deleteDiagRun, @@ -63,14 +72,6 @@ export { markInterruptedDiagRuns, recordStageResult, } from "./diag-runs"; -export type { DiagAction, DiagActionKind, DiagActionStatus } from "./diag-actions"; -export { - claimDiagAction, - completeDiagAction, - failDiagAction, - findCompletedAction, - getDiagAction, -} from "./diag-actions"; export { createDomain, deleteDomain, @@ -95,13 +96,24 @@ export { getGithubSession, getGithubTokenFromCookie, } from "./github-sessions"; +export type { LlmKeyInput, LlmKeyStatus, LlmProvider } from "./llm-keys"; +export { + deleteLlmKey, + getDecryptedLlmKey, + getLlmKeyStatus, + LLM_PROVIDERS, + updateLlmModels, + upsertLlmKey, +} from "./llm-keys"; export { getPlatformSettings, setIngressServer } from "./platform-settings"; export type { ProjectCleanupInfo } from "./projects"; export { + captureProjectGithubToken, createProject, deleteProject, deleteProjectCascade, getProjectById, + getProjectGithubToken, listProjects, updateProject, updateProjectGithubToken, @@ -118,9 +130,9 @@ export { upsertRoute, } from "./routes"; export { - ScalingRangeError, deleteScalingPolicy, getScalingPolicy, + ScalingRangeError, upsertScalingPolicy, } from "./scaling"; export type { ServerConnection } from "./servers"; @@ -135,15 +147,6 @@ export { } from "./servers"; export type { SmtpSettingsData } from "./settings"; export { getBackupStorageSettings, getSmtpSettings, upsertBackupStorageSettings, upsertSmtpSettings } from "./settings"; -export type { LlmKeyInput, LlmKeyStatus, LlmProvider } from "./llm-keys"; -export { - LLM_PROVIDERS, - deleteLlmKey, - getDecryptedLlmKey, - getLlmKeyStatus, - updateLlmModels, - upsertLlmKey, -} from "./llm-keys"; export { createSharedEnvVar, deleteSharedEnvVar, diff --git a/apps/api/src/db/repo/projects.ts b/apps/api/src/db/repo/projects.ts index 66a2592..248e23d 100644 --- a/apps/api/src/db/repo/projects.ts +++ b/apps/api/src/db/repo/projects.ts @@ -1,6 +1,8 @@ import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import type { CreateProjectInput, Project } from "../../types"; +import { config } from "../../utils/config"; +import { decryptValue, encryptValue } from "../../utils/crypto"; import { getDb } from "../db-provider"; import { alerts, @@ -113,6 +115,45 @@ export const updateProjectGithubToken = async ( .execute(); }; +export const getProjectGithubToken = async (id: string): Promise => { + const db = await getDb(); + const [row] = await db + .select({ + encrypted: projects.githubTokenEncrypted, + iv: projects.githubTokenIv, + tag: projects.githubTokenTag, + }) + .from(projects) + .where(eq(projects.id, id)) + .execute(); + if (!row?.encrypted || !row?.iv || !row?.tag) return null; + try { + return decryptValue(row.encrypted, row.iv, row.tag, config.envEncryptionKey); + } catch { + return null; + } +}; + +export const captureProjectGithubToken = async ( + projectId: string, + gitUrl: string | null | undefined, + cookieHeader: string | null, +): Promise => { + try { + if (!gitUrl || !/^https:\/\/github\.com\//i.test(gitUrl.trim()) || !cookieHeader) return false; + const { getGithubSession } = await import("./github-sessions"); + const match = cookieHeader.match(/(?:^|;\s*)github_session=([^;]+)/); + if (!match) return false; + const token = await getGithubSession(match[1]); + if (!token) return false; + const enc = encryptValue(token, config.envEncryptionKey); + await updateProjectGithubToken(projectId, enc.encrypted, enc.iv, enc.tag); + return true; + } catch { + return false; + } +}; + export const listProjects = async (): Promise => { const db = await getDb(); const rows = await db.select().from(projects).orderBy(projects.name).execute(); diff --git a/apps/api/src/db/repo/restore-tests.ts b/apps/api/src/db/repo/restore-tests.ts new file mode 100644 index 0000000..7f909f6 --- /dev/null +++ b/apps/api/src/db/repo/restore-tests.ts @@ -0,0 +1,79 @@ +import { and, desc, eq } from "drizzle-orm"; +import { getDb } from "../db-provider"; +import { restoreTests } from "../schema"; + +export type RestoreEventKind = "test" | "restore"; + +export interface RestoreTestRecord { + id: string; + backupId: string; + targetId: string; + serverId: string | null; + kind: RestoreEventKind; + status: string; + error: string | null; + startedAt: Date; + completedAt: Date | null; +} + +export async function createRestoreTestRecord( + record: Pick & { + serverId?: string | null; + kind?: RestoreEventKind; + }, +): Promise { + const db = await getDb(); + const [inserted] = await db + .insert(restoreTests) + .values({ + id: record.id, + backupId: record.backupId, + targetId: record.targetId, + serverId: record.serverId ?? null, + kind: record.kind ?? "test", + status: "running", + startedAt: new Date(), + }) + .returning(); + return inserted as RestoreTestRecord; +} + +export async function updateRestoreTestRecord( + id: string, + updates: Partial>, +): Promise { + const db = await getDb(); + const [updated] = await db.update(restoreTests).set(updates).where(eq(restoreTests.id, id)).returning(); + return (updated as RestoreTestRecord) || null; +} + +export async function getRestoreTestRecord(id: string): Promise { + const db = await getDb(); + const [record] = await db.select().from(restoreTests).where(eq(restoreTests.id, id)).limit(1); + return (record as RestoreTestRecord) || null; +} + +export interface RestoreTestFilters { + backupId?: string; + targetId?: string; + status?: string; + kind?: string; + limit?: number; +} + +export async function listRestoreTests(filters: RestoreTestFilters = {}): Promise { + const db = await getDb(); + const conditions = []; + if (filters.backupId) conditions.push(eq(restoreTests.backupId, filters.backupId)); + if (filters.targetId) conditions.push(eq(restoreTests.targetId, filters.targetId)); + if (filters.status) conditions.push(eq(restoreTests.status, filters.status)); + if (filters.kind) conditions.push(eq(restoreTests.kind, filters.kind)); + let query = db + .select() + .from(restoreTests) + .where(conditions.length ? and(...conditions) : undefined) + .orderBy(desc(restoreTests.startedAt)) + .$dynamic(); + if (filters.limit) query = query.limit(filters.limit); + return query as Promise; +} diff --git a/apps/api/src/db/repo/servers.ts b/apps/api/src/db/repo/servers.ts index 4180f58..b9e8e6c 100644 --- a/apps/api/src/db/repo/servers.ts +++ b/apps/api/src/db/repo/servers.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import type { CreateServerInput, Server, ServerMode, ServerStatus } from "../../types"; import { config } from "../../utils/config"; -import { decryptValue, encryptValue } from "../../utils/crypto"; +import { decryptValue } from "../../utils/crypto"; import { getDb } from "../db-provider"; import { servers } from "../schema"; import { formatTimestamp, getRowsAffected, now } from "./helpers"; @@ -61,7 +61,6 @@ export const createServer = async (input: CreateServerInput): Promise => const id = randomUUID(); const timestamp = now(); const db = await getDb(); - const encrypted = input.sshKey ? encryptValue(input.sshKey, config.envEncryptionKey) : null; await db .insert(servers) .values({ @@ -71,9 +70,6 @@ export const createServer = async (input: CreateServerInput): Promise => port: input.port ?? 2375, authToken: input.authToken ?? "", sshUser: input.sshUser ?? null, - sshKey: encrypted?.encrypted ?? null, - sshKeyIv: encrypted?.iv ?? null, - sshKeyTag: encrypted?.tag ?? null, sshKeyId: input.sshKeyId ?? null, sshPassword: input.sshPassword ?? null, mode: input.mode ?? "ssh", diff --git a/apps/api/src/db/schema.ts b/apps/api/src/db/schema.ts index cea82ef..1b1795d 100644 --- a/apps/api/src/db/schema.ts +++ b/apps/api/src/db/schema.ts @@ -1,514 +1 @@ -import { - boolean, - foreignKey, - index, - integer, - jsonb, - pgTable, - real, - serial, - text, - timestamp, - uniqueIndex, -} from "drizzle-orm/pg-core"; - -export const githubSessions = pgTable("github_sessions", { - id: text().primaryKey(), - accessTokenEncrypted: text("access_token_encrypted").notNull(), - accessTokenIv: text("access_token_iv").notNull(), - accessTokenTag: text("access_token_tag").notNull(), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), -}); - -export const githubIntegrations = pgTable("github_integrations", { - id: text().primaryKey(), - clientId: text("client_id").notNull(), - clientSecret: text("client_secret").notNull(), - appName: text("app_name").notNull().default("Dequel"), - webhookSecret: text("webhook_secret"), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), -}); - -export const projects = pgTable("projects", { - id: text().primaryKey(), - serverId: text("server_id"), - name: text().notNull(), - description: text(), - repoUrl: text("repo_url"), - repoBranch: text("repo_branch"), - baseDomain: text("base_domain"), - cpuLimit: real("cpu_limit"), - memoryLimitMb: integer("memory_limit_mb"), - port: integer("port"), - sourceDir: text("source_dir"), - sourceType: text("source_type").default("git").notNull(), - projectType: text("project_type").default("web").notNull(), - buildType: text("build_type").default("railpack").notNull(), - composeService: text("compose_service"), - composePort: integer("compose_port"), - composeServices: jsonb("compose_services"), - buildCommand: text("build_command"), - installCommand: text("install_command"), - outputDir: text("output_dir"), - startCommand: text("start_command"), - githubTokenEncrypted: text("github_token_encrypted"), - githubTokenIv: text("github_token_iv"), - githubTokenTag: text("github_token_tag"), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), -}); - -export const deployments = pgTable("deployments", { - id: text().primaryKey(), - projectId: text("project_id"), - serverId: text("server_id"), - sourceType: text("source_type").notNull(), - sourceRef: text("source_ref").notNull(), - status: text().notNull().default("pending"), - imageTag: text("image_tag"), - containerName: text("container_name"), - routePath: text("route_path"), - liveUrl: text("live_url"), - branch: text(), - commitSha: text("commit_sha"), - replicas: integer().notNull().default(1), - environment: text(), - failureReason: text("failure_reason"), - clearCache: boolean("clear_cache").notNull().default(false), - finishedAt: timestamp("finished_at", { withTimezone: true }), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), -}); - -export const deploymentLogs = pgTable( - "deployment_logs", - { - id: serial().primaryKey(), - deploymentId: text("deployment_id").notNull(), - sequence: integer().notNull(), - stage: text().notNull(), - message: text().notNull(), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - }, - (table) => [ - foreignKey({ columns: [table.deploymentId], foreignColumns: [deployments.id], onDelete: "cascade" }), - uniqueIndex("idx_logs_dep_seq").on(table.deploymentId, table.sequence), - ], -); - -export const deploymentEvents = pgTable( - "deployment_events", - { - id: text().primaryKey(), - deploymentId: text("deployment_id").notNull(), - type: text().notNull(), - message: text(), - metadata: jsonb("metadata"), - sentAt: timestamp("sent_at", { withTimezone: true }), - attempts: integer().notNull().default(0), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - }, - (table) => [ - foreignKey({ columns: [table.deploymentId], foreignColumns: [deployments.id], onDelete: "cascade" }), - index("idx_events_dep_type").on(table.deploymentId, table.type), - ], -); - -export const environmentVariables = pgTable( - "environment_variables", - { - id: text().primaryKey(), - projectId: text("project_id").notNull(), - key: text().notNull(), - value: text().notNull(), - valueEncrypted: text("value_encrypted"), - valueIv: text("value_iv"), - valueTag: text("value_tag"), - environment: text().notNull().default("production"), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), - }, - (table) => [ - foreignKey({ columns: [table.projectId], foreignColumns: [projects.id], onDelete: "cascade" }), - index("idx_env_vars_project").on(table.projectId, table.environment), - ], -); - -export const volumes = pgTable( - "volumes", - { - id: text().primaryKey(), - projectId: text("project_id").notNull(), - mountPath: text("mount_path").notNull().default("/app/data"), - sizeMb: integer("size_mb"), - dockerVolumeName: text("docker_volume_name"), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - }, - (table) => [foreignKey({ columns: [table.projectId], foreignColumns: [projects.id], onDelete: "cascade" })], -); - -export const databases = pgTable( - "databases", - { - id: text().primaryKey(), - projectId: text("project_id"), - serverId: text("server_id"), - name: text().notNull(), - type: text().notNull(), - version: text(), - databaseName: text("database_name").notNull(), - username: text().notNull(), - password: text().notNull(), - internalHost: text("internal_host").notNull(), - internalPort: integer("internal_port").notNull(), - cpuLimit: real("cpu_limit"), - memoryLimitMb: integer("memory_limit_mb"), - storageLimitMb: integer("storage_limit_mb"), - storageUsedMb: integer("storage_used_mb").notNull().default(0), - publicAccess: boolean("public_access").notNull().default(true), - allowPublicAccessFromAnywhere: boolean("allow_public_access_from_anywhere").notNull().default(false), - allowedCidrs: jsonb("allowed_cidrs").notNull().default([]), - externalPort: integer("external_port"), - proxyContainerName: text("proxy_container_name"), - volumeName: text("volume_name").notNull(), - connectionString: text("connection_string").notNull(), - status: text().notNull().default("provisioning"), - containerName: text("container_name"), - backupEnabled: boolean("backup_enabled").notNull().default(true), - backupSchedule: text("backup_schedule").notNull().default("0 */6 * * *"), - backupRetention: integer("backup_retention").notNull().default(7), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), - }, - (table) => [foreignKey({ columns: [table.projectId], foreignColumns: [projects.id], onDelete: "set null" })], -); - -export const domains = pgTable( - "domains", - { - id: text().primaryKey(), - projectId: text("project_id").notNull(), - domain: text().notNull(), - type: text().notNull().default("custom"), - validationStatus: text("validation_status").notNull().default("pending"), - sslStatus: text("ssl_status").notNull().default("pending"), - targetService: text("target_service"), - targetPort: integer("target_port"), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), - }, - (table) => [foreignKey({ columns: [table.projectId], foreignColumns: [projects.id], onDelete: "cascade" })], -); - -export const scalingPolicies = pgTable( - "scaling_policies", - { - id: text().primaryKey(), - projectId: text("project_id").notNull().unique(), - minReplicas: integer("min_replicas").notNull().default(1), - maxReplicas: integer("max_replicas").notNull().default(5), - cpuThresholdPercent: integer("cpu_threshold_percent").notNull().default(70), - memoryThresholdPercent: integer("memory_threshold_percent").notNull().default(85), - cooldownSeconds: integer("cooldown_seconds").notNull().default(120), - enabled: boolean().notNull().default(true), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), - }, - (table) => [foreignKey({ columns: [table.projectId], foreignColumns: [projects.id], onDelete: "cascade" })], -); - -export const servers = pgTable("servers", { - id: text().primaryKey(), - name: text().notNull(), - host: text().notNull(), - port: integer().notNull().default(2375), - authToken: text("auth_token").notNull().default(""), - sshUser: text("ssh_user"), - sshKey: text("ssh_key"), - sshKeyIv: text("ssh_key_iv"), - sshKeyTag: text("ssh_key_tag"), - sshPassword: text("ssh_password"), - sshKeyId: text("ssh_key_id"), - mode: text().notNull().default("ssh"), - agentId: text("agent_id").unique(), - agentVersion: text("agent_version"), - peerIp: text("peer_ip"), - capabilities: jsonb().notNull().default({}), - labels: jsonb().notNull().default({}), - status: text().notNull().default("pending"), - cpuTotal: integer("cpu_total"), - memoryTotalMb: integer("memory_total_mb"), - diskTotalMb: integer("disk_total_mb"), - cpuUsedPercent: real("cpu_used_percent"), - memoryUsedMb: integer("memory_used_mb"), - lastHeartbeat: timestamp("last_heartbeat", { withTimezone: true }), - registeredAt: timestamp("registered_at", { withTimezone: true }), - revokedAt: timestamp("revoked_at", { withTimezone: true }), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), -}); - -export const agentRegistrationTokens = pgTable("agent_registration_tokens", { - id: text().primaryKey(), - tokenHash: text("token_hash").notNull().unique(), - serverName: text("server_name").notNull(), - labels: jsonb().notNull().default({}), - expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), - usedAt: timestamp("used_at", { withTimezone: true }), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), -}); - -export const agentCredentials = pgTable( - "agent_credentials", - { - id: text().primaryKey(), - serverId: text("server_id").notNull(), - credentialHash: text("credential_hash").notNull().unique(), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - lastUsedAt: timestamp("last_used_at", { withTimezone: true }), - revokedAt: timestamp("revoked_at", { withTimezone: true }), - }, - (table) => [foreignKey({ columns: [table.serverId], foreignColumns: [servers.id], onDelete: "cascade" })], -); - -export const agentJobs = pgTable( - "agent_jobs", - { - id: text().primaryKey(), - deploymentId: text("deployment_id"), - serverId: text("server_id").notNull(), - type: text().notNull(), - payload: jsonb().notNull(), - status: text().notNull().default("queued"), - attempts: integer().notNull().default(0), - leaseId: text("lease_id"), - leaseExpiresAt: timestamp("lease_expires_at", { withTimezone: true }), - idempotencyKey: text("idempotency_key").notNull().unique(), - failureReason: text("failure_reason"), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - startedAt: timestamp("started_at", { withTimezone: true }), - finishedAt: timestamp("finished_at", { withTimezone: true }), - }, - (table) => [ - foreignKey({ columns: [table.serverId], foreignColumns: [servers.id], onDelete: "cascade" }), - foreignKey({ columns: [table.deploymentId], foreignColumns: [deployments.id], onDelete: "cascade" }), - index("idx_agent_jobs_server_status").on(table.serverId, table.status), - ], -); - -export const refreshTokens = pgTable("refresh_tokens", { - id: text().primaryKey(), - username: text().notNull(), - tokenHash: text("token_hash").notNull().unique(), - expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - blacklistedAt: timestamp("blacklisted_at", { withTimezone: true }), -}); - -export const apiKeys = pgTable("api_keys", { - id: text().primaryKey(), - name: text().notNull(), - keyHash: text("key_hash").notNull(), - permissions: text().notNull().default("deploy:read"), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - lastUsedAt: timestamp("last_used_at", { withTimezone: true }), -}); - -export const alerts = pgTable( - "alerts", - { - id: text().primaryKey(), - projectId: text("project_id").notNull(), - type: text().notNull(), - threshold: real(), - durationSeconds: integer("duration_seconds"), - channel: text().notNull().default("email"), - destination: text(), - enabled: boolean().notNull().default(true), - createdAt: timestamp("created_at", { withTimezone: true }).defaultNow(), - }, - (table) => [foreignKey({ columns: [table.projectId], foreignColumns: [projects.id], onDelete: "cascade" })], -); - -export const smtpSettings = pgTable("smtp_settings", { - id: text().primaryKey(), - host: text().notNull(), - port: integer().notNull().default(587), - user: text().notNull().default(""), - passEncrypted: text("pass_encrypted"), - passIv: text("pass_iv"), - passTag: text("pass_tag"), - fromAddress: text("from_address").notNull().default("dequel@localhost"), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), -}); - -export const platformSettings = pgTable("platform_settings", { - id: text().primaryKey(), - ingressServerId: text("ingress_server_id"), - updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), -}); - -export const backupStorageSettings = pgTable("backup_storage_settings", { - id: text().primaryKey(), - type: text().notNull().default("local"), - path: text().default("/data/backups"), - s3Endpoint: text("s3_endpoint"), - s3AccessKeyId: text("s3_access_key_id"), - s3SecretAccessKeyEncrypted: text("s3_secret_access_key_encrypted"), - s3SecretAccessKeyIv: text("s3_secret_access_key_iv"), - s3SecretAccessKeyTag: text("s3_secret_access_key_tag"), - s3Bucket: text("s3_bucket"), - s3Region: text("s3_region").default("auto"), - systemBackupSchedule: text("system_backup_schedule").default("0 */6 * * *"), - systemBackupRetention: integer("system_backup_retention").default(7), - updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), -}); - -export const llmProviderKeys = pgTable("llm_provider_keys", { - provider: text().primaryKey(), - keyEncrypted: text("key_encrypted"), - keyIv: text("key_iv"), - keyTag: text("key_tag"), - baseUrl: text("base_url"), - models: jsonb().notNull().default([]), - updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), -}); - -export const diagRuns = pgTable( - "diag_runs", - { - id: text().primaryKey(), - deploymentId: text("deployment_id").notNull(), - commitSha: text("commit_sha").notNull().default(""), - provider: text().notNull(), - model: text().notNull(), - status: text().notNull().default("running"), - currentStage: text("current_stage"), - cause: text(), - report: jsonb(), - error: text(), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), - }, - (table) => [ - foreignKey({ columns: [table.deploymentId], foreignColumns: [deployments.id], onDelete: "cascade" }), - uniqueIndex("diag_runs_deployment_commit").on(table.deploymentId, table.commitSha), - ], -); - -export const diagStages = pgTable( - "diag_stages", - { - id: text().primaryKey(), - runId: text("run_id").notNull(), - stage: text().notNull(), - payload: jsonb(), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - }, - (table) => [ - foreignKey({ columns: [table.runId], foreignColumns: [diagRuns.id], onDelete: "cascade" }), - uniqueIndex("diag_stages_run_stage").on(table.runId, table.stage), - ], -); - -export const diagActions = pgTable( - "diag_actions", - { - key: text().primaryKey(), - runId: text("run_id").notNull(), - kind: text().notNull(), - status: text().notNull().default("requested"), - result: jsonb(), - error: text(), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), - }, - (table) => [foreignKey({ columns: [table.runId], foreignColumns: [diagRuns.id], onDelete: "cascade" })], -); - -export const sharedEnvVars = pgTable("shared_env_vars", { - id: text().primaryKey(), - key: text().notNull(), - value: text().notNull(), - valueEncrypted: text("value_encrypted"), - valueIv: text("value_iv"), - valueTag: text("value_tag"), - environment: text().notNull().default("production"), - description: text(), - tags: jsonb().notNull().default([]), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), -}); - -export const projectSharedEnvLinks = pgTable( - "project_shared_env_links", - { - id: text().primaryKey(), - projectId: text("project_id").notNull(), - sharedEnvVarId: text("shared_env_var_id").notNull(), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - }, - (table) => [ - foreignKey({ columns: [table.projectId], foreignColumns: [projects.id], onDelete: "cascade" }), - foreignKey({ columns: [table.sharedEnvVarId], foreignColumns: [sharedEnvVars.id], onDelete: "cascade" }), - uniqueIndex("idx_proj_shared_var").on(table.projectId, table.sharedEnvVarId), - ], -); - -export const sshKeys = pgTable("ssh_keys", { - id: text().primaryKey(), - name: text().notNull(), - fingerprint: text().notNull().unique(), - privateKeyEncrypted: text("private_key_encrypted").notNull(), - privateKeyIv: text("private_key_iv").notNull(), - privateKeyTag: text("private_key_tag").notNull(), - publicKey: text("public_key"), - tags: jsonb().notNull().default([]), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), -}); - -export const routes = pgTable( - "routes", - { - id: text().primaryKey(), - serverId: text("server_id"), - deploymentId: text("deployment_id"), - projectId: text("project_id"), - hostname: text().notNull(), - routeFile: text("route_file").notNull(), - port: integer().notNull(), - targetContainers: jsonb("target_containers").notNull(), - upstreamHost: text("upstream_host"), - status: text().notNull().default("pending"), - lastError: text("last_error"), - confirmedAt: timestamp("confirmed_at", { withTimezone: true }), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), - }, - (table) => [uniqueIndex("idx_routes_hostname_server").on(table.hostname, table.serverId)], -); - -export const backups = pgTable( - "backups", - { - id: text().primaryKey(), - targetId: text("target_id").notNull(), - targetType: text("target_type").notNull(), - engine: text().notNull(), - filename: text(), - storageType: text("storage_type").notNull(), - storagePath: text("storage_path"), - sizeBytes: integer("size_bytes"), - error: text(), - status: text().notNull().default("pending"), - createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), - completedAt: timestamp("completed_at", { withTimezone: true }), - }, - (table) => [ - index("idx_backups_target").on(table.targetId), - index("idx_backups_created").on(table.createdAt), - index("idx_backups_status").on(table.status), - ], -); +export * from "./schema/index"; diff --git a/apps/api/src/db/schema/audit.ts b/apps/api/src/db/schema/audit.ts new file mode 100644 index 0000000..1094c16 --- /dev/null +++ b/apps/api/src/db/schema/audit.ts @@ -0,0 +1,22 @@ +import { index, jsonb, pgTable, text, timestamp } from "drizzle-orm/pg-core"; + +export const auditLogs = pgTable( + "audit_logs", + { + id: text().primaryKey(), + actorType: text("actor_type").notNull(), + actorId: text("actor_id").notNull(), + action: text().notNull(), + resourceType: text("resource_type").notNull(), + resourceId: text("resource_id"), + details: jsonb().notNull().default({}), + ipAddress: text("ip_address"), + userAgent: text("user_agent"), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => [ + index("idx_audit_logs_actor").on(table.actorType, table.actorId), + index("idx_audit_logs_resource").on(table.resourceType, table.resourceId), + index("idx_audit_logs_created_at").on(table.createdAt), + ], +); diff --git a/apps/api/src/db/schema/auth.ts b/apps/api/src/db/schema/auth.ts new file mode 100644 index 0000000..093cd7d --- /dev/null +++ b/apps/api/src/db/schema/auth.ts @@ -0,0 +1,40 @@ +import { pgTable, text, timestamp } from "drizzle-orm/pg-core"; + +export const githubSessions = pgTable("github_sessions", { + id: text().primaryKey(), + accessTokenEncrypted: text("access_token_encrypted").notNull(), + accessTokenIv: text("access_token_iv").notNull(), + accessTokenTag: text("access_token_tag").notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), +}); + +export const githubIntegrations = pgTable("github_integrations", { + id: text().primaryKey(), + clientId: text("client_id").notNull(), + clientSecret: text("client_secret").notNull(), + clientSecretIv: text("client_secret_iv"), + clientSecretTag: text("client_secret_tag"), + appName: text("app_name").notNull().default("Dequel"), + webhookSecret: text("webhook_secret"), + webhookSecretIv: text("webhook_secret_iv"), + webhookSecretTag: text("webhook_secret_tag"), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), +}); + +export const refreshTokens = pgTable("refresh_tokens", { + id: text().primaryKey(), + username: text().notNull(), + tokenHash: text("token_hash").notNull().unique(), + expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + blacklistedAt: timestamp("blacklisted_at", { withTimezone: true }), +}); + +export const apiKeys = pgTable("api_keys", { + id: text().primaryKey(), + name: text().notNull(), + keyHash: text("key_hash").notNull(), + permissions: text().notNull().default("deploy:read"), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + lastUsedAt: timestamp("last_used_at", { withTimezone: true }), +}); diff --git a/apps/api/src/db/schema/databases.ts b/apps/api/src/db/schema/databases.ts new file mode 100644 index 0000000..cf1da50 --- /dev/null +++ b/apps/api/src/db/schema/databases.ts @@ -0,0 +1,81 @@ +import { boolean, foreignKey, index, integer, jsonb, pgTable, real, text, timestamp } from "drizzle-orm/pg-core"; +import { projects } from "./projects"; + +export const databases = pgTable( + "databases", + { + id: text().primaryKey(), + projectId: text("project_id"), + serverId: text("server_id"), + name: text().notNull(), + type: text().notNull(), + version: text(), + databaseName: text("database_name").notNull(), + username: text().notNull(), + password: text(), + passwordEncrypted: text("password_encrypted"), + passwordIv: text("password_iv"), + passwordTag: text("password_tag"), + internalHost: text("internal_host").notNull(), + internalPort: integer("internal_port").notNull(), + cpuLimit: real("cpu_limit"), + memoryLimitMb: integer("memory_limit_mb"), + storageLimitMb: integer("storage_limit_mb"), + storageUsedMb: integer("storage_used_mb").notNull().default(0), + publicAccess: boolean("public_access").notNull().default(true), + allowPublicAccessFromAnywhere: boolean("allow_public_access_from_anywhere").notNull().default(false), + allowedCidrs: jsonb("allowed_cidrs").notNull().default([]), + externalPort: integer("external_port"), + proxyContainerName: text("proxy_container_name"), + volumeName: text("volume_name").notNull(), + connectionString: text("connection_string").notNull(), + status: text().notNull().default("provisioning"), + containerName: text("container_name"), + backupEnabled: boolean("backup_enabled").notNull().default(true), + backupSchedule: text("backup_schedule").notNull().default("0 */6 * * *"), + backupRetention: integer("backup_retention").notNull().default(7), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => [foreignKey({ columns: [table.projectId], foreignColumns: [projects.id], onDelete: "set null" })], +); + +export const backups = pgTable( + "backups", + { + id: text().primaryKey(), + targetId: text("target_id").notNull(), + targetType: text("target_type").notNull(), + engine: text().notNull(), + filename: text(), + storageType: text("storage_type").notNull(), + storagePath: text("storage_path"), + sizeBytes: integer("size_bytes"), + isEncrypted: boolean("is_encrypted").notNull().default(false), + error: text(), + status: text().notNull().default("pending"), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + completedAt: timestamp("completed_at", { withTimezone: true }), + }, + (table) => [ + index("idx_backups_target").on(table.targetId), + index("idx_backups_created").on(table.createdAt), + index("idx_backups_status").on(table.status), + ], +); + +export const backupStorageSettings = pgTable("backup_storage_settings", { + id: text().primaryKey(), + type: text().notNull().default("local"), + path: text().default("/data/backups"), + s3Endpoint: text("s3_endpoint"), + s3AccessKeyId: text("s3_access_key_id"), + s3SecretAccessKeyEncrypted: text("s3_secret_access_key_encrypted"), + s3SecretAccessKeyIv: text("s3_secret_access_key_iv"), + s3SecretAccessKeyTag: text("s3_secret_access_key_tag"), + s3Bucket: text("s3_bucket"), + s3Region: text("s3_region").default("auto"), + systemBackupSchedule: text("system_backup_schedule").default("0 */6 * * *"), + systemBackupRetention: integer("system_backup_retention").default(7), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), +}); diff --git a/apps/api/src/db/schema/deployments.ts b/apps/api/src/db/schema/deployments.ts new file mode 100644 index 0000000..6be0874 --- /dev/null +++ b/apps/api/src/db/schema/deployments.ts @@ -0,0 +1,86 @@ +import { + boolean, + foreignKey, + index, + integer, + jsonb, + pgTable, + serial, + text, + timestamp, + uniqueIndex, +} from "drizzle-orm/pg-core"; +import { projects } from "./projects"; + +export const deployments = pgTable("deployments", { + id: text().primaryKey(), + projectId: text("project_id"), + serverId: text("server_id"), + sourceType: text("source_type").notNull(), + sourceRef: text("source_ref").notNull(), + status: text().notNull().default("pending"), + imageTag: text("image_tag"), + containerName: text("container_name"), + routePath: text("route_path"), + liveUrl: text("live_url"), + branch: text(), + commitSha: text("commit_sha"), + replicas: integer().notNull().default(1), + environment: text(), + failureReason: text("failure_reason"), + clearCache: boolean("clear_cache").notNull().default(false), + finishedAt: timestamp("finished_at", { withTimezone: true }), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), +}); + +export const deploymentLogs = pgTable( + "deployment_logs", + { + id: serial().primaryKey(), + deploymentId: text("deployment_id").notNull(), + sequence: integer().notNull(), + stage: text().notNull(), + message: text().notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => [ + foreignKey({ columns: [table.deploymentId], foreignColumns: [deployments.id], onDelete: "cascade" }), + uniqueIndex("idx_logs_dep_seq").on(table.deploymentId, table.sequence), + ], +); + +export const deploymentEvents = pgTable( + "deployment_events", + { + id: text().primaryKey(), + deploymentId: text("deployment_id").notNull(), + type: text().notNull(), + message: text(), + metadata: jsonb("metadata"), + sentAt: timestamp("sent_at", { withTimezone: true }), + attempts: integer().notNull().default(0), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => [ + foreignKey({ columns: [table.deploymentId], foreignColumns: [deployments.id], onDelete: "cascade" }), + index("idx_events_dep_type").on(table.deploymentId, table.type), + ], +); + +export const scalingPolicies = pgTable( + "scaling_policies", + { + id: text().primaryKey(), + projectId: text("project_id").notNull().unique(), + minReplicas: integer("min_replicas").notNull().default(1), + maxReplicas: integer("max_replicas").notNull().default(5), + cpuThresholdPercent: integer("cpu_threshold_percent").notNull().default(70), + memoryThresholdPercent: integer("memory_threshold_percent").notNull().default(85), + cooldownSeconds: integer("cooldown_seconds").notNull().default(120), + enabled: boolean().notNull().default(true), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => [foreignKey({ columns: [table.projectId], foreignColumns: [projects.id], onDelete: "cascade" })], +); diff --git a/apps/api/src/db/schema/diagnostics.ts b/apps/api/src/db/schema/diagnostics.ts new file mode 100644 index 0000000..ed7812b --- /dev/null +++ b/apps/api/src/db/schema/diagnostics.ts @@ -0,0 +1,54 @@ +import { foreignKey, jsonb, pgTable, text, timestamp, uniqueIndex } from "drizzle-orm/pg-core"; +import { deployments } from "./deployments"; + +export const diagRuns = pgTable( + "diag_runs", + { + id: text().primaryKey(), + deploymentId: text("deployment_id").notNull(), + commitSha: text("commit_sha").notNull().default(""), + provider: text().notNull(), + model: text().notNull(), + status: text().notNull().default("running"), + currentStage: text("current_stage"), + cause: text(), + report: jsonb(), + error: text(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => [ + foreignKey({ columns: [table.deploymentId], foreignColumns: [deployments.id], onDelete: "cascade" }), + uniqueIndex("diag_runs_deployment_commit").on(table.deploymentId, table.commitSha), + ], +); + +export const diagStages = pgTable( + "diag_stages", + { + id: text().primaryKey(), + runId: text("run_id").notNull(), + stage: text().notNull(), + payload: jsonb(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => [ + foreignKey({ columns: [table.runId], foreignColumns: [diagRuns.id], onDelete: "cascade" }), + uniqueIndex("diag_stages_run_stage").on(table.runId, table.stage), + ], +); + +export const diagActions = pgTable( + "diag_actions", + { + key: text().primaryKey(), + runId: text("run_id").notNull(), + kind: text().notNull(), + status: text().notNull().default("requested"), + result: jsonb(), + error: text(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => [foreignKey({ columns: [table.runId], foreignColumns: [diagRuns.id], onDelete: "cascade" })], +); diff --git a/apps/api/src/db/schema/env-vars.ts b/apps/api/src/db/schema/env-vars.ts new file mode 100644 index 0000000..1fbf54a --- /dev/null +++ b/apps/api/src/db/schema/env-vars.ts @@ -0,0 +1,51 @@ +import { foreignKey, index, jsonb, pgTable, text, timestamp, uniqueIndex } from "drizzle-orm/pg-core"; +import { projects } from "./projects"; + +export const environmentVariables = pgTable( + "environment_variables", + { + id: text().primaryKey(), + projectId: text("project_id").notNull(), + key: text().notNull(), + value: text().notNull(), + valueEncrypted: text("value_encrypted"), + valueIv: text("value_iv"), + valueTag: text("value_tag"), + environment: text().notNull().default("production"), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => [ + foreignKey({ columns: [table.projectId], foreignColumns: [projects.id], onDelete: "cascade" }), + index("idx_env_vars_project").on(table.projectId, table.environment), + ], +); + +export const sharedEnvVars = pgTable("shared_env_vars", { + id: text().primaryKey(), + key: text().notNull(), + value: text().notNull(), + valueEncrypted: text("value_encrypted"), + valueIv: text("value_iv"), + valueTag: text("value_tag"), + environment: text().notNull().default("production"), + description: text(), + tags: jsonb().notNull().default([]), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), +}); + +export const projectSharedEnvLinks = pgTable( + "project_shared_env_links", + { + id: text().primaryKey(), + projectId: text("project_id").notNull(), + sharedEnvVarId: text("shared_env_var_id").notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => [ + foreignKey({ columns: [table.projectId], foreignColumns: [projects.id], onDelete: "cascade" }), + foreignKey({ columns: [table.sharedEnvVarId], foreignColumns: [sharedEnvVars.id], onDelete: "cascade" }), + uniqueIndex("idx_proj_shared_var").on(table.projectId, table.sharedEnvVarId), + ], +); diff --git a/apps/api/src/db/schema/index.ts b/apps/api/src/db/schema/index.ts new file mode 100644 index 0000000..b3939f6 --- /dev/null +++ b/apps/api/src/db/schema/index.ts @@ -0,0 +1,11 @@ +export * from "./audit"; +export * from "./auth"; +export * from "./databases"; +export * from "./deployments"; +export * from "./diagnostics"; +export * from "./env-vars"; +export * from "./networking"; +export * from "./projects"; +export * from "./restore-tests"; +export * from "./servers"; +export * from "./settings"; diff --git a/apps/api/src/db/schema/networking.ts b/apps/api/src/db/schema/networking.ts new file mode 100644 index 0000000..34a8b0c --- /dev/null +++ b/apps/api/src/db/schema/networking.ts @@ -0,0 +1,41 @@ +import { boolean, foreignKey, integer, jsonb, pgTable, text, timestamp, uniqueIndex } from "drizzle-orm/pg-core"; +import { projects } from "./projects"; + +export const domains = pgTable( + "domains", + { + id: text().primaryKey(), + projectId: text("project_id").notNull(), + domain: text().notNull(), + type: text().notNull().default("custom"), + validationStatus: text("validation_status").notNull().default("pending"), + sslStatus: text("ssl_status").notNull().default("pending"), + targetService: text("target_service"), + targetPort: integer("target_port"), + cloudflareProxied: boolean("cloudflare_proxied").notNull().default(false), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => [foreignKey({ columns: [table.projectId], foreignColumns: [projects.id], onDelete: "cascade" })], +); + +export const routes = pgTable( + "routes", + { + id: text().primaryKey(), + serverId: text("server_id"), + deploymentId: text("deployment_id"), + projectId: text("project_id"), + hostname: text().notNull(), + routeFile: text("route_file").notNull(), + port: integer().notNull(), + targetContainers: jsonb("target_containers").notNull(), + upstreamHost: text("upstream_host"), + status: text().notNull().default("pending"), + lastError: text("last_error"), + confirmedAt: timestamp("confirmed_at", { withTimezone: true }), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => [uniqueIndex("idx_routes_hostname_server").on(table.hostname, table.serverId)], +); diff --git a/apps/api/src/db/schema/projects.ts b/apps/api/src/db/schema/projects.ts new file mode 100644 index 0000000..50ba207 --- /dev/null +++ b/apps/api/src/db/schema/projects.ts @@ -0,0 +1,43 @@ +import { foreignKey, integer, jsonb, pgTable, real, text, timestamp } from "drizzle-orm/pg-core"; + +export const projects = pgTable("projects", { + id: text().primaryKey(), + serverId: text("server_id"), + name: text().notNull(), + description: text(), + repoUrl: text("repo_url"), + repoBranch: text("repo_branch"), + baseDomain: text("base_domain"), + cpuLimit: real("cpu_limit"), + memoryLimitMb: integer("memory_limit_mb"), + port: integer("port"), + sourceDir: text("source_dir"), + sourceType: text("source_type").default("git").notNull(), + projectType: text("project_type").default("web").notNull(), + buildType: text("build_type").default("railpack").notNull(), + composeService: text("compose_service"), + composePort: integer("compose_port"), + composeServices: jsonb("compose_services"), + buildCommand: text("build_command"), + installCommand: text("install_command"), + outputDir: text("output_dir"), + startCommand: text("start_command"), + githubTokenEncrypted: text("github_token_encrypted"), + githubTokenIv: text("github_token_iv"), + githubTokenTag: text("github_token_tag"), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), +}); + +export const volumes = pgTable( + "volumes", + { + id: text().primaryKey(), + projectId: text("project_id").notNull(), + mountPath: text("mount_path").notNull().default("/app/data"), + sizeMb: integer("size_mb"), + dockerVolumeName: text("docker_volume_name"), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => [foreignKey({ columns: [table.projectId], foreignColumns: [projects.id], onDelete: "cascade" })], +); diff --git a/apps/api/src/db/schema/restore-tests.ts b/apps/api/src/db/schema/restore-tests.ts new file mode 100644 index 0000000..629f6a7 --- /dev/null +++ b/apps/api/src/db/schema/restore-tests.ts @@ -0,0 +1,23 @@ +import { index, pgTable, text, timestamp } from "drizzle-orm/pg-core"; + +export const restoreTests = pgTable( + "restore_tests", + { + id: text().primaryKey(), + backupId: text("backup_id").notNull(), + targetId: text("target_id").notNull(), + serverId: text("server_id"), + kind: text().notNull().default("test"), + status: text().notNull().default("running"), + error: text(), + startedAt: timestamp("started_at", { withTimezone: true }).notNull().defaultNow(), + completedAt: timestamp("completed_at", { withTimezone: true }), + }, + (table) => [ + index("idx_restore_tests_backup").on(table.backupId), + index("idx_restore_tests_target").on(table.targetId), + index("idx_restore_tests_status").on(table.status), + ], +); + +export type RestoreTest = typeof restoreTests.$inferSelect; diff --git a/apps/api/src/db/schema/servers.ts b/apps/api/src/db/schema/servers.ts new file mode 100644 index 0000000..5d65e84 --- /dev/null +++ b/apps/api/src/db/schema/servers.ts @@ -0,0 +1,94 @@ +import { foreignKey, index, integer, jsonb, pgTable, real, text, timestamp } from "drizzle-orm/pg-core"; +import { deployments } from "./deployments"; + +export const servers = pgTable("servers", { + id: text().primaryKey(), + name: text().notNull(), + host: text().notNull(), + port: integer().notNull().default(2375), + authToken: text("auth_token").notNull().default(""), + sshUser: text("ssh_user"), + sshKey: text("ssh_key"), + sshKeyIv: text("ssh_key_iv"), + sshKeyTag: text("ssh_key_tag"), + sshPassword: text("ssh_password"), + sshKeyId: text("ssh_key_id"), + mode: text().notNull().default("ssh"), + agentId: text("agent_id").unique(), + agentVersion: text("agent_version"), + peerIp: text("peer_ip"), + capabilities: jsonb().notNull().default({}), + labels: jsonb().notNull().default({}), + status: text().notNull().default("pending"), + cpuTotal: integer("cpu_total"), + memoryTotalMb: integer("memory_total_mb"), + diskTotalMb: integer("disk_total_mb"), + cpuUsedPercent: real("cpu_used_percent"), + memoryUsedMb: integer("memory_used_mb"), + lastHeartbeat: timestamp("last_heartbeat", { withTimezone: true }), + registeredAt: timestamp("registered_at", { withTimezone: true }), + revokedAt: timestamp("revoked_at", { withTimezone: true }), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), +}); + +export const agentRegistrationTokens = pgTable("agent_registration_tokens", { + id: text().primaryKey(), + tokenHash: text("token_hash").notNull().unique(), + serverName: text("server_name").notNull(), + labels: jsonb().notNull().default({}), + expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), + usedAt: timestamp("used_at", { withTimezone: true }), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), +}); + +export const agentCredentials = pgTable( + "agent_credentials", + { + id: text().primaryKey(), + serverId: text("server_id").notNull(), + credentialHash: text("credential_hash").notNull().unique(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + lastUsedAt: timestamp("last_used_at", { withTimezone: true }), + revokedAt: timestamp("revoked_at", { withTimezone: true }), + }, + (table) => [foreignKey({ columns: [table.serverId], foreignColumns: [servers.id], onDelete: "cascade" })], +); + +export const agentJobs = pgTable( + "agent_jobs", + { + id: text().primaryKey(), + deploymentId: text("deployment_id"), + serverId: text("server_id").notNull(), + type: text().notNull(), + payload: jsonb().notNull(), + status: text().notNull().default("queued"), + attempts: integer().notNull().default(0), + leaseId: text("lease_id"), + leaseExpiresAt: timestamp("lease_expires_at", { withTimezone: true }), + idempotencyKey: text("idempotency_key").notNull().unique(), + failureReason: text("failure_reason"), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + startedAt: timestamp("started_at", { withTimezone: true }), + finishedAt: timestamp("finished_at", { withTimezone: true }), + }, + (table) => [ + foreignKey({ columns: [table.serverId], foreignColumns: [servers.id], onDelete: "cascade" }), + foreignKey({ columns: [table.deploymentId], foreignColumns: [deployments.id], onDelete: "cascade" }), + index("idx_agent_jobs_server_status").on(table.serverId, table.status), + ], +); + +export const sshKeys = pgTable("ssh_keys", { + id: text().primaryKey(), + name: text().notNull(), + fingerprint: text().notNull().unique(), + privateKeyEncrypted: text("private_key_encrypted").notNull(), + privateKeyIv: text("private_key_iv").notNull(), + privateKeyTag: text("private_key_tag").notNull(), + publicKey: text("public_key"), + tags: jsonb().notNull().default([]), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), +}); diff --git a/apps/api/src/db/schema/settings.ts b/apps/api/src/db/schema/settings.ts new file mode 100644 index 0000000..4a2f525 --- /dev/null +++ b/apps/api/src/db/schema/settings.ts @@ -0,0 +1,47 @@ +import { boolean, foreignKey, integer, jsonb, pgTable, real, text, timestamp } from "drizzle-orm/pg-core"; +import { projects } from "./projects"; + +export const alerts = pgTable( + "alerts", + { + id: text().primaryKey(), + projectId: text("project_id").notNull(), + type: text().notNull(), + threshold: real(), + durationSeconds: integer("duration_seconds"), + channel: text().notNull().default("email"), + destination: text(), + enabled: boolean().notNull().default(true), + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow(), + }, + (table) => [foreignKey({ columns: [table.projectId], foreignColumns: [projects.id], onDelete: "cascade" })], +); + +export const smtpSettings = pgTable("smtp_settings", { + id: text().primaryKey(), + host: text().notNull(), + port: integer().notNull().default(587), + user: text().notNull().default(""), + passEncrypted: text("pass_encrypted"), + passIv: text("pass_iv"), + passTag: text("pass_tag"), + fromAddress: text("from_address").notNull().default("dequel@localhost"), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), +}); + +export const platformSettings = pgTable("platform_settings", { + id: text().primaryKey(), + ingressServerId: text("ingress_server_id"), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), +}); + +export const llmProviderKeys = pgTable("llm_provider_keys", { + provider: text().primaryKey(), + keyEncrypted: text("key_encrypted"), + keyIv: text("key_iv"), + keyTag: text("key_tag"), + baseUrl: text("base_url"), + models: jsonb().notNull().default([]), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), +}); diff --git a/apps/api/src/db/test-helper.ts b/apps/api/src/db/test-helper.ts index c56d0e1..1e5b9b7 100644 --- a/apps/api/src/db/test-helper.ts +++ b/apps/api/src/db/test-helper.ts @@ -11,6 +11,9 @@ const TABLE_NAMES = [ "agent_registration_tokens", "alerts", "api_keys", + "audit_logs", + "backup_storage_settings", + "backups", "deployment_events", "deployment_logs", "deployments", @@ -23,12 +26,15 @@ const TABLE_NAMES = [ "github_integrations", "llm_provider_keys", "platform_settings", + "project_shared_env_links", "projects", "refresh_tokens", "routes", "scaling_policies", "servers", + "shared_env_vars", "smtp_settings", + "ssh_keys", "volumes", ]; @@ -42,9 +48,8 @@ export const setupTestDb = async () => { }; export const truncateAllTables = async (pool: Pool) => { - for (const name of TABLE_NAMES) { - await pool.query(`TRUNCATE TABLE "${name}" CASCADE`); - } + const tables = TABLE_NAMES.map((name) => `"${name}"`).join(", "); + await pool.query(`TRUNCATE TABLE ${tables} CASCADE`); }; export const teardownTestDb = async (pool: Pool) => { diff --git a/apps/api/src/executors/__tests__/ssh-compose-script.test.ts b/apps/api/src/executors/__tests__/ssh-compose-script.test.ts new file mode 100644 index 0000000..0efce7d --- /dev/null +++ b/apps/api/src/executors/__tests__/ssh-compose-script.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, test } from "bun:test"; +import { buildRemoteComposeScript } from "../ssh-compose-script"; + +const input = { + deploymentId: "abc123", + workspaceRoot: "/home/admin/.dequel/workspace", + gitUrl: "https://github.com/owner/repo", + commitSha: null, + projectName: "test", + dockerNetwork: "dequel_net", + environmentVariables: [{ key: "PASS", value: "pa$w0rd" }], +}; + +describe("buildRemoteComposeScript", () => { + test("escapes dollar signs before writing the env override so compose interpolation cannot eat them", () => { + const script = buildRemoteComposeScript(input); + expect(script).toContain('json.dumps(v.replace("$", "$$"))'); + }); + + test("emits a shell-parseable script", () => { + expect(() => buildRemoteComposeScript(input)).not.toThrow(); + expect(buildRemoteComposeScript(input)).toContain("git clone --depth 1"); + }); +}); diff --git a/apps/api/src/executors/ssh-build-script.ts b/apps/api/src/executors/ssh-build-script.ts index 72dee50..2c897c1 100644 --- a/apps/api/src/executors/ssh-build-script.ts +++ b/apps/api/src/executors/ssh-build-script.ts @@ -1,3 +1,5 @@ +import { gitAuthScriptLines, redactGitUrl } from "../orchestrator/source"; + export interface SshBuildScriptInput { deploymentId: string; workspaceRoot: string; @@ -83,8 +85,9 @@ export const buildRemoteDeployScript = (input: SshBuildScriptInput): string => { "fi", 'export BUILDKIT_HOST="docker-container://buildkit"', "", - `echo "[build] Cloning repository ${input.gitUrl}"`, - `git clone --depth 1 ${sh(input.gitUrl)} .`, + `echo "[build] Cloning repository " ${sh(redactGitUrl(input.gitUrl))}`, + ...gitAuthScriptLines(input.gitUrl), + `git clone --depth 1 ${sh(redactGitUrl(input.gitUrl))} .`, checkoutSha, "", 'if command -v bun >/dev/null 2>&1 || [ -x "$HOME/.bun/bin/bun" ]; then', diff --git a/apps/api/src/executors/ssh-compose-script.ts b/apps/api/src/executors/ssh-compose-script.ts index e0b0092..1bc6111 100644 --- a/apps/api/src/executors/ssh-compose-script.ts +++ b/apps/api/src/executors/ssh-compose-script.ts @@ -1,3 +1,5 @@ +import { gitAuthScriptLines, redactGitUrl } from "../orchestrator/source"; + export interface RemoteComposeScriptInput { deploymentId: string; workspaceRoot: string; @@ -22,6 +24,7 @@ export const buildRemoteComposeScript = (input: RemoteComposeScriptInput): strin const remoteWorkspace = input.workspaceRoot.startsWith("/app") ? "$HOME/.dequel/workspace" : input.workspaceRoot; const envLines = input.environmentVariables.map((v) => `${v.key}=${v.value}`).join("\n"); + const envRecord = JSON.stringify(Object.fromEntries(input.environmentVariables.map((v) => [v.key, v.value]))); const sourceDirLine = input.sourceDir ? `cd ${sh(input.sourceDir)}` : ""; @@ -44,8 +47,9 @@ export const buildRemoteComposeScript = (input: RemoteComposeScriptInput): strin " exit 1", "fi", "", - `echo "[compose] Cloning repository ${input.gitUrl}"`, - `git clone --depth 1 ${sh(input.gitUrl)} .`, + `echo "[compose] Cloning repository " ${sh(redactGitUrl(input.gitUrl))}`, + ...gitAuthScriptLines(input.gitUrl), + `git clone --depth 1 ${sh(redactGitUrl(input.gitUrl))} .`, checkoutSha, "", sourceDirLine, @@ -57,9 +61,39 @@ export const buildRemoteComposeScript = (input: RemoteComposeScriptInput): strin "done", 'if [ -z "$COMPOSE_FILE" ]; then echo "[compose] ERROR: no compose file found"; exit 1; fi', "", - `echo "[compose] Building and starting stack (project: deploy-${input.deploymentId})"`, - `docker compose -f "$COMPOSE_FILE" -p deploy-${input.deploymentId} build`, - `docker compose -f "$COMPOSE_FILE" -p deploy-${input.deploymentId} up -d`, + ...(envLines + ? [ + `export DEQUEL_EXTRA_ENV=${sh(envRecord)}`, + "", + "if command -v python3 >/dev/null 2>&1; then", + ' DEQUEL_COMPOSE_FILE="$COMPOSE_FILE" python3 - <<\'DEQUEL_OVERRIDE_EOF\' || echo "[compose] WARNING: could not generate env override file"', + "import json, os, subprocess", + 'envs = json.loads(os.environ["DEQUEL_EXTRA_ENV"])', + 'services = subprocess.check_output(["docker","compose","-f",os.environ["DEQUEL_COMPOSE_FILE"],"config","--services"]).decode().split()', + 'lines = ["services:"]', + "for svc in services:", + ' lines.append(" %s:" % svc)', + ' lines.append(" environment:")', + " for k, v in envs.items():", + ' lines.append(" %s: %s" % (k, json.dumps(v.replace("$", "$$"))))', + 'open("dequel-env-override.yml", "w").write("\\n".join(lines) + "\\n")', + 'print("[compose] Injected %d env var(s) into %d service(s)" % (len(envs), len(services)))', + "DEQUEL_OVERRIDE_EOF", + "else:", + ' echo "[compose] WARNING: python3 not found; platform env vars only apply to compose interpolation"', + "fi", + "", + 'OVERRIDE_ARGS=""', + 'if [ -f dequel-env-override.yml ]; then OVERRIDE_ARGS="-f dequel-env-override.yml"; fi', + `echo "[compose] Building and starting stack (project: deploy-${input.deploymentId})"`, + `docker compose -f "$COMPOSE_FILE" $OVERRIDE_ARGS -p deploy-${input.deploymentId} build`, + `docker compose -f "$COMPOSE_FILE" $OVERRIDE_ARGS -p deploy-${input.deploymentId} up -d`, + ] + : [ + `echo "[compose] Building and starting stack (project: deploy-${input.deploymentId})"`, + `docker compose -f "$COMPOSE_FILE" -p deploy-${input.deploymentId} build`, + `docker compose -f "$COMPOSE_FILE" -p deploy-${input.deploymentId} up -d`, + ]), "", `echo "[compose] Connecting containers to network ${input.dockerNetwork}"`, `for cid in $(docker compose -p deploy-${input.deploymentId} ps -q); do`, diff --git a/apps/api/src/executors/ssh.ts b/apps/api/src/executors/ssh.ts index fa9771e..000ea69 100644 --- a/apps/api/src/executors/ssh.ts +++ b/apps/api/src/executors/ssh.ts @@ -94,11 +94,12 @@ const deployComposeRemote = async (deployment: Deployment, project: Project, ser await emitLog(deployment.id, "system", `Deploying compose stack on server ${server.name} over SSH`); const envVars = await listEnvironmentVariablesForDeploy(project.id, deployment.environment ?? undefined); + const { resolveProjectGitUrl } = await import("../orchestrator/source"); const script = buildRemoteComposeScript({ deploymentId: deployment.id, workspaceRoot: config.workspaceRoot, - gitUrl: deployment.sourceRef, + gitUrl: await resolveProjectGitUrl(project.id, deployment.sourceRef), branch: deployment.branch, commitSha: deployment.commitSha, projectName: project.name, @@ -320,11 +321,12 @@ export const sshExecutor: DeploymentExecutor = { const imageTag = `${slugify(project.name)}-${deployment.id.slice(0, 8)}:latest`; const envVars = await listEnvironmentVariablesForDeploy(project.id, deployment.environment ?? undefined); + const { resolveProjectGitUrl } = await import("../orchestrator/source"); const pgidEcho = `echo "DEQUEL_PGID:$(ps -o pgid= -p $$ | tr -d ' ')"`; const script = `${pgidEcho}\n${buildRemoteDeployScript({ deploymentId: deployment.id, workspaceRoot: config.workspaceRoot, - gitUrl: deployment.sourceRef, + gitUrl: await resolveProjectGitUrl(project.id, deployment.sourceRef), branch: deployment.branch, commitSha: deployment.commitSha, imageTag, diff --git a/apps/api/src/orchestrator/__tests__/cleanup.test.ts b/apps/api/src/orchestrator/__tests__/cleanup.test.ts index 838320d..fc10dd7 100644 --- a/apps/api/src/orchestrator/__tests__/cleanup.test.ts +++ b/apps/api/src/orchestrator/__tests__/cleanup.test.ts @@ -25,6 +25,7 @@ mock.module(fileUrl("../runtime"), () => ({ mock.module(fileUrl("../../utils/config"), () => ({ config: { redisUrl: "redis://localhost:6379", + envEncryptionKey: "dev-env-key-change-me", }, })); diff --git a/apps/api/src/orchestrator/__tests__/compose-env.test.ts b/apps/api/src/orchestrator/__tests__/compose-env.test.ts new file mode 100644 index 0000000..d5a2d28 --- /dev/null +++ b/apps/api/src/orchestrator/__tests__/compose-env.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, test } from "bun:test"; +import { buildComposeEnvOverrideYaml } from "../compose-env"; + +describe("buildComposeEnvOverrideYaml", () => { + test("returns empty string when no env vars", () => { + expect(buildComposeEnvOverrideYaml(["web"], {})).toBe(""); + }); + + test("returns empty string when no services", () => { + expect(buildComposeEnvOverrideYaml([], { FOO: "bar" })).toBe(""); + }); + + test("injects env vars into every service", () => { + const yaml = buildComposeEnvOverrideYaml(["web", "worker"], { FOO: "bar" }); + expect(yaml).toContain(' web:\n environment:\n FOO: "bar"\n'); + expect(yaml).toContain(' worker:\n environment:\n FOO: "bar"\n'); + }); + + test("skips blank service names", () => { + const yaml = buildComposeEnvOverrideYaml(["web", " ", ""], { FOO: "bar" }); + expect((yaml.match(/environment:/g) ?? []).length).toBe(1); + }); + + test("pushes empty string for blank values", () => { + const yaml = buildComposeEnvOverrideYaml(["web"], { EMPTY: "" }); + expect(yaml).toContain(' EMPTY: ""'); + }); + + test("escapes literal dollar signs for compose interpolation", () => { + const yaml = buildComposeEnvOverrideYaml(["web"], { TOKEN: "adm$GITHUB" }); + expect(yaml).toContain('TOKEN: "adm$$GITHUB"'); + }); + + test("escapes plain dollar signs", () => { + const yaml = buildComposeEnvOverrideYaml(["web"], { PASS: "pa$$w0rd" }); + expect(yaml).toContain('PASS: "pa$$$$w0rd"'); + }); +}); diff --git a/apps/api/src/orchestrator/__tests__/compose.test.ts b/apps/api/src/orchestrator/__tests__/compose.test.ts index 50b3206..8de6cc2 100644 --- a/apps/api/src/orchestrator/__tests__/compose.test.ts +++ b/apps/api/src/orchestrator/__tests__/compose.test.ts @@ -122,7 +122,7 @@ services: }); }); -describe("spawnComposeCommand env isolation", () => { +describe("spawnComposeCommand env merge with project env vars", () => { let originalSpawn: any; afterEach(() => { @@ -132,10 +132,9 @@ describe("spawnComposeCommand env isolation", () => { } }); - test("does not pass process.env to docker compose subprocess", async () => { - process.env.DATABASE_URL = "postgresql://dequel:secret@localhost:5432/dequel"; - process.env.PORT = "3001"; - process.env.WORKSPACE_ROOT = "./workspace"; + test("passes project env vars alongside process.env to docker compose subprocess", async () => { + process.env.IDEK_DATABASE_URL = "postgresql://dequel:secret@localhost:5432/dequel"; + process.env.IDEK_PORT = "3001"; let capturedEnv: Record | undefined; originalSpawn = (await import("node:child_process")).spawn; @@ -156,7 +155,7 @@ describe("spawnComposeCommand env isolation", () => { ); const { buildWithCompose } = await import("../compose"); - const projectEnv = { MY_APP_VAR: "hello", ANOTHER_VAR: "world" }; + const projectEnv = { MY_APP_VAR: "hello", ANOTHER_VAR: "world", IDEK_PORT: "4000" }; try { await buildWithCompose(dir, "test-project", async () => {}, null, projectEnv); @@ -165,21 +164,19 @@ describe("spawnComposeCommand env isolation", () => { } expect(capturedEnv).toBeDefined(); - expect(capturedEnv).not.toHaveProperty("DATABASE_URL"); - expect(capturedEnv).not.toHaveProperty("PORT"); - expect(capturedEnv).not.toHaveProperty("WORKSPACE_ROOT"); expect(capturedEnv).toHaveProperty("MY_APP_VAR", "hello"); expect(capturedEnv).toHaveProperty("ANOTHER_VAR", "world"); + expect(capturedEnv).toHaveProperty("IDEK_DATABASE_URL", "postgresql://dequel:secret@localhost:5432/dequel"); + expect(capturedEnv).toHaveProperty("PATH"); + expect(capturedEnv).toHaveProperty("IDEK_PORT", "4000"); } finally { rmSync(dir, { recursive: true, force: true }); - delete process.env.DATABASE_URL; - delete process.env.PORT; - delete process.env.WORKSPACE_ROOT; + delete process.env.IDEK_DATABASE_URL; + delete process.env.IDEK_PORT; } }); - - test("passes empty env when no project env vars provided", async () => { - process.env.DATABASE_URL = "postgresql://dequel:secret@localhost:5432/dequel"; + test("passes process.env through when no project env vars provided", async () => { + process.env.IDEK_DATABASE_URL = "postgresql://dequel:secret@localhost:5432/dequel"; let capturedEnv: Record | undefined; const originalSpawn = (await import("node:child_process")).spawn; @@ -208,10 +205,11 @@ describe("spawnComposeCommand env isolation", () => { } expect(capturedEnv).toBeDefined(); - expect(capturedEnv).toEqual({}); + expect(capturedEnv).toHaveProperty("IDEK_DATABASE_URL", "postgresql://dequel:secret@localhost:5432/dequel"); + expect(capturedEnv).toHaveProperty("PATH"); } finally { rmSync(dir, { recursive: true, force: true }); - delete process.env.DATABASE_URL; + delete process.env.IDEK_DATABASE_URL; } }); }); diff --git a/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts b/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts index 311c3ee..f3beac6 100644 --- a/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts +++ b/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts @@ -45,6 +45,7 @@ const mockDb = { } return { sequence: 1 }; }), + getProjectGithubToken: mock(() => Promise.resolve(null)), listEnvironmentVariablesForDeploy: mock(() => Promise.resolve([])), listVolumes: mock(() => Promise.resolve([])), listDeployments: mock(() => Promise.resolve([])), diff --git a/apps/api/src/orchestrator/__tests__/railpack-config.test.ts b/apps/api/src/orchestrator/__tests__/railpack-config.test.ts index 6ef2940..9543a45 100644 --- a/apps/api/src/orchestrator/__tests__/railpack-config.test.ts +++ b/apps/api/src/orchestrator/__tests__/railpack-config.test.ts @@ -56,4 +56,142 @@ describe("generateDynamicRailpackJson static subdirectory detection", () => { cleanup(); } }); + + describe("vite SPA without server entry", () => { + test("serves the build output statically instead of node dist/index.js", async () => { + const { dir, cleanup } = setupWorkspace(); + try { + writeFileSync( + join(dir, "client", "package.json"), + JSON.stringify({ + name: "spa", + scripts: { build: "vite build" }, + devDependencies: { vite: "^5.0.0", react: "^18.0.0" }, + }), + ); + const logs: string[] = []; + await generateDynamicRailpackJson(dir, "client", null, null, null, async (line) => logs.push(line)); + const parsedConfig = JSON.parse(readFileSync(join(dir, "railpack.json"), "utf8")); + expect(parsedConfig.deploy?.startCommand).toBe("bun dequel-serve.js"); + expect(existsSync(join(dir, "dequel-serve.js"))).toBe(true); + expect(parsedConfig.steps?.build?.commands?.join(" ")).toContain("run build"); + } finally { + cleanup(); + } + }); + + test("keeps node dist/index.js fallback for vite apps with a server framework", async () => { + const { dir, cleanup } = setupWorkspace(); + try { + writeFileSync( + join(dir, "client", "package.json"), + JSON.stringify({ + name: "ssr", + scripts: { build: "vite build" }, + dependencies: { vite: "^5.0.0", express: "^4.0.0" }, + }), + ); + await generateDynamicRailpackJson(dir, "client", null, null, null, async () => {}); + const parsedConfig = JSON.parse(readFileSync(join(dir, "railpack.json"), "utf8")); + expect(parsedConfig.deploy?.startCommand).toBe("cd client && node dist/index.js"); + } finally { + cleanup(); + } + }); + + test("keeps node dist/index.js fallback for node apps that build but have no vite", async () => { + const { dir, cleanup } = setupWorkspace(); + try { + writeFileSync( + join(dir, "client", "package.json"), + JSON.stringify({ name: "plain", scripts: { build: "tsc" } }), + ); + await generateDynamicRailpackJson(dir, "client", null, null, null, async () => {}); + const parsedConfig = JSON.parse(readFileSync(join(dir, "railpack.json"), "utf8")); + expect(parsedConfig.deploy?.startCommand).toBe("cd client && node dist/index.js"); + } finally { + cleanup(); + } + }); + + test("still serves statically when only client libraries like socket.io-client are present", async () => { + const { dir, cleanup } = setupWorkspace(); + try { + writeFileSync( + join(dir, "client", "package.json"), + JSON.stringify({ + name: "spa", + scripts: { build: "vite build" }, + dependencies: { vite: "^5.0.0", "socket.io-client": "^4.0.0" }, + }), + ); + await generateDynamicRailpackJson(dir, "client", null, null, null, async () => {}); + const parsedConfig = JSON.parse(readFileSync(join(dir, "railpack.json"), "utf8")); + expect(parsedConfig.deploy?.startCommand).toBe("bun dequel-serve.js"); + } finally { + cleanup(); + } + }); + + test("still serves statically when only client libraries like next-themes are present", async () => { + const { dir, cleanup } = setupWorkspace(); + try { + writeFileSync( + join(dir, "client", "package.json"), + JSON.stringify({ + name: "spa", + scripts: { build: "vite build" }, + dependencies: { vite: "^5.0.0", "next-themes": "^0.3.0" }, + }), + ); + await generateDynamicRailpackJson(dir, "client", null, null, null, async () => {}); + const parsedConfig = JSON.parse(readFileSync(join(dir, "railpack.json"), "utf8")); + expect(parsedConfig.deploy?.startCommand).toBe("bun dequel-serve.js"); + } finally { + cleanup(); + } + }); + + test("keeps node dist/index.js for vite SSR builds configured with --ssr", async () => { + const { dir, cleanup } = setupWorkspace(); + try { + writeFileSync( + join(dir, "client", "package.json"), + JSON.stringify({ + name: "ssr", + scripts: { build: "vite build --ssr src/entry-server.ts" }, + dependencies: { vite: "^5.0.0" }, + }), + ); + await generateDynamicRailpackJson(dir, "client", null, null, null, async () => {}); + const parsedConfig = JSON.parse(readFileSync(join(dir, "railpack.json"), "utf8")); + expect(parsedConfig.deploy?.startCommand).toBe("cd client && node dist/index.js"); + } finally { + cleanup(); + } + }); + + test("keeps node dist/index.js for vite SSR builds configured via config file", async () => { + const { dir, cleanup } = setupWorkspace(); + try { + writeFileSync( + join(dir, "client", "package.json"), + JSON.stringify({ + name: "ssr", + scripts: { build: "vite build" }, + dependencies: { vite: "^5.0.0" }, + }), + ); + writeFileSync( + join(dir, "client", "vite.config.ts"), + "export default { build: { ssr: 'src/entry-server.ts', outDir: 'dist/server' } };", + ); + await generateDynamicRailpackJson(dir, "client", null, null, null, async () => {}); + const parsedConfig = JSON.parse(readFileSync(join(dir, "railpack.json"), "utf8")); + expect(parsedConfig.deploy?.startCommand).toBe("cd client && node dist/index.js"); + } finally { + cleanup(); + } + }); + }); }); diff --git a/apps/api/src/orchestrator/__tests__/runtime-instance.test.ts b/apps/api/src/orchestrator/__tests__/runtime-instance.test.ts new file mode 100644 index 0000000..5c238c9 --- /dev/null +++ b/apps/api/src/orchestrator/__tests__/runtime-instance.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, test } from "bun:test"; +import { evaluateInstanceState } from "../runtime"; + +describe("evaluateInstanceState", () => { + test("retries when container does not exist yet", () => { + const verdict = evaluateInstanceState({ exists: false, status: null }); + expect(verdict.action).toBe("retry"); + }); + + test("returns when running", () => { + const verdict = evaluateInstanceState({ exists: true, status: "running" }); + expect(verdict.action).toBe("return"); + }); + + test("fatal with logs when exited during initial wait", () => { + const verdict = evaluateInstanceState({ exists: true, status: "exited", logs: "Error: MODULE_NOT_FOUND" }); + expect(verdict.action).toBe("fatal"); + if (verdict.action === "fatal") expect(verdict.error.message).toContain("MODULE_NOT_FOUND"); + }); + + test("fatal with trimmed logs and placeholder when exited without output", () => { + const verdict = evaluateInstanceState({ exists: true, status: "exited", logs: "\n\n" }); + expect(verdict.action).toBe("fatal"); + if (verdict.action === "fatal") expect(verdict.error.message).toContain("*No logs produced*"); + }); + + test("fatal wording differs for stability-phase crash", () => { + const verdict = evaluateInstanceState({ exists: true, status: "exited", logs: "boom" }, "stability"); + expect(verdict.action).toBe("fatal"); + if (verdict.action === "fatal") expect(verdict.error.message).toContain("crashed shortly after starting"); + }); + + test("retries with note while still initializing", () => { + const verdict = evaluateInstanceState({ exists: true, status: "created" }); + expect(verdict.action).toBe("retry"); + if (verdict.action === "retry") expect(verdict.note).toContain("initializing"); + }); + + test("retries on unknown states like restarting", () => { + const verdict = evaluateInstanceState({ exists: true, status: "restarting" }); + expect(verdict.action).toBe("retry"); + expect(evaluateInstanceState({ exists: true, status: "running" }, "restart").action).toBe("return"); + }); +}); diff --git a/apps/api/src/orchestrator/__tests__/source-git-auth.test.ts b/apps/api/src/orchestrator/__tests__/source-git-auth.test.ts new file mode 100644 index 0000000..b207096 --- /dev/null +++ b/apps/api/src/orchestrator/__tests__/source-git-auth.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, test } from "bun:test"; +import { + buildAuthenticatedGitUrl, + buildGitAuthEnv, + gitAuthScriptLines, + isGithubHttpsUrl, + redactGitUrl, +} from "../source"; + +describe("git URL helpers", () => { + test("identifies github https urls", () => { + expect(isGithubHttpsUrl("https://github.com/owner/repo")).toBe(true); + expect(isGithubHttpsUrl("HTTPS://GITHUB.COM/owner/repo")).toBe(true); + expect(isGithubHttpsUrl("https://gitlab.com/owner/repo")).toBe(false); + expect(isGithubHttpsUrl("git@github.com:owner/repo.git")).toBe(false); + }); + + test("redacts credential-bearing urls", () => { + expect(redactGitUrl("https://x-access-token:abc123@github.com/owner/repo")).toBe("https://github.com/owner/repo"); + expect(redactGitUrl("https://github.com/owner/repo")).toBe("https://github.com/owner/repo"); + }); + + test("builds authenticated github urls only for https github repos", () => { + expect(buildAuthenticatedGitUrl("https://github.com/owner/repo", "tok")).toBe( + "https://x-access-token:tok@github.com/owner/repo", + ); + expect(buildAuthenticatedGitUrl("https://github.com/owner/repo", null)).toBe("https://github.com/owner/repo"); + expect(buildAuthenticatedGitUrl("https://gitlab.com/owner/repo", "tok")).toBe("https://gitlab.com/owner/repo"); + expect(buildAuthenticatedGitUrl("https://user@github.com/owner/repo", "tok")).toBe( + "https://user@github.com/owner/repo", + ); + }); +}); + +describe("buildGitAuthEnv", () => { + test("rewrites clean urls over the tokenized one via env instead of repo config", () => { + const auth = buildGitAuthEnv("https://x-access-token:tok@github.com/owner/repo"); + expect(auth?.cleanUrl).toBe("https://github.com/owner/repo"); + expect(auth?.env.GIT_CONFIG_COUNT).toBe("1"); + expect(auth?.env.GIT_CONFIG_KEY_0).toBe("url.https://x-access-token:tok@github.com/owner/repo.insteadOf"); + expect(auth?.env.GIT_CONFIG_VALUE_0).toBe("https://github.com/owner/repo"); + }); + + test("returns null when the url carries no credentials", () => { + expect(buildGitAuthEnv("https://github.com/owner/repo")).toBeNull(); + expect(buildGitAuthEnv("https://gitlab.com/owner/repo")).toBeNull(); + }); +}); + +describe("gitAuthScriptLines", () => { + test("emits quoted env exports for authenticated urls", () => { + const lines = gitAuthScriptLines("https://x-access-token:to'k@github.com/owner/repo"); + expect(lines).toHaveLength(3); + expect(lines[0]).toBe('export GIT_CONFIG_COUNT="1"'); + expect(lines[1]).toContain("url.https://x-access-token:to"); + expect(lines[1]).toContain(".insteadOf"); + expect(lines[2]).toContain("https://github.com/owner/repo"); + }); + + test("emits nothing for credential-free urls", () => { + expect(gitAuthScriptLines("https://github.com/owner/repo")).toEqual([]); + }); +}); diff --git a/apps/api/src/orchestrator/compose-deploy.ts b/apps/api/src/orchestrator/compose-deploy.ts index 218f7e1..fbbd773 100644 --- a/apps/api/src/orchestrator/compose-deploy.ts +++ b/apps/api/src/orchestrator/compose-deploy.ts @@ -4,6 +4,7 @@ import { config } from "../utils/config"; import { dockerBin } from "../utils/docker-bin"; import { buildCaddySnippet } from "../utils/domain-verifier"; import { deployWithCompose, destroyComposeStack, getComposeContainerNames, parseAllComposeServices } from "./compose"; +import { buildComposeEnvOverrideYaml } from "./compose-env"; import { reloadCaddy, tryRun } from "./runtime"; export interface ComposeDeployResult { @@ -47,10 +48,25 @@ export const deployComposeStack = async (options: ComposeDeployOptions): Promise await destroyComposeStack(`deploy-${oldDeploymentId}`); } - await deployWithCompose(workspacePath, `deploy-${deploymentId}`, onLog, sourceDir, envVars, signal); - const allServices = parseAllComposeServices(workspacePath, sourceDir, composeService, composePort); + let overridePath: string | null = null; + if (envVars && Object.keys(envVars).length > 0) { + const overrideYaml = buildComposeEnvOverrideYaml( + allServices.map((s) => s.serviceName), + envVars, + ); + if (overrideYaml) { + overridePath = join(workspacePath, "dequel-env-override.yml"); + await writeFile(overridePath, overrideYaml); + await onLog( + `Injecting ${Object.keys(envVars).length} env var(s) into ${allServices.length} compose service(s) via override file.`, + ); + } + } + + await deployWithCompose(workspacePath, `deploy-${deploymentId}`, onLog, sourceDir, envVars, signal, overridePath); + const composeContainers = await getComposeContainerNames(`deploy-${deploymentId}`); const containerFor = (serviceName: string) => composeContainers.get(serviceName) || `deploy-${deploymentId}-${serviceName}-1`; diff --git a/apps/api/src/orchestrator/compose-env.ts b/apps/api/src/orchestrator/compose-env.ts new file mode 100644 index 0000000..3ae2eb7 --- /dev/null +++ b/apps/api/src/orchestrator/compose-env.ts @@ -0,0 +1,16 @@ +const yamlScalar = (value: string): string => JSON.stringify(value).split("$").join("$$"); + +export const buildComposeEnvOverrideYaml = (serviceNames: string[], envVars: Record): string => { + const keys = Object.keys(envVars); + if (serviceNames.length === 0 || keys.length === 0) return ""; + const lines = ["services:"]; + for (const serviceName of serviceNames) { + if (!serviceName.trim()) continue; + lines.push(` ${serviceName}:`); + lines.push(" environment:"); + for (const key of keys) { + lines.push(` ${key}: ${yamlScalar(envVars[key])}`); + } + } + return `${lines.join("\n")}\n`; +}; diff --git a/apps/api/src/orchestrator/compose.ts b/apps/api/src/orchestrator/compose.ts index de13f74..d79d40e 100644 --- a/apps/api/src/orchestrator/compose.ts +++ b/apps/api/src/orchestrator/compose.ts @@ -206,7 +206,7 @@ const spawnComposeCommand = ( const isPosix = process.platform !== "win32"; const child = spawn(dockerBin, ["compose", ...args], { cwd, - env: envVars || {}, + env: { ...process.env, ...(envVars || {}) }, stdio: ["ignore", "pipe", "pipe"], detached: true, }); @@ -310,6 +310,7 @@ export const deployWithCompose = async ( sourceDir?: string | null, envVars?: Record, signal?: AbortSignal, + overrideFilePath?: string | null, ): Promise => { const composeFile = findComposeFilePath(workspacePath, sourceDir); if (!composeFile) { @@ -319,13 +320,11 @@ export const deployWithCompose = async ( const cwd = dirname(composeFile); await onLog(`Starting Docker Compose services (project: ${projectName})...`); - const res = await spawnComposeCommand( - ["-f", composeFile, "-p", projectName, "up", "-d"], - cwd, - envVars, - onLog, - signal, - ); + const args = ["-f", composeFile]; + if (overrideFilePath) args.push("-f", overrideFilePath); + args.push("-p", projectName, "up", "-d"); + + const res = await spawnComposeCommand(args, cwd, envVars, onLog, signal); if (res.code !== 0) { throw new Error(`docker compose up failed: ${res.stderr || res.stdout}`); } diff --git a/apps/api/src/orchestrator/pipeline.ts b/apps/api/src/orchestrator/pipeline.ts index e65dbde..a481b44 100644 --- a/apps/api/src/orchestrator/pipeline.ts +++ b/apps/api/src/orchestrator/pipeline.ts @@ -8,6 +8,7 @@ import { deleteRoutesByDeployment, getDeploymentById, getProjectById, + getProjectGithubToken, getServerById, listAllDatabases, listDeployments, @@ -249,6 +250,7 @@ export class PipelineOrchestrator { deployment.sourceRef, deployment.branch ?? undefined, deployment.commitSha ?? undefined, + deployment.projectId ? await getProjectGithubToken(deployment.projectId).catch(() => null) : null, ); const sha = await getHeadSha(workspacePath); if (sha) { diff --git a/apps/api/src/orchestrator/railpack-config-utils.ts b/apps/api/src/orchestrator/railpack-config-utils.ts index 77789ac..2e94b77 100644 --- a/apps/api/src/orchestrator/railpack-config-utils.ts +++ b/apps/api/src/orchestrator/railpack-config-utils.ts @@ -64,6 +64,61 @@ const fixPnpmWorkspaceIfNeeded = async (workspacePath: string, onLog: (line: str } }; +const SERVER_FRAMEWORK_NAMES = new Set([ + "express", + "fastify", + "koa", + "hono", + "elysia", + "polka", + "socket.io", + "nitropack", + "next", + "nuxt", +]); + +const SERVER_FRAMEWORK_SCOPES = ["@hono/", "@elysiajs/", "@remix-run/"]; + +const isServerFrameworkDep = (name: string): boolean => { + if (name.startsWith("@")) return SERVER_FRAMEWORK_SCOPES.some((scope) => name.startsWith(scope)); + return SERVER_FRAMEWORK_NAMES.has(name); +}; + +const packageDependsOnServerFramework = (packageJson: any): boolean => { + const sections = [packageJson.dependencies, packageJson.devDependencies]; + for (const section of sections) { + if (!section) continue; + for (const name of Object.keys(section)) { + if (isServerFrameworkDep(name)) return true; + } + } + return false; +}; + +const packageHasVite = (packageJson: any): boolean => { + const sections = [packageJson.dependencies, packageJson.devDependencies]; + for (const section of sections) { + if (!section) continue; + for (const name of Object.keys(section)) { + if (name === "vite" || name.startsWith("@vitejs/") || name.startsWith("vite-")) return true; + } + } + return false; +}; + +const VITE_SSR_SCRIPT_PATTERN = /--ssr(?=\s|$)/; +const VITE_SSR_CONFIG_PATTERN = /(^|[\s.;,(])ssr\s*[:=]/; + +const looksLikeViteSsr = async (buildDir: string, buildScript: string): Promise => { + if (VITE_SSR_SCRIPT_PATTERN.test(buildScript)) return true; + for (const name of ["vite.config.ts", "vite.config.js", "vite.config.mts", "vite.config.mjs"]) { + const file = Bun.file(join(buildDir, name)); + if (!(await file.exists())) continue; + if (VITE_SSR_CONFIG_PATTERN.test(await file.text())) return true; + } + return false; +}; + export const generateDynamicRailpackJson = async ( workspace: string, sourceDir: string | null, @@ -89,6 +144,7 @@ export const generateDynamicRailpackJson = async ( }; let configured = false; + let forceStaticServe = false; if (cleanSourceDir && !(await Bun.file(join(workspace, "package.json")).exists())) { const sourcePkgExists = await Bun.file(join(workspace, cleanSourceDir, "package.json")).exists(); @@ -177,9 +233,20 @@ export const generateDynamicRailpackJson = async ( } else if (scripts.server) { config.deploy.startCommand = cleanSourceDir ? `cd ${cleanSourceDir} && ${pm} run server` : `${pm} run server`; } else { - config.deploy.startCommand = cleanSourceDir - ? `cd ${cleanSourceDir} && node dist/index.js` - : "node dist/index.js"; + const viteBuildScript = scripts.build ? String(scripts.build) : ""; + if ( + scripts.build && + packageHasVite(packageJson) && + !packageDependsOnServerFramework(packageJson) && + !(await looksLikeViteSsr(buildDir, viteBuildScript)) + ) { + forceStaticServe = true; + await onLog("Detected Vite frontend without a server entry — serving the build output statically"); + } else { + config.deploy.startCommand = cleanSourceDir + ? `cd ${cleanSourceDir} && node dist/index.js` + : "node dist/index.js"; + } } configured = true; @@ -326,6 +393,7 @@ export const generateDynamicRailpackJson = async ( if ( projectType === "static" || + forceStaticServe || (!configured && ((await Bun.file(join(buildDir, "index.html")).exists()) || (await Bun.file(join(workspace, "index.html")).exists()))) diff --git a/apps/api/src/orchestrator/runtime.ts b/apps/api/src/orchestrator/runtime.ts index ff2cfd8..3413616 100644 --- a/apps/api/src/orchestrator/runtime.ts +++ b/apps/api/src/orchestrator/runtime.ts @@ -48,6 +48,30 @@ const getCaddyContainer = async (): Promise => { return name; }; +export interface ContainerInstanceState { + exists: boolean; + status: string | null; + logs?: string | null; +} + +export type Phase = "initial" | "stability" | "restart"; + +export type InstanceVerdict = + | { action: "return" } + | { action: "retry"; note?: string } + | { action: "fatal"; error: Error }; + +export const evaluateInstanceState = (state: ContainerInstanceState, phase: Phase = "initial"): InstanceVerdict => { + if (!state.exists) return { action: "retry", note: "not created yet" }; + const logs = (state.logs ?? "").trim() || "*No logs produced*"; + if (state.status === "running") return { action: "return" }; + if (state.status === "exited") { + const reason = phase === "stability" ? "crashed shortly after starting" : "exited immediately"; + return { action: "fatal", error: new Error(`Container ${reason}. Logs:\n${logs}`) }; + } + return { action: "retry", note: state.status === "created" ? "still initializing" : undefined }; +}; + const waitForRunningContainer = async ( containerName: string, retries: number, @@ -58,26 +82,34 @@ const waitForRunningContainer = async ( const exists = await run(dockerBin, ["inspect", "-f", "{{.Id}}", containerName]) .then(() => true) .catch(() => false); - if (!exists) { - await new Promise((r) => setTimeout(r, 500)); - continue; - } - const status = (await run(dockerBin, ["inspect", "-f", "{{.State.Status}}", containerName])).trim(); - if (status === "running") { + const status = exists + ? (await run(dockerBin, ["inspect", "-f", "{{.State.Status}}", containerName])).trim() + : null; + const logs = + exists && status === "exited" + ? await run(dockerBin, ["logs", "--tail", "50", containerName]).catch(() => null) + : null; + const verdict = evaluateInstanceState({ exists, status, logs }); + if (verdict.action === "return") { await new Promise((r) => setTimeout(r, 2000)); const stabilityStatus = (await run(dockerBin, ["inspect", "-f", "{{.State.Status}}", containerName])).trim(); + const stabilityLogs = + stabilityStatus !== "running" + ? await run(dockerBin, ["logs", "--tail", "50", containerName]).catch(() => null) + : null; + const stabilityVerdict = evaluateInstanceState( + { exists: true, status: stabilityStatus, logs: stabilityLogs }, + "stability", + ); + if (stabilityVerdict.action === "fatal") throw stabilityVerdict.error; if (stabilityStatus !== "running") { - const logs = await run(dockerBin, ["logs", "--tail", "50", containerName]).catch(() => "no logs available"); + const logs = (stabilityLogs ?? "").trim() || "*No logs produced*"; throw new Error(`Container crashed shortly after starting. Logs:\n${logs}`); } return; } - if (status === "exited") { - let logs = await run(dockerBin, ["logs", "--tail", "50", containerName]).catch(() => "no logs available"); - logs = logs.trim() || "*No logs produced*"; - throw new Error(`Container ${containerName} exited immediately. Logs:\n${logs}`); - } - if (status === "created") await onLog(`Container ${containerName} still initializing...`); + if (verdict.action === "fatal") throw verdict.error; + if (verdict.note) await onLog(`Container ${containerName} ${verdict.note}...`); } catch (e: any) { if (e.message?.includes("exited immediately") || e.message?.includes("crashed shortly")) throw e; } @@ -87,6 +119,39 @@ const waitForRunningContainer = async ( await tryRun(dockerBin, ["network", "disconnect", "-f", config.dockerNetwork, containerName]); await tryRun(dockerBin, ["start", containerName]); await tryRun(dockerBin, ["network", "connect", config.dockerNetwork, containerName]); + await verifyRunningAfterStart(containerName, onLog); +}; + +const verifyRunningAfterStart = async ( + containerName: string, + onLog: (line: string) => Promise, +): Promise => { + let lastStatus: string | null = null; + let lastLogs: string | null = null; + for (let attempt = 0; attempt < 10; attempt++) { + await new Promise((r) => setTimeout(r, 1000)); + try { + const exists = await run(dockerBin, ["inspect", "-f", "{{.Id}}", containerName]) + .then(() => true) + .catch(() => false); + lastStatus = exists ? (await run(dockerBin, ["inspect", "-f", "{{.State.Status}}", containerName])).trim() : null; + if (exists && lastStatus !== "running") { + lastLogs = await run(dockerBin, ["logs", "--tail", "50", containerName]).catch(() => lastLogs); + } + const verdict = evaluateInstanceState({ exists, status: lastStatus, logs: lastLogs }, "restart"); + if (verdict.action === "fatal") throw verdict.error; + } catch (e: any) { + if (e.message?.includes("exited immediately") || e.message?.includes("crashed shortly")) throw e; + } + } + if (lastStatus === "running") { + await onLog(`Container ${containerName} is running after restart`); + return "running"; + } + const logs = (lastLogs ?? "").trim() || "*No logs produced*"; + throw new Error( + `Container ${containerName} did not reach running state even after docker start (status: ${lastStatus ?? "unknown"}). Logs:\n${logs}`, + ); }; export const ensureContainerRunning = async (containerName: string) => { diff --git a/apps/api/src/orchestrator/source.ts b/apps/api/src/orchestrator/source.ts index d84fae4..f9ea9a2 100644 --- a/apps/api/src/orchestrator/source.ts +++ b/apps/api/src/orchestrator/source.ts @@ -13,8 +13,8 @@ const ensureSingleRoot = async (root: string): Promise => { return root; }; -const run = async (cmd: string, args: string[], cwd?: string) => { - const res = await safeSpawn(cmd, args, { cwd, timeoutMs: 120_000 }); +const run = async (cmd: string, args: string[], cwd?: string, env?: Record) => { + const res = await safeSpawn(cmd, args, { cwd, timeoutMs: 120_000, env }); if (res.code !== 0) { throw new Error(`${cmd} exited with code ${res.code}: ${res.stderr || res.stdout}`); } @@ -22,28 +22,80 @@ const run = async (cmd: string, args: string[], cwd?: string) => { const isValidSha = (s: string) => /^[0-9a-f]{7,40}$/i.test(s); +export const isGithubHttpsUrl = (gitUrl: string) => /^https:\/\/github\.com\//i.test(gitUrl.trim()); + +export const redactGitUrl = (gitUrl: string) => gitUrl.replace(/^(https:\/\/)[^/@]+@([^/]+)/i, "$1$2"); + +export const buildAuthenticatedGitUrl = (gitUrl: string, token?: string | null) => { + if (!token) return gitUrl; + const trimmed = gitUrl.trim(); + if (!isGithubHttpsUrl(trimmed)) return gitUrl; + if (trimmed.includes("@")) return gitUrl; + return trimmed.replace(/^https:\/\/github\.com\//i, `https://x-access-token:${token}@github.com/`); +}; + +export const resolveProjectGitUrl = async (projectId: string | null | undefined, gitUrl: string): Promise => { + if (!projectId || !isGithubHttpsUrl(gitUrl)) return gitUrl; + try { + const { getProjectGithubToken } = await import("../db/repo/projects"); + return buildAuthenticatedGitUrl(gitUrl, await getProjectGithubToken(projectId)); + } catch { + return gitUrl; + } +}; + +const shq = (value: string) => `'${value.replace(/'/g, `'\\''`)}'`; + +export const buildGitAuthEnv = (authedUrl: string): { env: Record; cleanUrl: string } | null => { + const cleanUrl = redactGitUrl(authedUrl); + if (cleanUrl === authedUrl) return null; + if (!isGithubHttpsUrl(cleanUrl)) return null; + return { + env: { + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: `url.${authedUrl}.insteadOf`, + GIT_CONFIG_VALUE_0: cleanUrl, + }, + cleanUrl, + }; +}; + +export const gitAuthScriptLines = (authedUrl: string): string[] => { + const auth = buildGitAuthEnv(authedUrl); + if (!auth) return []; + return [ + 'export GIT_CONFIG_COUNT="1"', + `export GIT_CONFIG_KEY_0=${shq(`url.${authedUrl}.insteadOf`)}`, + `export GIT_CONFIG_VALUE_0=${shq(auth.cleanUrl)}`, + ]; +}; + export const prepareSourceWorkspace = async ( deploymentId: string, gitUrl: string, branch?: string, commitSha?: string, + token?: string | null, ) => { const root = join(config.workspaceRoot, deploymentId); await rm(root, { recursive: true, force: true }); await mkdir(root, { recursive: true }); + const authedUrl = buildAuthenticatedGitUrl(gitUrl, token); + const auth = buildGitAuthEnv(authedUrl); + const remoteUrl = auth?.cleanUrl ?? authedUrl; if (commitSha) { if (!isValidSha(commitSha)) { throw new Error(`Invalid commit SHA: ${commitSha}`); } - await run("git", ["init"], root); - await run("git", ["remote", "add", "origin", gitUrl], root); - await run("git", ["fetch", "--depth", "1", "origin", commitSha], root); + await run("git", ["init"], root, auth?.env); + await run("git", ["remote", "add", "origin", remoteUrl], root, auth?.env); + await run("git", ["fetch", "--depth", "1", "origin", commitSha], root, auth?.env); await run("git", ["checkout", "FETCH_HEAD"], root); } else { const args = ["clone", "--depth", "1"]; if (branch) args.push("--branch", branch); - args.push(gitUrl, root); - await run("git", args); + args.push(remoteUrl, root); + await run("git", args, undefined, auth?.env); } return root; }; @@ -75,10 +127,12 @@ export const getHeadSha = async (repoPath: string): Promise => { } }; -export const getRemoteSha = async (gitUrl: string, branch?: string): Promise => { +export const getRemoteSha = async (gitUrl: string, branch?: string, token?: string | null): Promise => { try { const ref = branch ? `refs/heads/${branch}` : "HEAD"; - const res = await safeSpawn("git", ["ls-remote", gitUrl, ref], { timeoutMs: 15_000 }); + const res = await safeSpawn("git", ["ls-remote", buildAuthenticatedGitUrl(gitUrl, token), ref], { + timeoutMs: 15_000, + }); if (res.code !== 0) return null; const match = res.stdout.trim().split(/\s+/)[0]; return match || null; diff --git a/apps/api/src/scaling/__tests__/engine.test.ts b/apps/api/src/scaling/__tests__/engine.test.ts index 4273c98..e6daf3b 100644 --- a/apps/api/src/scaling/__tests__/engine.test.ts +++ b/apps/api/src/scaling/__tests__/engine.test.ts @@ -140,6 +140,7 @@ mock.module(fileUrl("../../utils/config"), () => ({ caddyBaseDomain: "localhost", appInternalPort: 3000, dockerNetwork: "dequel_net", + envEncryptionKey: "dev-env-key-change-me", }, })); @@ -196,6 +197,25 @@ describe("getCurrentReplicas", () => { const result = await (scalingEngine as any).getCurrentReplicas("proj-1"); expect(result).toBe(3); }); + + it("counts only the primary site block when custom-domain blocks exist", async () => { + mockReadFileContent = [ + "verify-app.custom-domain.com {", + " reverse_proxy some-other-container:3000", + "}", + "proj-1.localhost:80 {", + " reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000", + "}", + ].join("\n"); + const result = await (scalingEngine as any).getCurrentReplicas("proj-1"); + expect(result).toBe(2); + }); + + it("falls back to whole-file counting when no primary block matches", async () => { + mockReadFileContent = "other.localhost:80 {\n reverse_proxy a:3000\n}\n"; + const result = await (scalingEngine as any).getCurrentReplicas("proj-1"); + expect(result).toBe(1); + }); }); describe("evaluate", () => { @@ -272,6 +292,47 @@ describe("evaluate", () => { expect(mockWriteFilePath).toBeNull(); }); + it("does not advance lastScaleUp when already at max replicas", async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, maxReplicas: 2, cooldownSeconds: 0 }); + const lastScaleUpBefore = String(Date.now() - 60_000); + mockRedisData = { highCpuSince: String(Date.now() - 10000), lastScaleUp: lastScaleUpBefore }; + mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n"; + mockRunImpl = async () => '{"CPUPerc":"85%","MemUsage":"128MiB / 512MiB"}'; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockRedisData.lastScaleUp).toBe(lastScaleUpBefore); + expect(mockRedisData.highCpuSince).toBeFalsy(); + }); + + it("keeps accruing the low-CPU timer between threshold and 30%", async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cpuThresholdPercent: 1, cooldownSeconds: 0 }); + const lowCpuSinceBefore = String(Date.now() - 240_000); + mockRedisData = { lowCpuSince: lowCpuSinceBefore, highCpuSince: "0", lastScaleUp: "0" }; + mockRunImpl = async () => '{"CPUPerc":"22%","MemUsage":"64MiB / 512MiB"}'; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockRedisData.lowCpuSince).toBe(lowCpuSinceBefore); + }); + + it("scales down in the overlap zone (threshold < cpu < 30) once idle for 5 min", async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cpuThresholdPercent: 1, cooldownSeconds: 0, minReplicas: 1 }); + mockRedisData = { lowCpuSince: String(Date.now() - 310_000), lastScaleDown: "0" }; + mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n"; + mockTryRunImpl = async () => ""; + mockRunImpl = async (_cmd: string, args: string[]) => { + if (args.includes("stats")) return '{"CPUPerc":"22%","MemUsage":"64MiB / 512MiB"}'; + if (args[0] === "stop") return ""; + if (args[0] === "rm") return ""; + if (args[0] === "ps") return "caddy-abc123\n"; + if (args.includes("reload")) return ""; + return ""; + }; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockWriteFilePath).toBeTruthy(); + expect(mockWriteContent).not.toContain("replica-2"); + }); + it("does not scale down when CPU is moderate", async () => { mockRunImpl = async () => '{"CPUPerc":"50%","MemUsage":"128MiB / 512MiB"}'; await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); diff --git a/apps/api/src/scaling/engine.ts b/apps/api/src/scaling/engine.ts index 94d97cb..d33932f 100644 --- a/apps/api/src/scaling/engine.ts +++ b/apps/api/src/scaling/engine.ts @@ -30,6 +30,26 @@ interface Target { const AGENT_OFFLINE_MS = 90_000; +const countUpstreamContainers = (text: string): Set => { + const containers = new Set(); + for (const m of text.match(/reverse_proxy\s+([^\n]+)/g) ?? []) { + const parts = m.replace("reverse_proxy", "").trim().split(/\s+/); + for (const p of parts) { + if (!p.includes(":") || p.startsWith("{")) continue; + containers.add(p.split(":")[0]); + } + } + return containers; +}; + +export const countPrimarySiteReplicas = (content: string, slug: string): number | null => { + for (const block of content.matchAll(/^\s*([A-Za-z0-9._:-]+)\s*\{([^}]*)\}/gm)) { + if (!block[1].trim().startsWith(`${slug}.`)) continue; + return Math.max(1, countUpstreamContainers(block[2]).size); + } + return null; +}; + class ScalingEngine { private redis: Redis; private interval: ReturnType | null = null; @@ -112,12 +132,18 @@ class ScalingEngine { state.highCpuSince = now; } else if (now - state.highCpuSince > policy.cooldownSeconds * 1000) { if (now - state.lastScaleUp > policy.cooldownSeconds * 1000) { - await this.scaleUp(dep, policy.maxReplicas, slug, target, project?.cpuLimit, project?.memoryLimitMb); - state.lastScaleUp = now; + const scaled = await this.scaleUp( + dep, + policy.maxReplicas, + slug, + target, + project?.cpuLimit, + project?.memoryLimitMb, + ); + if (scaled) state.lastScaleUp = now; state.highCpuSince = null; } } - state.lowCpuSince = null; } else { state.highCpuSince = null; } @@ -267,18 +293,9 @@ class ScalingEngine { const routeFile = join(config.caddyRoutesDir, `${slug}.caddy`); try { const content = await readFile(routeFile, "utf8"); - const matches = content.match(/reverse_proxy\s+([^\n]+)/g); - if (!matches) return 1; - // Count unique container names - const containers = new Set(); - for (const m of matches) { - const parts = m.replace("reverse_proxy", "").trim().split(/\s+/); - for (const p of parts) { - if (!p.includes(":") || p.startsWith("{")) continue; - containers.add(p.split(":")[0]); - } - } - return Math.max(1, containers.size); + const primary = countPrimarySiteReplicas(content, slug); + if (primary !== null) return primary; + return Math.max(1, countUpstreamContainers(content).size); } catch { return 1; } @@ -306,9 +323,9 @@ class ScalingEngine { target: Target, cpuLimit?: number | null, memoryLimitMb?: number | null, - ) { + ): Promise { const currentReplicas = await this.getCurrentReplicas(slug, target, dep); - if (currentReplicas >= maxReplicas) return; + if (currentReplicas >= maxReplicas) return false; const newReplicaNum = currentReplicas + 1; const containerName = `deploy-${dep.id}-replica-${newReplicaNum}`; @@ -317,7 +334,7 @@ class ScalingEngine { if (target.mode === "agent") { await this.enqueueScaleJob(dep, target.server!, "up", newReplicaNum, slug); - return; + return true; } // Get the original container's image @@ -328,7 +345,7 @@ class ScalingEngine { await execDockerSshCommand(target.server!, ["inspect", "-f", "{{.Config.Image}}", dep.containerName!]) ).stdout.trim()) : await run(dockerBin, ["inspect", "-f", "{{.Config.Image}}", dep.containerName!]).catch(() => ""); - if (!imageTag) return; + if (!imageTag) return false; // Get env vars from original const envJson = @@ -389,12 +406,13 @@ class ScalingEngine { } } catch (err) { console.error(`[Scaling] Failed to create replica ${containerName}:`, err); - return; + return false; } // Update Caddy route to include the new replica await this.updateCaddyRoute(slug, dep, newReplicaNum, target); console.log(`[Scaling] Replica ${containerName} started`); + return true; } private async scaleDown( @@ -514,8 +532,10 @@ class ScalingEngine { // Build proxy targets: primary + all replicas const primaryName = dep.containerName || `${slug}-${dep.id.slice(0, 8)}`; const targets = [`${primaryName}:${port}`]; + const containerNames = [primaryName]; for (let i = 2; i <= replicaCount; i++) { targets.push(`deploy-${dep.id}-replica-${i}:${port}`); + containerNames.push(`deploy-${dep.id}-replica-${i}`); } const baseDomain = config.caddyBaseDomain === "localhost" ? `${config.caddyBaseDomain}:80` : config.caddyBaseDomain; @@ -523,15 +543,10 @@ class ScalingEngine { await import("../utils/ingress"); const ingressServer = await getIngressServer(); const viaIngress = shouldRouteViaIngress(target.server ?? null, ingressServer); - const { caddyReverseProxy, caddySite } = await import("../utils/caddy-site"); + const { buildCaddySnippet } = await import("../utils/domain-verifier"); const caddySnippet = viaIngress - ? projectServerSite( - `${slug}.${baseDomain}`, - port, - targets.map((t) => t.split(":")[0]), - true, - ) - : caddySite(`${slug}.${baseDomain}`, caddyReverseProxy(targets.join(" "))); + ? projectServerSite(`${slug}.${baseDomain}`, port, containerNames, true) + : await buildCaddySnippet(slug, primaryName, dep.projectId, undefined, port, { containerNames }); if (target.mode === "ssh") { await syncRemoteCaddyRoute(target.server!, `${slug}.caddy`, caddySnippet); diff --git a/apps/api/src/tunnels/gateway.ts b/apps/api/src/tunnels/gateway.ts index bf19e1c..738add4 100644 --- a/apps/api/src/tunnels/gateway.ts +++ b/apps/api/src/tunnels/gateway.ts @@ -203,6 +203,14 @@ const createPoolLookup = ); const row = result.rows[0]; if (!row) return null; + const isRemote = Boolean( + row.server_id && row.server_mode && row.server_id !== "local" && row.server_mode !== "local", + ); + const ssh = await resolveSshTarget(pool, row); + if (isRemote && !ssh) { + console.warn(`[Gateway] Remote database on server ${row.server_id} cannot resolve SSH credentials`); + return null; + } return { internalHost: row.internal_host, internalPort: row.internal_port, @@ -212,7 +220,7 @@ const createPoolLookup = allowedCidrs: Array.isArray(row.allowed_cidrs) ? row.allowed_cidrs.filter((value): value is string => typeof value === "string") : [], - ssh: await resolveSshTarget(pool, row), + ssh, }; }; diff --git a/apps/api/src/tunnels/ssh-tunnel.ts b/apps/api/src/tunnels/ssh-tunnel.ts index 8752a7b..29343fd 100644 --- a/apps/api/src/tunnels/ssh-tunnel.ts +++ b/apps/api/src/tunnels/ssh-tunnel.ts @@ -1,5 +1,5 @@ import { spawn } from "node:child_process"; -import { existsSync, mkdirSync, writeFileSync } from "node:fs"; +import { chmodSync, existsSync, mkdirSync, unlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { Duplex } from "node:stream"; @@ -18,9 +18,21 @@ const KEYS_DIR = join(tmpdir(), "dequel_gateway_keys"); const safeName = (value: string) => value.replace(/[^a-zA-Z0-9_-]/g, "_"); export const writeIdentityFile = (target: SshEngineTarget): string => { - if (!existsSync(KEYS_DIR)) mkdirSync(KEYS_DIR, { recursive: true, mode: 0o700 }); + if (!existsSync(KEYS_DIR)) { + mkdirSync(KEYS_DIR, { recursive: true, mode: 0o700 }); + } + try { + chmodSync(KEYS_DIR, 0o700); + } catch {} + const path = join(KEYS_DIR, `id_${safeName(target.serverId)}`); + try { + if (existsSync(path)) unlinkSync(path); + } catch {} writeFileSync(path, `${target.key.trim()}\n`, { mode: 0o600 }); + try { + chmodSync(path, 0o600); + } catch {} return path; }; @@ -45,7 +57,16 @@ export const buildSshArgs = (target: SshEngineTarget, keyPath: string): string[] ]; export const forwardViaSsh = (stream: Duplex, target: SshEngineTarget) => { - const child = spawn("ssh", buildSshArgs(target, writeIdentityFile(target)), { + let keyPath: string; + try { + keyPath = writeIdentityFile(target); + } catch (err) { + console.error("[Gateway SSH] Failed to write identity file:", err); + stream.destroy(err as Error); + return null; + } + + const child = spawn("ssh", buildSshArgs(target, keyPath), { stdio: ["pipe", "pipe", "ignore"], }); const kill = () => { @@ -60,6 +81,10 @@ export const forwardViaSsh = (stream: Duplex, target: SshEngineTarget) => { stream.destroy(); return child; } + child.stdin.on("error", () => { + kill(); + stream.destroy(); + }); stream.pipe(child.stdin); child.stdout.pipe(stream); return child; diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts index 89f9b99..4ca1e57 100644 --- a/apps/api/src/types.ts +++ b/apps/api/src/types.ts @@ -1,510 +1 @@ -export type DeploymentStatus = "pending" | "building" | "deploying" | "running" | "failed" | "inactive"; -export type SourceType = "git" | "upload" | "image" | "compose"; -export type LogStage = "build" | "deploy" | "system"; -export type DatabaseType = "postgresql" | "mysql" | "redis" | "mongodb"; -export type DatabaseStatus = - | "provisioning" - | "running" - | "stopped" - | "restarting" - | "deleting" - | "deletion_failed" - | "failed"; -export type DomainType = "base" | "custom"; -export type DomainValidationStatus = "pending" | "verified" | "failed"; -export type SslStatus = "pending" | "provisioned" | "failed"; -export type ServerStatus = "pending" | "connected" | "disconnected" | "failed"; -export type ServerMode = "local" | "ssh" | "agent" | "docker_tcp"; -export type AlertChannel = "email" | "slack" | "webhook"; -export type AlertType = "cpu" | "memory" | "downtime" | "cert_expiry"; - -export interface Project { - id: string; - serverId: string | null; - name: string; - description: string | null; - repoUrl: string | null; - repoBranch: string | null; - baseDomain: string | null; - cpuLimit: number | null; - memoryLimitMb: number | null; - port: number | null; - sourceDir: string | null; - sourceType: string; - projectType: string; - buildType: string; - composeService: string | null; - composePort: number | null; - composeServices: string | null; - buildCommand: string | null; - installCommand: string | null; - outputDir: string | null; - startCommand: string | null; - hasGithubToken: boolean; - createdAt: string; - updatedAt: string; -} - -export interface GithubIntegration { - id: string; - clientId: string; - clientSecret: string; - appName: string; - webhookSecret: string | null; - createdAt: string; -} - -export interface CreateProjectInput { - name: string; - serverId?: string | null; - description?: string; - repoUrl?: string; - repoBranch?: string; - baseDomain?: string; - cpuLimit?: number | null; - memoryLimitMb?: number | null; - port?: number | null; - sourceDir?: string | null; - sourceType?: string; - projectType?: string; - buildType?: string; - composeService?: string | null; - composePort?: number | null; - composeServices?: string | null; - buildCommand?: string | null; - installCommand?: string | null; - outputDir?: string | null; - startCommand?: string | null; -} - -export interface EnvironmentVariable { - id: string; - projectId: string; - key: string; - value: string; - environment: string; - createdAt: string; - updatedAt: string; -} - -export interface CreateEnvironmentVariableInput { - projectId: string; - key: string; - value: string; - environment?: string; -} - -export interface Volume { - id: string; - projectId: string; - mountPath: string; - sizeMb: number | null; - dockerVolumeName: string | null; - createdAt: string; -} - -export interface CreateVolumeInput { - projectId: string; - mountPath?: string; -} - -export interface Database { - id: string; - projectId: string | null; - serverId: string | null; - name: string; - type: DatabaseType; - version: string | null; - databaseName: string; - username: string; - password: string; - internalHost: string; - internalPort: number; - cpuLimit: number | null; - memoryLimitMb: number | null; - storageLimitMb: number | null; - storageUsedMb: number; - publicAccess: boolean; - allowPublicAccessFromAnywhere: boolean; - allowedCidrs: string[]; - externalPort: number | null; - proxyContainerName: string | null; - volumeName: string; - connectionString: string; - status: DatabaseStatus; - containerName: string | null; - backupEnabled: boolean; - backupSchedule: string; - backupRetention: number; - createdAt: string; - updatedAt: string; -} - -export interface CreateDatabaseInput { - projectId?: string | null; - serverId?: string | null; - name: string; - type: DatabaseType; - version?: string; - cpuLimit?: number | null; - memoryLimitMb?: number | null; - storageLimitMb?: number | null; - publicAccess?: boolean; - allowPublicAccessFromAnywhere?: boolean; - allowedCidrs?: string[]; - backupEnabled?: boolean; - backupSchedule?: string; - backupRetention?: number; -} - -export interface BackupStorageSettingsData { - type: "local" | "s3"; - path?: string; - s3Endpoint?: string; - s3AccessKeyId?: string; - s3SecretAccessKey?: string; - s3Bucket?: string; - s3Region?: string; - systemBackupSchedule?: string; - systemBackupRetention?: number; -} - -export interface Domain { - id: string; - projectId: string; - domain: string; - type: DomainType; - validationStatus: DomainValidationStatus; - sslStatus: SslStatus; - targetService: string | null; - targetPort: number | null; - createdAt: string; - updatedAt: string; -} - -export interface CreateDomainInput { - projectId: string; - domain: string; - type: DomainType; - targetService?: string | null; - targetPort?: number | null; -} - -export interface ScalingPolicy { - id: string; - projectId: string; - minReplicas: number; - maxReplicas: number; - cpuThresholdPercent: number; - memoryThresholdPercent: number; - cooldownSeconds: number; - enabled: boolean; - createdAt: string; - updatedAt: string; -} - -export interface CreateScalingPolicyInput { - projectId: string; - minReplicas?: number; - maxReplicas?: number; - cpuThresholdPercent?: number; - memoryThresholdPercent?: number; - cooldownSeconds?: number; - enabled?: boolean; -} - -export interface Server { - id: string; - name: string; - host: string; - port: number; - mode: ServerMode; - sshUser?: string | null; - sshKey?: string | null; - sshKeyId?: string | null; - sshPassword?: string | null; - sshKeyEncrypted?: string | null; - agentId: string | null; - agentVersion: string | null; - capabilities: Record; - labels: Record; - status: ServerStatus; - cpuTotal: number | null; - memoryTotalMb: number | null; - diskTotalMb: number | null; - cpuUsedPercent: number | null; - memoryUsedMb: number | null; - lastHeartbeat: string | null; - registeredAt: string | null; - revokedAt: string | null; - createdAt: string; - updatedAt: string; -} - -export interface CreateServerInput { - name: string; - host: string; - port?: number; - authToken?: string; - mode?: ServerMode; - sshUser?: string; - sshKey?: string; - sshKeyId?: string; - sshPassword?: string; -} - -export interface ApiKey { - id: string; - name: string; - keyHash: string; - permissions: string; - createdAt: string; - lastUsedAt: string | null; -} - -export interface CreateApiKeyInput { - name: string; - permissions?: string; -} - -export interface Alert { - id: string; - projectId: string; - type: AlertType; - threshold: number | null; - durationSeconds: number | null; - channel: AlertChannel; - destination: string | null; - enabled: boolean; - createdAt: string; -} - -export interface CreateAlertInput { - projectId: string; - type: AlertType; - threshold?: number; - durationSeconds?: number; - channel: AlertChannel; - destination?: string; -} - -export type FailureSource = "pipeline" | "rollback" | "ssh" | "agent" | "job-channel" | "reconciler" | "dispatch"; - -export interface RecordFailureInput { - deploymentId: string; - reason: string; - source: FailureSource; - cancel?: boolean; -} - -export interface RecordFailureOutcome { - claimed: boolean; - eventId: string | null; -} - -export interface FailureNotificationContext { - eventId: string; - deploymentId: string; - projectId: string | null; - projectName: string; - failureReason: string | null; - commitSha: string | null; - sourceRef: string; - finishedAt: string | null; - attempt: number; -} - -export type MailDelivery = - | { status: "sent" } - | { status: "skipped"; reason: "no_smtp" | "no_recipient" } - | { status: "failed"; error: string }; - -export type HealthStatus = "ok" | "warn" | "fail" | "unknown"; -export type OverallHealth = "healthy" | "degraded" | "down"; - -export interface HealthCheck { - name: "server" | "containers" | "ingress" | "http"; - status: HealthStatus; - detail: string | null; -} - -export interface StatusFailure { - deploymentId: string; - message: string | null; - commitSha: string | null; - sourceRef: string; - finishedAt: string | null; - recovered: boolean; - notifiedAt: string | null; -} - -export interface StatusHistoryEntry { - deploymentId: string; - type: string; - message: string | null; - at: string; -} - -export interface HttpErrorRate { - source: "loki"; - available: boolean; - windowSeconds: number; - totalRequests: number; - errorRequests: number; - errorRate: number | null; - byStatus: { status: string; count: number }[]; -} - -export interface ProjectStatus { - projectId: string; - windowSeconds: number; - health: { overall: OverallHealth; checks: HealthCheck[] }; - failures: StatusFailure[]; - history: StatusHistoryEntry[]; - replicas: { current: number } | null; - resources: { - server: { - status: string; - cpuUsedPercent: number | null; - memoryTotalMb: number | null; - lastHeartbeatAt: string | null; - } | null; - containers: { name: string; cpuPercent: number; memoryMb: number }[]; - }; - http: HttpErrorRate; -} - -export interface Deployment { - id: string; - projectId: string | null; - serverId: string | null; - sourceType: SourceType; - sourceRef: string; - status: DeploymentStatus; - imageTag: string | null; - containerName: string | null; - routePath: string | null; - liveUrl: string | null; - branch: string | null; - commitSha: string | null; - replicas: number; - environment: string | null; - failureReason: string | null; - clearCache: boolean; - finishedAt: string | null; - createdAt: string; - updatedAt: string; -} - -export interface PaginatedResult { - items: T[]; - total: number; - offset: number; - limit: number; -} - -export interface CreateDeploymentInput { - projectId?: string; - serverId?: string | null; - sourceType: SourceType; - sourceRef: string; - branch?: string; - commitSha?: string; - environment?: string; - clearCache?: boolean; -} - -export interface DeploymentLog { - id: number; - deploymentId: string; - sequence: number; - stage: LogStage; - message: string; - createdAt: string; -} - -export interface LogEvent { - deploymentId: string; - sequence: number; - stage: LogStage; - message: string; - timestamp: string; -} - -export type RouteStatus = "pending" | "active" | "failed" | "removed"; - -export interface Route { - id: string; - serverId: string | null; - deploymentId: string | null; - projectId: string | null; - hostname: string; - routeFile: string; - port: number; - targetContainers: string[]; - upstreamHost: string | null; - status: RouteStatus; - lastError: string | null; - confirmedAt: string | null; - createdAt: string; - updatedAt: string; -} - -export interface UpsertRouteInput { - serverId?: string | null; - deploymentId?: string | null; - projectId?: string | null; - hostname: string; - routeFile: string; - port: number; - targetContainers: string[]; - upstreamHost?: string | null; - status?: RouteStatus; - lastError?: string | null; -} - -export interface SharedEnvVar { - id: string; - key: string; - value: string; - environment: string; - description: string | null; - tags: string[]; - createdAt: string; - updatedAt: string; -} - -export interface CreateSharedEnvVarInput { - key: string; - value: string; - environment?: string; - description?: string; - tags?: string[]; -} - -export interface SshKey { - id: string; - name: string; - fingerprint: string; - publicKey: string | null; - tags: string[]; - createdAt: string; - updatedAt: string; -} - -export interface CreateSshKeyInput { - name: string; - privateKey: string; - tags?: string[]; -} - -export interface Permission { - action: string; - resource: string; -} - -export interface AuthContext { - type: "session" | "api-key"; - apiKeyId?: string; - permissions?: Permission[]; -} +export * from "./types/index"; diff --git a/apps/api/src/types/audit.ts b/apps/api/src/types/audit.ts new file mode 100644 index 0000000..07ab1c4 --- /dev/null +++ b/apps/api/src/types/audit.ts @@ -0,0 +1,35 @@ +export type AuditActorType = "user" | "api_key" | "system"; + +export interface AuditLog { + id: string; + actorType: AuditActorType; + actorId: string; + action: string; + resourceType: string; + resourceId: string | null; + details: Record; + ipAddress: string | null; + userAgent: string | null; + createdAt: string; +} + +export interface CreateAuditLogInput { + actorType: AuditActorType; + actorId: string; + action: string; + resourceType: string; + resourceId?: string | null; + details?: Record; + ipAddress?: string | null; + userAgent?: string | null; +} + +export interface ListAuditLogsFilters { + actorType?: AuditActorType; + actorId?: string; + action?: string; + resourceType?: string; + resourceId?: string; + limit?: number; + offset?: number; +} diff --git a/apps/api/src/types/auth.ts b/apps/api/src/types/auth.ts new file mode 100644 index 0000000..b8a6f1d --- /dev/null +++ b/apps/api/src/types/auth.ts @@ -0,0 +1,33 @@ +export interface GithubIntegration { + id: string; + clientId: string; + clientSecret: string; + appName: string; + webhookSecret: string | null; + createdAt: string; +} + +export interface ApiKey { + id: string; + name: string; + keyHash: string; + permissions: string; + createdAt: string; + lastUsedAt: string | null; +} + +export interface CreateApiKeyInput { + name: string; + permissions?: string; +} + +export interface Permission { + action: string; + resource: string; +} + +export interface AuthContext { + type: "session" | "api-key"; + apiKeyId?: string; + permissions?: Permission[]; +} diff --git a/apps/api/src/types/databases.ts b/apps/api/src/types/databases.ts new file mode 100644 index 0000000..4302bd1 --- /dev/null +++ b/apps/api/src/types/databases.ts @@ -0,0 +1,70 @@ +export type DatabaseType = "postgresql" | "mysql" | "redis" | "mongodb"; +export type DatabaseStatus = + | "provisioning" + | "running" + | "stopped" + | "restarting" + | "deleting" + | "deletion_failed" + | "failed"; + +export interface Database { + id: string; + projectId: string | null; + serverId: string | null; + name: string; + type: DatabaseType; + version: string | null; + databaseName: string; + username: string; + password: string; + internalHost: string; + internalPort: number; + cpuLimit: number | null; + memoryLimitMb: number | null; + storageLimitMb: number | null; + storageUsedMb: number; + publicAccess: boolean; + allowPublicAccessFromAnywhere: boolean; + allowedCidrs: string[]; + externalPort: number | null; + proxyContainerName: string | null; + volumeName: string; + connectionString: string; + status: DatabaseStatus; + containerName: string | null; + backupEnabled: boolean; + backupSchedule: string; + backupRetention: number; + createdAt: string; + updatedAt: string; +} + +export interface CreateDatabaseInput { + projectId?: string | null; + serverId?: string | null; + name: string; + type: DatabaseType; + version?: string; + cpuLimit?: number | null; + memoryLimitMb?: number | null; + storageLimitMb?: number | null; + publicAccess?: boolean; + allowPublicAccessFromAnywhere?: boolean; + allowedCidrs?: string[]; + backupEnabled?: boolean; + backupSchedule?: string; + backupRetention?: number; +} + +export interface BackupStorageSettingsData { + type: "local" | "s3"; + path?: string; + s3Endpoint?: string; + s3AccessKeyId?: string; + s3SecretAccessKey?: string; + s3Bucket?: string; + s3Region?: string; + systemBackupSchedule?: string; + systemBackupRetention?: number; +} diff --git a/apps/api/src/types/deployments.ts b/apps/api/src/types/deployments.ts new file mode 100644 index 0000000..83ea682 --- /dev/null +++ b/apps/api/src/types/deployments.ts @@ -0,0 +1,109 @@ +export type DeploymentStatus = "pending" | "building" | "deploying" | "running" | "failed" | "inactive"; +export type SourceType = "git" | "upload" | "image" | "compose"; +export type LogStage = "build" | "deploy" | "system"; + +export interface Deployment { + id: string; + projectId: string | null; + serverId: string | null; + sourceType: SourceType; + sourceRef: string; + status: DeploymentStatus; + imageTag: string | null; + containerName: string | null; + routePath: string | null; + liveUrl: string | null; + branch: string | null; + commitSha: string | null; + replicas: number; + environment: string | null; + failureReason: string | null; + clearCache: boolean; + finishedAt: string | null; + createdAt: string; + updatedAt: string; +} + +export interface PaginatedResult { + items: T[]; + total: number; + offset: number; + limit: number; +} + +export interface CreateDeploymentInput { + projectId?: string; + serverId?: string | null; + sourceType: SourceType; + sourceRef: string; + branch?: string; + commitSha?: string; + environment?: string; + clearCache?: boolean; +} + +export interface DeploymentLog { + id: number; + deploymentId: string; + sequence: number; + stage: LogStage; + message: string; + createdAt: string; +} + +export interface LogEvent { + deploymentId: string; + sequence: number; + stage: LogStage; + message: string; + timestamp: string; +} + +export interface ScalingPolicy { + id: string; + projectId: string; + minReplicas: number; + maxReplicas: number; + cpuThresholdPercent: number; + memoryThresholdPercent: number; + cooldownSeconds: number; + enabled: boolean; + createdAt: string; + updatedAt: string; +} + +export interface CreateScalingPolicyInput { + projectId: string; + minReplicas?: number; + maxReplicas?: number; + cpuThresholdPercent?: number; + memoryThresholdPercent?: number; + cooldownSeconds?: number; + enabled?: boolean; +} + +export type FailureSource = "pipeline" | "rollback" | "ssh" | "agent" | "job-channel" | "reconciler" | "dispatch"; + +export interface RecordFailureInput { + deploymentId: string; + reason: string; + source: FailureSource; + cancel?: boolean; +} + +export interface RecordFailureOutcome { + claimed: boolean; + eventId: string | null; +} + +export interface FailureNotificationContext { + eventId: string; + deploymentId: string; + projectId: string | null; + projectName: string; + failureReason: string | null; + commitSha: string | null; + sourceRef: string; + finishedAt: string | null; + attempt: number; +} diff --git a/apps/api/src/types/index.ts b/apps/api/src/types/index.ts new file mode 100644 index 0000000..6bcdc4a --- /dev/null +++ b/apps/api/src/types/index.ts @@ -0,0 +1,8 @@ +export * from "./projects"; +export * from "./deployments"; +export * from "./databases"; +export * from "./networking"; +export * from "./servers"; +export * from "./settings"; +export * from "./auth"; +export * from "./audit"; diff --git a/apps/api/src/types/networking.ts b/apps/api/src/types/networking.ts new file mode 100644 index 0000000..6d6239b --- /dev/null +++ b/apps/api/src/types/networking.ts @@ -0,0 +1,58 @@ +export type DomainType = "base" | "custom"; +export type DomainValidationStatus = "pending" | "verified" | "failed"; +export type SslStatus = "pending" | "provisioned" | "failed"; + +export interface Domain { + id: string; + projectId: string; + domain: string; + type: DomainType; + validationStatus: DomainValidationStatus; + sslStatus: SslStatus; + targetService: string | null; + targetPort: number | null; + cloudflareProxied: boolean; + createdAt: string; + updatedAt: string; +} + +export interface CreateDomainInput { + projectId: string; + domain: string; + type: DomainType; + targetService?: string | null; + targetPort?: number | null; + cloudflareProxied?: boolean; +} + +export type RouteStatus = "pending" | "active" | "failed" | "removed"; + +export interface Route { + id: string; + serverId: string | null; + deploymentId: string | null; + projectId: string | null; + hostname: string; + routeFile: string; + port: number; + targetContainers: string[]; + upstreamHost: string | null; + status: RouteStatus; + lastError: string | null; + confirmedAt: string | null; + createdAt: string; + updatedAt: string; +} + +export interface UpsertRouteInput { + serverId?: string | null; + deploymentId?: string | null; + projectId?: string | null; + hostname: string; + routeFile: string; + port: number; + targetContainers: string[]; + upstreamHost?: string | null; + status?: RouteStatus; + lastError?: string | null; +} diff --git a/apps/api/src/types/projects.ts b/apps/api/src/types/projects.ts new file mode 100644 index 0000000..6e518b2 --- /dev/null +++ b/apps/api/src/types/projects.ts @@ -0,0 +1,99 @@ +export interface Project { + id: string; + serverId: string | null; + name: string; + description: string | null; + repoUrl: string | null; + repoBranch: string | null; + baseDomain: string | null; + cpuLimit: number | null; + memoryLimitMb: number | null; + port: number | null; + sourceDir: string | null; + sourceType: string; + projectType: string; + buildType: string; + composeService: string | null; + composePort: number | null; + composeServices: string | null; + buildCommand: string | null; + installCommand: string | null; + outputDir: string | null; + startCommand: string | null; + hasGithubToken: boolean; + createdAt: string; + updatedAt: string; +} + +export interface CreateProjectInput { + name: string; + serverId?: string | null; + description?: string; + repoUrl?: string; + repoBranch?: string; + baseDomain?: string; + cpuLimit?: number | null; + memoryLimitMb?: number | null; + port?: number | null; + sourceDir?: string | null; + sourceType?: string; + projectType?: string; + buildType?: string; + composeService?: string | null; + composePort?: number | null; + composeServices?: string | null; + buildCommand?: string | null; + installCommand?: string | null; + outputDir?: string | null; + startCommand?: string | null; +} + +export interface EnvironmentVariable { + id: string; + projectId: string; + key: string; + value: string; + environment: string; + createdAt: string; + updatedAt: string; +} + +export interface CreateEnvironmentVariableInput { + projectId: string; + key: string; + value: string; + environment?: string; +} + +export interface Volume { + id: string; + projectId: string; + mountPath: string; + sizeMb: number | null; + dockerVolumeName: string | null; + createdAt: string; +} + +export interface CreateVolumeInput { + projectId: string; + mountPath?: string; +} + +export interface SharedEnvVar { + id: string; + key: string; + value: string; + environment: string; + description: string | null; + tags: string[]; + createdAt: string; + updatedAt: string; +} + +export interface CreateSharedEnvVarInput { + key: string; + value: string; + environment?: string; + description?: string; + tags?: string[]; +} diff --git a/apps/api/src/types/servers.ts b/apps/api/src/types/servers.ts new file mode 100644 index 0000000..4c8945c --- /dev/null +++ b/apps/api/src/types/servers.ts @@ -0,0 +1,57 @@ +export type ServerStatus = "pending" | "connected" | "disconnected" | "failed"; +export type ServerMode = "local" | "ssh" | "agent" | "docker_tcp"; + +export interface Server { + id: string; + name: string; + host: string; + port: number; + mode: ServerMode; + sshUser?: string | null; + sshKey?: string | null; + sshKeyId?: string | null; + sshPassword?: string | null; + sshKeyEncrypted?: string | null; + agentId: string | null; + agentVersion: string | null; + capabilities: Record; + labels: Record; + status: ServerStatus; + cpuTotal: number | null; + memoryTotalMb: number | null; + diskTotalMb: number | null; + cpuUsedPercent: number | null; + memoryUsedMb: number | null; + lastHeartbeat: string | null; + registeredAt: string | null; + revokedAt: string | null; + createdAt: string; + updatedAt: string; +} + +export interface CreateServerInput { + name: string; + host: string; + port?: number; + authToken?: string; + mode?: ServerMode; + sshUser?: string; + sshKeyId?: string; + sshPassword?: string; +} + +export interface SshKey { + id: string; + name: string; + fingerprint: string; + publicKey: string | null; + tags: string[]; + createdAt: string; + updatedAt: string; +} + +export interface CreateSshKeyInput { + name: string; + privateKey: string; + tags?: string[]; +} diff --git a/apps/api/src/types/settings.ts b/apps/api/src/types/settings.ts new file mode 100644 index 0000000..6cdad50 --- /dev/null +++ b/apps/api/src/types/settings.ts @@ -0,0 +1,83 @@ +export type AlertChannel = "email" | "slack" | "webhook"; +export type AlertType = "cpu" | "memory" | "downtime" | "cert_expiry"; + +export interface Alert { + id: string; + projectId: string; + type: AlertType; + threshold: number | null; + durationSeconds: number | null; + channel: AlertChannel; + destination: string | null; + enabled: boolean; + createdAt: string; +} + +export interface CreateAlertInput { + projectId: string; + type: AlertType; + threshold?: number; + durationSeconds?: number; + channel: AlertChannel; + destination?: string; +} + +export type MailDelivery = + | { status: "sent" } + | { status: "skipped"; reason: "no_smtp" | "no_recipient" } + | { status: "failed"; error: string }; + +export type HealthStatus = "ok" | "warn" | "fail" | "unknown"; +export type OverallHealth = "healthy" | "degraded" | "down"; + +export interface HealthCheck { + name: "server" | "containers" | "ingress" | "http"; + status: HealthStatus; + detail: string | null; +} + +export interface StatusFailure { + deploymentId: string; + message: string | null; + commitSha: string | null; + sourceRef: string; + finishedAt: string | null; + recovered: boolean; + notifiedAt: string | null; +} + +export interface StatusHistoryEntry { + deploymentId: string; + type: string; + message: string | null; + at: string; +} + +export interface HttpErrorRate { + source: "loki"; + available: boolean; + windowSeconds: number; + totalRequests: number; + errorRequests: number; + errorRate: number | null; + byStatus: { status: string; count: number }[]; +} + +export interface ProjectStatus { + projectId: string; + windowSeconds: number; + health: { overall: OverallHealth; checks: HealthCheck[] }; + failures: StatusFailure[]; + history: StatusHistoryEntry[]; + replicas: { current: number } | null; + resources: { + server: { + status: string; + cpuUsedPercent: number | null; + memoryTotalMb: number | null; + lastHeartbeatAt: string | null; + } | null; + containers: { name: string; cpuPercent: number; memoryMb: number }[]; + }; + http: HttpErrorRate; +} diff --git a/apps/api/src/utils/__tests__/domain-verifier.test.ts b/apps/api/src/utils/__tests__/domain-verifier.test.ts index 50d97d8..c34f787 100644 --- a/apps/api/src/utils/__tests__/domain-verifier.test.ts +++ b/apps/api/src/utils/__tests__/domain-verifier.test.ts @@ -210,4 +210,55 @@ describe("buildCaddySnippet", () => { const snippet = await buildCaddySnippet("my-app", "deploy-abc", "proj-3", noDomains, APP_PORT, snippetOpts); expect(snippet).toContain(":3000"); }); + + it("injects trusted_proxies when custom domain is cloudflare proxied", async () => { + const listFn = mock().mockResolvedValue([ + { + id: "1", + projectId: "proj-1", + domain: "cf.example.com", + type: "custom", + validationStatus: "verified", + sslStatus: "provisioned", + targetService: null, + targetPort: 3000, + cloudflareProxied: true, + createdAt: "", + updatedAt: "", + }, + ]); + const snippet = await buildCaddySnippet("my-app", "deploy-abc", "proj-1", listFn, APP_PORT, snippetOpts); + expect(snippet).toContain("trusted_proxies private_ranges 173.245.48.0/20"); + }); + + it("injects trusted_proxies when global CLOUDFLARE_PROXIED is enabled", async () => { + process.env.CLOUDFLARE_PROXIED = "true"; + try { + const snippet = await buildCaddySnippet("my-app", "deploy-abc", "proj-1", noDomains, APP_PORT, snippetOpts); + expect(snippet).toContain("trusted_proxies private_ranges 173.245.48.0/20"); + } finally { + delete process.env.CLOUDFLARE_PROXIED; + } + }); + + it("retains trusted_proxies for custom domain without target overrides when global is disabled", async () => { + const listFn = mock().mockResolvedValue([ + { + id: "d-1", + domain: "custom.example.com", + type: "custom", + validationStatus: "verified", + sslStatus: "provisioned", + targetService: null, + targetPort: null, + cloudflareProxied: true, + createdAt: "", + updatedAt: "", + }, + ]); + const snippet = await buildCaddySnippet("my-app", "deploy-abc", "proj-1", listFn, APP_PORT, snippetOpts); + expect(snippet).toContain( + "custom.example.com {\n log {\n output stdout\n format json\n }\n reverse_proxy deploy-abc:3000 {\n header_up Host {upstream_hostport}\n trusted_proxies private_ranges 173.245.48.0/20", + ); + }); }); diff --git a/apps/api/src/utils/caddy-site.ts b/apps/api/src/utils/caddy-site.ts index 6898bc7..2dfd3d0 100644 --- a/apps/api/src/utils/caddy-site.ts +++ b/apps/api/src/utils/caddy-site.ts @@ -1,10 +1,42 @@ +import { CADDY_CLOUDFLARE_TRUSTED_PROXIES } from "./cloudflare"; + export const CADDY_ACCESS_LOG_BLOCK = ` log { output stdout format json }`; -export const caddySite = (hosts: string, reverseProxy: string): string => - `${hosts} {\n${CADDY_ACCESS_LOG_BLOCK}\n${reverseProxy}\n}\n`; +export const CADDY_SECURITY_HEADERS_BLOCK = ` header { + Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" + X-Content-Type-Options "nosniff" + X-Frame-Options "DENY" + Referrer-Policy "strict-origin-when-cross-origin" + Permissions-Policy "camera=(), microphone=(), geolocation=()" + -Server + }`; + +export const CADDY_REQUEST_LIMITS_BLOCK = ` request_body { + max_size 100MB + }`; + +export interface CaddySiteOptions { + includeSecurityHeaders?: boolean; + includeRequestLimits?: boolean; +} + +export interface CaddyReverseProxyOptions { + cloudflareProxied?: boolean; +} + +export const caddySite = (hosts: string, reverseProxy: string, options?: CaddySiteOptions): string => { + const secHeaders = options?.includeSecurityHeaders ? `\n${CADDY_SECURITY_HEADERS_BLOCK}` : ""; + const reqLimits = options?.includeRequestLimits ? `\n${CADDY_REQUEST_LIMITS_BLOCK}` : ""; + return `${hosts} {\n${CADDY_ACCESS_LOG_BLOCK}${secHeaders}${reqLimits}\n${reverseProxy}\n}\n`; +}; + +export const caddyReverseProxy = (targets: string, options?: CaddyReverseProxyOptions): string => { + const trustedProxies = options?.cloudflareProxied + ? `\n trusted_proxies private_ranges ${CADDY_CLOUDFLARE_TRUSTED_PROXIES}\n header_up X-Real-IP {client_ip}` + : ""; -export const caddyReverseProxy = (targets: string): string => - ` reverse_proxy ${targets} {\n header_up Host {upstream_hostport}\n }`; + return ` reverse_proxy ${targets} {\n header_up Host {upstream_hostport}${trustedProxies}\n }`; +}; diff --git a/apps/api/src/utils/cloudflare.ts b/apps/api/src/utils/cloudflare.ts new file mode 100644 index 0000000..371ac39 --- /dev/null +++ b/apps/api/src/utils/cloudflare.ts @@ -0,0 +1,41 @@ +export const CLOUDFLARE_IPV4_CIDRS = [ + "173.245.48.0/20", + "103.21.244.0/22", + "103.22.200.0/22", + "103.31.4.0/22", + "141.101.64.0/18", + "108.162.192.0/18", + "190.93.240.0/20", + "188.114.96.0/20", + "197.234.240.0/22", + "198.41.128.0/17", + "162.158.0.0/15", + "104.16.0.0/13", + "104.24.0.0/14", + "172.64.0.0/13", + "131.0.72.0/22", +]; + +export const CLOUDFLARE_IPV6_CIDRS = [ + "2400:cb00::/32", + "2606:4700::/32", + "2803:f800::/32", + "2405:b500::/32", + "2405:8100::/32", + "2a06:98c0::/29", + "2c0f:f248::/32", +]; + +export const CLOUDFLARE_ALL_CIDRS = [...CLOUDFLARE_IPV4_CIDRS, ...CLOUDFLARE_IPV6_CIDRS]; + +export const CADDY_CLOUDFLARE_TRUSTED_PROXIES = CLOUDFLARE_ALL_CIDRS.join(" "); + +export const isGlobalCloudflareProxied = (): boolean => + process.env.CLOUDFLARE_PROXIED === "true" || process.env.CLOUDFLARE_PROXIED === "1"; + +export const isCloudflareProxied = (domainProxiedFlag?: boolean): boolean => { + if (domainProxiedFlag !== undefined && domainProxiedFlag !== null) { + return Boolean(domainProxiedFlag); + } + return isGlobalCloudflareProxied(); +}; diff --git a/apps/api/src/utils/domain-verifier.ts b/apps/api/src/utils/domain-verifier.ts index 05c7d2f..e044fe6 100644 --- a/apps/api/src/utils/domain-verifier.ts +++ b/apps/api/src/utils/domain-verifier.ts @@ -6,6 +6,7 @@ import { getProjectById, listDomains, listEnvironmentVariablesForDeploy, updateD import { domains } from "../db/schema"; import { reloadCaddy } from "../orchestrator/runtime"; import { caddyReverseProxy, caddySite } from "./caddy-site"; +import { isCloudflareProxied, isGlobalCloudflareProxied } from "./cloudflare"; import { config } from "./config"; import { resolveServerIp, validateDomain } from "./dns"; @@ -174,6 +175,7 @@ export const removeFromCaddyRoute = async ( export interface BuildSnippetOpts { baseDomain?: string; listEnvVarsFn?: typeof listEnvironmentVariablesForDeploy; + containerNames?: string[]; } export const buildCaddySnippet = async ( @@ -189,6 +191,7 @@ export const buildCaddySnippet = async ( const listEnvVars = opts?.listEnvVarsFn ?? listEnvironmentVariablesForDeploy; let defaultDomains = [`${slug}.${baseDomain}`]; let port = appPort ?? config.appInternalPort; + const isBaseProxied = isGlobalCloudflareProxied(); const customBlocks: string[] = []; if (projectId) { @@ -196,7 +199,8 @@ export const buildCaddySnippet = async ( const verified = projectDomains.filter((d) => d.validationStatus === "verified"); for (const d of verified) { const entryDomain = d.domain.includes("localhost") ? `${d.domain}:80` : d.domain; - if (d.targetService || d.targetPort) { + const isProxied = isCloudflareProxied((d as any).cloudflareProxied); + if (d.targetService || d.targetPort || isProxied !== isBaseProxied) { let targetContainer = containerName; if (d.targetService) { const parts = containerName.split("-"); @@ -206,7 +210,9 @@ export const buildCaddySnippet = async ( } } const tPort = d.targetPort || port; - customBlocks.push(caddySite(entryDomain, caddyReverseProxy(`${targetContainer}:${tPort}`))); + customBlocks.push( + caddySite(entryDomain, caddyReverseProxy(`${targetContainer}:${tPort}`, { cloudflareProxied: isProxied })), + ); } else { if (!defaultDomains.includes(entryDomain)) defaultDomains.push(entryDomain); } @@ -245,7 +251,12 @@ export const buildCaddySnippet = async ( } } - const primaryBlock = caddySite(defaultDomains.join(", "), caddyReverseProxy(`${containerName}:${port}`)); + const upstreamNames = opts?.containerNames?.length ? opts.containerNames : [containerName]; + const upstreams = upstreamNames.map((n) => `${n}:${port}`).join(" "); + const primaryBlock = caddySite( + defaultDomains.join(", "), + caddyReverseProxy(upstreams, { cloudflareProxied: isBaseProxied }), + ); return [primaryBlock, ...customBlocks].join("\n"); }; diff --git a/apps/docs/blume.config.ts b/apps/docs/blume.config.ts index c70d4ea..17204d0 100644 --- a/apps/docs/blume.config.ts +++ b/apps/docs/blume.config.ts @@ -82,4 +82,30 @@ export default defineConfig({ deployment: { site: "https://dequel.intrep.xyz", }, + seo: { + og: { + fonts: [], + palette: { + accent: "#ea580c", + background: "#0a0a0c", + border: "#222228", + foreground: "#f4f4f5", + muted: "#a1a1aa", + }, + titles: { + "/": "Dequel — Self-Hosted PaaS Engine & Container Platform", + }, + }, + software: { + name: "Dequel", + operatingSystem: "Linux", + applicationCategory: "DeveloperApplication", + description: + "Self-hosted PaaS container platform. Deploy apps from Git, ZIP, or Docker Compose with zero infrastructure setup.", + }, + organization: { + name: "Dequel", + url: "https://dequel.intrep.xyz", + }, + }, }); diff --git a/apps/docs/components/SEO.astro b/apps/docs/components/SEO.astro new file mode 100644 index 0000000..3634a2c --- /dev/null +++ b/apps/docs/components/SEO.astro @@ -0,0 +1,130 @@ +--- +interface Props { + title?: string; + description?: string; + canonicalUrl?: string; + ogImage?: string; + ogType?: string; + keywords?: string[]; + themeColor?: string; +} + +const { + title = "Dequel — Self-Hosted PaaS Engine & Container Platform", + description = "The open-source, self-hosted PaaS container platform. Deploy apps from Git, ZIP, or Docker Compose with automatic SSL, multi-server scaling, and zero infrastructure lock-in.", + canonicalUrl = "https://dequel.intrep.xyz/", + ogImage = "https://dequel.intrep.xyz/og/index.png", + ogType = "website", + keywords = [ + "Dequel", + "self-hosted PaaS", + "PaaS platform", + "Docker deployment", + "container platform", + "DevOps", + "Caddy reverse proxy", + "automatic SSL", + "Let's Encrypt", + "BuildKit", + "Railpack", + "zero vendor lock-in", + "homelab", + "multi-server deployment", + "microservices", + ], + themeColor = "#0a0a0c", +} = Astro.props; + +const siteName = "Dequel"; +const author = "Dequel"; + +const structuredData = { + "@context": "https://schema.org", + "@graph": [ + { + "@type": "WebSite", + "@id": "https://dequel.intrep.xyz/#website", + "name": "Dequel", + "url": "https://dequel.intrep.xyz/", + "description": description, + "publisher": { + "@type": "Organization", + "name": "Dequel", + "url": "https://dequel.intrep.xyz/", + "logo": { + "@type": "ImageObject", + "url": "https://dequel.intrep.xyz/logo.svg" + } + } + }, + { + "@type": "SoftwareApplication", + "@id": "https://dequel.intrep.xyz/#software", + "name": "Dequel", + "applicationCategory": "DeveloperApplication", + "operatingSystem": "Linux", + "offers": { + "@type": "Offer", + "price": "0", + "priceCurrency": "USD" + }, + "url": "https://dequel.intrep.xyz/", + "description": description, + "image": ogImage, + "screenshot": "https://dequel.intrep.xyz/screenshots/dashboard.png", + "featureList": [ + "Git push-to-deploy and ZIP uploads", + "Docker Compose deployment", + "Automatic SSL certificate provisioning via Caddy and Let's Encrypt", + "Dynamic multi-server container clustering", + "Zero-dependency self-hosted architecture", + "Full observability with Prometheus, cAdvisor, and Loki", + "Automatic horizontal scaling and auto-healing" + ] + } + ] +}; +--- + + + +{title} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +