From d7981053fad1a78ddb828a9a403b386b8385a542 Mon Sep 17 00:00:00 2001 From: Lftobs Date: Sat, 17 Oct 2026 03:06:14 +0100 Subject: [PATCH 1/5] feat: restore-test records, pool-only server keys, and env-injected compose deploys - restore_tests gains a kind column (test|restore) and query filters; real restores are now recorded as events (migration 0037) - backup panel adds a Test button beside Restore with live status chips - server addition requires pool-sourced auth: inline sshKey rejected, SSH mode needs sshKeyId or sshPassword (BUG-013) - github_integrations secrets encrypted at rest, legacy rows bootstrapped at boot (migration 0038) (BUG-004) - compose deploys inject platform env vars into service containers via a generated override file on both executors (BUG-012) - scale-event Caddy rewrites keep custom domains and replica upstreams (BUG-007) - idle scale-down hysteresis decoupled from the scale-up threshold; capped scale-ups stop refreshing lastScaleUp (BUG-010) - fallback docker start re-verified for 10s before reporting running; crash reproduced with container logs on failure (BUG-008) - vite-only builds default to static serve instead of node dist/index.js (BUG-009) - authenticated, token-redacted repository clones on remote SSH executors --- apps/api/src/api/backups/index.ts | 67 ++++++++ apps/api/src/api/deployments/index.ts | 4 + apps/api/src/api/projects/index.ts | 4 +- apps/api/src/api/servers/index.ts | 20 +-- apps/api/src/api/servers/validate.ts | 21 +++ apps/api/src/backup/restore-test.ts | 150 ++++++++++++++++++ .../github-integration-encryption.test.ts | 86 ++++++++++ apps/api/src/db/encryption-migration.ts | 75 ++++++++- .../src/db/migrations/0037_restore_tests.sql | 17 ++ .../0038_github_integration_encryption.sql | 4 + apps/api/src/db/migrations/meta/_journal.json | 14 ++ apps/api/src/db/repo/github.ts | 33 +++- apps/api/src/db/repo/index.ts | 38 ++--- apps/api/src/db/repo/projects.ts | 40 +++++ apps/api/src/db/repo/restore-tests.ts | 79 +++++++++ apps/api/src/db/repo/servers.ts | 4 - apps/api/src/db/schema.ts | 24 +++ apps/api/src/executors/ssh-build-script.ts | 4 +- apps/api/src/executors/ssh-compose-script.ts | 41 ++++- apps/api/src/executors/ssh.ts | 6 +- .../__tests__/compose-env.test.ts | 23 +++ .../orchestrator/__tests__/compose.test.ts | 29 ++-- .../__tests__/railpack-config.test.ts | 58 +++++++ .../__tests__/runtime-instance.test.ts | 44 +++++ apps/api/src/orchestrator/compose-deploy.ts | 20 ++- apps/api/src/orchestrator/compose-env.ts | 16 ++ apps/api/src/orchestrator/compose.ts | 15 +- apps/api/src/orchestrator/pipeline.ts | 2 + .../src/orchestrator/railpack-config-utils.ts | 51 +++++- apps/api/src/orchestrator/runtime.ts | 91 +++++++++-- apps/api/src/orchestrator/source.ts | 34 +++- apps/api/src/scaling/__tests__/engine.test.ts | 41 +++++ apps/api/src/scaling/engine.ts | 36 +++-- apps/api/src/types.ts | 1 - apps/api/src/utils/domain-verifier.ts | 5 +- apps/web/src/api/databases.ts | 21 ++- .../databases/DatabaseBackupManager.tsx | 140 ++++++++++++++-- .../settings/servers/AddServerForm.tsx | 25 ++- apps/web/src/types/index.ts | 12 ++ 39 files changed, 1256 insertions(+), 139 deletions(-) create mode 100644 apps/api/src/api/servers/validate.ts create mode 100644 apps/api/src/backup/restore-test.ts create mode 100644 apps/api/src/db/__tests__/github-integration-encryption.test.ts create mode 100644 apps/api/src/db/migrations/0037_restore_tests.sql create mode 100644 apps/api/src/db/migrations/0038_github_integration_encryption.sql create mode 100644 apps/api/src/db/repo/restore-tests.ts create mode 100644 apps/api/src/orchestrator/__tests__/compose-env.test.ts create mode 100644 apps/api/src/orchestrator/__tests__/runtime-instance.test.ts create mode 100644 apps/api/src/orchestrator/compose-env.ts diff --git a/apps/api/src/api/backups/index.ts b/apps/api/src/api/backups/index.ts index b052cb0..254d986 100644 --- a/apps/api/src/api/backups/index.ts +++ b/apps/api/src/api/backups/index.ts @@ -1,9 +1,12 @@ +import { randomUUID } from "node:crypto"; import { Elysia } from "elysia"; import { BackupOrchestrator } from "../../backup/orchestrator"; +import { runRestoreTest } from "../../backup/restore-test"; import type { BackupTarget, StorageConfig } from "../../backup/types"; import { S3_BACKUP_PREFIX } from "../../backup/types"; import { deleteBackupRecord, getBackupRecord, listBackupRecords } from "../../db/repo/backups"; import { getDatabaseById } from "../../db/repo/databases"; +import { createRestoreTestRecord, listRestoreTests, updateRestoreTestRecord } from "../../db/repo/restore-tests"; import { getBackupStorageSettings } from "../../db/repo/settings"; import { created, fail, ok } from "../response"; @@ -117,13 +120,77 @@ export const backupRoutes = new Elysia({ prefix: "/backups" }) return ok({ deleted: true }); }) .post("/:id/restore", async ({ params, set }) => { + const record = await getBackupRecord(params.id); + if (!record) { + set.status = 404; + return fail("Backup not found"); + } + let serverId: string | null = null; + if (record.targetId !== "internal") { + const db = await getDatabaseById(record.targetId); + serverId = db?.serverId ?? null; + } + const event = await createRestoreTestRecord({ + id: randomUUID(), + backupId: record.id, + targetId: record.targetId, + serverId, + kind: "restore", + }); try { const storage = await loadStorageConfig(); const orchestrator = new BackupOrchestrator(storage); await orchestrator.restore(params.id, resolveTarget); + await updateRestoreTestRecord(event.id, { status: "passed", completedAt: new Date() }); return ok({ restored: true }); + } catch (error) { + await updateRestoreTestRecord(event.id, { + status: "failed", + error: String(error), + completedAt: new Date(), + }); + set.status = 500; + return fail(String(error)); + } + }) + .post("/:id/restore-test", async ({ params, body, set }: any) => { + try { + const record = await getBackupRecord(params.id); + if (!record) { + set.status = 404; + return fail("Backup not found"); + } + const test = await runRestoreTest({ backupId: params.id, serverId: body?.serverId ?? null }); + if (test?.status !== "passed") { + set.status = 500; + return fail(test?.error || "Restore test failed"); + } + return created(test); } catch (error) { set.status = 500; return fail(String(error)); } + }) + .get("/restore-tests", async ({ query }: any) => { + const limit = query?.limit ? Number(query.limit) : undefined; + return ok( + await listRestoreTests({ + backupId: query?.backupId, + targetId: query?.targetId, + status: query?.status, + kind: query?.kind, + limit: Number.isInteger(limit) && (limit as number) > 0 ? limit : undefined, + }), + ); + }) + .get("/:id/restore-tests", async ({ params, query }: any) => { + const limit = query?.limit ? Number(query.limit) : undefined; + return ok( + await listRestoreTests({ + backupId: params.id, + status: query?.status, + kind: query?.kind, + limit: Number.isInteger(limit) && (limit as number) > 0 ? limit : undefined, + }), + ); }); diff --git a/apps/api/src/api/deployments/index.ts b/apps/api/src/api/deployments/index.ts index 69c9d17..9d2b44e 100644 --- a/apps/api/src/api/deployments/index.ts +++ b/apps/api/src/api/deployments/index.ts @@ -3,6 +3,7 @@ import { basename, join } from "node:path"; import { Elysia } from "elysia"; import { validateRemoteDeployment } from "../../agents/deployments"; import { + captureProjectGithubToken, countDeployments, createDeployment, getDeploymentById, @@ -131,6 +132,9 @@ export const deploymentsRoutes = new Elysia() commitSha, clearCache, }); + if (projectId) { + await captureProjectGithubToken(projectId, gitUrl, request.headers.get("cookie")); + } await dispatchDeployment(deployment, project, server); return created(deployment); } diff --git a/apps/api/src/api/projects/index.ts b/apps/api/src/api/projects/index.ts index f1f6c9b..72c99a8 100644 --- a/apps/api/src/api/projects/index.ts +++ b/apps/api/src/api/projects/index.ts @@ -2,6 +2,7 @@ import { unlink } from "node:fs/promises"; import { join } from "node:path"; import { Elysia } from "elysia"; import { + captureProjectGithubToken, createProject, deleteProjectCascade, getProjectById, @@ -58,7 +59,7 @@ export const projectsRoutes = new Elysia() } return ok(project); }) - .post("/projects", async ({ body, set }: any) => { + .post("/projects", async ({ body, request, set }: any) => { if (!body?.name) { set.status = 400; return fail("name is required"); @@ -117,6 +118,7 @@ export const projectsRoutes = new Elysia() project_type: project.projectType, source_type: project.sourceType, }).catch(() => {}); + await captureProjectGithubToken(project.id, body.repoUrl, request?.headers?.get("cookie") ?? null); return created(project); }) .patch("/projects/:id", async ({ params: { id }, body, set }: any) => { diff --git a/apps/api/src/api/servers/index.ts b/apps/api/src/api/servers/index.ts index 8910a07..00b9fd1 100644 --- a/apps/api/src/api/servers/index.ts +++ b/apps/api/src/api/servers/index.ts @@ -3,8 +3,8 @@ import { createServer, deleteServer, getServerById, listServers } from "../../db import { serverEventBus } from "../../servers/event-bus"; import { isServerPreparing, prepareServer } from "../../servers/prepare"; import { testSshConnection } from "../../utils/ssh"; -import { isPort, SERVER_HOST_RE } from "../../utils/validate"; import { created, fail, ok } from "../response"; +import { validateServerCreate } from "./validate"; const redactServer = (server: T): T => { if (server.sshKey) server.sshKey = "[redacted]"; @@ -15,21 +15,10 @@ const redactServer = ok((await listServers()).map((server) => redactServer(server)))) .post("/servers", async ({ body, set }: any) => { - if (!body?.name || !body?.host) { + const validationError = validateServerCreate(body); + if (validationError) { set.status = 400; - return fail("name and host are required"); - } - if (!SERVER_HOST_RE.test(body.host)) { - set.status = 400; - return fail("host must be a valid hostname or IP address"); - } - if (body.mode && body.mode !== "ssh" && body.mode !== "agent") { - set.status = 400; - return fail("mode must be 'ssh' or 'agent'"); - } - if (body.port !== undefined && body.port !== null && !isPort(Number(body.port))) { - set.status = 400; - return fail("port must be between 1 and 65535"); + return fail(validationError); } return created( redactServer( @@ -39,7 +28,6 @@ export const serversRoutes = new Elysia() port: body.port ? Number(body.port) : body.mode === "ssh" ? 22 : 2375, mode: body.mode ?? "ssh", sshUser: body.sshUser ?? "root", - sshKey: body.sshKey, sshKeyId: body.sshKeyId, sshPassword: body.sshPassword, authToken: body.authToken, diff --git a/apps/api/src/api/servers/validate.ts b/apps/api/src/api/servers/validate.ts new file mode 100644 index 0000000..2c1209b --- /dev/null +++ b/apps/api/src/api/servers/validate.ts @@ -0,0 +1,21 @@ +import { isPort, SERVER_HOST_RE } from "../../utils/validate"; + +export const validateServerCreate = (body: any): string | null => { + if (!body?.name || !body?.host) return "name and host are required"; + if (typeof body.host !== "string" || !SERVER_HOST_RE.test(body.host)) { + return "host must be a valid hostname or IP address"; + } + const mode = body.mode ?? "ssh"; + if (mode !== "ssh" && mode !== "agent") return "mode must be 'ssh' or 'agent'"; + if (body.port !== undefined && body.port !== null && !isPort(Number(body.port))) { + return "port must be between 1 and 65535"; + } + if (body.sshKey) { + return "SSH keys cannot be attached to a server directly. Add the key to the SSH key pool, then pass its sshKeyId."; + } + if (body.sshKeyId && typeof body.sshKeyId !== "string") return "sshKeyId must be a string"; + if (mode === "ssh" && !body.sshKeyId && !body.sshPassword) { + return "SSH servers need a pool key (sshKeyId) or a password (sshPassword)"; + } + return null; +}; diff --git a/apps/api/src/backup/restore-test.ts b/apps/api/src/backup/restore-test.ts new file mode 100644 index 0000000..378e209 --- /dev/null +++ b/apps/api/src/backup/restore-test.ts @@ -0,0 +1,150 @@ +import { randomUUID } from "node:crypto"; +import { createGunzip } from "node:zlib"; +import { resolveDbImage } from "../databases/manager"; +import { getDatabaseById, getServerById } from "../db/repo"; +import { getBackupRecord } from "../db/repo/backups"; +import { createRestoreTestRecord, updateRestoreTestRecord } from "../db/repo/restore-tests"; +import { getBackupStorageSettings } from "../db/repo/settings"; +import type { Server } from "../types"; +import { dockerRun, ensureContainerRemoved } from "../utils/docker-run"; +import type { BackupContext } from "./adapter"; +import { getAdapter } from "./adapters"; +import { createStorage } from "./storage/index"; +import type { BackupTarget, DatabaseType, StorageConfig } from "./types"; +import { S3_BACKUP_PREFIX } from "./types"; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function loadStorageConfig(): Promise { + const settings = await getBackupStorageSettings(); + if (settings.type === "s3") { + return { + type: "s3", + endpoint: settings.s3Endpoint, + accessKeyId: settings.s3AccessKeyId, + secretAccessKey: settings.s3SecretAccessKey, + bucket: settings.s3Bucket, + region: settings.s3Region, + prefix: S3_BACKUP_PREFIX, + }; + } + return { type: "local", path: settings.path }; +} + +const scratchEnv = (engine: DatabaseType, username: string, password: string, databaseName: string): string[] => { + if (engine === "postgresql") { + return [`POSTGRES_USER=${username}`, `POSTGRES_PASSWORD=${password}`, `POSTGRES_DB=${databaseName}`]; + } + if (engine === "mysql") { + return [ + `MYSQL_ROOT_PASSWORD=${password}`, + `MYSQL_DATABASE=${databaseName}`, + `MYSQL_USER=${username}`, + `MYSQL_PASSWORD=${password}`, + ]; + } + if (engine === "mongodb") { + return [ + `MONGO_INITDB_ROOT_USERNAME=${username}`, + `MONGO_INITDB_ROOT_PASSWORD=${password}`, + `MONGO_INITDB_DATABASE=${databaseName}`, + ]; + } + return []; +}; + +const scratchCommand = (engine: DatabaseType, password: string): string[] => { + if (engine === "redis") return ["redis-server", "--requirepass", password]; + return []; +}; + +export interface RestoreTestOptions { + backupId: string; + serverId?: string | null; +} + +export const runRestoreTest = async (options: RestoreTestOptions) => { + const job = await getBackupRecord(options.backupId); + if (!job) throw new Error("Backup not found"); + if (job.status !== "completed" || !job.storagePath) { + throw new Error("Backup has not completed successfully"); + } + + const engine = job.engine as DatabaseType; + const adapter = getAdapter(engine); + + let version: string | null = null; + let databaseName = "dequel"; + if (job.targetId !== "internal") { + const db = await getDatabaseById(job.targetId); + if (!db) throw new Error(`Database ${job.targetId} not found`); + version = db.version; + databaseName = db.databaseName; + } + + let server: Server | null = null; + let serverId: string | null = null; + if (options.serverId) { + server = await getServerById(options.serverId); + if (!server) throw new Error("Restore-test server not found"); + serverId = server.id; + } + + const test = await createRestoreTestRecord({ + id: randomUUID(), + backupId: job.id, + targetId: job.targetId, + serverId, + }); + + const scratchName = `dequel-restore-test-${test.id.slice(0, 8)}`; + const username = "dequel"; + const password = randomUUID().replace(/-/g, ""); + const target: BackupTarget = { + id: `restore-test-${test.id}`, + type: "managed", + engine, + containerName: scratchName, + databaseName, + serverId, + credentials: { username, password }, + }; + const ctx: BackupContext = { target, server }; + + try { + const image = resolveDbImage(engine, version); + const envArgs = scratchEnv(engine, username, password, databaseName).flatMap((e) => ["-e", e]); + await dockerRun( + "docker", + ["run", "-d", "--name", scratchName, ...envArgs, image, ...scratchCommand(engine, password)], + server, + ); + + const deadline = Date.now() + 90_000; + let ready = false; + while (Date.now() < deadline) { + if (await adapter.ping(ctx)) { + ready = true; + break; + } + await sleep(2_000); + } + if (!ready) throw new Error(`Scratch ${engine} container did not become ready`); + + const storage = createStorage(await loadStorageConfig()); + const compressed = await storage.download(job.storagePath); + await adapter.restore(ctx, compressed.pipe(createGunzip())); + + if (!(await adapter.ping(ctx))) throw new Error("Restored container failed health check"); + + return updateRestoreTestRecord(test.id, { status: "passed", completedAt: new Date() }); + } catch (error) { + return updateRestoreTestRecord(test.id, { + status: "failed", + error: String(error), + completedAt: new Date(), + }); + } finally { + await ensureContainerRemoved(scratchName, server).catch(() => {}); + } +}; diff --git a/apps/api/src/db/__tests__/github-integration-encryption.test.ts b/apps/api/src/db/__tests__/github-integration-encryption.test.ts new file mode 100644 index 0000000..044ec7b --- /dev/null +++ b/apps/api/src/db/__tests__/github-integration-encryption.test.ts @@ -0,0 +1,86 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "bun:test"; +import { drizzle } from "drizzle-orm/node-postgres"; +import type { Pool } from "pg"; +import { setDbProvider } from "../db-provider"; +import * as schema from "../schema"; +import { createTestPool, truncateAllTables } from "../test-helper"; + +const TEST_SECRET = "test-jwt-secret-for-testing-purposes-only"; +let pool: Pool; + +beforeAll(async () => { + pool = createTestPool(); + const db = drizzle(pool, { schema }); + setDbProvider(async () => db); + const { initAuth } = await import("../../utils/auth"); + initAuth(TEST_SECRET); +}); + +beforeEach(async () => { + await truncateAllTables(pool); +}); + +afterAll(async () => { + await truncateAllTables(pool); +}); + +describe("github integrations store secrets encrypted", () => { + it("writes ciphertext for client and webhook secrets", async () => { + const gh = await import("../repo/github"); + await gh.setGithubIntegration({ + clientId: "Iv1.testapp", + clientSecret: "gh-canary-secret", + webhookSecret: "hook-canary-secret", + }); + + const [rawRow] = await pool + .query( + "SELECT client_secret, client_secret_iv, client_secret_tag, webhook_secret, webhook_secret_iv FROM github_integrations", + ) + .then((r: any) => r.rows); + expect(rawRow.client_secret).not.toBe("gh-canary-secret"); + expect(rawRow.client_secret).toContain(""); + expect(rawRow.client_secret_iv).toBeTruthy(); + expect(rawRow.client_secret_tag).toBeTruthy(); + expect(rawRow.webhook_secret).not.toBe("hook-canary-secret"); + expect(rawRow.webhook_secret_iv).toBeTruthy(); + + const read = await gh.getGithubIntegration(); + expect(read?.clientSecret).toBe("gh-canary-secret"); + expect(read?.webhookSecret).toBe("hook-canary-secret"); + }); + + it("keeps an existing webhook secret when the update omits one", async () => { + const gh = await import("../repo/github"); + await gh.setGithubIntegration({ + clientId: "Iv1.testapp", + clientSecret: "secret-a", + webhookSecret: "hook-keep-me", + }); + const second = await gh.setGithubIntegration({ + clientId: "Iv1.testapp2", + clientSecret: "secret-b", + }); + expect(second.clientSecret).toBe("secret-b"); + expect(second.webhookSecret).toBe("hook-keep-me"); + + const read = await gh.getGithubIntegration(); + expect(read?.webhookSecret).toBe("hook-keep-me"); + }); + + it("decrypts rows migrated from legacy plaintext", async () => { + const gh = await import("../repo/github"); + const db = drizzle(pool, { schema }); + + await db.insert(schema.githubIntegrations).values({ + id: "legacy-row", + clientId: "Iv1.legacy", + clientSecret: "gh-plan-secret", + appName: "Dequel", + createdAt: new Date(), + }); + + const read = await gh.getGithubIntegration(); + expect(read?.clientSecret).toBe("gh-plan-secret"); + }); +}); diff --git a/apps/api/src/db/encryption-migration.ts b/apps/api/src/db/encryption-migration.ts index b9ae84c..41b30c9 100644 --- a/apps/api/src/db/encryption-migration.ts +++ b/apps/api/src/db/encryption-migration.ts @@ -1,10 +1,11 @@ -import { and, eq, isNotNull } from "drizzle-orm"; +import { and, eq, isNotNull, isNull } from "drizzle-orm"; import { config } from "../utils/config"; import { DEFAULT_ENV_ENCRYPTION_KEY, decryptValue, encryptValue } from "../utils/crypto"; import { getDb } from "./db-provider"; import { backupStorageSettings, environmentVariables, + githubIntegrations, githubSessions, llmProviderKeys, projects, @@ -270,6 +271,78 @@ const STORES: Store[] = [ ); }, }, + { + name: "github_integrations", + run: async (activeKey) => { + const db = await getDb(); + const legacy = await db.select().from(githubIntegrations).where(isNull(githubIntegrations.clientSecretIv)); + let migrated = 0; + for (const row of legacy) { + if (!row.clientSecret) continue; + const client = encryptValue(row.clientSecret, activeKey); + const webhook = row.webhookSecret ? encryptValue(row.webhookSecret, activeKey) : null; + await db + .update(githubIntegrations) + .set({ + clientSecret: client.encrypted, + clientSecretIv: client.iv, + clientSecretTag: client.tag, + ...(webhook + ? { webhookSecret: webhook.encrypted, webhookSecretIv: webhook.iv, webhookSecretTag: webhook.tag } + : {}), + }) + .where(eq(githubIntegrations.id, row.id)) + .execute(); + migrated += 1; + } + const encrypted = await db + .select() + .from(githubIntegrations) + .where(and(isNotNull(githubIntegrations.clientSecretIv), isNotNull(githubIntegrations.clientSecretTag))); + let unknown = 0; + for (const row of encrypted) { + const triples: { read: StoredTriple; write: (next: Triple) => Promise }[] = []; + if (row.clientSecretIv) { + triples.push({ + read: { enc: row.clientSecret, iv: row.clientSecretIv, tag: row.clientSecretTag }, + write: (next) => + db + .update(githubIntegrations) + .set({ clientSecret: next.encrypted, clientSecretIv: next.iv, clientSecretTag: next.tag }) + .where(eq(githubIntegrations.id, row.id)) + .execute(), + }); + } + if (row.webhookSecretIv) { + triples.push({ + read: { enc: row.webhookSecret, iv: row.webhookSecretIv, tag: row.webhookSecretTag }, + write: (next) => + db + .update(githubIntegrations) + .set({ webhookSecret: next.encrypted, webhookSecretIv: next.iv, webhookSecretTag: next.tag }) + .where(eq(githubIntegrations.id, row.id)) + .execute(), + }); + } + for (const triple of triples) { + const { enc, iv, tag } = triple.read; + if (!enc || !iv || !tag) continue; + try { + decryptValue(enc, iv, tag, activeKey); + continue; + } catch {} + try { + const plain = decryptValue(enc, iv, tag, DEFAULT_ENV_ENCRYPTION_KEY); + await triple.write(encryptValue(plain, activeKey)); + migrated += 1; + } catch { + unknown += 1; + } + } + } + return { migrated, unknown }; + }, + }, { name: "projects", run: async (activeKey) => { diff --git a/apps/api/src/db/migrations/0037_restore_tests.sql b/apps/api/src/db/migrations/0037_restore_tests.sql new file mode 100644 index 0000000..f8d8166 --- /dev/null +++ b/apps/api/src/db/migrations/0037_restore_tests.sql @@ -0,0 +1,17 @@ +CREATE TABLE "restore_tests" ( + "id" text PRIMARY KEY NOT NULL, + "backup_id" text NOT NULL, + "target_id" text NOT NULL, + "server_id" text, + "kind" text DEFAULT 'test' NOT NULL, + "status" text DEFAULT 'running' NOT NULL, + "error" text, + "started_at" timestamp with time zone DEFAULT now() NOT NULL, + "completed_at" timestamp with time zone +); +--> statement-breakpoint +CREATE INDEX "idx_restore_tests_backup" ON "restore_tests" USING btree ("backup_id"); +--> statement-breakpoint +CREATE INDEX "idx_restore_tests_target" ON "restore_tests" USING btree ("target_id"); +--> statement-breakpoint +CREATE INDEX "idx_restore_tests_status" ON "restore_tests" USING btree ("status"); diff --git a/apps/api/src/db/migrations/0038_github_integration_encryption.sql b/apps/api/src/db/migrations/0038_github_integration_encryption.sql new file mode 100644 index 0000000..3c544dd --- /dev/null +++ b/apps/api/src/db/migrations/0038_github_integration_encryption.sql @@ -0,0 +1,4 @@ +ALTER TABLE "github_integrations" ADD COLUMN "client_secret_iv" text;--> statement-breakpoint +ALTER TABLE "github_integrations" ADD COLUMN "client_secret_tag" text;--> statement-breakpoint +ALTER TABLE "github_integrations" ADD COLUMN "webhook_secret_iv" text;--> statement-breakpoint +ALTER TABLE "github_integrations" ADD COLUMN "webhook_secret_tag" text; \ No newline at end of file diff --git a/apps/api/src/db/migrations/meta/_journal.json b/apps/api/src/db/migrations/meta/_journal.json index 850b822..cccd166 100644 --- a/apps/api/src/db/migrations/meta/_journal.json +++ b/apps/api/src/db/migrations/meta/_journal.json @@ -92,6 +92,20 @@ "when": 1790700000000, "tag": "0036_fixdiag", "breakpoints": true + }, + { + "idx": 13, + "version": "7", + "when": 1791861049522, + "tag": "0037_restore_tests", + "breakpoints": true + }, + { + "idx": 14, + "version": "7", + "when": 1792300000000, + "tag": "0038_github_integration_encryption", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/api/src/db/repo/github.ts b/apps/api/src/db/repo/github.ts index 1b5c308..8935c41 100644 --- a/apps/api/src/db/repo/github.ts +++ b/apps/api/src/db/repo/github.ts @@ -1,16 +1,27 @@ import { randomUUID } from "node:crypto"; import { desc, eq } from "drizzle-orm"; import type { GithubIntegration } from "../../types"; +import { config } from "../../utils/config"; +import { decryptValue, encryptValue } from "../../utils/crypto"; import { getDb } from "../db-provider"; import { githubIntegrations } from "../schema"; import { now } from "./helpers"; +const decryptOrRaw = (value: string, iv: string | null, tag: string | null): string => { + if (!iv || !tag) return value; + try { + return decryptValue(value, iv, tag, config.envEncryptionKey); + } catch { + return ""; + } +}; + const mapGithubIntegration = (row: typeof githubIntegrations.$inferSelect): GithubIntegration => ({ id: row.id, clientId: row.clientId, - clientSecret: row.clientSecret, + clientSecret: decryptOrRaw(row.clientSecret, row.clientSecretIv, row.clientSecretTag), appName: row.appName, - webhookSecret: row.webhookSecret ?? null, + webhookSecret: row.webhookSecret ? decryptOrRaw(row.webhookSecret, row.webhookSecretIv, row.webhookSecretTag) : null, createdAt: row.createdAt, }); @@ -32,6 +43,8 @@ export const setGithubIntegration = async (input: { webhookSecret?: string; }): Promise => { const db = await getDb(); + const client = encryptValue(input.clientSecret, config.envEncryptionKey); + const webhook = input.webhookSecret ? encryptValue(input.webhookSecret, config.envEncryptionKey) : null; return db.transaction(async (tx) => { const [existing] = await tx @@ -47,9 +60,13 @@ export const setGithubIntegration = async (input: { .update(githubIntegrations) .set({ clientId: input.clientId, - clientSecret: input.clientSecret, + clientSecret: client.encrypted, + clientSecretIv: client.iv, + clientSecretTag: client.tag, appName: input.appName ?? "Dequel", - webhookSecret: input.webhookSecret ?? existing.webhookSecret, + ...(webhook + ? { webhookSecret: webhook.encrypted, webhookSecretIv: webhook.iv, webhookSecretTag: webhook.tag } + : {}), }) .where(eq(githubIntegrations.id, existing.id)) .execute(); @@ -67,9 +84,13 @@ export const setGithubIntegration = async (input: { .values({ id, clientId: input.clientId, - clientSecret: input.clientSecret, + clientSecret: client.encrypted, + clientSecretIv: client.iv, + clientSecretTag: client.tag, appName: input.appName ?? "Dequel", - webhookSecret: input.webhookSecret ?? null, + webhookSecret: webhook?.encrypted ?? null, + webhookSecretIv: webhook?.iv ?? null, + webhookSecretTag: webhook?.tag ?? null, createdAt: timestamp, }) .execute(); diff --git a/apps/api/src/db/repo/index.ts b/apps/api/src/db/repo/index.ts index 7add6c0..99aace2 100644 --- a/apps/api/src/db/repo/index.ts +++ b/apps/api/src/db/repo/index.ts @@ -51,6 +51,14 @@ export { updateDeploymentCommitSha, updateDeploymentStatus, } from "./deployments"; +export type { DiagAction, DiagActionKind, DiagActionStatus } from "./diag-actions"; +export { + claimDiagAction, + completeDiagAction, + failDiagAction, + findCompletedAction, + getDiagAction, +} from "./diag-actions"; export { createDiagRun, deleteDiagRun, @@ -63,14 +71,6 @@ export { markInterruptedDiagRuns, recordStageResult, } from "./diag-runs"; -export type { DiagAction, DiagActionKind, DiagActionStatus } from "./diag-actions"; -export { - claimDiagAction, - completeDiagAction, - failDiagAction, - findCompletedAction, - getDiagAction, -} from "./diag-actions"; export { createDomain, deleteDomain, @@ -95,13 +95,24 @@ export { getGithubSession, getGithubTokenFromCookie, } from "./github-sessions"; +export type { LlmKeyInput, LlmKeyStatus, LlmProvider } from "./llm-keys"; +export { + deleteLlmKey, + getDecryptedLlmKey, + getLlmKeyStatus, + LLM_PROVIDERS, + updateLlmModels, + upsertLlmKey, +} from "./llm-keys"; export { getPlatformSettings, setIngressServer } from "./platform-settings"; export type { ProjectCleanupInfo } from "./projects"; export { + captureProjectGithubToken, createProject, deleteProject, deleteProjectCascade, getProjectById, + getProjectGithubToken, listProjects, updateProject, updateProjectGithubToken, @@ -118,9 +129,9 @@ export { upsertRoute, } from "./routes"; export { - ScalingRangeError, deleteScalingPolicy, getScalingPolicy, + ScalingRangeError, upsertScalingPolicy, } from "./scaling"; export type { ServerConnection } from "./servers"; @@ -135,15 +146,6 @@ export { } from "./servers"; export type { SmtpSettingsData } from "./settings"; export { getBackupStorageSettings, getSmtpSettings, upsertBackupStorageSettings, upsertSmtpSettings } from "./settings"; -export type { LlmKeyInput, LlmKeyStatus, LlmProvider } from "./llm-keys"; -export { - LLM_PROVIDERS, - deleteLlmKey, - getDecryptedLlmKey, - getLlmKeyStatus, - updateLlmModels, - upsertLlmKey, -} from "./llm-keys"; export { createSharedEnvVar, deleteSharedEnvVar, diff --git a/apps/api/src/db/repo/projects.ts b/apps/api/src/db/repo/projects.ts index 66a2592..bc025e2 100644 --- a/apps/api/src/db/repo/projects.ts +++ b/apps/api/src/db/repo/projects.ts @@ -1,6 +1,8 @@ import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import type { CreateProjectInput, Project } from "../../types"; +import { config } from "../../utils/config"; +import { decryptValue, encryptValue } from "../../utils/crypto"; import { getDb } from "../db-provider"; import { alerts, @@ -113,6 +115,44 @@ export const updateProjectGithubToken = async ( .execute(); }; +export const getProjectGithubToken = async (id: string): Promise => { + const db = await getDb(); + const [row] = await db + .select({ + encrypted: projects.githubTokenEncrypted, + iv: projects.githubTokenIv, + tag: projects.githubTokenTag, + }) + .from(projects) + .where(eq(projects.id, id)) + .execute(); + if (!row?.encrypted || !row?.iv || !row?.tag) return null; + try { + return decryptValue(row.encrypted, row.iv, row.tag, config.envEncryptionKey); + } catch { + return null; + } +}; + +export const captureProjectGithubToken = async ( + projectId: string, + gitUrl: string | null | undefined, + cookieHeader: string | null, +): Promise => { + try { + if (!gitUrl || !/^https:\/\/github\.com\//i.test(gitUrl.trim()) || !cookieHeader) return; + const { getGithubSession } = await import("./github-sessions"); + const match = cookieHeader.match(/(?:^|;\s*)github_session=([^;]+)/); + if (!match) return; + const token = await getGithubSession(match[1]); + if (!token) return; + const enc = encryptValue(token, config.envEncryptionKey); + await updateProjectGithubToken(projectId, enc.encrypted, enc.iv, enc.tag); + } catch { + return; + } +}; + export const listProjects = async (): Promise => { const db = await getDb(); const rows = await db.select().from(projects).orderBy(projects.name).execute(); diff --git a/apps/api/src/db/repo/restore-tests.ts b/apps/api/src/db/repo/restore-tests.ts new file mode 100644 index 0000000..7f909f6 --- /dev/null +++ b/apps/api/src/db/repo/restore-tests.ts @@ -0,0 +1,79 @@ +import { and, desc, eq } from "drizzle-orm"; +import { getDb } from "../db-provider"; +import { restoreTests } from "../schema"; + +export type RestoreEventKind = "test" | "restore"; + +export interface RestoreTestRecord { + id: string; + backupId: string; + targetId: string; + serverId: string | null; + kind: RestoreEventKind; + status: string; + error: string | null; + startedAt: Date; + completedAt: Date | null; +} + +export async function createRestoreTestRecord( + record: Pick & { + serverId?: string | null; + kind?: RestoreEventKind; + }, +): Promise { + const db = await getDb(); + const [inserted] = await db + .insert(restoreTests) + .values({ + id: record.id, + backupId: record.backupId, + targetId: record.targetId, + serverId: record.serverId ?? null, + kind: record.kind ?? "test", + status: "running", + startedAt: new Date(), + }) + .returning(); + return inserted as RestoreTestRecord; +} + +export async function updateRestoreTestRecord( + id: string, + updates: Partial>, +): Promise { + const db = await getDb(); + const [updated] = await db.update(restoreTests).set(updates).where(eq(restoreTests.id, id)).returning(); + return (updated as RestoreTestRecord) || null; +} + +export async function getRestoreTestRecord(id: string): Promise { + const db = await getDb(); + const [record] = await db.select().from(restoreTests).where(eq(restoreTests.id, id)).limit(1); + return (record as RestoreTestRecord) || null; +} + +export interface RestoreTestFilters { + backupId?: string; + targetId?: string; + status?: string; + kind?: string; + limit?: number; +} + +export async function listRestoreTests(filters: RestoreTestFilters = {}): Promise { + const db = await getDb(); + const conditions = []; + if (filters.backupId) conditions.push(eq(restoreTests.backupId, filters.backupId)); + if (filters.targetId) conditions.push(eq(restoreTests.targetId, filters.targetId)); + if (filters.status) conditions.push(eq(restoreTests.status, filters.status)); + if (filters.kind) conditions.push(eq(restoreTests.kind, filters.kind)); + let query = db + .select() + .from(restoreTests) + .where(conditions.length ? and(...conditions) : undefined) + .orderBy(desc(restoreTests.startedAt)) + .$dynamic(); + if (filters.limit) query = query.limit(filters.limit); + return query as Promise; +} diff --git a/apps/api/src/db/repo/servers.ts b/apps/api/src/db/repo/servers.ts index 4180f58..ed32e4f 100644 --- a/apps/api/src/db/repo/servers.ts +++ b/apps/api/src/db/repo/servers.ts @@ -61,7 +61,6 @@ export const createServer = async (input: CreateServerInput): Promise => const id = randomUUID(); const timestamp = now(); const db = await getDb(); - const encrypted = input.sshKey ? encryptValue(input.sshKey, config.envEncryptionKey) : null; await db .insert(servers) .values({ @@ -71,9 +70,6 @@ export const createServer = async (input: CreateServerInput): Promise => port: input.port ?? 2375, authToken: input.authToken ?? "", sshUser: input.sshUser ?? null, - sshKey: encrypted?.encrypted ?? null, - sshKeyIv: encrypted?.iv ?? null, - sshKeyTag: encrypted?.tag ?? null, sshKeyId: input.sshKeyId ?? null, sshPassword: input.sshPassword ?? null, mode: input.mode ?? "ssh", diff --git a/apps/api/src/db/schema.ts b/apps/api/src/db/schema.ts index cea82ef..75a9cbe 100644 --- a/apps/api/src/db/schema.ts +++ b/apps/api/src/db/schema.ts @@ -24,8 +24,12 @@ export const githubIntegrations = pgTable("github_integrations", { id: text().primaryKey(), clientId: text("client_id").notNull(), clientSecret: text("client_secret").notNull(), + clientSecretIv: text("client_secret_iv"), + clientSecretTag: text("client_secret_tag"), appName: text("app_name").notNull().default("Dequel"), webhookSecret: text("webhook_secret"), + webhookSecretIv: text("webhook_secret_iv"), + webhookSecretTag: text("webhook_secret_tag"), createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), }); @@ -512,3 +516,23 @@ export const backups = pgTable( index("idx_backups_status").on(table.status), ], ); + +export const restoreTests = pgTable( + "restore_tests", + { + id: text().primaryKey(), + backupId: text("backup_id").notNull(), + targetId: text("target_id").notNull(), + serverId: text("server_id"), + kind: text().notNull().default("test"), + status: text().notNull().default("running"), + error: text(), + startedAt: timestamp("started_at", { withTimezone: true }).notNull().defaultNow(), + completedAt: timestamp("completed_at", { withTimezone: true }), + }, + (table) => [ + index("idx_restore_tests_backup").on(table.backupId), + index("idx_restore_tests_target").on(table.targetId), + index("idx_restore_tests_status").on(table.status), + ], +); diff --git a/apps/api/src/executors/ssh-build-script.ts b/apps/api/src/executors/ssh-build-script.ts index 72dee50..cdc33dc 100644 --- a/apps/api/src/executors/ssh-build-script.ts +++ b/apps/api/src/executors/ssh-build-script.ts @@ -1,3 +1,5 @@ +import { redactGitUrl } from "../orchestrator/source"; + export interface SshBuildScriptInput { deploymentId: string; workspaceRoot: string; @@ -83,7 +85,7 @@ export const buildRemoteDeployScript = (input: SshBuildScriptInput): string => { "fi", 'export BUILDKIT_HOST="docker-container://buildkit"', "", - `echo "[build] Cloning repository ${input.gitUrl}"`, + `echo "[build] Cloning repository ${redactGitUrl(input.gitUrl)}"`, `git clone --depth 1 ${sh(input.gitUrl)} .`, checkoutSha, "", diff --git a/apps/api/src/executors/ssh-compose-script.ts b/apps/api/src/executors/ssh-compose-script.ts index e0b0092..1ea6522 100644 --- a/apps/api/src/executors/ssh-compose-script.ts +++ b/apps/api/src/executors/ssh-compose-script.ts @@ -1,3 +1,5 @@ +import { redactGitUrl } from "../orchestrator/source"; + export interface RemoteComposeScriptInput { deploymentId: string; workspaceRoot: string; @@ -22,6 +24,7 @@ export const buildRemoteComposeScript = (input: RemoteComposeScriptInput): strin const remoteWorkspace = input.workspaceRoot.startsWith("/app") ? "$HOME/.dequel/workspace" : input.workspaceRoot; const envLines = input.environmentVariables.map((v) => `${v.key}=${v.value}`).join("\n"); + const envRecord = JSON.stringify(Object.fromEntries(input.environmentVariables.map((v) => [v.key, v.value]))); const sourceDirLine = input.sourceDir ? `cd ${sh(input.sourceDir)}` : ""; @@ -44,7 +47,7 @@ export const buildRemoteComposeScript = (input: RemoteComposeScriptInput): strin " exit 1", "fi", "", - `echo "[compose] Cloning repository ${input.gitUrl}"`, + `echo "[compose] Cloning repository ${redactGitUrl(input.gitUrl)}"`, `git clone --depth 1 ${sh(input.gitUrl)} .`, checkoutSha, "", @@ -57,9 +60,39 @@ export const buildRemoteComposeScript = (input: RemoteComposeScriptInput): strin "done", 'if [ -z "$COMPOSE_FILE" ]; then echo "[compose] ERROR: no compose file found"; exit 1; fi', "", - `echo "[compose] Building and starting stack (project: deploy-${input.deploymentId})"`, - `docker compose -f "$COMPOSE_FILE" -p deploy-${input.deploymentId} build`, - `docker compose -f "$COMPOSE_FILE" -p deploy-${input.deploymentId} up -d`, + ...(envLines + ? [ + `export DEQUEL_EXTRA_ENV=${sh(envRecord)}`, + "", + "if command -v python3 >/dev/null 2>&1; then", + ' DEQUEL_COMPOSE_FILE="$COMPOSE_FILE" python3 - <<\'DEQUEL_OVERRIDE_EOF\' || echo "[compose] WARNING: could not generate env override file"', + "import json, os, subprocess", + 'envs = json.loads(os.environ["DEQUEL_EXTRA_ENV"])', + 'services = subprocess.check_output(["docker","compose","-f",os.environ["DEQUEL_COMPOSE_FILE"],"config","--services"]).decode().split()', + 'lines = ["services:"]', + "for svc in services:", + ' lines.append(" %s:" % svc)', + ' lines.append(" environment:")', + " for k, v in envs.items():", + ' lines.append(" %s: %s" % (k, json.dumps(v)))', + 'open("dequel-env-override.yml", "w").write("\\n".join(lines) + "\\n")', + 'print("[compose] Injected %d env var(s) into %d service(s)" % (len(envs), len(services)))', + "DEQUEL_OVERRIDE_EOF", + "else:", + ' echo "[compose] WARNING: python3 not found; platform env vars only apply to compose interpolation"', + "fi", + "", + 'OVERRIDE_ARGS=""', + 'if [ -f dequel-env-override.yml ]; then OVERRIDE_ARGS="-f dequel-env-override.yml"; fi', + `echo "[compose] Building and starting stack (project: deploy-${input.deploymentId})"`, + `docker compose -f "$COMPOSE_FILE" $OVERRIDE_ARGS -p deploy-${input.deploymentId} build`, + `docker compose -f "$COMPOSE_FILE" $OVERRIDE_ARGS -p deploy-${input.deploymentId} up -d`, + ] + : [ + `echo "[compose] Building and starting stack (project: deploy-${input.deploymentId})"`, + `docker compose -f "$COMPOSE_FILE" -p deploy-${input.deploymentId} build`, + `docker compose -f "$COMPOSE_FILE" -p deploy-${input.deploymentId} up -d`, + ]), "", `echo "[compose] Connecting containers to network ${input.dockerNetwork}"`, `for cid in $(docker compose -p deploy-${input.deploymentId} ps -q); do`, diff --git a/apps/api/src/executors/ssh.ts b/apps/api/src/executors/ssh.ts index fa9771e..000ea69 100644 --- a/apps/api/src/executors/ssh.ts +++ b/apps/api/src/executors/ssh.ts @@ -94,11 +94,12 @@ const deployComposeRemote = async (deployment: Deployment, project: Project, ser await emitLog(deployment.id, "system", `Deploying compose stack on server ${server.name} over SSH`); const envVars = await listEnvironmentVariablesForDeploy(project.id, deployment.environment ?? undefined); + const { resolveProjectGitUrl } = await import("../orchestrator/source"); const script = buildRemoteComposeScript({ deploymentId: deployment.id, workspaceRoot: config.workspaceRoot, - gitUrl: deployment.sourceRef, + gitUrl: await resolveProjectGitUrl(project.id, deployment.sourceRef), branch: deployment.branch, commitSha: deployment.commitSha, projectName: project.name, @@ -320,11 +321,12 @@ export const sshExecutor: DeploymentExecutor = { const imageTag = `${slugify(project.name)}-${deployment.id.slice(0, 8)}:latest`; const envVars = await listEnvironmentVariablesForDeploy(project.id, deployment.environment ?? undefined); + const { resolveProjectGitUrl } = await import("../orchestrator/source"); const pgidEcho = `echo "DEQUEL_PGID:$(ps -o pgid= -p $$ | tr -d ' ')"`; const script = `${pgidEcho}\n${buildRemoteDeployScript({ deploymentId: deployment.id, workspaceRoot: config.workspaceRoot, - gitUrl: deployment.sourceRef, + gitUrl: await resolveProjectGitUrl(project.id, deployment.sourceRef), branch: deployment.branch, commitSha: deployment.commitSha, imageTag, diff --git a/apps/api/src/orchestrator/__tests__/compose-env.test.ts b/apps/api/src/orchestrator/__tests__/compose-env.test.ts new file mode 100644 index 0000000..92562db --- /dev/null +++ b/apps/api/src/orchestrator/__tests__/compose-env.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, test } from "bun:test"; +import { buildComposeEnvOverrideYaml } from "../compose-env"; + +describe("buildComposeEnvOverrideYaml", () => { + test("returns empty string when no env vars", () => { + expect(buildComposeEnvOverrideYaml(["web"], {})).toBe(""); + }); + + test("returns empty string when no services", () => { + expect(buildComposeEnvOverrideYaml([], { FOO: "bar" })).toBe(""); + }); + + test("injects env vars into every service", () => { + const yaml = buildComposeEnvOverrideYaml(["web", "worker"], { FOO: "bar" }); + expect(yaml).toContain(' web:\n environment:\n FOO: "bar"\n'); + expect(yaml).toContain(' worker:\n environment:\n FOO: "bar"\n'); + }); + + test("skips blank service names", () => { + const yaml = buildComposeEnvOverrideYaml(["web", " ", ""], { FOO: "bar" }); + expect((yaml.match(/environment:/g) ?? []).length).toBe(1); + }); +}); diff --git a/apps/api/src/orchestrator/__tests__/compose.test.ts b/apps/api/src/orchestrator/__tests__/compose.test.ts index 50b3206..2b6b629 100644 --- a/apps/api/src/orchestrator/__tests__/compose.test.ts +++ b/apps/api/src/orchestrator/__tests__/compose.test.ts @@ -122,7 +122,7 @@ services: }); }); -describe("spawnComposeCommand env isolation", () => { +describe("spawnComposeCommand env merge with project env vars", () => { let originalSpawn: any; afterEach(() => { @@ -132,10 +132,9 @@ describe("spawnComposeCommand env isolation", () => { } }); - test("does not pass process.env to docker compose subprocess", async () => { - process.env.DATABASE_URL = "postgresql://dequel:secret@localhost:5432/dequel"; - process.env.PORT = "3001"; - process.env.WORKSPACE_ROOT = "./workspace"; + test("passes project env vars alongside process.env to docker compose subprocess", async () => { + process.env.IDEK_DATABASE_URL = "postgresql://dequel:secret@localhost:5432/dequel"; + process.env.IDEK_WORKSPACE_ROOT = "./workspace"; let capturedEnv: Record | undefined; originalSpawn = (await import("node:child_process")).spawn; @@ -165,21 +164,18 @@ describe("spawnComposeCommand env isolation", () => { } expect(capturedEnv).toBeDefined(); - expect(capturedEnv).not.toHaveProperty("DATABASE_URL"); - expect(capturedEnv).not.toHaveProperty("PORT"); - expect(capturedEnv).not.toHaveProperty("WORKSPACE_ROOT"); expect(capturedEnv).toHaveProperty("MY_APP_VAR", "hello"); expect(capturedEnv).toHaveProperty("ANOTHER_VAR", "world"); + expect(capturedEnv).toHaveProperty("IDEK_DATABASE_URL", "postgresql://dequel:secret@localhost:5432/dequel"); + expect(capturedEnv).toHaveProperty("PATH"); } finally { rmSync(dir, { recursive: true, force: true }); - delete process.env.DATABASE_URL; - delete process.env.PORT; - delete process.env.WORKSPACE_ROOT; + delete process.env.IDEK_DATABASE_URL; + delete process.env.IDEK_WORKSPACE_ROOT; } }); - - test("passes empty env when no project env vars provided", async () => { - process.env.DATABASE_URL = "postgresql://dequel:secret@localhost:5432/dequel"; + test("passes process.env through when no project env vars provided", async () => { + process.env.IDEK_DATABASE_URL = "postgresql://dequel:secret@localhost:5432/dequel"; let capturedEnv: Record | undefined; const originalSpawn = (await import("node:child_process")).spawn; @@ -208,10 +204,11 @@ describe("spawnComposeCommand env isolation", () => { } expect(capturedEnv).toBeDefined(); - expect(capturedEnv).toEqual({}); + expect(capturedEnv).toHaveProperty("IDEK_DATABASE_URL", "postgresql://dequel:secret@localhost:5432/dequel"); + expect(capturedEnv).toHaveProperty("PATH"); } finally { rmSync(dir, { recursive: true, force: true }); - delete process.env.DATABASE_URL; + delete process.env.IDEK_DATABASE_URL; } }); }); diff --git a/apps/api/src/orchestrator/__tests__/railpack-config.test.ts b/apps/api/src/orchestrator/__tests__/railpack-config.test.ts index 6ef2940..74bf0a9 100644 --- a/apps/api/src/orchestrator/__tests__/railpack-config.test.ts +++ b/apps/api/src/orchestrator/__tests__/railpack-config.test.ts @@ -56,4 +56,62 @@ describe("generateDynamicRailpackJson static subdirectory detection", () => { cleanup(); } }); + + describe("vite SPA without server entry", () => { + test("serves the build output statically instead of node dist/index.js", async () => { + const { dir, cleanup } = setupWorkspace(); + try { + writeFileSync( + join(dir, "client", "package.json"), + JSON.stringify({ + name: "spa", + scripts: { build: "vite build" }, + devDependencies: { vite: "^5.0.0", react: "^18.0.0" }, + }), + ); + const logs: string[] = []; + await generateDynamicRailpackJson(dir, "client", null, null, null, async (line) => logs.push(line)); + const parsedConfig = JSON.parse(readFileSync(join(dir, "railpack.json"), "utf8")); + expect(parsedConfig.deploy?.startCommand).toBe("bun dequel-serve.js"); + expect(existsSync(join(dir, "dequel-serve.js"))).toBe(true); + expect(parsedConfig.steps?.build?.commands?.join(" ")).toContain("run build"); + } finally { + cleanup(); + } + }); + + test("keeps node dist/index.js fallback for vite apps with a server framework", async () => { + const { dir, cleanup } = setupWorkspace(); + try { + writeFileSync( + join(dir, "client", "package.json"), + JSON.stringify({ + name: "ssr", + scripts: { build: "vite build" }, + dependencies: { vite: "^5.0.0", express: "^4.0.0" }, + }), + ); + await generateDynamicRailpackJson(dir, "client", null, null, null, async () => {}); + const parsedConfig = JSON.parse(readFileSync(join(dir, "railpack.json"), "utf8")); + expect(parsedConfig.deploy?.startCommand).toBe("cd client && node dist/index.js"); + } finally { + cleanup(); + } + }); + + test("keeps node dist/index.js fallback for node apps that build but have no vite", async () => { + const { dir, cleanup } = setupWorkspace(); + try { + writeFileSync( + join(dir, "client", "package.json"), + JSON.stringify({ name: "plain", scripts: { build: "tsc" } }), + ); + await generateDynamicRailpackJson(dir, "client", null, null, null, async () => {}); + const parsedConfig = JSON.parse(readFileSync(join(dir, "railpack.json"), "utf8")); + expect(parsedConfig.deploy?.startCommand).toBe("cd client && node dist/index.js"); + } finally { + cleanup(); + } + }); + }); }); diff --git a/apps/api/src/orchestrator/__tests__/runtime-instance.test.ts b/apps/api/src/orchestrator/__tests__/runtime-instance.test.ts new file mode 100644 index 0000000..5c238c9 --- /dev/null +++ b/apps/api/src/orchestrator/__tests__/runtime-instance.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, test } from "bun:test"; +import { evaluateInstanceState } from "../runtime"; + +describe("evaluateInstanceState", () => { + test("retries when container does not exist yet", () => { + const verdict = evaluateInstanceState({ exists: false, status: null }); + expect(verdict.action).toBe("retry"); + }); + + test("returns when running", () => { + const verdict = evaluateInstanceState({ exists: true, status: "running" }); + expect(verdict.action).toBe("return"); + }); + + test("fatal with logs when exited during initial wait", () => { + const verdict = evaluateInstanceState({ exists: true, status: "exited", logs: "Error: MODULE_NOT_FOUND" }); + expect(verdict.action).toBe("fatal"); + if (verdict.action === "fatal") expect(verdict.error.message).toContain("MODULE_NOT_FOUND"); + }); + + test("fatal with trimmed logs and placeholder when exited without output", () => { + const verdict = evaluateInstanceState({ exists: true, status: "exited", logs: "\n\n" }); + expect(verdict.action).toBe("fatal"); + if (verdict.action === "fatal") expect(verdict.error.message).toContain("*No logs produced*"); + }); + + test("fatal wording differs for stability-phase crash", () => { + const verdict = evaluateInstanceState({ exists: true, status: "exited", logs: "boom" }, "stability"); + expect(verdict.action).toBe("fatal"); + if (verdict.action === "fatal") expect(verdict.error.message).toContain("crashed shortly after starting"); + }); + + test("retries with note while still initializing", () => { + const verdict = evaluateInstanceState({ exists: true, status: "created" }); + expect(verdict.action).toBe("retry"); + if (verdict.action === "retry") expect(verdict.note).toContain("initializing"); + }); + + test("retries on unknown states like restarting", () => { + const verdict = evaluateInstanceState({ exists: true, status: "restarting" }); + expect(verdict.action).toBe("retry"); + expect(evaluateInstanceState({ exists: true, status: "running" }, "restart").action).toBe("return"); + }); +}); diff --git a/apps/api/src/orchestrator/compose-deploy.ts b/apps/api/src/orchestrator/compose-deploy.ts index 218f7e1..fbbd773 100644 --- a/apps/api/src/orchestrator/compose-deploy.ts +++ b/apps/api/src/orchestrator/compose-deploy.ts @@ -4,6 +4,7 @@ import { config } from "../utils/config"; import { dockerBin } from "../utils/docker-bin"; import { buildCaddySnippet } from "../utils/domain-verifier"; import { deployWithCompose, destroyComposeStack, getComposeContainerNames, parseAllComposeServices } from "./compose"; +import { buildComposeEnvOverrideYaml } from "./compose-env"; import { reloadCaddy, tryRun } from "./runtime"; export interface ComposeDeployResult { @@ -47,10 +48,25 @@ export const deployComposeStack = async (options: ComposeDeployOptions): Promise await destroyComposeStack(`deploy-${oldDeploymentId}`); } - await deployWithCompose(workspacePath, `deploy-${deploymentId}`, onLog, sourceDir, envVars, signal); - const allServices = parseAllComposeServices(workspacePath, sourceDir, composeService, composePort); + let overridePath: string | null = null; + if (envVars && Object.keys(envVars).length > 0) { + const overrideYaml = buildComposeEnvOverrideYaml( + allServices.map((s) => s.serviceName), + envVars, + ); + if (overrideYaml) { + overridePath = join(workspacePath, "dequel-env-override.yml"); + await writeFile(overridePath, overrideYaml); + await onLog( + `Injecting ${Object.keys(envVars).length} env var(s) into ${allServices.length} compose service(s) via override file.`, + ); + } + } + + await deployWithCompose(workspacePath, `deploy-${deploymentId}`, onLog, sourceDir, envVars, signal, overridePath); + const composeContainers = await getComposeContainerNames(`deploy-${deploymentId}`); const containerFor = (serviceName: string) => composeContainers.get(serviceName) || `deploy-${deploymentId}-${serviceName}-1`; diff --git a/apps/api/src/orchestrator/compose-env.ts b/apps/api/src/orchestrator/compose-env.ts new file mode 100644 index 0000000..ef04820 --- /dev/null +++ b/apps/api/src/orchestrator/compose-env.ts @@ -0,0 +1,16 @@ +const yamlScalar = (value: string): string => JSON.stringify(value); + +export const buildComposeEnvOverrideYaml = (serviceNames: string[], envVars: Record): string => { + const keys = Object.keys(envVars); + if (serviceNames.length === 0 || keys.length === 0) return ""; + const lines = ["services:"]; + for (const serviceName of serviceNames) { + if (!serviceName.trim()) continue; + lines.push(` ${serviceName}:`); + lines.push(" environment:"); + for (const key of keys) { + lines.push(` ${key}: ${yamlScalar(envVars[key])}`); + } + } + return `${lines.join("\n")}\n`; +}; diff --git a/apps/api/src/orchestrator/compose.ts b/apps/api/src/orchestrator/compose.ts index de13f74..81e95a7 100644 --- a/apps/api/src/orchestrator/compose.ts +++ b/apps/api/src/orchestrator/compose.ts @@ -206,7 +206,7 @@ const spawnComposeCommand = ( const isPosix = process.platform !== "win32"; const child = spawn(dockerBin, ["compose", ...args], { cwd, - env: envVars || {}, + env: { ...(envVars || {}), ...process.env }, stdio: ["ignore", "pipe", "pipe"], detached: true, }); @@ -310,6 +310,7 @@ export const deployWithCompose = async ( sourceDir?: string | null, envVars?: Record, signal?: AbortSignal, + overrideFilePath?: string | null, ): Promise => { const composeFile = findComposeFilePath(workspacePath, sourceDir); if (!composeFile) { @@ -319,13 +320,11 @@ export const deployWithCompose = async ( const cwd = dirname(composeFile); await onLog(`Starting Docker Compose services (project: ${projectName})...`); - const res = await spawnComposeCommand( - ["-f", composeFile, "-p", projectName, "up", "-d"], - cwd, - envVars, - onLog, - signal, - ); + const args = ["-f", composeFile]; + if (overrideFilePath) args.push("-f", overrideFilePath); + args.push("-p", projectName, "up", "-d"); + + const res = await spawnComposeCommand(args, cwd, envVars, onLog, signal); if (res.code !== 0) { throw new Error(`docker compose up failed: ${res.stderr || res.stdout}`); } diff --git a/apps/api/src/orchestrator/pipeline.ts b/apps/api/src/orchestrator/pipeline.ts index e65dbde..a481b44 100644 --- a/apps/api/src/orchestrator/pipeline.ts +++ b/apps/api/src/orchestrator/pipeline.ts @@ -8,6 +8,7 @@ import { deleteRoutesByDeployment, getDeploymentById, getProjectById, + getProjectGithubToken, getServerById, listAllDatabases, listDeployments, @@ -249,6 +250,7 @@ export class PipelineOrchestrator { deployment.sourceRef, deployment.branch ?? undefined, deployment.commitSha ?? undefined, + deployment.projectId ? await getProjectGithubToken(deployment.projectId).catch(() => null) : null, ); const sha = await getHeadSha(workspacePath); if (sha) { diff --git a/apps/api/src/orchestrator/railpack-config-utils.ts b/apps/api/src/orchestrator/railpack-config-utils.ts index 77789ac..8404a30 100644 --- a/apps/api/src/orchestrator/railpack-config-utils.ts +++ b/apps/api/src/orchestrator/railpack-config-utils.ts @@ -64,6 +64,44 @@ const fixPnpmWorkspaceIfNeeded = async (workspacePath: string, onLog: (line: str } }; +const SERVER_FRAMEWORK_PREFIXES = [ + "express", + "fastify", + "koa", + "hono", + "@hono/", + "elysia", + "@elysiajs/", + "polka", + "socket.io", + "@remix-run/", + "nitropack", + "next", + "nuxt", +]; + +const packageDependsOnServerFramework = (packageJson: any): boolean => { + const sections = [packageJson.dependencies, packageJson.devDependencies]; + for (const section of sections) { + if (!section) continue; + for (const name of Object.keys(section)) { + if (SERVER_FRAMEWORK_PREFIXES.some((prefix) => name === prefix || name.startsWith(prefix))) return true; + } + } + return false; +}; + +const packageHasVite = (packageJson: any): boolean => { + const sections = [packageJson.dependencies, packageJson.devDependencies]; + for (const section of sections) { + if (!section) continue; + for (const name of Object.keys(section)) { + if (name === "vite" || name.startsWith("@vitejs/") || name.startsWith("vite-")) return true; + } + } + return false; +}; + export const generateDynamicRailpackJson = async ( workspace: string, sourceDir: string | null, @@ -89,6 +127,7 @@ export const generateDynamicRailpackJson = async ( }; let configured = false; + let forceStaticServe = false; if (cleanSourceDir && !(await Bun.file(join(workspace, "package.json")).exists())) { const sourcePkgExists = await Bun.file(join(workspace, cleanSourceDir, "package.json")).exists(); @@ -177,9 +216,14 @@ export const generateDynamicRailpackJson = async ( } else if (scripts.server) { config.deploy.startCommand = cleanSourceDir ? `cd ${cleanSourceDir} && ${pm} run server` : `${pm} run server`; } else { - config.deploy.startCommand = cleanSourceDir - ? `cd ${cleanSourceDir} && node dist/index.js` - : "node dist/index.js"; + if (scripts.build && packageHasVite(packageJson) && !packageDependsOnServerFramework(packageJson)) { + forceStaticServe = true; + await onLog("Detected Vite frontend without a server entry — serving the build output statically"); + } else { + config.deploy.startCommand = cleanSourceDir + ? `cd ${cleanSourceDir} && node dist/index.js` + : "node dist/index.js"; + } } configured = true; @@ -326,6 +370,7 @@ export const generateDynamicRailpackJson = async ( if ( projectType === "static" || + forceStaticServe || (!configured && ((await Bun.file(join(buildDir, "index.html")).exists()) || (await Bun.file(join(workspace, "index.html")).exists()))) diff --git a/apps/api/src/orchestrator/runtime.ts b/apps/api/src/orchestrator/runtime.ts index ff2cfd8..832cec0 100644 --- a/apps/api/src/orchestrator/runtime.ts +++ b/apps/api/src/orchestrator/runtime.ts @@ -48,6 +48,30 @@ const getCaddyContainer = async (): Promise => { return name; }; +export interface ContainerInstanceState { + exists: boolean; + status: string | null; + logs?: string | null; +} + +export type Phase = "initial" | "stability" | "restart"; + +export type InstanceVerdict = + | { action: "return" } + | { action: "retry"; note?: string } + | { action: "fatal"; error: Error }; + +export const evaluateInstanceState = (state: ContainerInstanceState, phase: Phase = "initial"): InstanceVerdict => { + if (!state.exists) return { action: "retry", note: "not created yet" }; + const logs = (state.logs ?? "").trim() || "*No logs produced*"; + if (state.status === "running") return { action: "return" }; + if (state.status === "exited") { + const reason = phase === "stability" ? "crashed shortly after starting" : "exited immediately"; + return { action: "fatal", error: new Error(`Container ${reason}. Logs:\n${logs}`) }; + } + return { action: "retry", note: state.status === "created" ? "still initializing" : undefined }; +}; + const waitForRunningContainer = async ( containerName: string, retries: number, @@ -58,26 +82,30 @@ const waitForRunningContainer = async ( const exists = await run(dockerBin, ["inspect", "-f", "{{.Id}}", containerName]) .then(() => true) .catch(() => false); - if (!exists) { - await new Promise((r) => setTimeout(r, 500)); - continue; - } - const status = (await run(dockerBin, ["inspect", "-f", "{{.State.Status}}", containerName])).trim(); - if (status === "running") { + const status = exists + ? (await run(dockerBin, ["inspect", "-f", "{{.State.Status}}", containerName])).trim() + : null; + const logs = + exists && status === "exited" + ? await run(dockerBin, ["logs", "--tail", "50", containerName]).catch(() => null) + : null; + const verdict = evaluateInstanceState({ exists, status, logs }); + if (verdict.action === "return") { await new Promise((r) => setTimeout(r, 2000)); const stabilityStatus = (await run(dockerBin, ["inspect", "-f", "{{.State.Status}}", containerName])).trim(); - if (stabilityStatus !== "running") { - const logs = await run(dockerBin, ["logs", "--tail", "50", containerName]).catch(() => "no logs available"); - throw new Error(`Container crashed shortly after starting. Logs:\n${logs}`); - } + const stabilityLogs = + stabilityStatus !== "running" + ? await run(dockerBin, ["logs", "--tail", "50", containerName]).catch(() => null) + : null; + const stabilityVerdict = evaluateInstanceState( + { exists: true, status: stabilityStatus, logs: stabilityLogs }, + "stability", + ); + if (stabilityVerdict.action === "fatal") throw stabilityVerdict.error; return; } - if (status === "exited") { - let logs = await run(dockerBin, ["logs", "--tail", "50", containerName]).catch(() => "no logs available"); - logs = logs.trim() || "*No logs produced*"; - throw new Error(`Container ${containerName} exited immediately. Logs:\n${logs}`); - } - if (status === "created") await onLog(`Container ${containerName} still initializing...`); + if (verdict.action === "fatal") throw verdict.error; + if (verdict.note) await onLog(`Container ${containerName} ${verdict.note}...`); } catch (e: any) { if (e.message?.includes("exited immediately") || e.message?.includes("crashed shortly")) throw e; } @@ -87,6 +115,37 @@ const waitForRunningContainer = async ( await tryRun(dockerBin, ["network", "disconnect", "-f", config.dockerNetwork, containerName]); await tryRun(dockerBin, ["start", containerName]); await tryRun(dockerBin, ["network", "connect", config.dockerNetwork, containerName]); + await verifyRunningAfterStart(containerName, onLog); +}; + +const verifyRunningAfterStart = async ( + containerName: string, + onLog: (line: string) => Promise, +): Promise => { + let lastStatus: string | null = null; + let lastLogs: string | null = null; + for (let attempt = 0; attempt < 10; attempt++) { + await new Promise((r) => setTimeout(r, 1000)); + try { + const exists = await run(dockerBin, ["inspect", "-f", "{{.Id}}", containerName]) + .then(() => true) + .catch(() => false); + lastStatus = exists ? (await run(dockerBin, ["inspect", "-f", "{{.State.Status}}", containerName])).trim() : null; + if (lastStatus === "running") { + await onLog(`Container ${containerName} is running after restart`); + return "running"; + } + lastLogs = exists ? await run(dockerBin, ["logs", "--tail", "50", containerName]).catch(() => null) : null; + const verdict = evaluateInstanceState({ exists, status: lastStatus, logs: lastLogs }, "restart"); + if (verdict.action === "fatal") throw verdict.error; + } catch (e: any) { + if (e.message?.includes("exited immediately") || e.message?.includes("crashed shortly")) throw e; + } + } + const logs = (lastLogs ?? "").trim() || "*No logs produced*"; + throw new Error( + `Container ${containerName} did not reach running state even after docker start (status: ${lastStatus ?? "unknown"}). Logs:\n${logs}`, + ); }; export const ensureContainerRunning = async (containerName: string) => { diff --git a/apps/api/src/orchestrator/source.ts b/apps/api/src/orchestrator/source.ts index d84fae4..4ceb15b 100644 --- a/apps/api/src/orchestrator/source.ts +++ b/apps/api/src/orchestrator/source.ts @@ -22,27 +22,51 @@ const run = async (cmd: string, args: string[], cwd?: string) => { const isValidSha = (s: string) => /^[0-9a-f]{7,40}$/i.test(s); +export const isGithubHttpsUrl = (gitUrl: string) => /^https:\/\/github\.com\//i.test(gitUrl.trim()); + +export const redactGitUrl = (gitUrl: string) => gitUrl.replace(/^(https:\/\/)[^/@]+@([^/]+)/i, "$1$2"); + +export const buildAuthenticatedGitUrl = (gitUrl: string, token?: string | null) => { + if (!token) return gitUrl; + const trimmed = gitUrl.trim(); + if (!isGithubHttpsUrl(trimmed)) return gitUrl; + if (trimmed.includes("@")) return gitUrl; + return trimmed.replace(/^https:\/\/github\.com\//i, `https://x-access-token:${token}@github.com/`); +}; + +export const resolveProjectGitUrl = async (projectId: string | null | undefined, gitUrl: string): Promise => { + if (!projectId || !isGithubHttpsUrl(gitUrl)) return gitUrl; + try { + const { getProjectGithubToken } = await import("../db/repo/projects"); + return buildAuthenticatedGitUrl(gitUrl, await getProjectGithubToken(projectId)); + } catch { + return gitUrl; + } +}; + export const prepareSourceWorkspace = async ( deploymentId: string, gitUrl: string, branch?: string, commitSha?: string, + token?: string | null, ) => { const root = join(config.workspaceRoot, deploymentId); await rm(root, { recursive: true, force: true }); await mkdir(root, { recursive: true }); + const authedUrl = buildAuthenticatedGitUrl(gitUrl, token); if (commitSha) { if (!isValidSha(commitSha)) { throw new Error(`Invalid commit SHA: ${commitSha}`); } await run("git", ["init"], root); - await run("git", ["remote", "add", "origin", gitUrl], root); + await run("git", ["remote", "add", "origin", authedUrl], root); await run("git", ["fetch", "--depth", "1", "origin", commitSha], root); await run("git", ["checkout", "FETCH_HEAD"], root); } else { const args = ["clone", "--depth", "1"]; if (branch) args.push("--branch", branch); - args.push(gitUrl, root); + args.push(authedUrl, root); await run("git", args); } return root; @@ -75,10 +99,12 @@ export const getHeadSha = async (repoPath: string): Promise => { } }; -export const getRemoteSha = async (gitUrl: string, branch?: string): Promise => { +export const getRemoteSha = async (gitUrl: string, branch?: string, token?: string | null): Promise => { try { const ref = branch ? `refs/heads/${branch}` : "HEAD"; - const res = await safeSpawn("git", ["ls-remote", gitUrl, ref], { timeoutMs: 15_000 }); + const res = await safeSpawn("git", ["ls-remote", buildAuthenticatedGitUrl(gitUrl, token), ref], { + timeoutMs: 15_000, + }); if (res.code !== 0) return null; const match = res.stdout.trim().split(/\s+/)[0]; return match || null; diff --git a/apps/api/src/scaling/__tests__/engine.test.ts b/apps/api/src/scaling/__tests__/engine.test.ts index 4273c98..872d963 100644 --- a/apps/api/src/scaling/__tests__/engine.test.ts +++ b/apps/api/src/scaling/__tests__/engine.test.ts @@ -272,6 +272,47 @@ describe("evaluate", () => { expect(mockWriteFilePath).toBeNull(); }); + it("does not advance lastScaleUp when already at max replicas", async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, maxReplicas: 2, cooldownSeconds: 0 }); + const lastScaleUpBefore = String(Date.now() - 60_000); + mockRedisData = { highCpuSince: String(Date.now() - 10000), lastScaleUp: lastScaleUpBefore }; + mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n"; + mockRunImpl = async () => '{"CPUPerc":"85%","MemUsage":"128MiB / 512MiB"}'; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockRedisData.lastScaleUp).toBe(lastScaleUpBefore); + expect(mockRedisData.highCpuSince).toBeFalsy(); + }); + + it("keeps accruing the low-CPU timer between threshold and 30%", async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cpuThresholdPercent: 1, cooldownSeconds: 0 }); + const lowCpuSinceBefore = String(Date.now() - 240_000); + mockRedisData = { lowCpuSince: lowCpuSinceBefore, highCpuSince: "0", lastScaleUp: "0" }; + mockRunImpl = async () => '{"CPUPerc":"22%","MemUsage":"64MiB / 512MiB"}'; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockRedisData.lowCpuSince).toBe(lowCpuSinceBefore); + }); + + it("scales down in the overlap zone (threshold < cpu < 30) once idle for 5 min", async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cpuThresholdPercent: 1, cooldownSeconds: 0, minReplicas: 1 }); + mockRedisData = { lowCpuSince: String(Date.now() - 310_000), lastScaleDown: "0" }; + mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n"; + mockTryRunImpl = async () => ""; + mockRunImpl = async (_cmd: string, args: string[]) => { + if (args.includes("stats")) return '{"CPUPerc":"22%","MemUsage":"64MiB / 512MiB"}'; + if (args[0] === "stop") return ""; + if (args[0] === "rm") return ""; + if (args[0] === "ps") return "caddy-abc123\n"; + if (args.includes("reload")) return ""; + return ""; + }; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockWriteFilePath).toBeTruthy(); + expect(mockWriteContent).not.toContain("replica-2"); + }); + it("does not scale down when CPU is moderate", async () => { mockRunImpl = async () => '{"CPUPerc":"50%","MemUsage":"128MiB / 512MiB"}'; await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); diff --git a/apps/api/src/scaling/engine.ts b/apps/api/src/scaling/engine.ts index 94d97cb..3b3ab10 100644 --- a/apps/api/src/scaling/engine.ts +++ b/apps/api/src/scaling/engine.ts @@ -112,12 +112,18 @@ class ScalingEngine { state.highCpuSince = now; } else if (now - state.highCpuSince > policy.cooldownSeconds * 1000) { if (now - state.lastScaleUp > policy.cooldownSeconds * 1000) { - await this.scaleUp(dep, policy.maxReplicas, slug, target, project?.cpuLimit, project?.memoryLimitMb); - state.lastScaleUp = now; + const scaled = await this.scaleUp( + dep, + policy.maxReplicas, + slug, + target, + project?.cpuLimit, + project?.memoryLimitMb, + ); + if (scaled) state.lastScaleUp = now; state.highCpuSince = null; } } - state.lowCpuSince = null; } else { state.highCpuSince = null; } @@ -306,9 +312,9 @@ class ScalingEngine { target: Target, cpuLimit?: number | null, memoryLimitMb?: number | null, - ) { + ): Promise { const currentReplicas = await this.getCurrentReplicas(slug, target, dep); - if (currentReplicas >= maxReplicas) return; + if (currentReplicas >= maxReplicas) return false; const newReplicaNum = currentReplicas + 1; const containerName = `deploy-${dep.id}-replica-${newReplicaNum}`; @@ -317,7 +323,7 @@ class ScalingEngine { if (target.mode === "agent") { await this.enqueueScaleJob(dep, target.server!, "up", newReplicaNum, slug); - return; + return true; } // Get the original container's image @@ -328,7 +334,7 @@ class ScalingEngine { await execDockerSshCommand(target.server!, ["inspect", "-f", "{{.Config.Image}}", dep.containerName!]) ).stdout.trim()) : await run(dockerBin, ["inspect", "-f", "{{.Config.Image}}", dep.containerName!]).catch(() => ""); - if (!imageTag) return; + if (!imageTag) return false; // Get env vars from original const envJson = @@ -389,12 +395,13 @@ class ScalingEngine { } } catch (err) { console.error(`[Scaling] Failed to create replica ${containerName}:`, err); - return; + return false; } // Update Caddy route to include the new replica await this.updateCaddyRoute(slug, dep, newReplicaNum, target); console.log(`[Scaling] Replica ${containerName} started`); + return true; } private async scaleDown( @@ -514,8 +521,10 @@ class ScalingEngine { // Build proxy targets: primary + all replicas const primaryName = dep.containerName || `${slug}-${dep.id.slice(0, 8)}`; const targets = [`${primaryName}:${port}`]; + const containerNames = [primaryName]; for (let i = 2; i <= replicaCount; i++) { targets.push(`deploy-${dep.id}-replica-${i}:${port}`); + containerNames.push(`deploy-${dep.id}-replica-${i}`); } const baseDomain = config.caddyBaseDomain === "localhost" ? `${config.caddyBaseDomain}:80` : config.caddyBaseDomain; @@ -523,15 +532,10 @@ class ScalingEngine { await import("../utils/ingress"); const ingressServer = await getIngressServer(); const viaIngress = shouldRouteViaIngress(target.server ?? null, ingressServer); - const { caddyReverseProxy, caddySite } = await import("../utils/caddy-site"); + const { buildCaddySnippet } = await import("../utils/domain-verifier"); const caddySnippet = viaIngress - ? projectServerSite( - `${slug}.${baseDomain}`, - port, - targets.map((t) => t.split(":")[0]), - true, - ) - : caddySite(`${slug}.${baseDomain}`, caddyReverseProxy(targets.join(" "))); + ? projectServerSite(`${slug}.${baseDomain}`, port, containerNames, true) + : await buildCaddySnippet(slug, primaryName, dep.projectId, undefined, port, { containerNames }); if (target.mode === "ssh") { await syncRemoteCaddyRoute(target.server!, `${slug}.caddy`, caddySnippet); diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts index 89f9b99..dd6d784 100644 --- a/apps/api/src/types.ts +++ b/apps/api/src/types.ts @@ -248,7 +248,6 @@ export interface CreateServerInput { authToken?: string; mode?: ServerMode; sshUser?: string; - sshKey?: string; sshKeyId?: string; sshPassword?: string; } diff --git a/apps/api/src/utils/domain-verifier.ts b/apps/api/src/utils/domain-verifier.ts index 05c7d2f..0baceba 100644 --- a/apps/api/src/utils/domain-verifier.ts +++ b/apps/api/src/utils/domain-verifier.ts @@ -174,6 +174,7 @@ export const removeFromCaddyRoute = async ( export interface BuildSnippetOpts { baseDomain?: string; listEnvVarsFn?: typeof listEnvironmentVariablesForDeploy; + containerNames?: string[]; } export const buildCaddySnippet = async ( @@ -245,7 +246,9 @@ export const buildCaddySnippet = async ( } } - const primaryBlock = caddySite(defaultDomains.join(", "), caddyReverseProxy(`${containerName}:${port}`)); + const upstreamNames = opts?.containerNames?.length ? opts.containerNames : [containerName]; + const upstreams = upstreamNames.map((n) => `${n}:${port}`).join(" "); + const primaryBlock = caddySite(defaultDomains.join(", "), caddyReverseProxy(upstreams)); return [primaryBlock, ...customBlocks].join("\n"); }; diff --git a/apps/web/src/api/databases.ts b/apps/web/src/api/databases.ts index 4bc0918..9146827 100644 --- a/apps/web/src/api/databases.ts +++ b/apps/web/src/api/databases.ts @@ -1,4 +1,11 @@ -import type { BackupJob, BackupStorageSettingsData, Database, QueryExecResult, TableInfo } from "../types"; +import type { + BackupJob, + BackupStorageSettingsData, + Database, + QueryExecResult, + RestoreTestRecord, + TableInfo, +} from "../types"; import { apiFetch } from "./http"; export const listAllDatabases = () => apiFetch("/databases"); @@ -72,6 +79,18 @@ export const triggerBackup = (targetId = "internal") => }); export const restoreBackup = (id: string) => apiFetch<{ restored: boolean }>(`/backups/${id}/restore`, { method: "POST" }); +export const triggerRestoreTest = (backupId: string) => + apiFetch(`/backups/${backupId}/restore-test`, { method: "POST" }); +export const listRestoreTests = ( + params: { backupId?: string; targetId?: string; status?: string; kind?: string; limit?: number } = {}, +) => { + const qs = new URLSearchParams(); + for (const [key, value] of Object.entries(params)) { + if (value !== undefined) qs.set(key, String(value)); + } + const query = qs.toString(); + return apiFetch(`/backups/restore-tests${query ? `?${query}` : ""}`); +}; export const deleteBackup = (id: string) => apiFetch<{ deleted: boolean }>(`/backups/${id}`, { method: "DELETE" }); export const getBackupStorageSettings = () => apiFetch("/settings/backup"); export const updateBackupStorageSettings = (data: BackupStorageSettingsData) => diff --git a/apps/web/src/components/databases/DatabaseBackupManager.tsx b/apps/web/src/components/databases/DatabaseBackupManager.tsx index 624c05a..b41f513 100644 --- a/apps/web/src/components/databases/DatabaseBackupManager.tsx +++ b/apps/web/src/components/databases/DatabaseBackupManager.tsx @@ -1,8 +1,8 @@ import { useQuery } from "@tanstack/react-query"; -import { Clock, Download, HardDrive, RefreshCw, ShieldAlert, ShieldCheck, Trash2 } from "lucide-react"; +import { Clock, FlaskConical, HardDrive, RefreshCw, ShieldAlert, ShieldCheck, Trash2 } from "lucide-react"; import { useState } from "react"; import * as api from "../../api/client"; -import type { BackupJob, Database } from "../../types"; +import type { BackupJob, Database, RestoreTestRecord } from "../../types"; import { Badge } from "../ui/badge"; import { Button } from "../ui/button"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "../ui/card"; @@ -13,6 +13,24 @@ interface DatabaseBackupManagerProps { database: Database; } +const timeAgo = (iso: string | null | undefined): string => { + if (!iso) return ""; + const minutes = Math.floor((Date.now() - new Date(iso).getTime()) / 60_000); + if (minutes < 1) return "just now"; + if (minutes < 60) return `${minutes}m ago`; + const hours = Math.floor(minutes / 60); + if (hours < 24) return `${hours}h ago`; + return `${Math.floor(hours / 24)}d ago`; +}; + +const eventChipClass = (status: string, tone: "green" | "red" | "amber") => { + const base = "inline-flex items-center gap-1 rounded-md px-1.5 py-0.5 text-[10px] font-mono uppercase"; + if (status === "running") return `${base} border border-amber-500/30 bg-amber-500/10 text-amber-400`; + return tone === "green" + ? `${base} border border-emerald-500/30 bg-emerald-500/10 text-emerald-400` + : `${base} border border-red-500/30 bg-red-500/10 text-red-400`; +}; + export function DatabaseBackupManager({ database }: DatabaseBackupManagerProps) { const [isTriggering, setIsTriggering] = useState(false); const [restoringBackup, setRestoringBackup] = useState(null); @@ -21,6 +39,8 @@ export function DatabaseBackupManager({ database }: DatabaseBackupManagerProps) const [deletingBackup, setDeletingBackup] = useState(null); const [isDeleting, setIsDeleting] = useState(false); const [error, setError] = useState(null); + const [testingBackupId, setTestingBackupId] = useState(null); + const [testNotice, setTestNotice] = useState<{ ok: boolean; text: string } | null>(null); const [page, setPage] = useState(1); const pageSize = 5; @@ -30,6 +50,22 @@ export function DatabaseBackupManager({ database }: DatabaseBackupManagerProps) refetchInterval: 10_000, }); + const { data: restoreEvents = [], refetch: refetchRestoreEvents } = useQuery({ + queryKey: ["restore-tests", database.id], + queryFn: () => api.listRestoreTests({ targetId: database.id, limit: 100 }).catch(() => []), + refetchInterval: 10_000, + }); + + const latestByBackup = (kind: "test" | "restore") => { + const map: Record = {}; + for (const event of restoreEvents) { + if (event.kind === kind && !map[event.backupId]) map[event.backupId] = event; + } + return map; + }; + const latestTests = latestByBackup("test"); + const latestRestores = latestByBackup("restore"); + const instanceBackups = allBackups.filter((b) => b.targetId === database.id); const paginatedBackups = instanceBackups.slice((page - 1) * pageSize, page * pageSize); @@ -53,17 +89,39 @@ export function DatabaseBackupManager({ database }: DatabaseBackupManagerProps) try { await api.restoreBackup(restoringBackup.id); setRestoreMessage("Instance database dump restored successfully."); + refetchRestoreEvents(); setTimeout(() => { setRestoringBackup(null); setRestoreMessage(null); }, 2000); } catch (err: any) { setRestoreMessage(`Restore failed: ${err.message}`); + refetchRestoreEvents(); } finally { setIsRestoring(false); } }; + const handleRestoreTest = async (job: BackupJob) => { + setTestingBackupId(job.id); + setTestNotice(null); + try { + await api.triggerRestoreTest(job.id); + setTestNotice({ + ok: true, + text: `Restore test passed for ${job.filename || job.id.slice(0, 8)}`, + }); + } catch (err: any) { + setTestNotice({ + ok: false, + text: `Restore test failed: ${err.message}`, + }); + } finally { + setTestingBackupId(null); + refetchRestoreEvents(); + } + }; + const handleDeleteBackup = async () => { if (!deletingBackup) return; setIsDeleting(true); @@ -116,6 +174,18 @@ export function DatabaseBackupManager({ database }: DatabaseBackupManagerProps) {error} )} + {testNotice && ( +
+ {testNotice.ok ? : } + {testNotice.text} +
+ )}
Snapshots ({instanceBackups.length}) @@ -154,18 +224,68 @@ export function DatabaseBackupManager({ database }: DatabaseBackupManagerProps) {job.sizeBytes ? `${(job.sizeBytes / (1024 * 1024)).toFixed(2)} MB` : "Pending"} {new Date(job.createdAt).toLocaleString()}
+ {(latestTests[job.id] || latestRestores[job.id]) && ( +
+ {latestTests[job.id] && ( + + {latestTests[job.id].status === "running" + ? "Testing…" + : latestTests[job.id].status === "passed" + ? "Restorable" + : "Test failed"} + {latestTests[job.id].status !== "running" && + ` · ${timeAgo(latestTests[job.id].completedAt)}`} + + )} + {latestRestores[job.id] && ( + + {latestRestores[job.id].status === "running" + ? "Restoring…" + : latestRestores[job.id].status === "passed" + ? "Restored" + : "Restore failed"} + {latestRestores[job.id].status !== "running" && + ` · ${timeAgo(latestRestores[job.id].completedAt)}`} + + )} +
+ )}
{job.status === "completed" && ( - + <> + + + )}
{st && (
@@ -172,7 +181,9 @@ export function DomainsTab({ projectId }: DomainsTabProps) { - + TLS {st.tlsOk ? "Secured" : "Pending"} @@ -213,7 +224,17 @@ export function DomainsTab({ projectId }: DomainsTabProps) { - +
+ + {d.cloudflareProxied && ( + + Cloudflare + + )} +
{(() => { const st = domainStatuses?.find((s) => s.domain === d.domain); @@ -222,8 +243,12 @@ export function DomainsTab({ projectId }: DomainsTabProps) { {st ? ( - - + + {st.dnsOk ? "Resolved" : "Unresolved"} @@ -234,8 +259,12 @@ export function DomainsTab({ projectId }: DomainsTabProps) { {st ? ( - - + + {st.tlsOk ? "Secured" : "Pending"} @@ -275,6 +304,8 @@ export function DomainsTab({ projectId }: DomainsTabProps) { targetPort={targetPort} setTargetPort={setTargetPort} isCompose={(project as any)?.buildType === "compose"} + cloudflareProxied={cloudflareProxied} + setCloudflareProxied={setCloudflareProxied} isAdding={isAdding} onAdd={add} /> diff --git a/apps/web/src/types/index.ts b/apps/web/src/types/index.ts index 9042a56..1983a81 100644 --- a/apps/web/src/types/index.ts +++ b/apps/web/src/types/index.ts @@ -142,6 +142,7 @@ export interface Domain { type: DomainType; validationStatus: DomainValidationStatus; sslStatus: SslStatus; + cloudflareProxied?: boolean; createdAt: string; updatedAt: string; } diff --git a/docker-compose.yml b/docker-compose.yml index f859b96..e48d520 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: volumes: - buildkit-data:/var/lib/buildkit networks: - dequel: + control: aliases: - buildkit healthcheck: @@ -81,6 +81,9 @@ services: retries: 100 start_period: 10s networks: + control: + aliases: + - api dequel: aliases: - api @@ -122,7 +125,7 @@ services: - /etc/shadow:/host-etc/shadow:ro - /etc/group:/host-etc/group:ro networks: - dequel: + control: aliases: - pam-auth healthcheck: @@ -190,6 +193,7 @@ services: retries: 5 start_period: 5s networks: + control: dequel: caddy: @@ -229,7 +233,7 @@ services: volumes: - redis-data:/data networks: - dequel: + control: aliases: - redis healthcheck: @@ -248,7 +252,7 @@ services: volumes: - postgres-data:/var/lib/postgresql/data networks: - dequel: + control: aliases: - postgres healthcheck: @@ -270,7 +274,7 @@ services: devices: - /dev/kmsg networks: - dequel: + control: aliases: - cadvisor healthcheck: @@ -303,6 +307,9 @@ services: cadvisor: condition: service_started networks: + control: + aliases: + - prometheus dequel: aliases: - prometheus @@ -329,7 +336,7 @@ services: - ./infra/monitoring/loki-config.yml:/etc/loki/loki-config.yml:ro - loki-data:/loki networks: - dequel: + control: aliases: - loki healthcheck: @@ -360,6 +367,7 @@ services: loki: condition: service_started networks: + control: dequel: grafana: @@ -381,6 +389,9 @@ services: loki: condition: service_started networks: + control: + aliases: + - grafana dequel: aliases: - grafana @@ -401,6 +412,8 @@ services: networks: dequel: name: dequel_net + control: + name: dequel_control volumes: buildkit-data: diff --git a/infra/caddy/Caddyfile b/infra/caddy/Caddyfile index 5d87cf5..fdc7382 100644 --- a/infra/caddy/Caddyfile +++ b/infra/caddy/Caddyfile @@ -2,9 +2,25 @@ email {$CADDY_EMAIL} } +(security_headers) { + header { + Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" + X-Content-Type-Options "nosniff" + X-Frame-Options "DENY" + Referrer-Policy "strict-origin-when-cross-origin" + Permissions-Policy "camera=(), microphone=(), geolocation=()" + -Server + } + request_body { + max_size 100MB + } +} + import /etc/caddy/routes/*.caddy {$CADDY_BASE_DOMAIN:localhost} { + import security_headers + log { output stdout format json @@ -14,7 +30,7 @@ import /etc/caddy/routes/*.caddy handle /api/* { reverse_proxy api:3001 { - trusted_proxies private_ranges + trusted_proxies private_ranges 173.245.48.0/20 103.21.244.0/22 103.22.200.0/22 103.31.4.0/22 141.101.64.0/18 108.162.192.0/18 190.93.240.0/20 188.114.96.0/20 197.234.240.0/22 198.41.128.0/17 162.158.0.0/15 104.16.0.0/13 104.24.0.0/14 172.64.0.0/13 131.0.72.0/22 2400:cb00::/32 2606:4700::/32 2803:f800::/32 2405:b500::/32 2405:8100::/32 2a06:98c0::/29 2c0f:f248::/32 } } @@ -34,11 +50,13 @@ import /etc/caddy/routes/*.caddy } http:// { + import security_headers + encode zstd gzip handle /api/* { reverse_proxy api:3001 { - trusted_proxies private_ranges + trusted_proxies private_ranges 173.245.48.0/20 103.21.244.0/22 103.22.200.0/22 103.31.4.0/22 141.101.64.0/18 108.162.192.0/18 190.93.240.0/20 188.114.96.0/20 197.234.240.0/22 198.41.128.0/17 162.158.0.0/15 104.16.0.0/13 104.24.0.0/14 172.64.0.0/13 131.0.72.0/22 2400:cb00::/32 2606:4700::/32 2803:f800::/32 2405:b500::/32 2405:8100::/32 2a06:98c0::/29 2c0f:f248::/32 } } diff --git a/infra/monitoring/loki-config.yml b/infra/monitoring/loki-config.yml index b3ac39d..cc6a006 100644 --- a/infra/monitoring/loki-config.yml +++ b/infra/monitoring/loki-config.yml @@ -27,7 +27,10 @@ storage_config: compactor: working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem limits_config: + retention_period: 2160h reject_old_samples: false - reject_old_samples_max_age: 168h + reject_old_samples_max_age: 2160h diff --git a/scripts/dequel b/scripts/dequel index 7e06bef..9173316 100755 --- a/scripts/dequel +++ b/scripts/dequel @@ -361,6 +361,7 @@ cmd_help() { echo " backup Create a backup of the database" echo " backup list List all backups" echo " backup restore Restore from a backup" + echo " firewall Manage host firewall (status, setup)" echo " uninstall Remove Dequel completely (config, images, volumes)" echo " --version Show version" echo " --help Show this help" @@ -383,6 +384,19 @@ cmd_help() { echo " DEQUEL_HOME Config directory (default: auto-detected from install path)" } +cmd_firewall() { + shift + local script="$DEQUEL_HOME/scripts/firewall-setup.sh" + if [ ! -f "$script" ]; then + script="$(dirname "$SCRIPT_PATH")/firewall-setup.sh" + fi + if [ -f "$script" ]; then + bash "$script" "$@" + else + fail "firewall-setup.sh not found." + fi +} + case "${1:-}" in start) cmd_start ;; stop) cmd_stop ;; @@ -390,6 +404,7 @@ case "${1:-}" in status) cmd_status ;; logs) cmd_logs "$@" ;; update) cmd_update "$@" ;; + firewall) cmd_firewall "$@" ;; backup) shift case "${1:-}" in diff --git a/scripts/firewall-setup.sh b/scripts/firewall-setup.sh new file mode 100755 index 0000000..f1755f2 --- /dev/null +++ b/scripts/firewall-setup.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +set -euo pipefail + +SSH_PORT="${SSH_PORT:-22}" + +is_root() { + [ "$(id -u)" -eq 0 ] +} + +run_as_root() { + if is_root; then + "$@" + elif command -v sudo >/dev/null 2>&1; then + sudo "$@" + else + echo "Error: root privileges required" >&2 + exit 1 + fi +} + +check_ufw_installed() { + if ! command -v ufw >/dev/null 2>&1; then + echo "ufw is not installed. Installing ufw..." + if command -v apt-get >/dev/null 2>&1; then + run_as_root apt-get update -qq && run_as_root apt-get install -y -qq ufw + elif command -v yum >/dev/null 2>&1; then + run_as_root yum install -y ufw + else + echo "Package manager not supported for automatic ufw installation. Please install ufw manually." >&2 + exit 1 + fi + fi +} + +configure_firewall() { + check_ufw_installed + + echo "Configuring host firewall policies..." + run_as_root ufw default deny incoming + run_as_root ufw default allow outgoing + + echo "Allowing loopback interface..." + run_as_root ufw allow in on lo + + echo "Allowing SSH on port ${SSH_PORT}..." + run_as_root ufw allow "${SSH_PORT}/tcp" + + echo "Allowing HTTP (80) and HTTPS (443) ingress..." + run_as_root ufw allow 80/tcp + run_as_root ufw allow 443/tcp + + echo "Enabling UFW..." + run_as_root ufw --force enable + run_as_root ufw status verbose + echo "Host firewall configured successfully." +} + +show_status() { + check_ufw_installed + run_as_root ufw status verbose +} + +main() { + local action="${1:-setup}" + case "$action" in + setup|enable) + configure_firewall + ;; + status) + show_status + ;; + *) + echo "Usage: $0 [setup|status]" >&2 + exit 1 + ;; + esac +} + +main "$@" From 040e8bb928c8d985e274f8a2b1da80448d7e31c6 Mon Sep 17 00:00:00 2001 From: Lftobs Date: Thu, 8 Oct 2026 08:55:22 +0100 Subject: [PATCH 4/5] fix(security): address code review feedback - Authenticated AES-256-GCM streaming encryption with auth tag verification and error propagation for backups - Mask stored connection strings at rest and build authenticated URLs dynamically in repository - Prioritize sanitized X-Real-IP header from trusted reverse proxy over client-controlled headers - Validate and bound audit log query pagination (limit capped at 100, non-negative offset) - Retain per-domain Cloudflare proxy configuration for custom domains without target overrides - Include firewall setup script in dequel update and install download workflows - Automatically detect active SSH port and display rule inventory in host firewall setup - Restrict SSH key identity file permissions and reject unauthenticated remote SSH gateway routes --- apps/api/src/api/__tests__/rate-limit.test.ts | 4 +- apps/api/src/api/audit-logs/index.ts | 10 +- apps/api/src/api/rate-limit.ts | 8 +- apps/api/src/backup/__tests__/crypto.test.ts | 32 +++++ apps/api/src/backup/crypto.ts | 102 ++++++++++---- apps/api/src/backup/orchestrator.ts | 28 +++- .../src/db/__tests__/databases-repo-runner.ts | 5 +- apps/api/src/db/__tests__/databases.test.ts | 4 +- apps/api/src/db/repo/databases.ts | 129 +++++++++++------- apps/api/src/db/test-helper.ts | 5 +- apps/api/src/tunnels/gateway.ts | 10 +- apps/api/src/tunnels/ssh-tunnel.ts | 31 ++++- .../utils/__tests__/domain-verifier.test.ts | 21 +++ apps/api/src/utils/caddy-site.ts | 2 +- apps/api/src/utils/domain-verifier.ts | 6 +- infra/caddy/Caddyfile | 2 + scripts/dequel | 6 + scripts/firewall-setup.sh | 36 ++++- scripts/install.sh | 2 + 19 files changed, 335 insertions(+), 108 deletions(-) diff --git a/apps/api/src/api/__tests__/rate-limit.test.ts b/apps/api/src/api/__tests__/rate-limit.test.ts index 38d3389..198c3a4 100644 --- a/apps/api/src/api/__tests__/rate-limit.test.ts +++ b/apps/api/src/api/__tests__/rate-limit.test.ts @@ -10,7 +10,7 @@ describe("Rate Limiting Middleware", () => { it("extracts client IP from headers with priority", () => { const req1 = new Request("http://localhost/test", { - headers: { "cf-connecting-ip": "1.1.1.1", "x-forwarded-for": "2.2.2.2" }, + headers: { "x-real-ip": "1.1.1.1", "cf-connecting-ip": "2.2.2.2" }, }); expect(getClientIp(req1)).toBe("1.1.1.1"); @@ -20,7 +20,7 @@ describe("Rate Limiting Middleware", () => { expect(getClientIp(req2)).toBe("2.2.2.2"); const req3 = new Request("http://localhost/test", { - headers: { "x-real-ip": "4.4.4.4" }, + headers: { "cf-connecting-ip": "4.4.4.4" }, }); expect(getClientIp(req3)).toBe("4.4.4.4"); diff --git a/apps/api/src/api/audit-logs/index.ts b/apps/api/src/api/audit-logs/index.ts index 1d578a2..0b05b47 100644 --- a/apps/api/src/api/audit-logs/index.ts +++ b/apps/api/src/api/audit-logs/index.ts @@ -3,15 +3,19 @@ import { countAuditLogs, listAuditLogs } from "../../db/repo"; import { ok } from "../response"; export const auditLogsRoutes = new Elysia().get("/audit-logs", async ({ query }: any) => { + const rawLimit = Number(query?.limit); + const limit = Number.isFinite(rawLimit) && rawLimit > 0 ? Math.min(Math.floor(rawLimit), 100) : 50; + const rawOffset = Number(query?.offset); + const offset = Number.isFinite(rawOffset) && rawOffset >= 0 ? Math.floor(rawOffset) : 0; const filters = { actorType: query?.actorType, actorId: query?.actorId, action: query?.action, resourceType: query?.resourceType, resourceId: query?.resourceId, - limit: query?.limit ? Number(query.limit) : 50, - offset: query?.offset ? Number(query.offset) : 0, + limit, + offset, }; const [items, total] = await Promise.all([listAuditLogs(filters), countAuditLogs(filters)]); - return ok({ items, total, limit: filters.limit, offset: filters.offset }); + return ok({ items, total, limit, offset }); }); diff --git a/apps/api/src/api/rate-limit.ts b/apps/api/src/api/rate-limit.ts index ebffea8..d33d213 100644 --- a/apps/api/src/api/rate-limit.ts +++ b/apps/api/src/api/rate-limit.ts @@ -13,12 +13,12 @@ export const RATE_LIMIT_RULES: Record = { }; export const getClientIp = (request: Request): string => { - const cfIp = request.headers.get("cf-connecting-ip"); - if (cfIp) return cfIp.trim(); - const forwarded = request.headers.get("x-forwarded-for"); - if (forwarded) return forwarded.split(",")[0].trim(); const realIp = request.headers.get("x-real-ip"); if (realIp) return realIp.trim(); + const forwarded = request.headers.get("x-forwarded-for"); + if (forwarded) return forwarded.split(",")[0].trim(); + const cfIp = request.headers.get("cf-connecting-ip"); + if (cfIp) return cfIp.trim(); return "127.0.0.1"; }; diff --git a/apps/api/src/backup/__tests__/crypto.test.ts b/apps/api/src/backup/__tests__/crypto.test.ts index 02a02e5..f70f33c 100644 --- a/apps/api/src/backup/__tests__/crypto.test.ts +++ b/apps/api/src/backup/__tests__/crypto.test.ts @@ -31,4 +31,36 @@ describe("Backup Crypto Streams", () => { expect(decryptedText).toBe(plainText); }); + + it("fails authentication when encrypted data is tampered with", async () => { + const plainText = "SELECT * FROM secrets;"; + const source = Readable.from([Buffer.from(plainText, "utf-8")]); + + const { input: encInput, output: encOutput } = createBackupEncryptStream(); + source.pipe(encInput); + + const chunks: Buffer[] = []; + for await (const chunk of encOutput) { + chunks.push(chunk as Buffer); + } + const encryptedBytes = Buffer.concat(chunks); + + // Tamper with one byte in the ciphertext + encryptedBytes[15] ^= 0xff; + + const encSource = Readable.from([encryptedBytes]); + const { input: decInput, output: decOutput } = createBackupDecryptStream(); + encSource.pipe(decInput); + + let errored = false; + try { + for await (const _ of decOutput) { + // drain + } + } catch { + errored = true; + } + + expect(errored).toBe(true); + }); }); diff --git a/apps/api/src/backup/crypto.ts b/apps/api/src/backup/crypto.ts index bf99ec2..923e0bf 100644 --- a/apps/api/src/backup/crypto.ts +++ b/apps/api/src/backup/crypto.ts @@ -1,54 +1,102 @@ import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto"; -import { PassThrough, Transform } from "node:stream"; +import { Transform } from "node:stream"; import { config } from "../utils/config"; import { deriveKey } from "../utils/crypto"; -const ALGO = "aes-256-cbc"; -const IV_LEN = 16; +const ALGO = "aes-256-gcm"; +const IV_LEN = 12; +const TAG_LEN = 16; + +const getEncryptionKey = (): string => { + const key = config.envEncryptionKey; + if (!key || (process.env.NODE_ENV === "production" && key === "dev-env-key-change-me")) { + throw new Error("ENV_ENCRYPTION_KEY must be configured in production"); + } + return key; +}; export const createBackupEncryptStream = () => { + const key = deriveKey(getEncryptionKey()); const iv = randomBytes(IV_LEN); - const key = deriveKey(config.envEncryptionKey || "dev-env-key-change-me"); const cipher = createCipheriv(ALGO, key, iv); - const output = new PassThrough(); + let ivSent = false; - output.write(iv); - cipher.pipe(output); + const transform = new Transform({ + transform(chunk: Buffer, _encoding, callback) { + try { + if (!ivSent) { + this.push(iv); + ivSent = true; + } + const data = cipher.update(chunk); + if (data.length > 0) this.push(data); + callback(); + } catch (err) { + callback(err as Error); + } + }, + flush(callback) { + try { + if (!ivSent) { + this.push(iv); + ivSent = true; + } + const finalData = cipher.final(); + if (finalData.length > 0) this.push(finalData); + const tag = cipher.getAuthTag(); + this.push(tag); + callback(); + } catch (err) { + callback(err as Error); + } + }, + }); - return { input: cipher, output }; + return { input: transform, output: transform }; }; export const createBackupDecryptStream = () => { - let ivBuffer = Buffer.alloc(0); + const key = deriveKey(getEncryptionKey()); + let buffer = Buffer.alloc(0); let decipher: ReturnType | null = null; - const output = new PassThrough(); const transform = new Transform({ transform(chunk: Buffer, _encoding, callback) { - if (!decipher) { - ivBuffer = Buffer.concat([ivBuffer, chunk]); - if (ivBuffer.length >= IV_LEN) { - const iv = ivBuffer.subarray(0, IV_LEN); - const remaining = ivBuffer.subarray(IV_LEN); - const key = deriveKey(config.envEncryptionKey || "dev-env-key-change-me"); - decipher = createDecipheriv(ALGO, key, iv); - decipher.pipe(output); - if (remaining.length > 0) { - decipher.write(remaining); + try { + buffer = Buffer.concat([buffer, chunk]); + if (!decipher) { + if (buffer.length < IV_LEN + TAG_LEN) { + return callback(); } + const iv = buffer.subarray(0, IV_LEN); + decipher = createDecipheriv(ALGO, key, iv); + buffer = buffer.subarray(IV_LEN); } - } else { - decipher.write(chunk); + if (buffer.length > TAG_LEN) { + const cipherText = buffer.subarray(0, buffer.length - TAG_LEN); + buffer = buffer.subarray(buffer.length - TAG_LEN); + const plain = decipher.update(cipherText); + if (plain.length > 0) this.push(plain); + } + callback(); + } catch (err) { + callback(err as Error); } - callback(); }, flush(callback) { - if (decipher) { - decipher.end(); + try { + if (!decipher || buffer.length !== TAG_LEN) { + return callback(new Error("Invalid backup payload: missing or truncated authentication tag")); + } + decipher.setAuthTag(buffer); + const finalPlain = decipher.final(); + if (finalPlain.length > 0) this.push(finalPlain); + callback(); + } catch (err) { + callback(err as Error); } - callback(); }, }); - return { input: transform, output }; + return { input: transform, output: transform }; }; diff --git a/apps/api/src/backup/orchestrator.ts b/apps/api/src/backup/orchestrator.ts index 5613682..54246e4 100644 --- a/apps/api/src/backup/orchestrator.ts +++ b/apps/api/src/backup/orchestrator.ts @@ -44,10 +44,18 @@ export class BackupOrchestrator { const dump = await adapter.dump(ctx); await this.updateJob(job.id, { status: "compressing" }); - const compressed = dump.pipe(createGzip()); - + const gzip = createGzip(); const { input: encInput, output: encOutput } = createBackupEncryptStream(); - compressed.pipe(encInput); + + dump.on("error", (err) => { + gzip.destroy(err); + encInput.destroy(err); + }); + gzip.on("error", (err) => { + encInput.destroy(err); + }); + + dump.pipe(gzip).pipe(encInput); await this.updateJob(job.id, { status: "uploading" }); const filename = `${target.id}-${new Date().toISOString().replace(/[:.]/g, "-")}.sql.gz.enc`; @@ -79,15 +87,27 @@ export class BackupOrchestrator { const downloaded = await this.storage.download(job.storagePath!); const { createGunzip } = await import("node:zlib"); + const gunzip = createGunzip(); let streamToGunzip: any = downloaded; if (job.isEncrypted) { const { input: decInput, output: decOutput } = createBackupDecryptStream(); + downloaded.on("error", (err) => { + decInput.destroy(err); + gunzip.destroy(err); + }); + decOutput.on("error", (err) => { + gunzip.destroy(err); + }); downloaded.pipe(decInput); streamToGunzip = decOutput; + } else { + downloaded.on("error", (err) => { + gunzip.destroy(err); + }); } - const dump = streamToGunzip.pipe(createGunzip()); + const dump = streamToGunzip.pipe(gunzip); await adapter.restore(ctx, dump); } diff --git a/apps/api/src/db/__tests__/databases-repo-runner.ts b/apps/api/src/db/__tests__/databases-repo-runner.ts index 1c2491c..c4fb1c5 100644 --- a/apps/api/src/db/__tests__/databases-repo-runner.ts +++ b/apps/api/src/db/__tests__/databases-repo-runner.ts @@ -32,9 +32,8 @@ const db = drizzle(pool, { schema }); setDbProvider(async () => db); const cleanup = async () => { - for (const name of TABLE_NAMES) { - await pool.query(`TRUNCATE TABLE "${name}" CASCADE`); - } + const tables = TABLE_NAMES.map((name) => `"${name}"`).join(", "); + await pool.query(`TRUNCATE TABLE ${tables} CASCADE`); }; try { diff --git a/apps/api/src/db/__tests__/databases.test.ts b/apps/api/src/db/__tests__/databases.test.ts index 2c13f14..64c49d8 100644 --- a/apps/api/src/db/__tests__/databases.test.ts +++ b/apps/api/src/db/__tests__/databases.test.ts @@ -35,7 +35,7 @@ describe("Database Repository Tests", () => { expect(result.standalone.volumeName).toMatch(/^db-[a-f0-9-]{12}$/); expect(result.standalone.connectionPrefix).toBe("postgresql:"); expect(result.standalone.hasPassword).toBe(true); - }); + }, 30000); it("persists project attachment, allowlist, and lifecycle updates", () => { const result = runRepoScenarios(); @@ -50,5 +50,5 @@ describe("Database Repository Tests", () => { expect(result.updated.storageUsedMb).toBe(256); expect(result.deleted).toBe(true); expect(result.total).toBe(1); - }); + }, 30000); }); diff --git a/apps/api/src/db/repo/databases.ts b/apps/api/src/db/repo/databases.ts index 314a460..b3e5355 100644 --- a/apps/api/src/db/repo/databases.ts +++ b/apps/api/src/db/repo/databases.ts @@ -7,16 +7,31 @@ import { getDb } from "../db-provider"; import { databases } from "../schema"; import { getRowsAffected, now } from "./helpers"; -const mapDatabase = (row: typeof databases.$inferSelect): Database => ({ - id: row.id, - projectId: row.projectId ?? null, - serverId: row.serverId ?? null, - name: row.name, - type: row.type as Database["type"], - version: row.version, - databaseName: row.databaseName, - username: row.username, - password: (() => { +export const buildConnectionString = ( + type: Database["type"], + username: string, + password: string, + host: string, + port: number, + databaseName: string, +): string => { + if (type === "redis") { + return password ? `redis://:${password}@${host}:${port}` : `redis://${host}:${port}`; + } + if (type === "mongodb") { + const auth = password ? `${username}:${password}@` : ""; + return `mongodb://${auth}${host}:${port}/${databaseName}?authSource=admin`; + } + if (type === "mysql") { + const auth = password ? `${username}:${password}@` : `${username}@`; + return `mysql://${auth}${host}:${port}/${databaseName}`; + } + const auth = password ? `${username}:${password}@` : `${username}@`; + return `postgresql://${auth}${host}:${port}/${databaseName}`; +}; + +const mapDatabase = (row: typeof databases.$inferSelect): Database => { + const password = (() => { if (row.passwordEncrypted && row.passwordIv && row.passwordTag) { try { return decryptValue(row.passwordEncrypted, row.passwordIv, row.passwordTag, config.envEncryptionKey); @@ -25,36 +40,57 @@ const mapDatabase = (row: typeof databases.$inferSelect): Database => ({ } } return row.password ?? ""; - })(), - internalHost: row.internalHost, - internalPort: row.internalPort, - cpuLimit: row.cpuLimit, - memoryLimitMb: row.memoryLimitMb, - storageLimitMb: row.storageLimitMb, - storageUsedMb: row.storageUsedMb, - publicAccess: Boolean(row.publicAccess), - allowPublicAccessFromAnywhere: Boolean(row.allowPublicAccessFromAnywhere), - allowedCidrs: Array.isArray(row.allowedCidrs) - ? row.allowedCidrs - : (() => { - try { - return JSON.parse(row.allowedCidrs || "[]"); - } catch { - return []; - } - })(), - externalPort: row.externalPort, - proxyContainerName: row.proxyContainerName, - volumeName: row.volumeName, - connectionString: row.connectionString, - status: row.status as DatabaseStatus, - containerName: row.containerName, - backupEnabled: Boolean(row.backupEnabled), - backupSchedule: row.backupSchedule ?? "0 */6 * * *", - backupRetention: row.backupRetention ?? 7, - createdAt: row.createdAt, - updatedAt: row.updatedAt, -}); + })(); + + return { + id: row.id, + projectId: row.projectId ?? null, + serverId: row.serverId ?? null, + name: row.name, + type: row.type as Database["type"], + version: row.version, + databaseName: row.databaseName, + username: row.username, + password, + internalHost: row.internalHost, + internalPort: row.internalPort, + cpuLimit: row.cpuLimit, + memoryLimitMb: row.memoryLimitMb, + storageLimitMb: row.storageLimitMb, + storageUsedMb: row.storageUsedMb, + publicAccess: Boolean(row.publicAccess), + allowPublicAccessFromAnywhere: Boolean(row.allowPublicAccessFromAnywhere), + allowedCidrs: Array.isArray(row.allowedCidrs) + ? row.allowedCidrs + : (() => { + try { + return JSON.parse(row.allowedCidrs || "[]"); + } catch { + return []; + } + })(), + externalPort: row.externalPort, + proxyContainerName: row.proxyContainerName, + volumeName: row.volumeName, + connectionString: password + ? buildConnectionString( + row.type as Database["type"], + row.username, + password, + row.internalHost, + row.internalPort, + row.databaseName, + ) + : row.connectionString, + status: row.status as DatabaseStatus, + containerName: row.containerName, + backupEnabled: Boolean(row.backupEnabled), + backupSchedule: row.backupSchedule ?? "0 */6 * * *", + backupRetention: row.backupRetention ?? 7, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }; +}; export const createDatabase = async (input: CreateDatabaseInput): Promise => { const id = randomUUID(); @@ -68,16 +104,7 @@ export const createDatabase = async (input: CreateDatabaseInput): Promise { }; export const truncateAllTables = async (pool: Pool) => { - for (const name of TABLE_NAMES) { - await pool.query(`TRUNCATE TABLE "${name}" CASCADE`); - } + const tables = TABLE_NAMES.map((name) => `"${name}"`).join(", "); + await pool.query(`TRUNCATE TABLE ${tables} CASCADE`); }; export const teardownTestDb = async (pool: Pool) => { diff --git a/apps/api/src/tunnels/gateway.ts b/apps/api/src/tunnels/gateway.ts index bf19e1c..738add4 100644 --- a/apps/api/src/tunnels/gateway.ts +++ b/apps/api/src/tunnels/gateway.ts @@ -203,6 +203,14 @@ const createPoolLookup = ); const row = result.rows[0]; if (!row) return null; + const isRemote = Boolean( + row.server_id && row.server_mode && row.server_id !== "local" && row.server_mode !== "local", + ); + const ssh = await resolveSshTarget(pool, row); + if (isRemote && !ssh) { + console.warn(`[Gateway] Remote database on server ${row.server_id} cannot resolve SSH credentials`); + return null; + } return { internalHost: row.internal_host, internalPort: row.internal_port, @@ -212,7 +220,7 @@ const createPoolLookup = allowedCidrs: Array.isArray(row.allowed_cidrs) ? row.allowed_cidrs.filter((value): value is string => typeof value === "string") : [], - ssh: await resolveSshTarget(pool, row), + ssh, }; }; diff --git a/apps/api/src/tunnels/ssh-tunnel.ts b/apps/api/src/tunnels/ssh-tunnel.ts index 8752a7b..29343fd 100644 --- a/apps/api/src/tunnels/ssh-tunnel.ts +++ b/apps/api/src/tunnels/ssh-tunnel.ts @@ -1,5 +1,5 @@ import { spawn } from "node:child_process"; -import { existsSync, mkdirSync, writeFileSync } from "node:fs"; +import { chmodSync, existsSync, mkdirSync, unlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { Duplex } from "node:stream"; @@ -18,9 +18,21 @@ const KEYS_DIR = join(tmpdir(), "dequel_gateway_keys"); const safeName = (value: string) => value.replace(/[^a-zA-Z0-9_-]/g, "_"); export const writeIdentityFile = (target: SshEngineTarget): string => { - if (!existsSync(KEYS_DIR)) mkdirSync(KEYS_DIR, { recursive: true, mode: 0o700 }); + if (!existsSync(KEYS_DIR)) { + mkdirSync(KEYS_DIR, { recursive: true, mode: 0o700 }); + } + try { + chmodSync(KEYS_DIR, 0o700); + } catch {} + const path = join(KEYS_DIR, `id_${safeName(target.serverId)}`); + try { + if (existsSync(path)) unlinkSync(path); + } catch {} writeFileSync(path, `${target.key.trim()}\n`, { mode: 0o600 }); + try { + chmodSync(path, 0o600); + } catch {} return path; }; @@ -45,7 +57,16 @@ export const buildSshArgs = (target: SshEngineTarget, keyPath: string): string[] ]; export const forwardViaSsh = (stream: Duplex, target: SshEngineTarget) => { - const child = spawn("ssh", buildSshArgs(target, writeIdentityFile(target)), { + let keyPath: string; + try { + keyPath = writeIdentityFile(target); + } catch (err) { + console.error("[Gateway SSH] Failed to write identity file:", err); + stream.destroy(err as Error); + return null; + } + + const child = spawn("ssh", buildSshArgs(target, keyPath), { stdio: ["pipe", "pipe", "ignore"], }); const kill = () => { @@ -60,6 +81,10 @@ export const forwardViaSsh = (stream: Duplex, target: SshEngineTarget) => { stream.destroy(); return child; } + child.stdin.on("error", () => { + kill(); + stream.destroy(); + }); stream.pipe(child.stdin); child.stdout.pipe(stream); return child; diff --git a/apps/api/src/utils/__tests__/domain-verifier.test.ts b/apps/api/src/utils/__tests__/domain-verifier.test.ts index 7b67461..c34f787 100644 --- a/apps/api/src/utils/__tests__/domain-verifier.test.ts +++ b/apps/api/src/utils/__tests__/domain-verifier.test.ts @@ -240,4 +240,25 @@ describe("buildCaddySnippet", () => { delete process.env.CLOUDFLARE_PROXIED; } }); + + it("retains trusted_proxies for custom domain without target overrides when global is disabled", async () => { + const listFn = mock().mockResolvedValue([ + { + id: "d-1", + domain: "custom.example.com", + type: "custom", + validationStatus: "verified", + sslStatus: "provisioned", + targetService: null, + targetPort: null, + cloudflareProxied: true, + createdAt: "", + updatedAt: "", + }, + ]); + const snippet = await buildCaddySnippet("my-app", "deploy-abc", "proj-1", listFn, APP_PORT, snippetOpts); + expect(snippet).toContain( + "custom.example.com {\n log {\n output stdout\n format json\n }\n reverse_proxy deploy-abc:3000 {\n header_up Host {upstream_hostport}\n trusted_proxies private_ranges 173.245.48.0/20", + ); + }); }); diff --git a/apps/api/src/utils/caddy-site.ts b/apps/api/src/utils/caddy-site.ts index a93fff2..2dfd3d0 100644 --- a/apps/api/src/utils/caddy-site.ts +++ b/apps/api/src/utils/caddy-site.ts @@ -35,7 +35,7 @@ export const caddySite = (hosts: string, reverseProxy: string, options?: CaddySi export const caddyReverseProxy = (targets: string, options?: CaddyReverseProxyOptions): string => { const trustedProxies = options?.cloudflareProxied - ? `\n trusted_proxies private_ranges ${CADDY_CLOUDFLARE_TRUSTED_PROXIES}` + ? `\n trusted_proxies private_ranges ${CADDY_CLOUDFLARE_TRUSTED_PROXIES}\n header_up X-Real-IP {client_ip}` : ""; return ` reverse_proxy ${targets} {\n header_up Host {upstream_hostport}${trustedProxies}\n }`; diff --git a/apps/api/src/utils/domain-verifier.ts b/apps/api/src/utils/domain-verifier.ts index 4d376f3..636eb25 100644 --- a/apps/api/src/utils/domain-verifier.ts +++ b/apps/api/src/utils/domain-verifier.ts @@ -190,6 +190,7 @@ export const buildCaddySnippet = async ( const listEnvVars = opts?.listEnvVarsFn ?? listEnvironmentVariablesForDeploy; let defaultDomains = [`${slug}.${baseDomain}`]; let port = appPort ?? config.appInternalPort; + const isBaseProxied = isGlobalCloudflareProxied(); const customBlocks: string[] = []; if (projectId) { @@ -197,7 +198,8 @@ export const buildCaddySnippet = async ( const verified = projectDomains.filter((d) => d.validationStatus === "verified"); for (const d of verified) { const entryDomain = d.domain.includes("localhost") ? `${d.domain}:80` : d.domain; - if (d.targetService || d.targetPort) { + const isProxied = isCloudflareProxied((d as any).cloudflareProxied); + if (d.targetService || d.targetPort || isProxied !== isBaseProxied) { let targetContainer = containerName; if (d.targetService) { const parts = containerName.split("-"); @@ -207,7 +209,6 @@ export const buildCaddySnippet = async ( } } const tPort = d.targetPort || port; - const isProxied = isCloudflareProxied((d as any).cloudflareProxied); customBlocks.push( caddySite(entryDomain, caddyReverseProxy(`${targetContainer}:${tPort}`, { cloudflareProxied: isProxied })), ); @@ -249,7 +250,6 @@ export const buildCaddySnippet = async ( } } - const isBaseProxied = isGlobalCloudflareProxied(); const primaryBlock = caddySite( defaultDomains.join(", "), caddyReverseProxy(`${containerName}:${port}`, { cloudflareProxied: isBaseProxied }), diff --git a/infra/caddy/Caddyfile b/infra/caddy/Caddyfile index fdc7382..63e014a 100644 --- a/infra/caddy/Caddyfile +++ b/infra/caddy/Caddyfile @@ -31,6 +31,7 @@ import /etc/caddy/routes/*.caddy handle /api/* { reverse_proxy api:3001 { trusted_proxies private_ranges 173.245.48.0/20 103.21.244.0/22 103.22.200.0/22 103.31.4.0/22 141.101.64.0/18 108.162.192.0/18 190.93.240.0/20 188.114.96.0/20 197.234.240.0/22 198.41.128.0/17 162.158.0.0/15 104.16.0.0/13 104.24.0.0/14 172.64.0.0/13 131.0.72.0/22 2400:cb00::/32 2606:4700::/32 2803:f800::/32 2405:b500::/32 2405:8100::/32 2a06:98c0::/29 2c0f:f248::/32 + header_up X-Real-IP {client_ip} } } @@ -57,6 +58,7 @@ http:// { handle /api/* { reverse_proxy api:3001 { trusted_proxies private_ranges 173.245.48.0/20 103.21.244.0/22 103.22.200.0/22 103.31.4.0/22 141.101.64.0/18 108.162.192.0/18 190.93.240.0/20 188.114.96.0/20 197.234.240.0/22 198.41.128.0/17 162.158.0.0/15 104.16.0.0/13 104.24.0.0/14 172.64.0.0/13 131.0.72.0/22 2400:cb00::/32 2606:4700::/32 2803:f800::/32 2405:b500::/32 2405:8100::/32 2a06:98c0::/29 2c0f:f248::/32 + header_up X-Real-IP {client_ip} } } diff --git a/scripts/dequel b/scripts/dequel index 9173316..39a56df 100755 --- a/scripts/dequel +++ b/scripts/dequel @@ -136,6 +136,7 @@ cmd_update() { curl -fsSL "$base_url/docker-compose.yml" -o "$tmp_dir/docker-compose.yml" || fail "Failed to download docker-compose.yml" curl -fsSL "$base_url/infra/caddy/Caddyfile" -o "$tmp_dir/Caddyfile" || fail "Failed to download Caddyfile" curl -fsSL "$base_url/scripts/dequel" -o "$tmp_dir/dequel" || fail "Failed to download dequel CLI" + curl -fsSL "$base_url/scripts/firewall-setup.sh" -o "$tmp_dir/firewall-setup.sh" 2>/dev/null || true curl -fsSL "$base_url/VERSION" -o "$tmp_dir/VERSION" 2>/dev/null || echo "${tag#v}" > "$tmp_dir/VERSION" mkdir -p "$tmp_dir/infra/monitoring/grafana/datasources" \ @@ -196,6 +197,11 @@ with open('$tmp_dir/docker-compose.yml', 'w') as f: for f in pam-server.py pam-verify.py; do [ -f "$tmp_dir/scripts/auth/$f" ] && mv "$tmp_dir/scripts/auth/$f" "$DEQUEL_HOME/scripts/auth/$f" done + if [ -f "$tmp_dir/firewall-setup.sh" ]; then + mkdir -p "$DEQUEL_HOME/scripts" + mv "$tmp_dir/firewall-setup.sh" "$DEQUEL_HOME/scripts/firewall-setup.sh" + chmod +x "$DEQUEL_HOME/scripts/firewall-setup.sh" 2>/dev/null || true + fi mv "$tmp_dir/VERSION" "$DEQUEL_HOME/VERSION" chmod +x "$tmp_dir/dequel" diff --git a/scripts/firewall-setup.sh b/scripts/firewall-setup.sh index f1755f2..f97bc7e 100755 --- a/scripts/firewall-setup.sh +++ b/scripts/firewall-setup.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -SSH_PORT="${SSH_PORT:-22}" +SSH_PORT="${SSH_PORT:-}" is_root() { [ "$(id -u)" -eq 0 ] @@ -32,8 +32,41 @@ check_ufw_installed() { fi } +detect_and_validate_ssh_port() { + if [ -z "${SSH_PORT:-}" ]; then + if [ -n "${SSH_CONNECTION:-}" ]; then + local conn_port + conn_port=$(echo "$SSH_CONNECTION" | awk '{print $4}') + if [ -n "$conn_port" ] && [ "$conn_port" -gt 0 ] 2>/dev/null; then + echo "Detected active SSH connection on port ${conn_port}." + SSH_PORT="$conn_port" + fi + fi + fi + if [ -z "${SSH_PORT:-}" ]; then + if command -v ss >/dev/null 2>&1; then + local ss_port + ss_port=$(ss -tlpn 2>/dev/null | grep -E 'sshd|/ssh' | awk '{print $4}' | awk -F: '{print $NF}' | head -n1 || true) + if [ -n "$ss_port" ] && [ "$ss_port" -gt 0 ] 2>/dev/null; then + echo "Detected listening SSH service on port ${ss_port}." + SSH_PORT="$ss_port" + fi + fi + fi + SSH_PORT="${SSH_PORT:-22}" + + if ! [[ "${SSH_PORT}" =~ ^[0-9]+$ ]] || [ "${SSH_PORT}" -lt 1 ] || [ "${SSH_PORT}" -gt 65535 ]; then + echo "Error: Invalid SSH port '${SSH_PORT}'. Must be an integer between 1 and 65535." >&2 + exit 1 + fi +} + configure_firewall() { check_ufw_installed + detect_and_validate_ssh_port + + echo "Inspecting existing firewall rules..." + run_as_root ufw status numbered || true echo "Configuring host firewall policies..." run_as_root ufw default deny incoming @@ -51,6 +84,7 @@ configure_firewall() { echo "Enabling UFW..." run_as_root ufw --force enable + echo "Updated firewall rules:" run_as_root ufw status verbose echo "Host firewall configured successfully." } diff --git a/scripts/install.sh b/scripts/install.sh index 646da4a..ddf5bc1 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -125,6 +125,8 @@ download_configs() { download_if_missing "$BASE_URL/infra/caddy/Caddyfile" "$INSTALL_DIR/infra/caddy/Caddyfile" download_if_missing "$BASE_URL/scripts/dequel" "$INSTALL_DIR/dequel" download_if_missing "$BASE_URL/scripts/auth/pam-server.py" "$INSTALL_DIR/scripts/auth/pam-server.py" + download_if_missing "$BASE_URL/scripts/firewall-setup.sh" "$INSTALL_DIR/scripts/firewall-setup.sh" + chmod +x "$INSTALL_DIR/scripts/firewall-setup.sh" 2>/dev/null || true for f in prometheus.yml loki-config.yml promtail-config.yml; do download_if_missing "$BASE_URL/infra/monitoring/$f" "$INSTALL_DIR/infra/monitoring/$f" From fe963975b635f082bbe9ede89dfb0738958363e0 Mon Sep 17 00:00:00 2001 From: Lftobs Date: Thu, 22 Oct 2026 19:38:36 +0100 Subject: [PATCH 5/5] feat(docs): ship OG SEO for landing page and serve install.sh --- apps/docs/blume.config.ts | 26 ++++ apps/docs/components/SEO.astro | 130 ++++++++++++++++ apps/docs/docs/installation.md | 6 + apps/docs/package.json | 1 + apps/docs/pages/index.astro | 19 +-- apps/docs/public/install.sh | 266 +++++++++++++++++++++++++++++++++ apps/docs/public/og.png | Bin 0 -> 29876 bytes apps/docs/vercel.json | 14 +- 8 files changed, 444 insertions(+), 18 deletions(-) create mode 100644 apps/docs/components/SEO.astro create mode 100755 apps/docs/public/install.sh create mode 100644 apps/docs/public/og.png diff --git a/apps/docs/blume.config.ts b/apps/docs/blume.config.ts index c70d4ea..17204d0 100644 --- a/apps/docs/blume.config.ts +++ b/apps/docs/blume.config.ts @@ -82,4 +82,30 @@ export default defineConfig({ deployment: { site: "https://dequel.intrep.xyz", }, + seo: { + og: { + fonts: [], + palette: { + accent: "#ea580c", + background: "#0a0a0c", + border: "#222228", + foreground: "#f4f4f5", + muted: "#a1a1aa", + }, + titles: { + "/": "Dequel — Self-Hosted PaaS Engine & Container Platform", + }, + }, + software: { + name: "Dequel", + operatingSystem: "Linux", + applicationCategory: "DeveloperApplication", + description: + "Self-hosted PaaS container platform. Deploy apps from Git, ZIP, or Docker Compose with zero infrastructure setup.", + }, + organization: { + name: "Dequel", + url: "https://dequel.intrep.xyz", + }, + }, }); diff --git a/apps/docs/components/SEO.astro b/apps/docs/components/SEO.astro new file mode 100644 index 0000000..3634a2c --- /dev/null +++ b/apps/docs/components/SEO.astro @@ -0,0 +1,130 @@ +--- +interface Props { + title?: string; + description?: string; + canonicalUrl?: string; + ogImage?: string; + ogType?: string; + keywords?: string[]; + themeColor?: string; +} + +const { + title = "Dequel — Self-Hosted PaaS Engine & Container Platform", + description = "The open-source, self-hosted PaaS container platform. Deploy apps from Git, ZIP, or Docker Compose with automatic SSL, multi-server scaling, and zero infrastructure lock-in.", + canonicalUrl = "https://dequel.intrep.xyz/", + ogImage = "https://dequel.intrep.xyz/og/index.png", + ogType = "website", + keywords = [ + "Dequel", + "self-hosted PaaS", + "PaaS platform", + "Docker deployment", + "container platform", + "DevOps", + "Caddy reverse proxy", + "automatic SSL", + "Let's Encrypt", + "BuildKit", + "Railpack", + "zero vendor lock-in", + "homelab", + "multi-server deployment", + "microservices", + ], + themeColor = "#0a0a0c", +} = Astro.props; + +const siteName = "Dequel"; +const author = "Dequel"; + +const structuredData = { + "@context": "https://schema.org", + "@graph": [ + { + "@type": "WebSite", + "@id": "https://dequel.intrep.xyz/#website", + "name": "Dequel", + "url": "https://dequel.intrep.xyz/", + "description": description, + "publisher": { + "@type": "Organization", + "name": "Dequel", + "url": "https://dequel.intrep.xyz/", + "logo": { + "@type": "ImageObject", + "url": "https://dequel.intrep.xyz/logo.svg" + } + } + }, + { + "@type": "SoftwareApplication", + "@id": "https://dequel.intrep.xyz/#software", + "name": "Dequel", + "applicationCategory": "DeveloperApplication", + "operatingSystem": "Linux", + "offers": { + "@type": "Offer", + "price": "0", + "priceCurrency": "USD" + }, + "url": "https://dequel.intrep.xyz/", + "description": description, + "image": ogImage, + "screenshot": "https://dequel.intrep.xyz/screenshots/dashboard.png", + "featureList": [ + "Git push-to-deploy and ZIP uploads", + "Docker Compose deployment", + "Automatic SSL certificate provisioning via Caddy and Let's Encrypt", + "Dynamic multi-server container clustering", + "Zero-dependency self-hosted architecture", + "Full observability with Prometheus, cAdvisor, and Loki", + "Automatic horizontal scaling and auto-healing" + ] + } + ] +}; +--- + + + +{title} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +