From ccf7a1dfeac0756ed7c262f15a58b6b54b0c3033 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Oct 2026 19:32:52 +0000 Subject: [PATCH 1/2] Initial plan From 065df023cd4a97c9933df7f6e6ec5ddf2d852510 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Oct 2026 19:38:06 +0000 Subject: [PATCH 2/2] Fix image hosting and await image delivery lifecycle Co-authored-by: sfreeman422 <16405652+sfreeman422@users.noreply.github.com> --- README.md | 51 +++++++++++ packages/backend/.env.example | 1 + packages/backend/src/ai/ai.constants.ts | 1 + packages/backend/src/ai/ai.controller.spec.ts | 12 +++ packages/backend/src/ai/ai.service.spec.ts | 84 ++++++++++++++++++- packages/backend/src/ai/ai.service.ts | 39 +++++---- .../backend/src/ai/image.controller.spec.ts | 57 +++++++++++++ packages/backend/src/ai/image.controller.ts | 13 +++ packages/backend/src/index.ts | 2 + 9 files changed, 242 insertions(+), 18 deletions(-) create mode 100644 packages/backend/src/ai/image.controller.spec.ts create mode 100644 packages/backend/src/ai/image.controller.ts diff --git a/README.md b/README.md index 0544292e..23a24b89 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ Add these slash commands with their request URLs: - `/counter` → `/counter` - `/repstats` → `/rep/get` - `/walkie` → `/walkie` +- `/image` → `/ai/image` **Important:** Check `Escape Channels, users and links sent to your app` for all commands. @@ -186,9 +187,59 @@ NODE_ENV=development # External APIs (optional, for AI features) OPENAI_API_KEY=sk-your-openai-key +GOOGLE_GEMINI_API_KEY=your-gemini-api-key GOOGLE_TRANSLATE_API_KEY=your-google-translate-key +IMAGE_DIR=/tmp/mocker-images +IMAGE_BASE_URL=https://api.muzzle.lol/images ``` +#### `/image` troubleshooting and diagnosis + +`/image` passes suppression, prompt-length, daily-limit, and in-flight checks before +`POST /ai/image` acknowledges the request. The backend calls Gemini's +`gemini-3-pro-image-preview` with image output enabled, writes the returned inline +image data to `IMAGE_DIR`, and posts an image block through Slack's `chat.postMessage`. +`GOOGLE_GEMINI_API_KEY` is required for image generation; an OpenAI or Google Translate +key is not a substitute. + +The investigation reproduced these defects with mocked Gemini/Slack responses and real +temporary-file writes (no paid API calls or workspace messages): + +- **Image hosting mismatch:** files were written to `/tmp/mocker-images` but the returned + URL was hard-coded to `https://muzzle.lol/images/...`. The backend had no image-serving + route; any separate reverse-proxy mapping was outside the repository and unverified. + The backend now serves generated PNG filenames at `GET /images/.png`, before + Slack signature verification so Slack can fetch them. `IMAGE_BASE_URL` defaults to + `https://api.muzzle.lol/images`; set it to your publicly reachable HTTPS backend + (or tunnel) image URL. This also applies to Moonbeam relaunch images. +- **Request lifecycle:** a deferred Slack response reproduced the lock being removed + before delivery, and an empty Gemini response removed it twice. Delivery is now awaited + and the lock is released once, after disk writing and Slack delivery/fallback finish. +- **Detached error handling:** a rejected channel post followed by a rejected fallback DM + could produce an unhandled rejection. Both are now awaited and logged; a failed channel + post refunds the daily request count once. + +Before the fixes, regression output included `AssertionError: expected 1 to be +0` +for premature lock removal and `Number of calls: 2` for empty-response lock cleanup. +Empty Gemini responses previously raised `No b64_json was returned for prompt: ...` +(an obsolete OpenAI field name); they now raise `Gemini returned no image data`. +Relevant structured log messages are `Gemini image generation returned no image data`, +`Failed to write AI image to disk`, and `Failed to send generated AI image to Slack`. +No production API error logs or stack traces were available during this investigation; +key validity, billing/quota, model availability, and reverse-proxy configuration still +require verification in the deployed environment. The inspected PR CI run required +approval and had no job logs, rather than a failing build. + +For deployment, ensure `IMAGE_DIR` is writable by the backend and that the public +`IMAGE_BASE_URL` routes to that backend's `/images` endpoint. Fetch a generated URL +without authentication and confirm a `200` response with `Content-Type: image/png`. +Generated images are public to anyone holding their random URL; do not put sensitive +files in this directory. `/tmp` is ephemeral, so use persistent/shared storage if images +must survive restarts or be fetched across multiple replicas, and arrange file retention +cleanup. If Slack rejects a channel post, check bot scopes and invite Moonbeam to private +channels. Inspect Gemini errors for key, quota, model access, or safety-blocked output +rather than assuming all failures are channel-permission problems. + #### Production rollout for dashboard query indexes If `TYPEORM_SYNCHRONIZE=false`, TypeORM will not apply the dashboard index decorators to an existing production database. diff --git a/packages/backend/.env.example b/packages/backend/.env.example index 0415a9be..1188e55e 100644 --- a/packages/backend/.env.example +++ b/packages/backend/.env.example @@ -64,4 +64,5 @@ MAX_JOKE_ATTEMPTS=20 # Optional: Runtime/storage # ========================= IMAGE_DIR=/tmp/mocker-images +IMAGE_BASE_URL=https://api.muzzle.lol/images REDIS_CONTAINER_NAME=localhost diff --git a/packages/backend/src/ai/ai.constants.ts b/packages/backend/src/ai/ai.constants.ts index 6b9a5a10..76911ac8 100644 --- a/packages/backend/src/ai/ai.constants.ts +++ b/packages/backend/src/ai/ai.constants.ts @@ -1,5 +1,6 @@ export const GPT_MODEL = 'gpt-6-luna'; export const GPT_IMAGE_MODEL = 'dall-e-3'; +export const DEFAULT_IMAGE_DIR = '/tmp/mocker-images'; export const MAX_AI_REQUESTS_PER_DAY = 5; export const GENERAL_TEXT_INSTRUCTIONS = 'Generate a response with a focus on being helpful and succinct.'; diff --git a/packages/backend/src/ai/ai.controller.spec.ts b/packages/backend/src/ai/ai.controller.spec.ts index 4a55f071..8f36a7f3 100644 --- a/packages/backend/src/ai/ai.controller.spec.ts +++ b/packages/backend/src/ai/ai.controller.spec.ts @@ -68,6 +68,18 @@ describe('aiController', () => { expect(generateImage).toHaveBeenCalledWith('U1', 'T1', 'C1', 'cat'); }); + it('acknowledges /image and reports asynchronous generation failures to the requesting user', async () => { + generateImage.mockRejectedValueOnce(new Error('Gemini returned no image data')); + + const response = await request(app) + .post('/image') + .send({ user_id: 'U1', team_id: 'T1', channel_id: 'C1', text: 'cat' }) + .expect(200); + + expect(response.text).toContain('Processing your request'); + expect(sendEphemeral).toHaveBeenCalledWith('C1', '`Sorry! Your request for cat failed. Please try again.`', 'U1'); + }); + it('handles /prompt-with-history', async () => { const body = { user_id: 'U1', team_id: 'T1', channel_id: 'C1', text: 'sum' }; await request(app).post('/prompt-with-history').send(body).expect(200); diff --git a/packages/backend/src/ai/ai.service.spec.ts b/packages/backend/src/ai/ai.service.spec.ts index 34e4f9ed..c9bd705e 100644 --- a/packages/backend/src/ai/ai.service.spec.ts +++ b/packages/backend/src/ai/ai.service.spec.ts @@ -141,6 +141,66 @@ describe('AIService', () => { }); describe('generateImage', () => { + it('keeps the request inflight until Slack delivery completes', async () => { + (aiService.gemini.models.generateContent as Mock).mockResolvedValue({ + candidates: [{ content: { parts: [{ inlineData: { data: 'aW1hZ2U=' } }] } }], + }); + vi.spyOn(aiService, 'writeToDiskAndReturnUrl').mockResolvedValue('https://example.com/images/test.png'); + let finishDelivery!: () => void; + (aiService.webService.sendMessage as Mock).mockImplementation( + () => + new Promise((resolve) => { + finishDelivery = () => resolve({ ok: true }); + }), + ); + + const generation = aiService.generateImage('U1', 'T1', 'C1', 'draw cat'); + await vi.waitFor(() => expect(aiService.webService.sendMessage).toHaveBeenCalled()); + const removedBeforeDelivery = (aiService.redis.removeInflight as Mock).mock.calls.length; + finishDelivery(); + await generation; + + expect(removedBeforeDelivery).toBe(0); + expect(aiService.redis.removeInflight).toHaveBeenCalledExactlyOnceWith('U1', 'T1'); + }); + + it('refunds failed Gemini requests and clears the inflight lock', async () => { + (aiService.gemini.models.generateContent as Mock).mockRejectedValue(new Error('Gemini unavailable')); + + await expect(aiService.generateImage('U1', 'T1', 'C1', 'draw cat')).rejects.toThrow('Gemini unavailable'); + + expect(aiService.redis.decrementDailyRequests).toHaveBeenCalledExactlyOnceWith('U1', 'T1'); + expect(aiService.redis.removeInflight).toHaveBeenCalledExactlyOnceWith('U1', 'T1'); + }); + + it('refunds disk write failures and clears the inflight lock', async () => { + (aiService.gemini.models.generateContent as Mock).mockResolvedValue({ + candidates: [{ content: { parts: [{ inlineData: { data: 'aW1hZ2U=' } }] } }], + }); + vi.spyOn(aiService, 'writeToDiskAndReturnUrl').mockRejectedValue(new Error('EACCES')); + + await expect(aiService.generateImage('U1', 'T1', 'C1', 'draw cat')).rejects.toThrow('EACCES'); + + expect(aiService.redis.decrementDailyRequests).toHaveBeenCalledExactlyOnceWith('U1', 'T1'); + expect(aiService.redis.removeInflight).toHaveBeenCalledExactlyOnceWith('U1', 'T1'); + expect(aiService.webService.sendMessage).not.toHaveBeenCalled(); + }); + + it('handles channel and fallback DM failures without an unhandled rejection or duplicate refund', async () => { + (aiService.gemini.models.generateContent as Mock).mockResolvedValue({ + candidates: [{ content: { parts: [{ inlineData: { data: 'aW1hZ2U=' } }] } }], + }); + vi.spyOn(aiService, 'writeToDiskAndReturnUrl').mockResolvedValue('https://example.com/images/test.png'); + (aiService.webService.sendMessage as Mock).mockRejectedValue(new Error('channel_not_found')); + + await expect(aiService.generateImage('U1', 'T1', 'C1', 'draw cat')).resolves.toBeUndefined(); + + expect(aiService.webService.sendMessage).toHaveBeenCalledTimes(2); + expect(aiService.redis.decrementDailyRequests).toHaveBeenCalledExactlyOnceWith('U1', 'T1'); + expect(aiService.redis.removeInflight).toHaveBeenCalledExactlyOnceWith('U1', 'T1'); + expect(aiService.aiServiceLogger.error).toHaveBeenCalled(); + }); + it('sends generated image after writing it to disk', async () => { const generateContentSpy = aiService.gemini.models.generateContent as Mock; generateContentSpy.mockResolvedValue({ @@ -154,7 +214,7 @@ describe('AIService', () => { }); const diskSpy = vi.spyOn(aiService, 'writeToDiskAndReturnUrl').mockResolvedValue('https://muzzle.lol/image.png'); - const sendSpy = vi.spyOn(aiService, 'sendImage').mockImplementation(); + const sendSpy = vi.spyOn(aiService, 'sendImage').mockResolvedValue(); await aiService.generateImage('U1', 'T1', 'C1', 'draw cat'); @@ -171,6 +231,8 @@ describe('AIService', () => { await expect(aiService.generateImage('U1', 'T1', 'C1', 'draw cat')).rejects.toThrow(); expect(errSpy).toHaveBeenCalled(); + expect(aiService.redis.decrementDailyRequests).toHaveBeenCalledExactlyOnceWith('U1', 'T1'); + expect(aiService.redis.removeInflight).toHaveBeenCalledExactlyOnceWith('U1', 'T1'); }); }); @@ -187,6 +249,7 @@ describe('AIService', () => { const filename = imageUrl.split('/').pop(); expect(filename).toBeDefined(); + expect(imageUrl).toBe(`https://api.muzzle.lol/images/${filename}`); expect(fs.existsSync(imageDir)).toBe(true); expect(fs.readFileSync(path.join(imageDir, filename as string))).toEqual(Buffer.from('png-bytes')); } finally { @@ -199,6 +262,21 @@ describe('AIService', () => { fs.rmSync(tempRoot, { recursive: true, force: true }); } }); + + it('uses the configured public image URL without duplicate slashes', async () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ai-service-')); + vi.stubEnv('IMAGE_DIR', tempRoot); + vi.stubEnv('IMAGE_BASE_URL', 'https://example.com/images/'); + + try { + const imageUrl = await aiService.writeToDiskAndReturnUrl(Buffer.from('png-bytes').toString('base64')); + + expect(imageUrl).toMatch(/^https:\/\/example\.com\/images\/[^/]+\.png$/); + } finally { + vi.unstubAllEnvs(); + fs.rmSync(tempRoot, { recursive: true, force: true }); + } + }); }); describe('redeployMoonbeam', () => { @@ -824,9 +902,7 @@ describe('AIService', () => { const sendMock = aiService.webService.sendMessage as Mock; sendMock.mockRejectedValueOnce(new Error('slack fail')).mockResolvedValueOnce({ ok: true }); - aiService.sendImage('https://img', 'U1', 'T1', 'C1', 'draw cat'); - await Promise.resolve(); - await Promise.resolve(); + await aiService.sendImage('https://img', 'U1', 'T1', 'C1', 'draw cat'); expect(sendMock).toHaveBeenCalled(); expect(aiService.redis.decrementDailyRequests).toHaveBeenCalledWith('U1', 'T1'); diff --git a/packages/backend/src/ai/ai.service.ts b/packages/backend/src/ai/ai.service.ts index 33f33c63..5e96d324 100644 --- a/packages/backend/src/ai/ai.service.ts +++ b/packages/backend/src/ai/ai.service.ts @@ -11,6 +11,7 @@ import type { KnownBlock } from '@slack/web-api'; import { WebService } from '../shared/services/web/web.service'; import { CORPO_SPEAK_INSTRUCTIONS, + DEFAULT_IMAGE_DIR, GENERAL_TEXT_INSTRUCTIONS, MOONBEAM_SYSTEM_INSTRUCTIONS, getHistoryInstructions, @@ -107,7 +108,6 @@ const ensureSentenceCaseAndPunctuation = (text: string): string => { return `${capitalized}.`; }; -const DEFAULT_IMAGE_DIR = path.join('/tmp', 'mocker-images'); const execFileAsync = promisify(execFile); const RELEASE_METADATA_PATHS = [ path.resolve(process.cwd(), 'release-metadata.json'), @@ -239,7 +239,8 @@ export class AIService { try { await fs.promises.mkdir(dir, { recursive: true }); await fs.promises.writeFile(filePath, imageBytes); - return `https://muzzle.lol/images/${filename}`; + const baseUrl = process.env.IMAGE_BASE_URL || 'https://api.muzzle.lol/images'; + return `${baseUrl.replace(/\/+$/, '')}/${filename}`; } catch (error) { logError(this.aiServiceLogger, 'Failed to write AI image to disk', error, { imageDirectory: dir, @@ -385,12 +386,10 @@ export class AIService { return imageBytes.toString('base64'); }) .then(async (x) => { - await this.redis.removeInflight(userId, teamId); - if (x) { return this.writeToDiskAndReturnUrl(x); } else { - const error = new Error(`No b64_json was returned for prompt: ${text}`); + const error = new Error('Gemini returned no image data'); logError(this.aiServiceLogger, 'Gemini image generation returned no image data', error, { userId, teamId, @@ -401,7 +400,7 @@ export class AIService { } }) .then((imageUrl) => { - this.sendImage(imageUrl, userId, teamId, channel, text); + return this.sendImage(imageUrl, userId, teamId, channel, text); }) .catch(async (e) => { logError(this.aiServiceLogger, 'Failed to generate AI image response', e, { @@ -410,9 +409,11 @@ export class AIService { channelId: channel, prompt: text, }); - await this.redis.removeInflight(userId, teamId); await this.redis.decrementDailyRequests(userId, teamId); throw e; + }) + .finally(async () => { + await this.redis.removeInflight(userId, teamId); }); } @@ -720,7 +721,13 @@ export class AIService { }; } - sendImage(image: string | undefined, userId: string, teamId: string, channel: string, text: string): void { + async sendImage( + image: string | undefined, + userId: string, + teamId: string, + channel: string, + text: string, + ): Promise { if (image) { const blocks: KnownBlock[] = [ { @@ -738,18 +745,22 @@ export class AIService { ], }, ]; - this.webService.sendMessage(channel, text, blocks).catch((e) => { + await this.webService.sendMessage(channel, text, blocks).catch(async (e) => { logError(this.aiServiceLogger, 'Failed to send generated AI image to Slack', e, { userId, teamId, channelId: channel, prompt: text, }); - void this.decrementDaiyRequests(userId, teamId); - void this.webService.sendMessage( - userId, - 'Sorry, unable to send the requested image to Slack. You have been credited for your Moon Token. Perhaps you were trying to send in a private channel? If so, invite @MoonBeam and try again.', - ); + await this.decrementDaiyRequests(userId, teamId); + await this.webService + .sendMessage( + userId, + 'Sorry, unable to send the requested image to Slack. You have been credited for your Moon Token. Perhaps you were trying to send in a private channel? If so, invite @MoonBeam and try again.', + ) + .catch((error) => { + logError(this.aiServiceLogger, 'Failed to notify user of AI image delivery failure', error, { userId }); + }); }); } } diff --git a/packages/backend/src/ai/image.controller.spec.ts b/packages/backend/src/ai/image.controller.spec.ts new file mode 100644 index 00000000..82621dd4 --- /dev/null +++ b/packages/backend/src/ai/image.controller.spec.ts @@ -0,0 +1,57 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import express from 'express'; +import request from 'supertest'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { imageController } from './image.controller'; +import { AIService } from './ai.service'; +import { signatureVerificationMiddleware } from '../shared/middleware/signatureVerification'; + +describe('imageController', () => { + let directory: string; + const app = express(); + app.use('/images', imageController); + app.use(signatureVerificationMiddleware); + + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), 'mocker-images-')); + vi.stubEnv('IMAGE_DIR', directory); + vi.stubEnv('IMAGE_BASE_URL', 'https://example.com/images'); + vi.stubEnv('MUZZLE_BOT_SIGNING_SECRET', 'test-signing-secret'); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it('serves generated image bytes without a Slack signature', async () => { + const image = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aF9kAAAAASUVORK5CYII=', + 'base64', + ); + const url = await new AIService().writeToDiskAndReturnUrl(image.toString('base64')); + + const response = await request(app).get(new URL(url).pathname).expect(200); + + expect(response.headers['content-type']).toBe('image/png'); + expect(response.headers['x-content-type-options']).toBe('nosniff'); + expect(response.body).toEqual(image); + await request(app).head(new URL(url).pathname).expect(200); + }); + + it.each(['/.env', '/secret.txt', '/', '/%2e%2e%2fsecret.txt'])('does not expose non-image paths: %s', async (url) => { + fs.writeFileSync(path.join(directory, '.env'), 'private-config'); + fs.writeFileSync(path.join(directory, 'secret.txt'), 'private-config'); + + const response = await request(app).get(`/images${url}`); + + expect(response.status).not.toBe(200); + expect(response.text).not.toContain('private-config'); + }); + + it('does not bypass Slack signature verification for command requests', async () => { + await request(app).post('/ai/image').send({ text: 'cat' }).expect(400); + }); +}); diff --git a/packages/backend/src/ai/image.controller.ts b/packages/backend/src/ai/image.controller.ts new file mode 100644 index 00000000..fbd17521 --- /dev/null +++ b/packages/backend/src/ai/image.controller.ts @@ -0,0 +1,13 @@ +import express from 'express'; +import { DEFAULT_IMAGE_DIR } from './ai.constants'; + +export const imageController = express.Router(); + +imageController.get('/:filename([0-9a-fA-F-]{36}\\.png)', (req, res, next) => { + express.static(process.env.IMAGE_DIR ?? DEFAULT_IMAGE_DIR, { + dotfiles: 'deny', + index: false, + redirect: false, + setHeaders: (response) => response.setHeader('X-Content-Type-Options', 'nosniff'), + })(req, res, next); +}); diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 8abd55e7..f0bd771d 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -9,6 +9,7 @@ import { rateLimit } from 'express-rate-limit'; import { createConnection, getConnectionOptions } from 'typeorm'; import type { RequestWithRawBody } from './shared/models/express/RequestWithRawBody'; import { aiController } from './ai/ai.controller'; +import { imageController } from './ai/image.controller'; import { clapController } from './clap/clap.controller'; import { confessionController } from './confession/confession.controller'; import { counterController } from './counter/counter.controller'; @@ -101,6 +102,7 @@ app.use('/dashboard', searchCors, searchRateLimit, authMiddleware, dashboardCont app.use('/calendar', searchCors, searchRateLimit, authMiddleware, calendarController); app.use('/fantasy', searchCors, searchRateLimit, authMiddleware, fantasyController); app.use('/health', healthController); +app.use('/images', imageController); app.use(signatureVerificationMiddleware); app.use('/ai', aiController); app.use('/clap', clapController);