From 3ef11a192f6ff767b0972bb3c08fdd9227349d42 Mon Sep 17 00:00:00 2001 From: Tejas Narayan Date: Sat, 10 Oct 2026 11:35:08 +0000 Subject: [PATCH 01/40] feat: updating v1 --- app/api/v1/api.py | 2 + app/api/v1/routes/scenario_metrics.py | 147 ++++++++ app/core/license.py | 6 + .../testing/scenario_metric_generation.py | 306 ++++++++++++++++ .../components/enterprise-comparison.tsx | 5 + frontend/src/lib/api.ts | 28 ++ frontend/src/pages/scenarios/Scenarios.tsx | 333 +++++++++++++++++- tests/test_api/test_enterprise_gating.py | 11 + tests/test_api/test_scenario_metrics.py | 85 +++++ tests/test_core/test_license_offerings.py | 2 + .../test_scenario_metric_generation.py | 99 ++++++ 11 files changed, 1020 insertions(+), 4 deletions(-) create mode 100644 app/api/v1/routes/scenario_metrics.py create mode 100644 app/services/testing/scenario_metric_generation.py create mode 100644 tests/test_api/test_scenario_metrics.py create mode 100644 tests/test_services/test_testing/test_scenario_metric_generation.py diff --git a/app/api/v1/api.py b/app/api/v1/api.py index b5f651c8..d5d999d4 100644 --- a/app/api/v1/api.py +++ b/app/api/v1/api.py @@ -23,6 +23,7 @@ evaluators, evaluator_suites, metrics, + scenario_metrics, evaluator_results, evaluator_result_metric_clusters, chat, @@ -85,6 +86,7 @@ api_router.include_router(evaluators.router) api_router.include_router(evaluator_suites.router) api_router.include_router(metrics.router) +api_router.include_router(scenario_metrics.router) # Register before evaluator_results so `/evaluator-results/metric-clusters` # is not captured by `/evaluator-results/{id}` with id="metric-clusters". api_router.include_router(evaluator_result_metric_clusters.router) diff --git a/app/api/v1/routes/scenario_metrics.py b/app/api/v1/routes/scenario_metrics.py new file mode 100644 index 00000000..08f6a398 --- /dev/null +++ b/app/api/v1/routes/scenario_metrics.py @@ -0,0 +1,147 @@ +"""Enterprise scenario-to-metric generation routes.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Literal, Optional +from uuid import UUID, uuid4 + +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session + +from app.database import get_db +from app.dependencies import ( + get_api_key, + get_organization_id, + get_workspace_id, + require_enterprise_feature, +) +from app.services.ai.llm_resolver import get_llm_provider_and_model_for_request +from app.services.billing.flexprice_service import record_metrics_llm_assist +from app.services.testing.scenario_metric_generation import ( + ScenarioMetricInput, + generate_metrics_from_scenarios, +) +from app.services.usage.context import ( + LLMUsageContext, + LLMUsageProductSection, + llm_usage_context, +) + +router = APIRouter( + prefix="/metrics", + tags=["metrics"], + dependencies=[Depends(require_enterprise_feature("scenario_metrics"))], +) + + +class ScenarioMetricScenarioInput(BaseModel): + name: str = Field(..., min_length=1, max_length=255) + description: str = Field(..., min_length=1) + goal: Optional[str] = None + + +class GenerateMetricsFromScenariosRequest(BaseModel): + agent_name: str = Field(..., min_length=1, max_length=255) + production_prompt: str = Field(..., min_length=1) + call_medium: Optional[str] = None + scenarios: List[ScenarioMetricScenarioInput] = Field(..., min_length=1, max_length=10) + provider: Optional[str] = None + model: Optional[str] = None + credential_id: Optional[UUID] = None + llm_config: Optional[Dict[str, Any]] = None + + +class GeneratedScenarioMetricDraftResponse(BaseModel): + name: str + description: str + metric_type: Literal["rating", "boolean", "number", "text"] + custom_data_type: Optional[Literal["boolean", "enum", "number_range"]] = None + custom_config: Dict[str, Any] = {} + supported_surfaces: List[str] + enabled_surfaces: List[str] + tags: List[str] = [] + scenario_name: str + + +class GenerateMetricsFromScenariosResponse(BaseModel): + metrics: List[GeneratedScenarioMetricDraftResponse] + provider: str + model: str + + +@router.post("/generate-from-scenarios", response_model=GenerateMetricsFromScenariosResponse) +async def generate_metrics_from_scenarios_route( + data: GenerateMetricsFromScenariosRequest, + background_tasks: BackgroundTasks, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """Suggest one evaluation metric per scenario; does not persist metrics.""" + provider_enum, model_str = get_llm_provider_and_model_for_request( + organization_id, db, data.provider, data.model, data.credential_id + ) + + scenario_inputs = [ + ScenarioMetricInput( + name=item.name.strip(), + description=item.description.strip(), + goal=item.goal.strip() if item.goal else None, + ) + for item in data.scenarios + ] + + try: + with llm_usage_context( + LLMUsageContext( + organization_id=organization_id, + workspace_id=workspace_id, + product_section=LLMUsageProductSection.METRICS, + ) + ): + result = generate_metrics_from_scenarios( + agent_name=data.agent_name.strip(), + production_prompt=data.production_prompt, + call_medium=data.call_medium, + scenarios=scenario_inputs, + organization_id=organization_id, + workspace_id=workspace_id, + db=db, + llm_provider=provider_enum, + llm_model=model_str, + llm_config=data.llm_config, + credential_id=data.credential_id, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except Exception as exc: + raise HTTPException(status_code=502, detail=f"AI generation failed: {exc}") from exc + + background_tasks.add_task( + record_metrics_llm_assist, + organization_id, + uuid4(), + workspace_id=workspace_id, + mode="scenario_metrics", + ) + + return GenerateMetricsFromScenariosResponse( + metrics=[ + GeneratedScenarioMetricDraftResponse( + name=m.name, + description=m.description, + metric_type=m.metric_type, # type: ignore[arg-type] + custom_data_type=m.custom_data_type, # type: ignore[arg-type] + custom_config=m.custom_config, + supported_surfaces=m.supported_surfaces, + enabled_surfaces=m.enabled_surfaces, + tags=m.tags, + scenario_name=m.scenario_name, + ) + for m in result.metrics + ], + provider=result.provider, + model=result.model, + ) diff --git a/app/core/license.py b/app/core/license.py index fb6d1c9a..a90d24c7 100644 --- a/app/core/license.py +++ b/app/core/license.py @@ -54,6 +54,11 @@ "description": "Batch ad-hoc metric scoring runs against evaluation results and call imports.", "category": "evaluation", }, + "scenario_metrics": { + "title": "Scenario Metrics", + "description": "Auto-generate evaluation metrics from agent scenarios and push them into Metrics after review.", + "category": "evaluation", + }, "db_sharding": { "title": "Call Import DB Sharding", "description": "Horizontally shard call-import row storage across multiple database nodes.", @@ -119,6 +124,7 @@ DEFAULT_ENTERPRISE_OFFERINGS: List[str] = [ "alerts", "metric_studio", + "scenario_metrics", "db_sharding", "llm_gateway", ] diff --git a/app/services/testing/scenario_metric_generation.py b/app/services/testing/scenario_metric_generation.py new file mode 100644 index 00000000..5790fff4 --- /dev/null +++ b/app/services/testing/scenario_metric_generation.py @@ -0,0 +1,306 @@ +"""Generate evaluation metric drafts from scenario definitions and agent prompts.""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Sequence +from uuid import UUID + +from loguru import logger +from sqlalchemy.orm import Session + +from app.models.database import Metric +from app.models.enums import ModelProvider +from app.services.ai.llm_service import llm_service +from app.services.metrics.surfaces import ALLOWED_METRIC_SURFACES +from app.services.testing.scenario_generation_modality import scenario_modality_from_call_medium + + +@dataclass +class ScenarioMetricInput: + name: str + description: str + goal: Optional[str] = None + + +@dataclass +class ScenarioMetricDraft: + name: str + description: str + metric_type: str + custom_data_type: Optional[str] + custom_config: Dict[str, Any] + supported_surfaces: List[str] + enabled_surfaces: List[str] + tags: List[str] + scenario_name: str + + +@dataclass +class ScenarioMetricGenerationResult: + metrics: List[ScenarioMetricDraft] + provider: str + model: str + + +def evaluation_surface_for_call_medium(call_medium: Optional[str]) -> str: + return "chat_agent" if scenario_modality_from_call_medium(call_medium) == "chat" else "agent" + + +def _extract_json_array(text: str) -> List[Any]: + cleaned = (text or "").strip() + if cleaned.startswith("```"): + cleaned = re.sub(r"^```(?:json)?\s*", "", cleaned) + cleaned = re.sub(r"\s*```$", "", cleaned) + start = cleaned.find("[") + end = cleaned.rfind("]") + if start == -1 or end == -1 or end <= start: + raise ValueError("LLM response did not contain a JSON array") + return json.loads(cleaned[start : end + 1]) + + +def _build_messages( + *, + agent_name: str, + production_prompt: str, + surface: str, + scenarios: Sequence[ScenarioMetricInput], +) -> List[Dict[str, str]]: + scenario_blocks = [] + for index, scenario in enumerate(scenarios, start=1): + parts = [ + f"Scenario {index}: {scenario.name}", + f"Description:\n{scenario.description.strip()}", + ] + if scenario.goal: + parts.append(f"Goal: {scenario.goal.strip()}") + scenario_blocks.append("\n".join(parts)) + + schema_block = f""" +You MUST respond with ONLY a JSON array (no markdown, no commentary) with exactly {len(scenarios)} objects, in the same order as the scenarios below. Each object: +{{ + "name": str (concise, Title Case, <= 60 chars), + "description": str (1-3 sentences: LLM-judge rubric for whether the agent handled THIS scenario per the production prompt), + "metric_type": "rating" | "boolean" | "number" | "text", + "custom_data_type": "boolean" | "enum" | "number_range" | null, + "custom_config": {{}}, + "supported_surfaces": ["{surface}"], + "enabled_surfaces": ["{surface}"] +}} + +Rules: + - Prefer structured types (boolean, rating/enum, number) over free text. + - metric_type must align with custom_data_type (boolean/boolean, rating/enum, number/number_range). + - Each metric must be specific to its scenario's success criteria and edge cases. + - Do not reuse identical names across scenarios. +""" + + system_message = ( + "You are an expert evaluation designer. For each test scenario, define one " + "LLM-as-judge metric that scores whether an agent conversation satisfied that " + "scenario given the agent's production system prompt. Always respond with valid JSON only." + ) + + user_message = ( + f"Agent Name: {agent_name}\n" + f"Evaluation surface: {surface}\n\n" + f"Production system prompt:\n{production_prompt.strip()}\n\n" + f"Scenarios:\n\n" + + "\n\n---\n\n".join(scenario_blocks) + + schema_block + ) + + return [ + {"role": "system", "content": system_message}, + {"role": "user", "content": user_message}, + ] + + +def normalize_generated_metric_fields( + parsed: Dict[str, Any], + *, + surface: str, +) -> Dict[str, Any]: + allowed_surfaces = set(ALLOWED_METRIC_SURFACES) + supported = [s for s in (parsed.get("supported_surfaces") or []) if s in allowed_surfaces] + if surface not in supported: + supported = list({*supported, surface}) + enabled_surfaces = [s for s in (parsed.get("enabled_surfaces") or supported) if s in supported] + if not enabled_surfaces: + enabled_surfaces = list(supported) + + metric_type = (parsed.get("metric_type") or "rating").lower() + if metric_type not in {"rating", "boolean", "number", "text"}: + metric_type = "rating" + + custom_data_type = parsed.get("custom_data_type") + if custom_data_type not in {"boolean", "enum", "number_range", None}: + custom_data_type = None + + if metric_type == "text": + custom_data_type = None + custom_config: Dict[str, Any] = {} + else: + if custom_data_type is None: + custom_data_type = ( + "boolean" + if metric_type == "boolean" + else "number_range" + if metric_type == "number" + else "enum" + ) + custom_config = parsed.get("custom_config") or {} + if custom_data_type == "enum" and not isinstance(custom_config.get("options"), list): + custom_config = {"options": ["Excellent", "Good", "Neutral", "Poor"]} + if custom_data_type == "number_range": + custom_config = { + "min": float(custom_config.get("min", 0)), + "max": float(custom_config.get("max", 10)), + "step": float(custom_config.get("step", 1)), + } + if custom_data_type == "boolean": + custom_config = {} + + name = (parsed.get("name") or "Scenario Metric").strip()[:60] + description = (parsed.get("description") or "").strip()[:1000] + + return { + "name": name, + "description": description, + "metric_type": metric_type, + "custom_data_type": custom_data_type, + "custom_config": custom_config, + "supported_surfaces": supported, + "enabled_surfaces": enabled_surfaces, + } + + +def _dedupe_metric_name( + db: Session, + organization_id: UUID, + workspace_id: UUID, + name: str, +) -> str: + workspace_filter = Metric.workspace_id == workspace_id + existing = ( + db.query(Metric) + .filter( + Metric.name == name, + Metric.organization_id == organization_id, + workspace_filter, + Metric.parent_metric_id.is_(None), + ) + .first() + ) + if not existing: + return name + suffix = 2 + while True: + candidate = f"{name} ({suffix})"[:60] + collision = ( + db.query(Metric) + .filter( + Metric.name == candidate, + Metric.organization_id == organization_id, + workspace_filter, + Metric.parent_metric_id.is_(None), + ) + .first() + ) + if not collision: + return candidate + suffix += 1 + + +def _build_tags(agent_name: str, extra: Optional[List[str]] = None) -> List[str]: + reserved = {agent_name.strip(), "auto-generated"} + tags: List[str] = [] + for value in [agent_name.strip(), "auto-generated"]: + if value and value not in tags: + tags.append(value) + for item in extra or []: + text = str(item).strip() + if text and text not in reserved and text not in tags: + tags.append(text) + return tags[:12] + + +def generate_metrics_from_scenarios( + *, + agent_name: str, + production_prompt: str, + call_medium: Optional[str], + scenarios: Sequence[ScenarioMetricInput], + organization_id: UUID, + workspace_id: UUID, + db: Session, + llm_provider: ModelProvider, + llm_model: str, + llm_config: Optional[Dict[str, Any]] = None, + credential_id: Optional[UUID] = None, +) -> ScenarioMetricGenerationResult: + if not production_prompt.strip(): + raise ValueError("Production prompt is required") + if not scenarios: + raise ValueError("At least one scenario is required") + if len(scenarios) > 10: + raise ValueError("At most 10 scenarios per request") + + surface = evaluation_surface_for_call_medium(call_medium) + messages = _build_messages( + agent_name=agent_name, + production_prompt=production_prompt, + surface=surface, + scenarios=scenarios, + ) + + result = llm_service.generate_response( + messages=messages, + llm_provider=llm_provider, + llm_model=llm_model, + organization_id=organization_id, + db=db, + llm_config=llm_config, + task_defaults={"temperature": 0.4, "max_tokens": 8000}, + credential_id=credential_id, + ) + + try: + raw = _extract_json_array(result["text"]) + except (ValueError, json.JSONDecodeError) as exc: + logger.warning(f"[ScenarioMetrics] Failed to parse metric JSON array: {exc}") + raise ValueError("Could not parse generated metrics from LLM response") from exc + + if not isinstance(raw, list) or len(raw) < len(scenarios): + raise ValueError("LLM response did not contain enough metric definitions") + + drafts: List[ScenarioMetricDraft] = [] + for index, scenario in enumerate(scenarios): + item = raw[index] + if not isinstance(item, dict): + raise ValueError(f"Metric at index {index} is not an object") + normalized = normalize_generated_metric_fields(item, surface=surface) + name = _dedupe_metric_name( + db, organization_id, workspace_id, normalized["name"] + ) + drafts.append( + ScenarioMetricDraft( + name=name, + description=normalized["description"], + metric_type=normalized["metric_type"], + custom_data_type=normalized["custom_data_type"], + custom_config=normalized["custom_config"], + supported_surfaces=normalized["supported_surfaces"], + enabled_surfaces=normalized["enabled_surfaces"], + tags=_build_tags(agent_name, item.get("suggested_tags")), + scenario_name=scenario.name, + ) + ) + + return ScenarioMetricGenerationResult( + metrics=drafts, + provider=llm_provider.value, + model=llm_model, + ) diff --git a/docs-fumadocs/components/enterprise-comparison.tsx b/docs-fumadocs/components/enterprise-comparison.tsx index 5f3f8b1c..558909ae 100644 --- a/docs-fumadocs/components/enterprise-comparison.tsx +++ b/docs-fumadocs/components/enterprise-comparison.tsx @@ -74,6 +74,11 @@ const SECTIONS: ComparisonSection[] = [ oss: { type: 'no' }, enterprise: { type: 'yes' }, }, + { + feature: 'Scenario metrics', + oss: { type: 'no' }, + enterprise: { type: 'yes' }, + }, { feature: 'Alerts', oss: { type: 'no' }, diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 6db124f0..551d9495 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -5633,6 +5633,34 @@ class ApiClient { return response.data } + async generateMetricsFromScenarios(data: { + agent_name: string + production_prompt: string + call_medium?: string | null + scenarios: Array<{ name: string; description: string; goal?: string }> + provider?: string + model?: string + credential_id?: string + llm_config?: Record + }): Promise<{ + metrics: Array<{ + name: string + description: string + metric_type: 'rating' | 'boolean' | 'number' | 'text' + custom_data_type: 'boolean' | 'enum' | 'number_range' | null + custom_config: Record + supported_surfaces: string[] + enabled_surfaces: string[] + tags: string[] + scenario_name: string + }> + provider: string + model: string + }> { + const response = await this.client.post('/api/v1/metrics/generate-from-scenarios', data) + return response.data + } + async parseBulkMetric(data: { prompt: string surface: 'agent' | 'chat_agent' | 'voice_playground' diff --git a/frontend/src/pages/scenarios/Scenarios.tsx b/frontend/src/pages/scenarios/Scenarios.tsx index d0f55a84..1640ec38 100644 --- a/frontend/src/pages/scenarios/Scenarios.tsx +++ b/frontend/src/pages/scenarios/Scenarios.tsx @@ -1,6 +1,20 @@ import { useState, useMemo, useEffect, useRef, type ReactNode } from 'react' import { createPortal } from 'react-dom' -import { FileText, Tag, Plus, Sparkles, Trash2, X, Loader, Phone, Brain, ChevronDown, AlertCircle } from 'lucide-react' +import { + FileText, + Tag, + Plus, + Sparkles, + Trash2, + X, + Loader, + Phone, + Brain, + ChevronDown, + AlertCircle, + Lock, + BarChart3, +} from 'lucide-react' import { apiClient } from '../../lib/api' import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query' import Button from '../../components/Button' @@ -29,12 +43,46 @@ import { isChatMedium, type AgentMediumFilter, } from '../../lib/agentMedium' +import { useLicenseStore } from '../../store/licenseStore' + +interface GeneratedScenarioMetricDraft { + name: string + description: string + metric_type: 'rating' | 'boolean' | 'number' | 'text' + custom_data_type: 'boolean' | 'enum' | 'number_range' | null + custom_config: Record + supported_surfaces: string[] + enabled_surfaces: string[] + tags: string[] +} interface GeneratedScenarioDraft { id: string name: string description: string goal?: string + metric?: GeneratedScenarioMetricDraft + metricPushed?: boolean +} + +function mergeRequiredMetricTags(tags: string[], agentName: string): string[] { + const required = [agentName.trim(), 'auto-generated'].filter(Boolean) + const seen = new Set() + const merged: string[] = [] + for (const tag of required) { + if (!seen.has(tag)) { + seen.add(tag) + merged.push(tag) + } + } + for (const tag of tags) { + const trimmed = tag.trim() + if (trimmed && !seen.has(trimmed)) { + seen.add(trimmed) + merged.push(trimmed) + } + } + return merged } type CreateMode = 'agent_prompt' | 'call' | 'custom' | null @@ -42,6 +90,8 @@ type CreateMode = 'agent_prompt' | 'call' | 'custom' | null export default function Scenarios() { const queryClient = useQueryClient() const { showToast, ToastContainer } = useToast() + const { isFeatureEnabled, isLoaded: licenseLoaded, getFeatureMeta } = useLicenseStore() + const scenarioMetricsEnabled = licenseLoaded && isFeatureEnabled('scenario_metrics') const [showMainModal, setShowMainModal] = useState(false) const [createMode, setCreateMode] = useState(null) const [showDetailsModal, setShowDetailsModal] = useState(false) @@ -72,7 +122,9 @@ export default function Scenarios() { const [additionalAgentPromptContext, setAdditionalAgentPromptContext] = useState('') const [generatedScenarioDrafts, setGeneratedScenarioDrafts] = useState([]) const [isGeneratingFromAgentPrompt, setIsGeneratingFromAgentPrompt] = useState(false) + const [isGeneratingScenarioMetrics, setIsGeneratingScenarioMetrics] = useState(false) const [savingDraftIds, setSavingDraftIds] = useState>(new Set()) + const [pushingMetricDraftIds, setPushingMetricDraftIds] = useState>(new Set()) const [editGeneratePrompt, setEditGeneratePrompt] = useState('') const [isGeneratingEditDescription, setIsGeneratingEditDescription] = useState(false) @@ -427,6 +479,143 @@ export default function Scenarios() { setGeneratedScenarioDrafts((prev) => prev.filter((draft) => draft.id !== draftId)) } + const resolveAgentProductionPrompt = (agent: AgentOption) => { + const chatAgent = isChatMedium(agent.call_medium) + return ( + chatAgent + ? (agent.provider_prompt || agent.description || '').trim() + : (agent.description || '').trim() + ) + } + + const updateGeneratedDraftMetric = ( + draftId: string, + updates: Partial + ) => { + setGeneratedScenarioDrafts((prev) => + prev.map((draft) => + draft.id === draftId && draft.metric + ? { ...draft, metric: { ...draft.metric, ...updates } } + : draft + ) + ) + } + + const handleGenerateMetricsForDrafts = async () => { + if (!scenarioMetricsEnabled) { + const title = getFeatureMeta('scenario_metrics')?.title ?? 'Scenario Metrics' + showToast(`${title} requires an EfficientAI Enterprise license.`, 'error') + return + } + const selectedAgent = availableAgents.find((a) => a.id === selectedAgentIdForGeneration) + if (!selectedAgent) { + showToast('Please select an agent', 'error') + return + } + if (!selectedAIProvider || !selectedModel) { + showToast('Please select an AI provider and model', 'error') + return + } + if (generatedScenarioDrafts.length === 0) { + showToast('Generate scenarios first', 'error') + return + } + + const agentPrompt = resolveAgentProductionPrompt(selectedAgent) + if (!agentPrompt) { + showToast('Selected agent has no production prompt', 'error') + return + } + + setIsGeneratingScenarioMetrics(true) + try { + const response = await apiClient.generateMetricsFromScenarios({ + agent_name: selectedAgent.name, + production_prompt: agentPrompt, + call_medium: selectedAgent.call_medium, + scenarios: generatedScenarioDrafts.map((draft) => ({ + name: draft.name.trim(), + description: draft.description.trim(), + goal: draft.goal?.trim() || undefined, + })), + provider: selectedAIProvider, + model: selectedModel, + ...(llmConfig ? { llm_config: llmConfig } : {}), + }) + + setGeneratedScenarioDrafts((prev) => + prev.map((draft, index) => { + const generated = response.metrics[index] + if (!generated) return draft + return { + ...draft, + metricPushed: false, + metric: { + name: generated.name, + description: generated.description, + metric_type: generated.metric_type, + custom_data_type: generated.custom_data_type, + custom_config: generated.custom_config ?? {}, + supported_surfaces: generated.supported_surfaces, + enabled_surfaces: generated.enabled_surfaces, + tags: generated.tags ?? [], + }, + } + }) + ) + showToast(`Generated ${response.metrics.length} metric drafts`, 'success') + } catch (error: any) { + showToast( + `Failed to generate metrics: ${error.response?.data?.detail || error.message}`, + 'error' + ) + } finally { + setIsGeneratingScenarioMetrics(false) + } + } + + const pushMetricForDraft = async (draft: GeneratedScenarioDraft) => { + if (!draft.metric || draft.metricPushed) return + const selectedAgent = availableAgents.find((a) => a.id === selectedAgentIdForGeneration) + const agentName = selectedAgent?.name?.trim() || 'Agent' + const metric = draft.metric + const tags = mergeRequiredMetricTags(metric.tags, agentName) + + setPushingMetricDraftIds((prev) => new Set(prev).add(draft.id)) + try { + await apiClient.createMetric({ + name: metric.name.trim(), + description: metric.description.trim() || undefined, + metric_type: metric.metric_type, + metric_origin: 'custom', + supported_surfaces: metric.supported_surfaces, + enabled_surfaces: metric.enabled_surfaces, + custom_data_type: metric.custom_data_type ?? undefined, + custom_config: + metric.custom_config && Object.keys(metric.custom_config).length > 0 + ? metric.custom_config + : undefined, + tags, + scope: 'workspace', + }) + await queryClient.invalidateQueries({ queryKey: ['metrics'] }) + setGeneratedScenarioDrafts((prev) => + prev.map((item) => + item.id === draft.id ? { ...item, metricPushed: true, metric: { ...metric, tags } } : item + ) + ) + showToast(`Added metric "${metric.name}" to Metrics`, 'success') + } catch (error: any) { + showToast(`Failed to add metric: ${error.response?.data?.detail || error.message}`, 'error') + } finally { + setPushingMetricDraftIds((prev) => { + const next = new Set(prev) + next.delete(draft.id) + return next + }) + } + } + const saveGeneratedDraft = async (draft: GeneratedScenarioDraft) => { if (!draft.name.trim()) { showToast('Scenario name cannot be empty', 'error') @@ -487,6 +676,8 @@ export default function Scenarios() { setAdditionalAgentPromptContext('') setGeneratedScenarioDrafts([]) setSavingDraftIds(new Set()) + setPushingMetricDraftIds(new Set()) + setIsGeneratingScenarioMetrics(false) setShowProviderDropdown(false) setScenarioMediumFilter('voice') resetForm() @@ -990,9 +1181,42 @@ export default function Scenarios() { {generatedScenarioDrafts.length > 0 && (
-
- Generated Scenarios ({generatedScenarioDrafts.length}) -
+
+
+ Generated Scenarios ({generatedScenarioDrafts.length}) +
+ +
+ {!scenarioMetricsEnabled && licenseLoaded && ( +

+ {getFeatureMeta('scenario_metrics')?.title ?? 'Scenario Metrics'} is an + Enterprise feature. Set{' '} + + EFFICIENTAI_LICENSE + {' '} + on your server to generate and push metrics from these scenarios. +

+ )} {generatedScenarioDrafts.map((draft) => (
@@ -1019,6 +1243,107 @@ export default function Scenarios() { placeholder="Scenario description (markdown supported)..." />
+ {draft.metric && ( +
+
+

+ Generated metric +

+ {draft.metricPushed && ( + + In Metrics + + )} +
+
+ + + updateGeneratedDraftMetric(draft.id, { name: e.target.value }) + } + className="w-full px-3 py-2 border border-gray-300 rounded-lg focus:ring-2 focus:ring-primary-500 focus:border-transparent bg-white disabled:bg-gray-100" + /> +
+
+ +