From 456bd843454bab7750a8ad3684b14aca021f6f96 Mon Sep 17 00:00:00 2001 From: WenhuaXia Date: Sat, 10 Oct 2026 18:44:04 +0800 Subject: [PATCH] =?UTF-8?q?test(custom):=20=E9=A6=96=E4=B8=AA=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E8=87=AA=E5=8A=A8=E8=AE=BE=E9=BB=98=E8=AE=A4=20LLM=20?= =?UTF-8?q?=E5=9B=9E=E5=BD=92=E6=B5=8B=E8=AF=95=20(=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=E5=B7=B2=E7=94=B1=E4=B8=8A=E6=B8=B8=20#54=20=E5=90=88=E5=85=A5?= =?UTF-8?q?)=20/=20regression=20tests=20for=20first-model=20auto-default?= =?UTF-8?q?=20(implementation=20upstreamed=20via=20PR=20#54)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 上游 origin/main 的 fix(models): set first configured model as default LLM (PR #54) 已实现 _maybe_promote_first_model_to_default，且比本地原实现 多解析 legacy model 配置 + webui 创建后刷新默认模型。 本分支保留 8 个回归测试覆盖同一行为契约： 首个模型+凭据→设默认; inline key→设默认; 无凭据→不设; 第二个模型→不覆盖; 其它 provider 已有默认→不覆盖。 Upstream PR #54 implements the same fix (plus legacy `model` resolution and a WebUI refresh after create). This branch now carries only the regression tests pinning the behavior contract. --- .../routes/test_custom_provider_routes.py | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/tests/server/routes/test_custom_provider_routes.py b/tests/server/routes/test_custom_provider_routes.py index d2b8c60cd..87955ebb8 100644 --- a/tests/server/routes/test_custom_provider_routes.py +++ b/tests/server/routes/test_custom_provider_routes.py @@ -159,3 +159,126 @@ async def test_create_custom_model_defaults_reasoning_on( assert raw is not None assert raw["models"]["custom-reasoning-default"]["supports_reasoning"] is True assert Provider._models["custom-reasoning-default"].capabilities.supports_reasoning is True + + +# ==================== auto-promote first model to default LLM ==================== +# Regression: a user adding a model from the model-management page (not the +# onboarding wizard) ends up with a configured provider but no default LLM. +# /api/onboarding/status then reports completed=False and the WebUI re-opens +# the setup wizard on every login. create_model now promotes the FIRST model +# to default when no default exists and the provider already has credentials. + + +class _StubSecretManager: + def __init__(self, values): + self._values = values + + def get(self, key): + return self._values.get(key) + + def has(self, key): + return key in self._values + + +@pytest.fixture +def _credential_stub(monkeypatch): + """Stub the SecretManager exactly where credential.get_api_key resolves it. + + credential.get_api_key does `from flocks.security import get_secret_manager` + at call time, so the name lives in the flocks.security __init__ namespace + (not the flocks.security.secrets submodule). + """ + import flocks.security as security_module + + return lambda values: monkeypatch.setattr( + security_module, "get_secret_manager", lambda: _StubSecretManager(values) + ) + + +async def _create_model(client, model_id="first-model"): + response = await client.post( + "/api/custom/models/custom-tb-inner", + json={ + "model_id": model_id, + "name": model_id, + "context_window": 32768, + "max_output_tokens": 4096, + }, + ) + assert response.status_code == 201, response.text + return response.json() + + +@pytest.mark.asyncio +async def test_first_model_auto_becomes_default_when_credential_exists( + client: AsyncClient, temp_custom_provider_project, _credential_stub +): + from flocks.config.config_writer import ConfigWriter + + _credential_stub({"custom-tb-inner_llm_key": "sk-test-first"}) + + await _create_model(client) + + default = ConfigWriter.get_default_model("llm") + assert default == {"provider_id": "custom-tb-inner", "model_id": "first-model"} + + +@pytest.mark.asyncio +async def test_first_model_with_inline_api_key_auto_becomes_default( + client: AsyncClient, temp_custom_provider_project, _credential_stub +): + from flocks.config.config_writer import ConfigWriter + + # No secret, but flocks.json holds a plain-text inline apiKey. + _credential_stub({}) + ConfigWriter.update_provider_field("custom-tb-inner", "options.apiKey", "sk-inline-test") + + await _create_model(client) + + default = ConfigWriter.get_default_model("llm") + assert default == {"provider_id": "custom-tb-inner", "model_id": "first-model"} + + +@pytest.mark.asyncio +async def test_first_model_without_credential_does_not_become_default( + client: AsyncClient, temp_custom_provider_project, _credential_stub +): + from flocks.config.config_writer import ConfigWriter + + # Secret reference in config, but the secret value is absent -> no credential. + _credential_stub({}) + + await _create_model(client) + + assert ConfigWriter.get_default_model("llm") is None + + +@pytest.mark.asyncio +async def test_second_model_does_not_override_existing_default( + client: AsyncClient, temp_custom_provider_project, _credential_stub +): + from flocks.config.config_writer import ConfigWriter + + _credential_stub({"custom-tb-inner_llm_key": "sk-test-second"}) + + await _create_model(client, model_id="first-model") + await _create_model(client, model_id="second-model") + + default = ConfigWriter.get_default_model("llm") + assert default == {"provider_id": "custom-tb-inner", "model_id": "first-model"} + + +@pytest.mark.asyncio +async def test_create_model_does_not_override_an_existing_default_from_another_provider( + client: AsyncClient, temp_custom_provider_project, _credential_stub +): + from flocks.config.config_writer import ConfigWriter + + _credential_stub({"custom-tb-inner_llm_key": "sk-test-override"}) + # A default from a different provider (e.g. configured via onboarding) must be kept. + ConfigWriter.set_default_model("llm", "threatbook-cn-llm", "qwen3-max") + + await _create_model(client) + + default = ConfigWriter.get_default_model("llm") + assert default == {"provider_id": "threatbook-cn-llm", "model_id": "qwen3-max"}