diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f9632f42a..c8bdc397d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,6 +97,18 @@ jobs: # once the issue is resolved it should be able to be re-floated # https://github.com/Azure/azure-cli/issues/32980. # This can be removed once https://github.com/Azure/azure-cli/issues/32869 is supported. + - name: Set Python 3.13 (Windows) + if: matrix.os-name == 'Windows' && matrix.test-category == 'AzureServiceBus' + uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Install pinned Azure CLI on Python 3.13 (Windows) + if: matrix.os-name == 'Windows' && matrix.test-category == 'AzureServiceBus' + run: | + python -m pip install --upgrade pip + python -m pip install --user "azure-cli==2.64.0" + $userScripts = python -c "import sysconfig; print(sysconfig.get_path('scripts', 'nt_user'))" + echo $userScripts >> $Env:GITHUB_PATH - name: Set Python 3.13 (Linux) if: matrix.os-name == 'Linux' && matrix.test-category == 'AzureServiceBus' uses: actions/setup-python@v5 diff --git a/docs/README.md b/docs/README.md index f421eb3b1c..bbc7b5bb98 100644 --- a/docs/README.md +++ b/docs/README.md @@ -28,6 +28,7 @@ This section points to sources that explain why ServiceControl is designed the w - [Retries over Azure Storage Queues transport](retries-asq-transport.md) — transport-specific retry handling - [Data versioning design](data-versioning-design.md) — the cache-versioning invariant for API responses - [Event log design](eventlog-design.md) — what the event log is and what it records +- [Platform health API](platform-health.md) — how ServicePulse reads internal health independently from customer custom checks - [Multiple ServiceControl instances communication](multipleservicecontrolinstancescommunication.md) — how primary, audit, and monitoring instances talk to each other - [Handling unavailable runtime dependencies](handling-unavailable-runtime-dependencies.md) — how instances react when a dependency is unavailable - [Telemetry](telemetry.md) — telemetry configuration and emitted metrics diff --git a/docs/platform-health.md b/docs/platform-health.md new file mode 100644 index 0000000000..737d019ca6 --- /dev/null +++ b/docs/platform-health.md @@ -0,0 +1,77 @@ +# Platform Health API + +ServiceControl exposes `GET /api/platform-health` for the ServiceControl-owned data on ServicePulse's Platform Health page. The API root advertises its URL in `platform_health`. The response uses the existing snake_case JSON convention and omits unknown nullable fields. + +The public motivation is [ServiceControl #5860](https://github.com/Particular/ServiceControl/issues/5860). The consumer data requirements were checked against [ServicePulse's Platform Health store](https://github.com/Particular/ServicePulse/blob/e2688743d23fe5a4b835d4cff128ad12da87d34c/src/Frontend/src/stores/PlatformHealthStore.ts) and [platform model](https://github.com/Particular/ServicePulse/blob/e2688743d23fe5a4b835d4cff128ad12da87d34c/src/Frontend/src/resources/PlatformModel.ts). + +## Response + +The existing `status`, `severity`, and `alerts` fields remain, with additive `instances` and `license` sections. + +### Instances + +`instances` contains the primary followed by every distinct configured remote, even when no check has reported or a remote cannot be reached. Remotes are ordered by stable ID, not by the order that their requests complete. + +| Field | Meaning and source | +| --- | --- | +| `id` | Existing URL-derived ServiceControl instance ID; independent of display name and row position | +| `name` | Configured instance name; a never-observed remote falls back to its URI hostname | +| `kind`, `role` | `error` / `primary-error`, `error` / `remote-error`, `audit` / `remote-audit`, or `unknown` / `remote-unknown` | +| `api_url` | Request-facing primary URL, honoring forwarded scheme, host and prefix; configured remote URL with its virtual directory preserved | +| `version` | Installed local version or remote `X-Particular-Version`; absent when unknown, never replaced with the primary's version | +| `host_id` | Actual reporting host identity from the local NServiceBus host or remote configuration; absent on older remotes | +| `health` | `healthy` for reachable instances without an associated failure, `degraded` for reachable instances with failures, `unavailable` for failed probes | +| `observed_at` | UTC timestamp for the current refresh, from the injected clock | +| `metadata_observed_at` | Timestamp of the last successful metadata observation; differs from `observed_at` during an outage | +| `health_signals_status` | `reported`, `unreported`, `disabled`, or `ambiguous`; not a guarantee that every possible check has run | +| `last_reported_at` | Latest associated check timestamp, including successful reports; distinct from HTTP observation time | +| `issues` | Associated failed internal checks, with the same fields as root alerts | +| `transport_type`, `error_queue`, `error_log_queue`, `forward_error_messages` | Available transport configuration; a known `false` forwarding setting is preserved | +| `audit_queue`, `audit_log_queue`, `forward_audit_messages` | Available audit transport configuration | +| `error_retention_period`, `audit_retention_period` | Available retention durations in the existing TimeSpan JSON format, for example `14.00:00:00` | + +Primary and audit `/api/configuration` (also `/api/instance-info`) include `instance_type` and `host.host_id`. Primary configuration additionally reports `health_checks_enabled`. Older remotes without `instance_type` are identified only when their retention configuration establishes the type. A never-observed, unreachable remote is explicitly unknown, not assumed to be an audit instance. + +Remote probes use the registered named HTTP clients and their query timeout. Non-success status codes, empty or malformed configuration, and connection failures do not produce healthy rows. An outage retains the last successful metadata in memory, clearly dated by `metadata_observed_at`. Other rows and the license section still return. Caller cancellation propagates instead of returning partial success. No recursive platform-health requests are made to other primaries. + +### Issues and summary + +Each failed check has `id`, `check_id`, `category`, `message`, `reported_at`, `instance_name`, `host`, and `host_id`. An associated issue also has `instance_id`. + +Association uses case-insensitive instance name plus reporting host ID. A legacy remote without a host ID can use a name match only when there is one matching inventory row and one reporting host with that name. Ambiguous or unmatched reports remain in root `alerts` without `instance_id`; they are never assigned to several rows. Consumers should retain a place to display those unassigned alerts. + +The legacy summary describes captured checks, not the whole browser-visible platform: `status` is `unknown` before any internal report, `healthy` when none are failing, and `unhealthy` when at least one is failing. Its corresponding `severity` values are `unknown`, `none`, and `error`. ServicePulse should use per-instance health for page severity and combine it with its independently observed monitoring state. The legacy summary does not account for monitoring, browser connectivity, license expiry, or available upgrades. + +Check state is process-local. Reports older than a check's latest `reported_at` are ignored; a newer successful report clears that failure. Reports do not expire: different checks have different schedules, including one-shot checks. After restart, check observations and last-known remote metadata are initially empty. `healthy` therefore means reachable without a known associated failure, not proof of complete or fresh check coverage. An unreachable process cannot report its own browser-facing unavailability in a successful response. + +### License + +`license.availability` is `available` after a successful refresh and `unavailable` when license details cannot be refreshed. An unavailable license never claims to be valid and does not suppress instance health. + +The available summary includes `status`, `license_status`, `license_type`, `trial_license`, optional `expiration_date` and `upgrade_protection_expiration`, and `license_extension_url`. It preserves the existing license status values for subscription, trial and upgrade-protection gates. Renewal URLs share the `/api/license` mapping with `clientName=servicepulse`, including MassTransit evaluation/subscription links. `has_mass_transit_connector` reports connector presence. Customer registration, licensed products and endpoint-license metadata are not included. + +## ServicePulse integration + +The endpoint supplies primary/remote inventory, installed versions, configuration, issues, and the license summary. Updating this endpoint does not update the ServicePulse consumer automatically; the consumer must map `instances` and `license` into its stores and support unknown instance types and unassigned alerts. + +ServicePulse continues to own: + +- Its running frontend version and ServicePulse row. +- The browser-selected monitoring URL, monitoring requests, and monitoring row. +- Browser-to-primary connectivity failures, including when this endpoint cannot be reached. +- Release-feed requests, latest-version comparison, release links, upgrade badges, and outdated-only navigation state. Installed version and license validity are not a guarantee that an upgrade path is supported. +- The customer-check fetch for the support export. Export combines this response, browser-owned rows, and the existing custom-check results. Customer checks never affect platform health. + +Keep the legacy consumer fallback for supported ServiceControl versions without the advertised capability. Do not interpret `401`, `403`, a timeout, or a failed response as an absent capability. The existing custom-check API, classification, notifications, and integration events remain unchanged. Audit health still arrives through the current custom-check reporting transport; this increment does not remove that dependency or introduce replacement events. + +## Access + +The endpoint retains `error:customchecks:view`, granted by the existing reader, writer and admin roles. No permission or authentication behavior is changed. With authentication disabled it is anonymous. With authentication and RBAC enabled, anonymous callers receive `401` and authenticated callers without a read role receive `403`. + +Known shared-policy limitation: authentication enabled with RBAC disabled currently resolves named permissions to allow-all, so the expanded response, including the license summary, can be accessed anonymously. Fixing that policy is separate work. Container liveness and readiness remain separate at `/health` and `/health/ready`. + +## Verification + +For manual requests, use [PlatformHealth.http](../src/ServiceControl/PlatformHealth.http). Its authenticated request reads an existing bearer token from `SERVICECONTROL_ACCESS_TOKEN`; do not store credentials in the request file. + +`PlatformHealthStateTests` and `PlatformHealthApiTests` cover snapshot ordering, delayed reports, serialization, source projection, identity ambiguity, offline metadata, partial failures, license mapping and cancellation. Remote-client tests cover HTTP status, malformed responses and prefixed URLs. Shared acceptance scenarios exercise the real root/configuration/health responses and preserve custom-check behavior. The multi-instance `When_inspecting_platform_health` scenario exercises real audit check delivery, issue ownership, recovery and unavailable inventory. OIDC acceptance scenarios cover the existing read-role policy. \ No newline at end of file diff --git a/src/ServiceControl.AcceptanceTests/Monitoring/CustomChecks/When_custom_checks_are_classified.cs b/src/ServiceControl.AcceptanceTests/Monitoring/CustomChecks/When_custom_checks_are_classified.cs index 8b1995386e..da388165c1 100644 --- a/src/ServiceControl.AcceptanceTests/Monitoring/CustomChecks/When_custom_checks_are_classified.cs +++ b/src/ServiceControl.AcceptanceTests/Monitoring/CustomChecks/When_custom_checks_are_classified.cs @@ -2,6 +2,7 @@ namespace ServiceControl.AcceptanceTests.Monitoring.CustomChecks { using System; using System.Linq; + using System.Text.Json; using System.Threading; using System.Threading.Tasks; using AcceptanceTesting; @@ -11,6 +12,9 @@ namespace ServiceControl.AcceptanceTests.Monitoring.CustomChecks using NServiceBus.CustomChecks; using NUnit.Framework; using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.Api.Contracts; + using ServiceControl.Infrastructure; + using ApiSerializerOptions = global::ServiceControl.Infrastructure.WebApi.SerializerOptions; using CustomCheckView = global::ServiceControl.Contracts.CustomChecks.CustomCheckView; using CheckStatus = global::ServiceControl.Persistence.Status; @@ -28,7 +32,10 @@ public async Task Internal_checks_are_flagged_internal_and_endpoint_checks_are_n CustomCheckView internalCheck = null; CustomCheckView endpointCheck = null; + PlatformHealthView platformHealth = null; + RootUrls urls = null; string wireBody = null; + string healthWireBody = null; await Define() .WithEndpoint() @@ -47,10 +54,22 @@ await Define() wireBody = await raw.Content.ReadAsStringAsync(); } - return internalCheck != null && endpointCheck != null && wireBody != null; + if (internalCheck != null && endpointCheck != null && platformHealth == null) + { + urls = await this.TryGet("/api"); + using var response = await this.GetRaw("/api/platform-health"); + healthWireBody = await response.Content.ReadAsStringAsync(); + platformHealth = JsonSerializer.Deserialize(healthWireBody, ApiSerializerOptions.Default); + } + + return internalCheck != null && endpointCheck != null && wireBody != null && platformHealth != null; }) .Run(); + using var healthJson = JsonDocument.Parse(healthWireBody); + var instanceJson = healthJson.RootElement.GetProperty("instances")[0]; + var instance = platformHealth.Instances.Single(item => item.Role == "primary-error"); + using (Assert.EnterMultipleScope()) { Assert.That(internalCheck, Is.Not.Null, "primary internal checks report at startup; nothing was found"); @@ -58,6 +77,20 @@ await Define() Assert.That(endpointCheck, Is.Not.Null); Assert.That(endpointCheck.Internal, Is.False); + Assert.That(platformHealth.Alerts, Has.None.Matches(alert => alert.CheckId == "MyCustomCheckId")); + Assert.That(urls.PlatformHealth, Does.EndWith("/api/platform-health")); + Assert.That(instance.Id, Is.EqualTo(Settings.InstanceId)); + Assert.That(instance.Name, Is.EqualTo(Settings.InstanceName)); + Assert.That(instance.HostId, Is.EqualTo(internalCheck.OriginatingEndpoint.HostId)); + Assert.That(instance.HealthSignalsStatus, Is.EqualTo("reported")); + Assert.That(instance.Version, Is.EqualTo(ServiceControlVersion.GetFileVersion())); + Assert.That(instance.ApiUrl.TrimEnd('/'), Is.EqualTo(urls.PlatformHealth[..^"/platform-health".Length])); + Assert.That(instance.ErrorQueue, Is.EqualTo(Settings.ErrorQueue)); + Assert.That(instance.ErrorRetentionPeriod, Is.EqualTo(Settings.ErrorRetentionPeriod)); + Assert.That(instanceJson.GetProperty("health_signals_status").GetString(), Is.EqualTo("reported")); + Assert.That(instanceJson.GetProperty("forward_error_messages").GetBoolean(), Is.EqualTo(Settings.ForwardErrorMessages)); + Assert.That(healthJson.RootElement.GetProperty("license").GetProperty("availability").GetString(), Is.EqualTo("available")); + Assert.That(platformHealth.License.LicenseStatus, Is.Not.Null.And.Not.Empty); // What the wire actually carries: Assert.That(wireBody, Does.Contain("\"internal\":true"), "internal checks must render internal:true on the wire"); diff --git a/src/ServiceControl.AcceptanceTests/Security/OpenIdConnect/When_authentication_is_enabled.cs b/src/ServiceControl.AcceptanceTests/Security/OpenIdConnect/When_authentication_is_enabled.cs index 3795913478..16e252c37c 100644 --- a/src/ServiceControl.AcceptanceTests/Security/OpenIdConnect/When_authentication_is_enabled.cs +++ b/src/ServiceControl.AcceptanceTests/Security/OpenIdConnect/When_authentication_is_enabled.cs @@ -82,20 +82,19 @@ await OpenIdConnectAssertions.AssertAuthConfigurationResponse( expectedRoleBasedAuthorizationEnabled: true); } - [Test] - public async Task Should_reject_requests_without_bearer_token() + [TestCase("/api/errors")] + [TestCase("/api/platform-health")] + public async Task Should_reject_requests_without_bearer_token(string path) { HttpResponseMessage response = null; _ = await Define() .Done(async ctx => { - // Use /api/errors which does NOT have [AllowAnonymous] so it should require authentication - // Note: /api is marked [AllowAnonymous] for server-to-server configuration fetching response = await OpenIdConnectAssertions.SendRequestWithoutAuth( HttpClient, HttpMethod.Get, - "/api/errors"); + path); return response != null; }) .Run(); @@ -123,22 +122,21 @@ public async Task Should_reject_requests_with_invalid_bearer_token() OpenIdConnectAssertions.AssertUnauthorized(response); } - [Test] - public async Task Should_accept_requests_with_valid_bearer_token() + [TestCase("/api/errors")] + [TestCase("/api/platform-health")] + public async Task Should_accept_requests_with_valid_bearer_token(string path) { HttpResponseMessage response = null; _ = await Define() .Done(async ctx => { - // The "reader" role grants every :view permission, including error:messages:view - // required by /api/errors. Without a role-bearing claim the request would be 403. var validToken = mockOidcServer.GenerateToken( additionalClaims: new[] { new Claim("roles", "reader") }); response = await OpenIdConnectAssertions.SendRequestWithBearerToken( HttpClient, HttpMethod.Get, - "/api/errors", + path, validToken); return response != null; }) @@ -147,6 +145,23 @@ public async Task Should_accept_requests_with_valid_bearer_token() OpenIdConnectAssertions.AssertAuthenticated(response); } + [Test] + public async Task Should_forbid_platform_health_without_a_read_role() + { + HttpResponseMessage response = null; + + await Define() + .Done(async _ => + { + response = await OpenIdConnectAssertions.SendRequestWithBearerToken( + HttpClient, HttpMethod.Get, "/api/platform-health", mockOidcServer.GenerateToken()); + return response != null; + }) + .Run(); + + OpenIdConnectAssertions.AssertForbidden(response); + } + [Test] public async Task Should_reject_requests_with_expired_token() { diff --git a/src/ServiceControl.AcceptanceTests/WebApi/When_the_configuration_page_is_read.cs b/src/ServiceControl.AcceptanceTests/WebApi/When_the_configuration_page_is_read.cs index f9a6ffe0b1..64e39fb9e0 100644 --- a/src/ServiceControl.AcceptanceTests/WebApi/When_the_configuration_page_is_read.cs +++ b/src/ServiceControl.AcceptanceTests/WebApi/When_the_configuration_page_is_read.cs @@ -1,5 +1,6 @@ namespace ServiceControl.AcceptanceTests.WebApi { + using System; using System.IO; using System.IO.Compression; using System.Net; @@ -29,6 +30,7 @@ await Define() }) .Run(); + using var json = JsonDocument.Parse(configuration); using (Assert.EnterMultipleScope()) { Assert.That(configuration, Is.EqualTo(instanceInfo), @@ -36,6 +38,9 @@ await Define() Assert.That(configuration, Does.Contain(Settings.InstanceName), "The configuration page names the instance it is describing"); + Assert.That(json.RootElement.GetProperty("instance_type").GetString(), Is.EqualTo("error")); + Assert.That(json.RootElement.GetProperty("host").GetProperty("host_id").GetGuid(), Is.Not.EqualTo(Guid.Empty)); + Assert.That(json.RootElement.GetProperty("health_checks_enabled").GetBoolean(), Is.False); } } diff --git a/src/ServiceControl.Api/Contracts/PlatformHealthView.cs b/src/ServiceControl.Api/Contracts/PlatformHealthView.cs new file mode 100644 index 0000000000..20fc1708dd --- /dev/null +++ b/src/ServiceControl.Api/Contracts/PlatformHealthView.cs @@ -0,0 +1,66 @@ +namespace ServiceControl.Api.Contracts +{ + using System; + + public class PlatformHealthView + { + public string Status { get; set; } + public string Severity { get; set; } + public PlatformHealthAlert[] Alerts { get; set; } + public PlatformHealthInstance[] Instances { get; set; } = []; + public PlatformHealthLicense License { get; set; } = new(); + } + + public class PlatformHealthAlert + { + public Guid Id { get; set; } + public string InstanceId { get; set; } + public string CheckId { get; set; } + public string Category { get; set; } + public string Message { get; set; } + public DateTime ReportedAt { get; set; } + public string InstanceName { get; set; } + public string Host { get; set; } + public Guid HostId { get; set; } + } + +#nullable enable + public sealed record PlatformHealthInstance + { + public required string Id { get; init; } + public required string Name { get; init; } + public required string ApiUrl { get; init; } + public string Kind { get; init; } = "unknown"; + public string Role { get; init; } = "remote-unknown"; + public string? Version { get; init; } + public Guid? HostId { get; init; } + public string Health { get; init; } = "unavailable"; + public DateTimeOffset ObservedAt { get; init; } + public DateTimeOffset? MetadataObservedAt { get; init; } + public string HealthSignalsStatus { get; init; } = "unreported"; + public DateTimeOffset? LastReportedAt { get; init; } + public PlatformHealthAlert[] Issues { get; init; } = []; + public string? TransportType { get; init; } + public string? ErrorQueue { get; init; } + public string? ErrorLogQueue { get; init; } + public bool? ForwardErrorMessages { get; init; } + public string? AuditQueue { get; init; } + public string? AuditLogQueue { get; init; } + public bool? ForwardAuditMessages { get; init; } + public TimeSpan? ErrorRetentionPeriod { get; init; } + public TimeSpan? AuditRetentionPeriod { get; init; } + } + + public sealed record PlatformHealthLicense + { + public string Availability { get; init; } = "unavailable"; + public string? Status { get; init; } + public string? LicenseStatus { get; init; } + public string? LicenseType { get; init; } + public bool? TrialLicense { get; init; } + public DateTimeOffset? ExpirationDate { get; init; } + public DateTimeOffset? UpgradeProtectionExpiration { get; init; } + public string? LicenseExtensionUrl { get; init; } + public bool HasMassTransitConnector { get; init; } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Api/Contracts/RootUrls.cs b/src/ServiceControl.Api/Contracts/RootUrls.cs index 2e9a2aeb32..f5b3df37ae 100644 --- a/src/ServiceControl.Api/Contracts/RootUrls.cs +++ b/src/ServiceControl.Api/Contracts/RootUrls.cs @@ -21,5 +21,6 @@ public class RootUrls public string ArchivedGroupsUrl { get; set; } public string GetArchiveGroup { get; set; } public string MyRoutesUrl { get; set; } + public string PlatformHealth { get; set; } } } diff --git a/src/ServiceControl.Api/IPlatformHealthApi.cs b/src/ServiceControl.Api/IPlatformHealthApi.cs new file mode 100644 index 0000000000..aad4036471 --- /dev/null +++ b/src/ServiceControl.Api/IPlatformHealthApi.cs @@ -0,0 +1,11 @@ +namespace ServiceControl.Api +{ + using System.Threading; + using System.Threading.Tasks; + using Contracts; + + public interface IPlatformHealthApi + { + Task GetHealth(string baseUrl, CancellationToken cancellationToken = default); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Audit.UnitTests/API/APIApprovals.cs b/src/ServiceControl.Audit.UnitTests/API/APIApprovals.cs index 94e2302234..a545839986 100644 --- a/src/ServiceControl.Audit.UnitTests/API/APIApprovals.cs +++ b/src/ServiceControl.Audit.UnitTests/API/APIApprovals.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Reflection; using System.Text; + using System.Text.Json; using Audit.Infrastructure.Settings; using Audit.Infrastructure.WebApi; using Microsoft.AspNetCore.Authorization; @@ -13,6 +14,7 @@ using Microsoft.AspNetCore.Mvc.Controllers; using Microsoft.AspNetCore.Mvc.Routing; using Microsoft.AspNetCore.Routing; + using NServiceBus.Hosting; using NUnit.Framework; using Particular.Approvals; using ServiceControl.Hosting.Auth; @@ -30,7 +32,7 @@ public void RootPathValue() var settings = CreateTestSettings(); - var controller = new RootController(settings) + var controller = new RootController(settings, new HostInformation(Guid.Empty, "localhost")) { ControllerContext = controllerContext, Url = new UrlHelper(actionContext) @@ -41,6 +43,22 @@ public void RootPathValue() Approver.Verify(result.Value); } + [Test] + public void Configuration_reports_the_instance_type_and_reporting_host_id() + { + var hostId = Guid.NewGuid(); + var controller = new RootController(CreateTestSettings(), new HostInformation(hostId, "localhost")); + + using var json = JsonDocument.Parse(JsonSerializer.Serialize(controller.Config().Value, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower })); + + using (Assert.EnterMultipleScope()) + { + Assert.That(json.RootElement.GetProperty("instance_type").GetString(), Is.EqualTo("audit")); + Assert.That(json.RootElement.GetProperty("host").GetProperty("host_id").GetGuid(), Is.EqualTo(hostId)); + } + } + [Test] public void HttpApiRoutes() { diff --git a/src/ServiceControl.Audit/Infrastructure/WebApi/RootController.cs b/src/ServiceControl.Audit/Infrastructure/WebApi/RootController.cs index 6334d612a6..8a6f72d379 100644 --- a/src/ServiceControl.Audit/Infrastructure/WebApi/RootController.cs +++ b/src/ServiceControl.Audit/Infrastructure/WebApi/RootController.cs @@ -4,6 +4,7 @@ using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http.Extensions; using Microsoft.AspNetCore.Mvc; + using NServiceBus.Hosting; using Settings; // the /api endpoint is used for service-to-service communication. This currently needs to be anonymous @@ -12,9 +13,10 @@ [Route("api")] public class RootController : ControllerBase { - public RootController(Settings settings) + public RootController(Settings settings, HostInformation hostInformation) { this.settings = settings; + this.hostInformation = hostInformation; } [Route("")] @@ -50,9 +52,11 @@ public OkObjectResult Config() { object content = new { + InstanceType = "audit", Host = new { settings.InstanceName, + hostInformation.HostId, Logging = new { settings.LoggingSettings.LogPath, @@ -87,6 +91,7 @@ public OkObjectResult Config() } readonly Settings settings; + readonly HostInformation hostInformation; public class RootUrls { diff --git a/src/ServiceControl.MultiInstance.AcceptanceTests/Infrastructure/When_inspecting_platform_health.cs b/src/ServiceControl.MultiInstance.AcceptanceTests/Infrastructure/When_inspecting_platform_health.cs new file mode 100644 index 0000000000..de4cd93809 --- /dev/null +++ b/src/ServiceControl.MultiInstance.AcceptanceTests/Infrastructure/When_inspecting_platform_health.cs @@ -0,0 +1,99 @@ +namespace ServiceControl.MultiInstance.AcceptanceTests.Infrastructure; + +using System; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using AcceptanceTesting; +using Api.Contracts; +using Audit.Auditing; +using Microsoft.Extensions.DependencyInjection; +using NServiceBus.AcceptanceTesting; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using TestSupport; + +class When_inspecting_platform_health : AcceptanceTest +{ + [Test] + public async Task Should_show_audit_issues_recovery_and_unavailable_configured_instances() + { + const string checkId = "Audit Message Ingestion Process"; + const string failure = "Audit ingestion interrupted for the platform health scenario"; + var auditState = new AuditIngestionCustomCheck.State(); + auditState.ReportError(failure); + var offline = new RemoteInstanceSetting("http://offline:12121"); + + AuditHostBuilderCustomization = builder => builder.Services.AddSingleton(auditState); + CustomServiceControlPrimarySettings = settings => settings.RemoteInstances = [.. settings.RemoteInstances, offline]; + PrimaryHostBuilderCustomization = builder => builder.Services.AddKeyedSingleton>( + offline.InstanceId, () => new UnavailableHandler()); + + PlatformHealthView failing = null; + PlatformHealthView recovered = null; + + await Define() + .Do("Read the platform inventory and failing audit report", async context => + { + failing = await this.TryGet("/api/platform-health", instanceName: ServiceControlInstanceName); + context.LastAlerts = string.Join(", ", failing?.Alerts.Select(alert => $"{alert.InstanceName}: {alert.CheckId}") ?? []); + return failing?.Alerts.Any(alert => alert.CheckId == checkId) == true; + }) + .Do("Observe audit recovery in platform health", async context => + { + if (!context.RecoveryRequested) + { + auditState.Clear(); + context.RecoveryRequested = true; + } + + recovered = await this.TryGet("/api/platform-health", instanceName: ServiceControlInstanceName); + context.LastAlerts = string.Join(", ", recovered?.Alerts.Select(alert => $"{alert.InstanceName}: {alert.CheckId}") ?? []); + return recovered != null && recovered.Alerts.All(alert => alert.CheckId != checkId); + }) + .Done(_ => true) + .Run(); + + var primary = failing.Instances.Single(instance => instance.Role == "primary-error"); + var audit = failing.Instances.Single(instance => instance.Role == "remote-audit"); + var unavailable = failing.Instances.Single(instance => instance.Id == offline.InstanceId); + var issue = failing.Alerts.Single(alert => alert.CheckId == checkId); + var recoveredAudit = recovered.Instances.Single(instance => instance.Id == audit.Id); + + using (Assert.EnterMultipleScope()) + { + Assert.That(failing.Instances, Has.Length.EqualTo(3)); + Assert.That(primary.Name, Is.EqualTo(ServiceControlInstanceName)); + Assert.That(primary.Issues, Has.None.Matches(alert => alert.CheckId == checkId)); + Assert.That(audit.Name, Is.EqualTo(ServiceControlAuditInstanceName)); + Assert.That(audit.HostId, Is.EqualTo(issue.HostId)); + Assert.That(audit.Health, Is.EqualTo("degraded")); + Assert.That(audit.HealthSignalsStatus, Is.EqualTo("reported")); + Assert.That(audit.Version, Is.Not.Null.And.Not.Empty); + Assert.That(audit.AuditRetentionPeriod, Is.Not.Null); + Assert.That(audit.Issues, Has.Some.Matches(alert => alert.Id == issue.Id)); + Assert.That(issue.InstanceId, Is.EqualTo(audit.Id)); + Assert.That(issue.Message, Is.EqualTo(failure)); + Assert.That(unavailable.Kind, Is.EqualTo("unknown")); + Assert.That(unavailable.Health, Is.EqualTo("unavailable")); + Assert.That(unavailable.Version, Is.Null); + Assert.That(recoveredAudit.Health, Is.EqualTo("healthy")); + Assert.That(recoveredAudit.Issues, Is.Empty); + Assert.That(recovered.License.Availability, Is.EqualTo("available")); + } + } + + sealed class UnavailableHandler : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken = default) => + throw new HttpRequestException(HttpRequestError.ConnectionError); + } + + class Context : ScenarioContext, ISequenceContext + { + public int Step { get; set; } + public string LastAlerts { get; set; } + public bool RecoveryRequested { get; set; } + } +} \ No newline at end of file diff --git a/src/ServiceControl.UnitTests/API/APIApprovals.cs b/src/ServiceControl.UnitTests/API/APIApprovals.cs index 04d17fc409..d103264a2e 100644 --- a/src/ServiceControl.UnitTests/API/APIApprovals.cs +++ b/src/ServiceControl.UnitTests/API/APIApprovals.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Reflection; using System.Text; + using System.Text.Json; using System.Threading.Tasks; using Api.Contracts; using Microsoft.AspNetCore.Authorization; @@ -15,6 +16,7 @@ using Microsoft.AspNetCore.Routing; using Microsoft.Extensions.Logging.Abstractions; using NServiceBus.CustomChecks; + using NServiceBus.Hosting; using NUnit.Framework; using Particular.Approvals; using Particular.ServiceControl.Licensing; @@ -38,7 +40,8 @@ public async Task RootPathValue() new ActiveLicense(null, NullLogger.Instance) { IsValid = true }, new Settings(), null, - new MassTransitConnectorHeartbeatStatus()); + new MassTransitConnectorHeartbeatStatus(), + new HostInformation(Guid.Empty, "localhost")); var controller = new RootController(configurationApi) { @@ -51,6 +54,23 @@ public async Task RootPathValue() Approver.Verify(result); } + [Test] + public async Task Configuration_reports_the_instance_type_and_reporting_host_id() + { + var hostId = Guid.NewGuid(); + var configuration = new ConfigurationApi(null, new Settings { DisableHealthChecks = true }, null, + new MassTransitConnectorHeartbeatStatus(), new HostInformation(hostId, "localhost")); + + using var json = JsonDocument.Parse(JsonSerializer.Serialize(await configuration.GetConfig(), SerializerOptions.Default)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(json.RootElement.GetProperty("instance_type").GetString(), Is.EqualTo("error")); + Assert.That(json.RootElement.GetProperty("host").GetProperty("host_id").GetGuid(), Is.EqualTo(hostId)); + Assert.That(json.RootElement.GetProperty("health_checks_enabled").GetBoolean(), Is.False); + } + } + [Test] public void HttpApiRoutes() { diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt index 69498d4d04..994be1bead 100644 --- a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt +++ b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt @@ -57,6 +57,7 @@ PATCH /pendingretries/queues/resolve => ServiceControl.MessageFailures.Api.Resol POST /pendingretries/queues/retry => ServiceControl.MessageFailures.Api.PendingRetryMessagesController:RetryBy(PendingRetryRequest request, CancellationToken cancellationToken) PATCH /pendingretries/resolve => ServiceControl.MessageFailures.Api.ResolveMessagesController:ResolveBy(UniqueMessageIdsModel request, CancellationToken cancellationToken) POST /pendingretries/retry => ServiceControl.MessageFailures.Api.PendingRetryMessagesController:RetryBy(String[] ids, CancellationToken cancellationToken) +GET /platform-health => ServiceControl.PlatformHealth.PlatformHealthController:Get(CancellationToken cancellationToken) GET /recoverability/classifiers => ServiceControl.Recoverability.API.FailureGroupsController:GetSupportedClassifiers() GET /recoverability/groups/{classifier?} => ServiceControl.Recoverability.API.FailureGroupsController:GetAllGroups(String classifier, String classifierFilter, CancellationToken cancellationToken) DELETE /recoverability/groups/{groupId:required:minlength(1)}/comment => ServiceControl.Recoverability.API.FailureGroupsController:DeleteComment(String groupId, CancellationToken cancellationToken) diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.RootPathValue.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.RootPathValue.approved.txt index b73f751bba..7dd5575f82 100644 --- a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.RootPathValue.approved.txt +++ b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.RootPathValue.approved.txt @@ -17,5 +17,6 @@ "EventLogItems": "http://localhost/eventlogitems", "ArchivedGroupsUrl": "http://localhost/errors/groups/{classifier?}", "GetArchiveGroup": "http://localhost/archive/groups/id/{groupId}", - "MyRoutesUrl": "http://localhost/my/routes" + "MyRoutesUrl": "http://localhost/my/routes", + "PlatformHealth": "http://localhost/platform-health" } \ No newline at end of file diff --git a/src/ServiceControl.UnitTests/Licensing/ActiveLicenseTests.cs b/src/ServiceControl.UnitTests/Licensing/ActiveLicenseTests.cs index 00a93ffc04..7ce334c736 100644 --- a/src/ServiceControl.UnitTests/Licensing/ActiveLicenseTests.cs +++ b/src/ServiceControl.UnitTests/Licensing/ActiveLicenseTests.cs @@ -4,9 +4,14 @@ using System.Threading; using System.Threading.Tasks; using LicenseManagement; + using Microsoft.Extensions.Logging.Abstractions; using NUnit.Framework; using Particular.ServiceControl.Licensing; using Persistence; + using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.Connector.MassTransit; + using ServiceControl.Licensing; + using ServiceControl.Monitoring.HeartbeatMonitoring; [TestFixture] public class ActiveLicenseTests @@ -60,6 +65,46 @@ public async Task Accepts_license_base_on_the_db_value_only() Assert.That(checkedDetails.HasLicenseExpired, Is.False); } + [TestCase(false, true, "https://particular.net/extend-your-trial?p=servicepulse")] + [TestCase(true, true, "https://particular.net/license/mt?p=servicepulse&t=0")] + [TestCase(true, false, "https://particular.net/license/mt?p=servicepulse&t=1")] + public async Task License_information_preserves_the_existing_mapping_and_renewal_links(bool massTransit, bool evaluation, string expectedUrl) + { + var details = LicenseDetails.TrialFromEndDate(new DateOnly(2026, 9, 30)); + var active = new ActiveLicense(null, NullLogger.Instance) + { + Details = details, + IsValid = false, + IsEvaluation = evaluation + }; + var connector = new MassTransitConnectorHeartbeatStatus(); + if (massTransit) + { + connector.Update(new MassTransitConnectorHeartbeat + { + Version = "1.0.0", + ErrorQueues = [], + Logs = [], + SentDateTimeOffset = DateTimeOffset.MinValue + }); + } + + var provider = new LicenseInfoProvider(active, new Settings { InstanceName = "Primary" }, connector); + var result = await provider.GetLicense(false, "servicepulse"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.LicenseExtensionUrl, Is.EqualTo(expectedUrl)); + Assert.That(result.LicenseStatus, Is.EqualTo(details.Status)); + Assert.That(result.Status, Is.EqualTo("invalid")); + Assert.That(result.TrialLicense, Is.True); + Assert.That(result.LicenseType, Is.EqualTo(details.LicenseType)); + Assert.That(result.ExpirationDate, Is.EqualTo(details.ExpirationDate?.ToString("O"))); + Assert.That(result.UpgradeProtectionExpiration, Is.Empty); + Assert.That(result.InstanceName, Is.EqualTo("Primary")); + } + } + class FakeDataProvider : ITrialLicenseDataProvider { TrialMetadata metadata; diff --git a/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthApiTests.cs b/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthApiTests.cs new file mode 100644 index 0000000000..b2bee5bde4 --- /dev/null +++ b/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthApiTests.cs @@ -0,0 +1,420 @@ +namespace ServiceControl.UnitTests.PlatformHealth; + +using System; +using System.Linq; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Time.Testing; +using NServiceBus.Hosting; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Api; +using ServiceControl.Api.Contracts; +using ServiceControl.Contracts.CustomChecks; +using ServiceControl.Infrastructure; +using ServiceControl.Licensing; +using ServiceControl.Monitoring.HeartbeatMonitoring; +using ServiceControl.Operations; +using ServiceControl.PlatformHealth; + +[TestFixture] +class PlatformHealthApiTests +{ + [SetUp] + public void SetUp() + { + settings = new Settings + { + InstanceName = "Primary", + TransportType = "RabbitMQ", + ErrorQueue = "error", + ErrorLogQueue = "error.log", + ForwardErrorMessages = false, + RemoteInstances = [] + }; + state = new PlatformHealthState(); + configuration = new FakeConfigurationApi(); + licensing = new FakeLicenseInfoProvider(); + clock = new FakeTimeProvider(new DateTimeOffset(2026, 9, 30, 12, 0, 0, TimeSpan.Zero)); + api = new PlatformHealthApi(settings, new HostInformation(PrimaryHostId, "primary-host"), state, + configuration, licensing, new MassTransitConnectorHeartbeatStatus(), clock); + } + + [Test] + public async Task Includes_primary_inventory_configuration_and_version_before_checks_report() + { + var result = await api.GetHealth("https://public/servicecontrol/api"); + var primary = result.Instances.Single(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Status, Is.EqualTo("unknown")); + Assert.That(primary.Id, Is.EqualTo(settings.InstanceId)); + Assert.That(primary.Name, Is.EqualTo("Primary")); + Assert.That(primary.Kind, Is.EqualTo("error")); + Assert.That(primary.Role, Is.EqualTo("primary-error")); + Assert.That(primary.ApiUrl, Is.EqualTo("https://public/servicecontrol/api/")); + Assert.That(primary.Version, Is.EqualTo(ServiceControlVersion.GetFileVersion())); + Assert.That(primary.Health, Is.EqualTo("healthy")); + Assert.That(primary.HealthSignalsStatus, Is.EqualTo("unreported")); + Assert.That(primary.LastReportedAt, Is.Null); + Assert.That(primary.ObservedAt, Is.EqualTo(clock.GetUtcNow())); + Assert.That(primary.HostId, Is.EqualTo(PrimaryHostId)); + Assert.That(primary.TransportType, Is.EqualTo("RabbitMQ")); + Assert.That(primary.ErrorQueue, Is.EqualTo("error")); + Assert.That(primary.ErrorLogQueue, Is.EqualTo("error.log")); + Assert.That(primary.ForwardErrorMessages, Is.False); + Assert.That(primary.ErrorRetentionPeriod, Is.EqualTo(settings.ErrorRetentionPeriod)); + Assert.That(licensing.Refresh, Is.True); + Assert.That(licensing.ClientName, Is.EqualTo("servicepulse")); + } + } + + [Test] + public async Task Controller_uses_the_public_scheme_host_and_proxy_prefix() + { + var context = new DefaultHttpContext(); + context.Request.Scheme = "https"; + context.Request.Host = new HostString("public.example", 8443); + context.Request.PathBase = "/servicecontrol"; + var controller = new PlatformHealthController(api) { ControllerContext = new ControllerContext { HttpContext = context } }; + + var result = await controller.Get(); + + Assert.That(result.Instances[0].ApiUrl, Is.EqualTo("https://public.example:8443/servicecontrol/api/")); + } + + [Test] + public async Task Correlates_issues_by_host_identity_and_clears_them_after_recovery() + { + var remote = new RemoteInstanceSetting("https://audit"); + settings.RemoteInstances = [remote]; + configuration.Remotes = [Remote(remote, "Primary", AuditHostId)]; + var report = Report("Primary", AuditHostId); + state.Record(report); + + var failing = await api.GetHealth("https://primary/api/"); + var audit = failing.Instances.Single(instance => instance.Id == remote.InstanceId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(failing.Instances[0].Issues, Is.Empty); + Assert.That(audit.Kind, Is.EqualTo("audit")); + Assert.That(audit.Role, Is.EqualTo("remote-audit")); + Assert.That(audit.Health, Is.EqualTo("degraded")); + Assert.That(audit.HealthSignalsStatus, Is.EqualTo("reported")); + Assert.That(audit.AuditRetentionPeriod, Is.EqualTo(TimeSpan.FromDays(7))); + Assert.That(audit.Issues, Has.Length.EqualTo(1)); + Assert.That(audit.Issues[0].InstanceId, Is.EqualTo(remote.InstanceId)); + Assert.That(failing.Alerts[0].InstanceId, Is.EqualTo(remote.InstanceId)); + } + + report.HasFailed = false; + report.ReportedAt = report.ReportedAt.AddMinutes(1); + state.Record(report); + var recovered = await api.GetHealth("https://primary/api/"); + + Assert.That(recovered.Instances.Single(instance => instance.Id == remote.InstanceId).Health, Is.EqualTo("healthy")); + Assert.That(recovered.Alerts, Is.Empty); + } + + [Test] + public async Task Keeps_configured_offline_instances_and_last_observed_metadata() + { + var remote = new RemoteInstanceSetting("https://audit/prefix"); + settings.RemoteInstances = [remote]; + var first = await api.GetHealth("https://primary/api/"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(first.Instances, Has.Length.EqualTo(2)); + Assert.That(first.Instances[1].Health, Is.EqualTo("unavailable")); + Assert.That(first.Instances[1].Kind, Is.EqualTo("unknown")); + Assert.That(first.Instances[1].Version, Is.Null); + } + + configuration.Remotes = [Remote(remote, "Audit", AuditHostId)]; + var online = await api.GetHealth("https://primary/api/"); + configuration.Remotes = []; + clock.Advance(TimeSpan.FromMinutes(1)); + var offline = await api.GetHealth("https://primary/api/"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(offline.Instances[1].Name, Is.EqualTo("Audit")); + Assert.That(offline.Instances[1].Kind, Is.EqualTo("audit")); + Assert.That(offline.Instances[1].Version, Is.EqualTo("6.10.0")); + Assert.That(offline.Instances[1].ApiUrl, Is.EqualTo("https://audit/prefix/api/")); + Assert.That(offline.Instances[1].Health, Is.EqualTo("unavailable")); + Assert.That(offline.Instances[1].MetadataObservedAt, Is.EqualTo(online.Instances[1].ObservedAt)); + Assert.That(offline.Instances[1].ObservedAt, Is.EqualTo(clock.GetUtcNow())); + } + } + + [Test] + public async Task Remote_error_instances_keep_configuration_and_recover_connectivity() + { + var remote = new RemoteInstanceSetting("https://remote-error/prefix"); + settings.RemoteInstances = [remote]; + var metadata = Remote(remote, "Remote error", AuditHostId); + metadata.Configuration["instance_type"] = "error"; + metadata.Configuration["data_retention"] = JsonNode.Parse("""{"error_retention_period":"21.00:00:00"}"""); + metadata.Configuration["transport"] = JsonNode.Parse(""" + {"transport_type":"RabbitMQ","error_queue":"remote.error","error_log_queue":"remote.log","forward_error_messages":false} + """); + configuration.Remotes = [metadata]; + state.Record(Report("Remote error", AuditHostId)); + + var result = await api.GetHealth("https://primary/api/"); + var instance = result.Instances[1]; + using (Assert.EnterMultipleScope()) + { + Assert.That(instance.Kind, Is.EqualTo("error")); + Assert.That(instance.Role, Is.EqualTo("remote-error")); + Assert.That(instance.Health, Is.EqualTo("degraded")); + Assert.That(instance.ErrorQueue, Is.EqualTo("remote.error")); + Assert.That(instance.ErrorLogQueue, Is.EqualTo("remote.log")); + Assert.That(instance.ForwardErrorMessages, Is.False); + Assert.That(instance.ErrorRetentionPeriod, Is.EqualTo(TimeSpan.FromDays(21))); + Assert.That(instance.AuditRetentionPeriod, Is.Null); + } + + metadata.Status = "unavailable"; + Assert.That((await api.GetHealth("https://primary/api/")).Instances[1].Health, Is.EqualTo("unavailable")); + metadata.Status = "online"; + metadata.Version = "6.11.0"; + clock.Advance(TimeSpan.FromMinutes(1)); + var recovered = (await api.GetHealth("https://primary/api/")).Instances[1]; + using (Assert.EnterMultipleScope()) + { + Assert.That(recovered.Health, Is.EqualTo("degraded")); + Assert.That(recovered.Version, Is.EqualTo("6.11.0")); + Assert.That(recovered.MetadataObservedAt, Is.EqualTo(clock.GetUtcNow())); + } + } + + [Test] + public async Task Malformed_optional_remote_metadata_does_not_hide_other_instances() + { + var malformed = new RemoteInstanceSetting("https://malformed"); + var valid = new RemoteInstanceSetting("https://valid"); + settings.RemoteInstances = [malformed, valid]; + var badMetadata = Remote(malformed, "Malformed audit", AuditHostId); + badMetadata.Configuration["data_retention"]["audit_retention_period"] = "not a duration"; + configuration.Remotes = [badMetadata, Remote(valid, "Valid audit", Guid.NewGuid())]; + + var result = await api.GetHealth("https://primary/api/"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Instances, Has.Length.EqualTo(3)); + Assert.That(result.Instances.Single(instance => instance.Id == malformed.InstanceId).Health, Is.EqualTo("unavailable")); + Assert.That(result.Instances.Single(instance => instance.Id == valid.InstanceId).Health, Is.EqualTo("healthy")); + Assert.That(result.License.Availability, Is.EqualTo("available")); + } + } + + [TestCase(false)] + [TestCase(true)] + public async Task Legacy_name_matching_requires_unique_inventory_and_reporting_host(bool duplicateInventory) + { + var first = new RemoteInstanceSetting("https://first"); + var second = new RemoteInstanceSetting("https://second"); + settings.RemoteInstances = duplicateInventory ? [first, second] : [first]; + configuration.Remotes = duplicateInventory ? [Remote(first, "Audit"), Remote(second, "Audit")] : [Remote(first, "Audit")]; + state.Record(Report("Audit", AuditHostId)); + if (!duplicateInventory) + { + state.Record(Report("Audit", Guid.NewGuid())); + } + + var result = await api.GetHealth("https://primary/api/"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Instances.Skip(1).Select(instance => instance.HealthSignalsStatus), Is.All.EqualTo("ambiguous")); + Assert.That(result.Instances.SelectMany(instance => instance.Issues), Is.Empty); + Assert.That(result.Alerts.Select(alert => alert.InstanceId), Is.All.Null); + } + } + + [Test] + public async Task Legacy_configuration_infers_kind_and_matches_unique_names_without_inventing_versions() + { + var remote = new RemoteInstanceSetting("https://audit"); + settings.RemoteInstances = [remote]; + var legacy = Remote(remote, "Audit"); + legacy.Configuration.AsObject().Remove("instance_type"); + legacy.Version = "Unknown"; + configuration.Remotes = [legacy]; + state.Record(Report("audit", AuditHostId)); + + var result = await api.GetHealth("https://primary/api/"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Instances[1].Kind, Is.EqualTo("audit")); + Assert.That(result.Instances[1].Version, Is.Null); + Assert.That(result.Instances[1].Health, Is.EqualTo("degraded")); + Assert.That(result.Alerts[0].InstanceId, Is.EqualTo(remote.InstanceId)); + } + } + + [Test] + public async Task License_failure_and_remote_failure_do_not_hide_local_health() + { + settings.RemoteInstances = [new RemoteInstanceSetting("https://offline")]; + configuration.Failure = new InvalidOperationException("Remote unavailable"); + licensing.Failure = new InvalidOperationException("License unavailable"); + state.Record(Report("Primary", PrimaryHostId)); + + var result = await api.GetHealth("https://primary/api/"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Instances, Has.Length.EqualTo(2)); + Assert.That(result.Instances[0].Health, Is.EqualTo("degraded")); + Assert.That(result.Instances[1].Health, Is.EqualTo("unavailable")); + Assert.That(result.License.Availability, Is.EqualTo("unavailable")); + Assert.That(result.License.Status, Is.Null); + Assert.That(result.License.LicenseStatus, Is.Null); + } + } + + [TestCase("Valid")] + [TestCase("ValidWithExpiringTrial")] + [TestCase("InvalidDueToExpiredTrial")] + [TestCase("InvalidDueToExpiredSubscription")] + [TestCase("InvalidDueToExpiredUpgradeProtection")] + public async Task License_status_and_renewal_fields_preserve_the_license_api_values(string licenseStatus) + { + licensing.Info.LicenseStatus = licenseStatus; + licensing.Info.ExpirationDate = "2026-10-01T00:00:00.0000000Z"; + licensing.Info.UpgradeProtectionExpiration = ""; + + var result = await api.GetHealth("https://primary/api/"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.License.Availability, Is.EqualTo("available")); + Assert.That(result.License.LicenseStatus, Is.EqualTo(licenseStatus)); + Assert.That(result.License.LicenseType, Is.EqualTo(licensing.Info.LicenseType)); + Assert.That(result.License.TrialLicense, Is.EqualTo(licensing.Info.TrialLicense)); + Assert.That(result.License.LicenseExtensionUrl, Is.EqualTo(licensing.Info.LicenseExtensionUrl)); + Assert.That(result.License.ExpirationDate, Is.EqualTo(new DateTimeOffset(2026, 10, 1, 0, 0, 0, TimeSpan.Zero))); + Assert.That(result.License.UpgradeProtectionExpiration, Is.Null); + } + } + + [Test] + public async Task Disabled_checks_are_explicit_and_not_mistaken_for_reports() + { + settings.DisableHealthChecks = true; + + var result = await api.GetHealth("https://primary/api/"); + + Assert.That(result.Instances[0].HealthSignalsStatus, Is.EqualTo("disabled")); + } + + [Test] + public void Caller_cancellation_is_not_converted_to_partial_success() + { + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + Assert.That(async () => await api.GetHealth("https://primary/api/", cancellation.Token), Throws.InstanceOf()); + } + + [Test] + public void Cancellation_is_forwarded_to_dependencies_and_propagated_during_refresh() + { + settings.RemoteInstances = [new RemoteInstanceSetting("https://audit")]; + using var cancellation = new CancellationTokenSource(); + licensing.BeforeRead = cancellation.Cancel; + licensing.Failure = new OperationCanceledException(cancellation.Token); + + Assert.That(async () => await api.GetHealth("https://primary/api/", cancellation.Token), Throws.InstanceOf()); + using (Assert.EnterMultipleScope()) + { + Assert.That(configuration.CancellationToken, Is.EqualTo(cancellation.Token)); + Assert.That(licensing.CancellationToken, Is.EqualTo(cancellation.Token)); + } + } + + static RemoteConfiguration Remote(RemoteInstanceSetting setting, string name, Guid? hostId = null) + { + var configuration = JsonNode.Parse(""" + {"instance_type":"audit","host":{},"data_retention":{"audit_retention_period":"7.00:00:00"}} + """); + configuration["host"]["instance_name"] = name; + if (hostId.HasValue) + { + configuration["host"]["host_id"] = hostId.Value.ToString(); + } + return new RemoteConfiguration { ApiUri = setting.BaseAddress, Configuration = configuration, Status = "online", Version = "6.10.0" }; + } + + CustomCheckDetail Report(string name, Guid hostId) => new() + { + CustomCheckId = "Audit Message Ingestion", + Category = "ServiceControl Health", + HasFailed = true, + FailureReason = "Ingestion failed", + ReportedAt = clock.GetUtcNow().UtcDateTime, + OriginatingEndpoint = new EndpointDetails { Name = name, Host = "host", HostId = hostId } + }; + + sealed class FakeConfigurationApi : IConfigurationApi + { + public RemoteConfiguration[] Remotes { get; set; } = []; + public Exception Failure { get; set; } + public CancellationToken CancellationToken { get; private set; } + + public Task GetRemoteConfigs(CancellationToken cancellationToken = default) + { + CancellationToken = cancellationToken; + return Failure == null ? Task.FromResult(Remotes) : Task.FromException(Failure); + } + + public Task GetConfig(CancellationToken cancellationToken = default) => throw new NotSupportedException(); + public Task GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotSupportedException(); + } + + sealed class FakeLicenseInfoProvider : ILicenseInfoProvider + { + public LicenseInfo Info { get; } = new() + { + Status = "valid", + LicenseStatus = "Valid", + LicenseType = "Trial", + TrialLicense = true, + LicenseExtensionUrl = "https://particular.net/extend-your-trial?p=servicepulse" + }; + public Exception Failure { get; set; } + public bool Refresh { get; private set; } + public string ClientName { get; private set; } + public CancellationToken CancellationToken { get; private set; } + public Action BeforeRead { get; set; } + + public Task GetLicense(bool refresh, string clientName, CancellationToken cancellationToken = default) + { + Refresh = refresh; + ClientName = clientName; + CancellationToken = cancellationToken; + BeforeRead?.Invoke(); + return Failure == null ? Task.FromResult(Info) : Task.FromException(Failure); + } + } + + Settings settings; + PlatformHealthState state; + FakeConfigurationApi configuration; + FakeLicenseInfoProvider licensing; + FakeTimeProvider clock; + PlatformHealthApi api; + static readonly Guid PrimaryHostId = Guid.Parse("BD444A23-93E3-42E5-A9C2-15CD7436756E"); + static readonly Guid AuditHostId = Guid.Parse("627A66F4-F7C5-4D18-8793-0D8C385A5744"); +} \ No newline at end of file diff --git a/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthStateTests.cs b/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthStateTests.cs new file mode 100644 index 0000000000..11418d14e2 --- /dev/null +++ b/src/ServiceControl.UnitTests/PlatformHealth/PlatformHealthStateTests.cs @@ -0,0 +1,207 @@ +namespace ServiceControl.UnitTests.PlatformHealth +{ + using System; + using System.Text.Json; + using NUnit.Framework; + using ServiceControl.Api.Contracts; + using ServiceControl.Contracts.CustomChecks; + using ServiceControl.Infrastructure.WebApi; + using ServiceControl.Operations; + using ServiceControl.PlatformHealth; + + [TestFixture] + class PlatformHealthStateTests + { + [Test] + public void Health_is_unknown_until_an_internal_check_reports() + { + var health = new PlatformHealthState().GetHealth(); + + Assert.That(health.Status, Is.EqualTo("unknown")); + Assert.That(health.Severity, Is.EqualTo("unknown")); + Assert.That(health.Alerts, Is.Empty); + } + + [Test] + public void Failed_internal_check_is_returned_as_an_alert_and_cleared_on_recovery() + { + var state = new PlatformHealthState(); + var detail = Detail("ServiceControl Primary Instance", hasFailed: true); + + state.Record(detail); + + var failingHealth = state.GetHealth(); + Assert.That(failingHealth.Status, Is.EqualTo("unhealthy")); + Assert.That(failingHealth.Severity, Is.EqualTo("error")); + Assert.That(failingHealth.Alerts, Has.Length.EqualTo(1)); + Assert.That(failingHealth.Alerts[0].CheckId, Is.EqualTo(detail.CustomCheckId)); + Assert.That(failingHealth.Alerts[0].Message, Is.EqualTo(detail.FailureReason)); + Assert.That(failingHealth.Alerts[0].InstanceName, Is.EqualTo(detail.OriginatingEndpoint.Name)); + Assert.That(failingHealth.Alerts[0].HostId, Is.EqualTo(detail.OriginatingEndpoint.HostId)); + + detail.HasFailed = false; + detail.FailureReason = null; + state.Record(detail); + + var recoveredHealth = state.GetHealth(); + Assert.That(recoveredHealth.Status, Is.EqualTo("healthy")); + Assert.That(recoveredHealth.Severity, Is.EqualTo("none")); + Assert.That(recoveredHealth.Alerts, Is.Empty); + } + + [Test] + public void Customer_custom_checks_do_not_affect_platform_health() + { + var state = new PlatformHealthState(); + state.Record(Detail("Customer check", hasFailed: true)); + + var health = state.GetHealth(); + + Assert.That(health.Status, Is.EqualTo("unknown")); + Assert.That(health.Alerts, Is.Empty); + } + + [Test] + public void Audit_internal_checks_are_included_with_their_originating_instance() + { + var state = new PlatformHealthState(); + var detail = Detail("Audit Message Ingestion", hasFailed: true); + detail.OriginatingEndpoint.Name = "ServiceControl.Audit"; + + state.Record(detail); + + var health = state.GetHealth(); + Assert.That(health.Status, Is.EqualTo("unhealthy")); + Assert.That(health.Alerts, Has.Length.EqualTo(1)); + Assert.That(health.Alerts[0].InstanceName, Is.EqualTo("ServiceControl.Audit")); + } + + [Test] + public void Delayed_failure_does_not_replace_a_newer_recovery() + { + var state = new PlatformHealthState(); + var recovery = Detail("ServiceControl Primary Instance", hasFailed: false); + recovery.ReportedAt = recovery.ReportedAt.AddMinutes(1); + + state.Record(recovery); + state.Record(Detail("ServiceControl Primary Instance", hasFailed: true)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(state.GetHealth().Status, Is.EqualTo("healthy")); + Assert.That(state.GetChecks()[0].ReportedAt, Is.EqualTo(recovery.ReportedAt)); + } + } + + [Test] + public void Snapshots_include_passes_and_do_not_change_with_later_reports() + { + var state = new PlatformHealthState(); + var detail = Detail("ServiceControl Primary Instance", hasFailed: false); + state.Record(detail); + var snapshot = state.GetChecks(); + + detail.HasFailed = true; + detail.FailureReason = "Later failure"; + detail.ReportedAt = detail.ReportedAt.AddMinutes(1); + state.Record(detail); + + using (Assert.EnterMultipleScope()) + { + Assert.That(snapshot, Has.Length.EqualTo(1)); + Assert.That(snapshot[0].HasFailed, Is.False); + Assert.That(snapshot[0].Message, Is.Null); + Assert.That(state.GetChecks()[0].HasFailed, Is.True); + Assert.That(state.GetChecks()[0].Id, Is.EqualTo(snapshot[0].Id)); + } + } + + [Test] + public void Same_named_checks_on_different_hosts_have_distinct_stably_ordered_ids() + { + var state = new PlatformHealthState(); + var first = Detail("Audit Message Ingestion", hasFailed: true); + var second = Detail("Audit Message Ingestion", hasFailed: false); + second.OriginatingEndpoint.HostId = Guid.Parse("627A66F4-F7C5-4D18-8793-0D8C385A5744"); + + state.Record(second); + state.Record(first); + var snapshot = state.GetChecks(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(snapshot, Has.Length.EqualTo(2)); + Assert.That(snapshot[0].Id, Is.Not.EqualTo(snapshot[1].Id)); + Assert.That(snapshot, Is.Ordered.By(nameof(PlatformHealthState.CheckState.Id))); + Assert.That(state.GetHealth().Alerts, Has.Length.EqualTo(1)); + } + } + + [Test] + public void Expanded_contract_preserves_wire_names_and_false_values_but_omits_unknown_fields() + { + var health = new PlatformHealthView + { + Status = "unknown", + Severity = "unknown", + Alerts = [], + Instances = [new PlatformHealthInstance + { + Id = "primary", + Name = "ServiceControl", + ApiUrl = "https://localhost/servicecontrol/api/", + Kind = "error", + Role = "primary-error", + Version = "6.10.0", + Health = "healthy", + ObservedAt = new DateTimeOffset(2026, 9, 30, 12, 0, 0, TimeSpan.Zero), + ForwardErrorMessages = false, + ErrorRetentionPeriod = TimeSpan.FromDays(14) + }], + License = new PlatformHealthLicense + { + Availability = "available", + LicenseStatus = "InvalidDueToExpiredSubscription", + TrialLicense = false, + LicenseExtensionUrl = "https://particular.net/extend-your-trial?p=servicepulse" + } + }; + + using var json = JsonDocument.Parse(JsonSerializer.Serialize(health, SerializerOptions.Default)); + var instance = json.RootElement.GetProperty("instances")[0]; + var license = json.RootElement.GetProperty("license"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(json.RootElement.GetProperty("status").GetString(), Is.EqualTo("unknown")); + Assert.That(json.RootElement.GetProperty("alerts").GetArrayLength(), Is.Zero); + Assert.That(instance.GetProperty("api_url").GetString(), Is.EqualTo("https://localhost/servicecontrol/api/")); + Assert.That(instance.GetProperty("version").GetString(), Is.EqualTo("6.10.0")); + Assert.That(instance.GetProperty("forward_error_messages").GetBoolean(), Is.False); + Assert.That(instance.GetProperty("error_retention_period").GetString(), Is.EqualTo("14.00:00:00")); + Assert.That(instance.GetProperty("observed_at").GetDateTimeOffset(), Is.EqualTo(health.Instances[0].ObservedAt)); + Assert.That(instance.GetProperty("health_signals_status").GetString(), Is.EqualTo("unreported")); + Assert.That(instance.TryGetProperty("audit_retention_period", out _), Is.False); + Assert.That(instance.TryGetProperty("last_reported_at", out _), Is.False); + Assert.That(license.GetProperty("license_status").GetString(), Is.EqualTo("InvalidDueToExpiredSubscription")); + Assert.That(license.GetProperty("trial_license").GetBoolean(), Is.False); + Assert.That(license.TryGetProperty("expiration_date", out _), Is.False); + } + } + + static CustomCheckDetail Detail(string checkId, bool hasFailed) => new() + { + CustomCheckId = checkId, + Category = "ServiceControl Health", + HasFailed = hasFailed, + FailureReason = hasFailed ? "Check failed" : null, + ReportedAt = new DateTime(2026, 9, 30, 12, 0, 0, DateTimeKind.Utc), + OriginatingEndpoint = new EndpointDetails + { + Name = "ServiceControl", + Host = "localhost", + HostId = Guid.Parse("82E379F4-A5BD-4D83-8B64-70488BC6ED3A") + } + }; + } +} \ No newline at end of file diff --git a/src/ServiceControl.UnitTests/ScatterGather/RemoteInstanceHttpClientTests.cs b/src/ServiceControl.UnitTests/ScatterGather/RemoteInstanceHttpClientTests.cs index fbe124508d..e3064ff5ae 100644 --- a/src/ServiceControl.UnitTests/ScatterGather/RemoteInstanceHttpClientTests.cs +++ b/src/ServiceControl.UnitTests/ScatterGather/RemoteInstanceHttpClientTests.cs @@ -1,11 +1,17 @@ namespace ServiceControl.UnitTests.ScatterGather; using System; +using System.Net; using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; +using NServiceBus.Hosting; using NUnit.Framework; using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Infrastructure.Api; using ServiceControl.Infrastructure.WebApi; +using ServiceControl.Monitoring.HeartbeatMonitoring; using ServiceControl.Persistence; [TestFixture] @@ -28,5 +34,105 @@ public void The_remote_client_waits_no_longer_than_the_query_time_limit() Assert.That(client.Timeout, Is.EqualTo(TimeSpan.FromMinutes(5))); } + [Test] + public async Task Configuration_requests_preserve_virtual_directory_prefixes() + { + var settings = new Settings { RemoteInstances = [new RemoteInstanceSetting("https://audit/servicecontrol/api")] }; + using var handler = new StubHandler(_ => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("""{"host":{"instance_name":"Audit"}}""") + })); + var services = new ServiceCollection(); + services.AddSingleton(new TestPersistenceSettings()); + services.AddRemoteInstancesHttpClients(settings); + services.AddHttpClient(settings.RemoteInstances[0].InstanceId).ConfigurePrimaryHttpMessageHandler(() => handler); + using var provider = services.BuildServiceProvider(); + var api = CreateApi(settings, provider.GetRequiredService()); + + var remotes = await api.GetRemoteConfigs(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(handler.RequestUri, Is.EqualTo(new Uri("https://audit/servicecontrol/api/configuration"))); + Assert.That(remotes[0].Status, Is.EqualTo("online")); + } + } + + [TestCase(HttpStatusCode.Unauthorized)] + [TestCase(HttpStatusCode.Forbidden)] + [TestCase(HttpStatusCode.InternalServerError)] + public async Task Failed_http_responses_cannot_be_reported_as_online(HttpStatusCode statusCode) + { + using var handler = new StubHandler(_ => Task.FromResult(new HttpResponseMessage(statusCode) + { + Content = new StringContent("""{"host":{"instance_name":"Audit"}}""") + })); + using var client = new HttpClient(handler) { BaseAddress = new Uri("https://audit/") }; + + var settings = new Settings { RemoteInstances = [new RemoteInstanceSetting("https://audit")] }; + var remotes = await CreateApi(settings, new StubClientFactory(client)).GetRemoteConfigs(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(remotes[0].Status, Is.Not.EqualTo("online")); + Assert.That(remotes[0].Configuration, Is.Null); + } + } + + [TestCase("null")] + [TestCase("[]")] + [TestCase("{}")] + [TestCase("{broken}")] + public async Task Malformed_or_missing_configuration_is_not_online(string body) + { + using var handler = new StubHandler(_ => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(body) + })); + using var client = new HttpClient(handler) { BaseAddress = new Uri("https://audit/") }; + + var settings = new Settings { RemoteInstances = [new RemoteInstanceSetting("https://audit")] }; + var remotes = await CreateApi(settings, new StubClientFactory(client)).GetRemoteConfigs(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(remotes[0].Status, Is.EqualTo("error")); + Assert.That(remotes[0].Configuration, Is.Null); + } + } + + [Test] + public async Task A_timeout_returns_an_unavailable_remote_but_caller_cancellation_propagates() + { + using var client = new HttpClient(new StubHandler(_ => Task.FromException(new TaskCanceledException()))) + { BaseAddress = new Uri("https://audit/") }; + var api = CreateApi(new Settings { RemoteInstances = [new RemoteInstanceSetting("https://audit")] }, new StubClientFactory(client)); + + Assert.That((await api.GetRemoteConfigs())[0].Status, Is.EqualTo("unavailable")); + + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + Assert.That(async () => await api.GetRemoteConfigs(cancellation.Token), Throws.InstanceOf()); + } + + static ConfigurationApi CreateApi(Settings settings, IHttpClientFactory clientFactory) => + new(null, settings, clientFactory, new MassTransitConnectorHeartbeatStatus(), new HostInformation(Guid.Empty, "localhost")); + + class StubClientFactory(HttpClient client) : IHttpClientFactory + { + public HttpClient CreateClient(string name) => client; + } + + class StubHandler(Func> respond) : HttpMessageHandler + { + public Uri RequestUri { get; private set; } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken = default) + { + RequestUri = request.RequestUri; + return respond(cancellationToken); + } + } + class TestPersistenceSettings : PersistenceSettings; } diff --git a/src/ServiceControl/CustomChecks/CustomCheckResultProcessor.cs b/src/ServiceControl/CustomChecks/CustomCheckResultProcessor.cs index a4677006fb..f768593b63 100644 --- a/src/ServiceControl/CustomChecks/CustomCheckResultProcessor.cs +++ b/src/ServiceControl/CustomChecks/CustomCheckResultProcessor.cs @@ -6,19 +6,23 @@ namespace ServiceControl.CustomChecks using Contracts.CustomChecks; using Infrastructure.DomainEvents; using Microsoft.Extensions.Logging; + using PlatformHealth; using ServiceControl.Persistence; class CustomCheckResultProcessor { - public CustomCheckResultProcessor(IDomainEvents domainEvents, ICustomChecksDataStore store, ILogger logger) + public CustomCheckResultProcessor(IDomainEvents domainEvents, ICustomChecksDataStore store, ILogger logger, PlatformHealthState platformHealthState = null) { this.domainEvents = domainEvents; this.store = store; this.logger = logger; + this.platformHealthState = platformHealthState; } public async Task ProcessResult(CustomCheckDetail checkDetail, CancellationToken cancellationToken = default) { + platformHealthState?.Record(checkDetail); + try { var statusChange = await store.UpdateCustomCheckStatus(checkDetail, cancellationToken); @@ -84,5 +88,6 @@ await domainEvents.Raise(new CustomCheckSucceeded int lastCount; readonly ILogger logger; + readonly PlatformHealthState platformHealthState; } } \ No newline at end of file diff --git a/src/ServiceControl/CustomChecks/CustomChecksComponent.cs b/src/ServiceControl/CustomChecks/CustomChecksComponent.cs index 5e4b20e731..4f83f3b098 100644 --- a/src/ServiceControl/CustomChecks/CustomChecksComponent.cs +++ b/src/ServiceControl/CustomChecks/CustomChecksComponent.cs @@ -6,7 +6,9 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Particular.ServiceControl; + using PlatformHealth; using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.Api; using Transports; class CustomChecksComponent : ServiceControlComponent @@ -32,8 +34,10 @@ public override void Configure(Settings settings, ITransportCustomization transp if (!settings.ErrorIngestionOnly) { hostBuilder.Services.AddPlatformConnectionProvider(); + hostBuilder.Services.AddSingleton(); } hostBuilder.Services.AddSingleton(); + hostBuilder.Services.AddSingleton(); } } } \ No newline at end of file diff --git a/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs b/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs index 4d61e23ca8..793c561525 100644 --- a/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs +++ b/src/ServiceControl/Infrastructure/Api/ConfigurationApi.cs @@ -4,17 +4,19 @@ using System.IO; using System.Linq; using System.Net.Http; +using System.Text.Json; using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using Configuration; using Monitoring.HeartbeatMonitoring; +using NServiceBus.Hosting; using Particular.ServiceControl.Licensing; using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Api; using ServiceControl.Api.Contracts; -class ConfigurationApi(ActiveLicense license, Settings settings, IHttpClientFactory httpClientFactory, MassTransitConnectorHeartbeatStatus connectorHeartbeatStatus) : IConfigurationApi +class ConfigurationApi(ActiveLicense license, Settings settings, IHttpClientFactory httpClientFactory, MassTransitConnectorHeartbeatStatus connectorHeartbeatStatus, HostInformation hostInformation) : IConfigurationApi { public Task GetUrls(string baseUrl, CancellationToken cancellationToken = default) { @@ -44,6 +46,7 @@ public Task GetUrls(string baseUrl, CancellationToken cancellationToke ArchivedGroupsUrl = baseUrl + "errors/groups/{classifier?}", GetArchiveGroup = baseUrl + "archive/groups/id/{groupId}", MyRoutesUrl = baseUrl + "my/routes", + PlatformHealth = baseUrl + "platform-health", }; return Task.FromResult(model); @@ -54,9 +57,12 @@ public Task GetConfig(CancellationToken cancellationToken = default) { object content = new { + InstanceType = "error", + HealthChecksEnabled = !settings.DisableHealthChecks, Host = new { settings.InstanceName, + hostInformation.HostId, Logging = new { settings.LoggingSettings.LogPath, @@ -103,7 +109,8 @@ public async Task GetRemoteConfigs(CancellationToken canc try { - using var response = await httpClient.GetAsync("/api/configuration", cancellationToken); + using var response = await httpClient.GetAsync("api/configuration", cancellationToken); + response.EnsureSuccessStatusCode(); if (response.Headers.TryGetValues("X-Particular-Version", out var values)) { @@ -112,6 +119,13 @@ public async Task GetRemoteConfigs(CancellationToken canc await using Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken); config = await JsonNode.ParseAsync(stream, cancellationToken: cancellationToken); + if (config is not JsonObject configuration || + configuration["host"] is not JsonObject host || + host["instance_name"] is not JsonValue instanceName || + !instanceName.TryGetValue(out var name) || string.IsNullOrWhiteSpace(name)) + { + throw new JsonException("Remote response is not an instance configuration."); + } } catch (HttpRequestException ex) { @@ -121,6 +135,10 @@ public async Task GetRemoteConfigs(CancellationToken canc { throw; } + catch (OperationCanceledException) + { + status = "unavailable"; + } catch (Exception) { status = "error"; @@ -131,7 +149,7 @@ public async Task GetRemoteConfigs(CancellationToken canc ApiUri = remote.BaseAddress, Version = version, Status = status, - Configuration = config + Configuration = status == "online" ? config : null }; }); diff --git a/src/ServiceControl/Infrastructure/WebApi/RemoteInstanceServiceCollectionExtensions.cs b/src/ServiceControl/Infrastructure/WebApi/RemoteInstanceServiceCollectionExtensions.cs index 6daf32d9a8..145afbd037 100644 --- a/src/ServiceControl/Infrastructure/WebApi/RemoteInstanceServiceCollectionExtensions.cs +++ b/src/ServiceControl/Infrastructure/WebApi/RemoteInstanceServiceCollectionExtensions.cs @@ -37,7 +37,7 @@ public static void AddRemoteInstancesHttpClients(this IServiceCollection service { client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); // Application settings might contain remote URLs with /api. We strip that away to be a real base address. - client.BaseAddress = new Uri(remoteInstance.BaseAddress); + client.BaseAddress = new Uri(remoteInstance.BaseAddress.TrimEnd('/') + "/"); // This instance's query time limit bounds the whole composite: a remote that has not answered by then // is reported as missing, whatever its own limit is. Its own limit still ends the query on its side. client.Timeout = serviceProvider.GetRequiredService().QueryTimeout; diff --git a/src/ServiceControl/Licensing/LicenseController.cs b/src/ServiceControl/Licensing/LicenseController.cs index a2c734b261..666a4b9669 100644 --- a/src/ServiceControl/Licensing/LicenseController.cs +++ b/src/ServiceControl/Licensing/LicenseController.cs @@ -10,46 +10,19 @@ namespace ServiceControl.Licensing using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; - using Monitoring.HeartbeatMonitoring; using Particular.LicensingComponent.Contracts; using Particular.LicensingComponent.Persistence; using Particular.ServiceControl.Licensing; - using ServiceBus.Management.Infrastructure.Settings; [ApiController] [Route("api")] - public class LicenseController(ActiveLicense activeLicense, Settings settings, MassTransitConnectorHeartbeatStatus connectorHeartbeatStatus, ILicensingDataStore dataStore) : ControllerBase + public class LicenseController(ActiveLicense activeLicense, ILicenseInfoProvider licenseInfoProvider, ILicensingDataStore dataStore) : ControllerBase { [Authorize(Policy = Permissions.ErrorLicensingView)] [HttpGet] [Route("license")] - public async Task> License(bool refresh, string clientName, CancellationToken cancellationToken = default) - { - if (refresh) - { - await activeLicense.Refresh(cancellationToken); - } - - var licenseInfo = new LicenseInfo - { - TrialLicense = activeLicense.Details.IsTrialLicense, - Edition = activeLicense.Details.Edition ?? string.Empty, - RegisteredTo = activeLicense.Details.RegisteredTo ?? string.Empty, - UpgradeProtectionExpiration = activeLicense.Details.UpgradeProtectionExpiration?.ToString("O") ?? string.Empty, - ExpirationDate = activeLicense.Details.ExpirationDate?.ToString("O") ?? string.Empty, - Status = activeLicense.IsValid ? "valid" : "invalid", - LicenseType = activeLicense.Details.LicenseType ?? string.Empty, - InstanceName = settings.InstanceName ?? string.Empty, - LicenseStatus = activeLicense.Details.Status, - Products = activeLicense.Details.Products, - HasEndpointMetadata = activeLicense.Details.HasEndpointMetadata, - LicenseExtensionUrl = connectorHeartbeatStatus.LastHeartbeat == null - ? $"https://particular.net/extend-your-trial?p={clientName}" - : $"https://particular.net/license/mt?p={clientName}&t={(activeLicense.IsEvaluation ? 0 : 1)}" - }; - - return licenseInfo; - } + public async Task> License(bool refresh, string clientName, CancellationToken cancellationToken = default) => + await licenseInfoProvider.GetLicense(refresh, clientName, cancellationToken); [Authorize(Policy = Permissions.ErrorThroughputView)] [HttpGet] diff --git a/src/ServiceControl/Licensing/LicenseInfoProvider.cs b/src/ServiceControl/Licensing/LicenseInfoProvider.cs new file mode 100644 index 0000000000..e1b88f7afd --- /dev/null +++ b/src/ServiceControl/Licensing/LicenseInfoProvider.cs @@ -0,0 +1,44 @@ +namespace ServiceControl.Licensing; + +using System; +using System.Threading; +using System.Threading.Tasks; +using Monitoring.HeartbeatMonitoring; +using Particular.ServiceControl.Licensing; +using ServiceBus.Management.Infrastructure.Settings; + +public interface ILicenseInfoProvider +{ + Task GetLicense(bool refresh, string clientName, CancellationToken cancellationToken = default); +} + +sealed class LicenseInfoProvider(ActiveLicense activeLicense, Settings settings, MassTransitConnectorHeartbeatStatus connectorHeartbeatStatus) : ILicenseInfoProvider +{ + public async Task GetLicense(bool refresh, string clientName, CancellationToken cancellationToken = default) + { + if (refresh) + { + await activeLicense.Refresh(cancellationToken); + } + + var details = activeLicense.Details ?? throw new InvalidOperationException("License details are unavailable."); + + return new LicenseInfo + { + TrialLicense = details.IsTrialLicense, + Edition = details.Edition ?? string.Empty, + RegisteredTo = details.RegisteredTo ?? string.Empty, + UpgradeProtectionExpiration = details.UpgradeProtectionExpiration?.ToString("O") ?? string.Empty, + ExpirationDate = details.ExpirationDate?.ToString("O") ?? string.Empty, + Status = activeLicense.IsValid ? "valid" : "invalid", + LicenseType = details.LicenseType ?? string.Empty, + InstanceName = settings.InstanceName ?? string.Empty, + LicenseStatus = details.Status, + Products = details.Products, + HasEndpointMetadata = details.HasEndpointMetadata, + LicenseExtensionUrl = connectorHeartbeatStatus.LastHeartbeat == null + ? $"https://particular.net/extend-your-trial?p={clientName}" + : $"https://particular.net/license/mt?p={clientName}&t={(activeLicense.IsEvaluation ? 0 : 1)}" + }; + } +} \ No newline at end of file diff --git a/src/ServiceControl/PlatformHealth.http b/src/ServiceControl/PlatformHealth.http new file mode 100644 index 0000000000..72fbdab048 --- /dev/null +++ b/src/ServiceControl/PlatformHealth.http @@ -0,0 +1,14 @@ +@baseUrl = http://localhost:33333/api + +### Discover platform health +GET {{baseUrl}} +Accept: application/json + +### Read platform health with authentication disabled +GET {{baseUrl}}/platform-health +Accept: application/json + +### Read platform health with an existing bearer token +GET {{baseUrl}}/platform-health +Accept: application/json +Authorization: Bearer {{$processEnv SERVICECONTROL_ACCESS_TOKEN}} \ No newline at end of file diff --git a/src/ServiceControl/PlatformHealth/PlatformHealthApi.cs b/src/ServiceControl/PlatformHealth/PlatformHealthApi.cs new file mode 100644 index 0000000000..cf9bc03121 --- /dev/null +++ b/src/ServiceControl/PlatformHealth/PlatformHealthApi.cs @@ -0,0 +1,277 @@ +namespace ServiceControl.PlatformHealth; + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Api; +using Api.Contracts; +using Infrastructure; +using Infrastructure.WebApi; +using Licensing; +using Monitoring.HeartbeatMonitoring; +using NServiceBus.Hosting; +using NServiceBus.Logging; +using ServiceBus.Management.Infrastructure.Settings; + +sealed class PlatformHealthApi( + Settings settings, + HostInformation hostInformation, + PlatformHealthState state, + IConfigurationApi configurationApi, + ILicenseInfoProvider licenseInfoProvider, + MassTransitConnectorHeartbeatStatus connectorHeartbeatStatus, + TimeProvider timeProvider) : IPlatformHealthApi +{ + public async Task GetHealth(string baseUrl, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + var observedAt = timeProvider.GetUtcNow(); + var remoteTask = GetRemoteConfigurations(cancellationToken); + var licenseTask = GetLicense(cancellationToken); + await Task.WhenAll(remoteTask, licenseTask); + cancellationToken.ThrowIfCancellationRequested(); + + var remotes = await remoteTask; + var instances = new List + { + new() + { + Id = settings.InstanceId, + Name = settings.InstanceName, + Kind = "error", + Role = "primary-error", + ApiUrl = baseUrl.TrimEnd('/') + "/", + Version = ServiceControlVersion.GetFileVersion(), + HostId = hostInformation.HostId, + Health = "healthy", + ObservedAt = observedAt, + MetadataObservedAt = observedAt, + HealthSignalsStatus = settings.DisableHealthChecks ? "disabled" : "unreported", + TransportType = settings.TransportType, + ErrorQueue = settings.ErrorQueue, + ErrorLogQueue = settings.ErrorLogQueue, + ForwardErrorMessages = settings.ForwardErrorMessages, + ErrorRetentionPeriod = settings.ErrorRetentionPeriod, + AuditRetentionPeriod = settings.AuditRetentionPeriod + } + }; + + foreach (var remote in settings.RemoteInstances.DistinctBy(remote => remote.InstanceId).OrderBy(remote => remote.InstanceId, StringComparer.Ordinal)) + { + var configuration = remotes.FirstOrDefault(candidate => candidate.ApiUri == remote.BaseAddress); + instances.Add(GetRemoteInstance(remote, configuration, observedAt)); + } + + var reports = state.GetChecks(); + var response = PlatformHealthState.GetHealth(reports); + var assignments = new Dictionary(); + var ambiguousInstances = new HashSet(StringComparer.Ordinal); + foreach (var report in reports) + { + var namedInstances = instances.Where(instance => string.Equals(instance.Name, report.InstanceName, StringComparison.OrdinalIgnoreCase)).ToArray(); + var exactMatches = namedInstances.Where(instance => instance.HostId == report.HostId).ToArray(); + if (exactMatches.Length == 1) + { + assignments[report.Id] = exactMatches[0].Id; + continue; + } + + var reportingHosts = reports.Where(candidate => string.Equals(candidate.InstanceName, report.InstanceName, StringComparison.OrdinalIgnoreCase)) + .Select(candidate => candidate.HostId).Distinct().Take(2).Count(); + if (exactMatches.Length == 0 && namedInstances.Length == 1 && namedInstances[0].HostId is null && reportingHosts == 1) + { + assignments[report.Id] = namedInstances[0].Id; + continue; + } + + foreach (var instance in namedInstances) + { + ambiguousInstances.Add(instance.Id); + } + } + + foreach (var alert in response.Alerts) + { + alert.InstanceId = assignments.GetValueOrDefault(alert.Id); + } + + response.Instances = instances.Select(instance => + { + var associated = reports.Where(report => assignments.GetValueOrDefault(report.Id) == instance.Id).ToArray(); + var issues = response.Alerts.Where(alert => alert.InstanceId == instance.Id).ToArray(); + return instance with + { + Health = instance.Health == "unavailable" ? "unavailable" : issues.Length == 0 ? "healthy" : "degraded", + HealthSignalsStatus = instance.HealthSignalsStatus == "disabled" ? "disabled" : + ambiguousInstances.Contains(instance.Id) ? "ambiguous" : associated.Length == 0 ? "unreported" : "reported", + LastReportedAt = associated.Length == 0 ? null : new DateTimeOffset(DateTime.SpecifyKind(associated.Max(report => report.ReportedAt), DateTimeKind.Utc)), + Issues = issues + }; + }).ToArray(); + response.License = await licenseTask; + return response; + } + + PlatformHealthInstance GetRemoteInstance(RemoteInstanceSetting remote, RemoteConfiguration configuration, DateTimeOffset observedAt) + { + if (configuration?.Status == "online" && configuration.Configuration != null) + { + try + { + var metadata = configuration.Configuration.Deserialize(SerializerOptions.Default); + if (!string.IsNullOrWhiteSpace(metadata?.Host?.InstanceName)) + { + var kind = metadata.InstanceType switch + { + "error" => "error", + "audit" => "audit", + null when metadata.DataRetention?.ErrorRetentionPeriod != null => "error", + null when metadata.DataRetention?.AuditRetentionPeriod != null => "audit", + _ => "unknown" + }; + var instance = new PlatformHealthInstance + { + Id = remote.InstanceId, + Name = metadata.Host.InstanceName, + Kind = kind, + Role = "remote-" + kind, + ApiUrl = remote.BaseAddress.TrimEnd('/') + "/api/", + Version = configuration.Version is null or "Unknown" or "Missing" or "" ? null : configuration.Version, + HostId = metadata.Host.HostId is null || metadata.Host.HostId == Guid.Empty ? null : metadata.Host.HostId, + Health = "healthy", + ObservedAt = observedAt, + MetadataObservedAt = observedAt, + HealthSignalsStatus = metadata.HealthChecksEnabled == false ? "disabled" : "unreported", + TransportType = metadata.Transport?.TransportType, + ErrorQueue = metadata.Transport?.ErrorQueue, + ErrorLogQueue = metadata.Transport?.ErrorLogQueue, + ForwardErrorMessages = metadata.Transport?.ForwardErrorMessages, + AuditQueue = metadata.Transport?.AuditQueue, + AuditLogQueue = metadata.Transport?.AuditLogQueue, + ForwardAuditMessages = metadata.Transport?.ForwardAuditMessages, + ErrorRetentionPeriod = metadata.DataRetention?.ErrorRetentionPeriod, + AuditRetentionPeriod = metadata.DataRetention?.AuditRetentionPeriod + }; + return lastKnownRemotes.AddOrUpdate(remote.InstanceId, instance, + (_, previous) => instance.MetadataObservedAt >= previous.MetadataObservedAt ? instance : previous); + } + } + catch (JsonException exception) + { + log.Warn("Unable to read remote instance metadata for platform health.", exception); + } + } + + if (lastKnownRemotes.TryGetValue(remote.InstanceId, out var lastKnown)) + { + return lastKnown with { Health = "unavailable", ObservedAt = observedAt }; + } + + return new PlatformHealthInstance + { + Id = remote.InstanceId, + Name = new Uri(remote.BaseAddress).Host, + ApiUrl = remote.BaseAddress.TrimEnd('/') + "/api/", + ObservedAt = observedAt + }; + } + + async Task GetRemoteConfigurations(CancellationToken cancellationToken) + { + if (settings.RemoteInstances.Length == 0) + { + return []; + } + + try + { + return await configurationApi.GetRemoteConfigs(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + log.Warn("Unable to refresh remote configurations for platform health.", exception); + return []; + } + } + + async Task GetLicense(CancellationToken cancellationToken) + { + var hasConnector = connectorHeartbeatStatus.LastHeartbeat != null; + try + { + var license = await licenseInfoProvider.GetLicense(true, "servicepulse", cancellationToken); + if (!string.IsNullOrWhiteSpace(license?.LicenseStatus)) + { + return new PlatformHealthLicense + { + Availability = "available", + Status = license.Status, + LicenseStatus = license.LicenseStatus, + LicenseType = license.LicenseType, + TrialLicense = license.TrialLicense, + ExpirationDate = ParseDate(license.ExpirationDate), + UpgradeProtectionExpiration = ParseDate(license.UpgradeProtectionExpiration), + LicenseExtensionUrl = license.LicenseExtensionUrl, + HasMassTransitConnector = hasConnector + }; + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + log.Warn("Unable to refresh license information for platform health.", exception); + } + + return new PlatformHealthLicense { HasMassTransitConnector = hasConnector }; + } + + static DateTimeOffset? ParseDate(string value) => + DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var date) ? date : null; + + readonly ConcurrentDictionary lastKnownRemotes = new(StringComparer.Ordinal); + static readonly ILog log = LogManager.GetLogger(); + + sealed class InstanceConfiguration + { + public string InstanceType { get; init; } + public bool? HealthChecksEnabled { get; init; } + public HostConfiguration Host { get; init; } + public RetentionConfiguration DataRetention { get; init; } + public TransportConfiguration Transport { get; init; } + } + + sealed class HostConfiguration + { + public string InstanceName { get; init; } + public Guid? HostId { get; init; } + } + + sealed class RetentionConfiguration + { + public TimeSpan? ErrorRetentionPeriod { get; init; } + public TimeSpan? AuditRetentionPeriod { get; init; } + } + + sealed class TransportConfiguration + { + public string TransportType { get; init; } + public string ErrorQueue { get; init; } + public string ErrorLogQueue { get; init; } + public bool? ForwardErrorMessages { get; init; } + public string AuditQueue { get; init; } + public string AuditLogQueue { get; init; } + public bool? ForwardAuditMessages { get; init; } + } +} \ No newline at end of file diff --git a/src/ServiceControl/PlatformHealth/PlatformHealthController.cs b/src/ServiceControl/PlatformHealth/PlatformHealthController.cs new file mode 100644 index 0000000000..9d289a48ae --- /dev/null +++ b/src/ServiceControl/PlatformHealth/PlatformHealthController.cs @@ -0,0 +1,24 @@ +namespace ServiceControl.PlatformHealth +{ + using System.Threading; + using System.Threading.Tasks; + using Microsoft.AspNetCore.Authorization; + using Microsoft.AspNetCore.Http; + using Microsoft.AspNetCore.Http.Extensions; + using Microsoft.AspNetCore.Mvc; + using ServiceControl.Api; + using ServiceControl.Api.Contracts; + using ServiceControl.Infrastructure.Auth; + + [ApiController] + [Route("api")] + public class PlatformHealthController(IPlatformHealthApi platformHealthApi) : ControllerBase + { + [Authorize(Policy = Permissions.ErrorCustomChecksView)] + [Route("platform-health")] + [HttpGet] + [ProducesResponseType(typeof(PlatformHealthView), StatusCodes.Status200OK)] + public Task Get(CancellationToken cancellationToken = default) => + platformHealthApi.GetHealth(UriHelper.BuildAbsolute(Request.Scheme, Request.Host, Request.PathBase, "/api/"), cancellationToken); + } +} \ No newline at end of file diff --git a/src/ServiceControl/PlatformHealth/PlatformHealthState.cs b/src/ServiceControl/PlatformHealth/PlatformHealthState.cs new file mode 100644 index 0000000000..8f363bfc22 --- /dev/null +++ b/src/ServiceControl/PlatformHealth/PlatformHealthState.cs @@ -0,0 +1,82 @@ +namespace ServiceControl.PlatformHealth +{ + using System; + using System.Collections.Concurrent; + using System.Linq; + using ServiceControl.Api.Contracts; + using ServiceControl.Contracts.CustomChecks; + + public class PlatformHealthState + { + internal void Record(CustomCheckDetail detail) + { + if (!InternalCustomCheckClassification.IsInternal(detail.CustomCheckId)) + { + return; + } + + var id = detail.GetDeterministicId(); + var report = new CheckState + { + Id = id, + CheckId = detail.CustomCheckId, + Category = detail.Category, + HasFailed = detail.HasFailed, + Message = detail.FailureReason, + ReportedAt = detail.ReportedAt, + InstanceName = detail.OriginatingEndpoint.Name, + Host = detail.OriginatingEndpoint.Host, + HostId = detail.OriginatingEndpoint.HostId + }; + + checks.AddOrUpdate(id, report, (_, previous) => report.ReportedAt >= previous.ReportedAt ? report : previous); + } + + internal CheckState[] GetChecks() => checks.Values + .OrderBy(check => check.InstanceName, StringComparer.OrdinalIgnoreCase) + .ThenBy(check => check.CheckId, StringComparer.OrdinalIgnoreCase) + .ThenBy(check => check.Id) + .ToArray(); + + public PlatformHealthView GetHealth() => GetHealth(GetChecks()); + + internal static PlatformHealthView GetHealth(CheckState[] currentChecks) + { + var failedChecks = currentChecks + .Where(check => check.HasFailed) + .ToArray(); + + return new PlatformHealthView + { + Status = currentChecks.Length == 0 ? "unknown" : failedChecks.Length == 0 ? "healthy" : "unhealthy", + Severity = currentChecks.Length == 0 ? "unknown" : failedChecks.Length == 0 ? "none" : "error", + Alerts = failedChecks.Select(check => new PlatformHealthAlert + { + Id = check.Id, + CheckId = check.CheckId, + Category = check.Category, + Message = check.Message, + ReportedAt = check.ReportedAt, + InstanceName = check.InstanceName, + Host = check.Host, + HostId = check.HostId + }).ToArray() + }; + } + + readonly ConcurrentDictionary checks = new(); + + internal sealed record CheckState + { + public Guid Id { get; init; } + public string CheckId { get; init; } + public string Category { get; init; } + public bool HasFailed { get; init; } + public string Message { get; init; } + public DateTime ReportedAt { get; init; } + public string InstanceName { get; init; } + public string Host { get; init; } + public Guid HostId { get; init; } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl/ServiceControlApiHostBuilderExtensions.cs b/src/ServiceControl/ServiceControlApiHostBuilderExtensions.cs index ad678bcef7..f08814c940 100644 --- a/src/ServiceControl/ServiceControlApiHostBuilderExtensions.cs +++ b/src/ServiceControl/ServiceControlApiHostBuilderExtensions.cs @@ -2,6 +2,7 @@ namespace Particular.ServiceControl { using global::ServiceControl.Api; using global::ServiceControl.Infrastructure.Api; + using global::ServiceControl.Licensing; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -10,6 +11,7 @@ static class ServiceControlApiHostBuilderExtensions public static void AddServiceControlApis(this IHostApplicationBuilder hostBuilder) { hostBuilder.Services.AddSingleton(); + hostBuilder.Services.AddSingleton(); hostBuilder.Services.AddSingleton(); hostBuilder.Services.AddSingleton(); hostBuilder.Services.AddSingleton();