Resource alerts: threshold alert rules for VMs, volumes, hosts and storage pools

.github/workflows/ci.yml

@@ -128,6 +128,7 @@ jobs:
                   smoke/test_nested_virtualization
                   smoke/test_set_sourcenat
                   smoke/test_webhook_lifecycle
+                  smoke/test_resource_alerts
                   smoke/test_purge_expunged_vms
                   smoke/test_extension_lifecycle
                   smoke/test_extension_custom_action_lifecycle

api/src/main/java/com/cloud/event/EventTypes.java

@@ -822,6 +822,11 @@ public class EventTypes {
     public static final String EVENT_QUOTA_TARIFF_DELETE = "QUOTA.TARIFF.DELETE";
     public static final String EVENT_QUOTA_TARIFF_UPDATE = "QUOTA.TARIFF.UPDATE";
 
+    // Resource alert rules
+    public static final String EVENT_RESOURCE_ALERT_RULE_CREATE = "RESOURCE.ALERT.RULE.CREATE";
+    public static final String EVENT_RESOURCE_ALERT_RULE_UPDATE = "RESOURCE.ALERT.RULE.UPDATE";
+    public static final String EVENT_RESOURCE_ALERT_RULE_DELETE = "RESOURCE.ALERT.RULE.DELETE";
+
     // Routing
     public static final String EVENT_ZONE_IP4_SUBNET_CREATE = "ZONE.IP4.SUBNET.CREATE";
     public static final String EVENT_ZONE_IP4_SUBNET_UPDATE = "ZONE.IP4.SUBNET.UPDATE";

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -1089,6 +1089,14 @@ public class ApiConstants {
     public static final String AGGR_FUNCTION = "aggrfunction";
     public static final String AGGR_VALUE = "aggrvalue";
     public static final String THRESHOLD = "threshold";
+    public static final String METRIC = "metric";
+    public static final String CONDITION = "condition";
+    public static final String SEVERITY = "severity";
+    public static final String RESET_INTERVAL = "resetinterval";
+    public static final String WEBHOOK_IDS = "webhookids";
+    public static final String WEBHOOK_NAMES = "webhooknames";
+    public static final String CLEANUP_WEBHOOKS = "cleanupwebhooks";
+    public static final String ALERT_RULE_ID = "alertruleid";
     public static final String RELATIONAL_OPERATOR = "relationaloperator";
     public static final String OTHER_DEPLOY_PARAMS = "otherdeployparams";
     public static final String MIN_MEMBERS = "minmembers";

client/pom.xml

@@ -432,6 +432,11 @@
             <artifactId>cloud-mom-webhook</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.cloudstack</groupId>
+            <artifactId>cloud-plugin-resource-alerts</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.cloudstack</groupId>
             <artifactId>cloud-framework-agent-lb</artifactId>

engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java

@@ -34,6 +34,11 @@ public interface VolumeDao extends GenericDao<VolumeVO, Long>, StateDao<Volume.S
 
     List<VolumeVO> findByAccount(long accountId);
 
+    /**
+     * Lists IDs of volumes that are detached or attached to user VMs, leaving out system VM and router volumes.
+     */
+    List<Long> listUserVolumeIdsByAccountOrDomainsAndState(Long accountId, List<Long> domainIds, Volume.State state);
+
     List<VolumeVO> findIncludingRemovedByAccount(long accountId);
 
     Pair<Long, Long> getCountAndTotalByPool(long poolId);

engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java

@@ -99,6 +99,9 @@ public class VolumeDaoImpl extends GenericDaoBase<VolumeVO, Long> implements Vol
 
     private static final String ORDER_POOLS_NUMBER_OF_VOLUMES_FOR_ACCOUNT_PART1 = "SELECT pool.id, SUM(IF(vol.state='Ready' AND vol.account_id = ?, 1, 0)) FROM `cloud`.`storage_pool` pool LEFT JOIN `cloud`.`volumes` vol ON pool.id = vol.pool_id WHERE pool.data_center_id = ? ";
     private static final String ORDER_POOLS_NUMBER_OF_VOLUMES_FOR_ACCOUNT_PART2 = " GROUP BY pool.id ORDER BY 2 ASC ";
+    private static final String LIST_USER_VOLUME_IDS = "SELECT vol.id FROM `cloud`.`volumes` vol "
+            + "LEFT JOIN `cloud`.`vm_instance` vm ON vm.id = vol.instance_id "
+            + "WHERE vol.removed IS NULL AND (vol.instance_id IS NULL OR vm.type = 'User')";
 
     private static final String ORDER_ZONE_WIDE_POOLS_NUMBER_OF_VOLUMES_FOR_ACCOUNT = "SELECT pool.id, SUM(IF(vol.state='Ready' AND vol.account_id = ?, 1, 0)) FROM `cloud`.`storage_pool` pool LEFT JOIN `cloud`.`volumes` vol ON pool.id = vol.pool_id WHERE pool.data_center_id = ? "
             + " AND pool.scope = 'ZONE' AND pool.status='Up' " + " GROUP BY pool.id ORDER BY 2 ASC ";
@@ -118,6 +121,47 @@ public List<VolumeVO> findByAccount(long accountId) {
         return listBy(sc);
     }
 
+    @Override
+    public List<Long> listUserVolumeIdsByAccountOrDomainsAndState(Long accountId, List<Long> domainIds, Volume.State state) {
+        if (domainIds != null && domainIds.isEmpty()) {
+            return new ArrayList<>();
+        }
+        StringBuilder sql = new StringBuilder(LIST_USER_VOLUME_IDS);
+        if (accountId != null) {
+            sql.append(" AND vol.account_id = ?");
+        }
+        if (domainIds != null) {
+            sql.append(" AND vol.domain_id IN (").append(String.join(",", Collections.nCopies(domainIds.size(), "?"))).append(")");
+        }
+        if (state != null) {
+            sql.append(" AND vol.state = ?");
+        }
+        List<Long> ids = new ArrayList<>();
+        TransactionLegacy txn = TransactionLegacy.currentTxn();
+        try (PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql.toString())) {
+            int i = 1;
+            if (accountId != null) {
+                pstmt.setLong(i++, accountId);
+            }
+            if (domainIds != null) {
+                for (Long domainId : domainIds) {
+                    pstmt.setLong(i++, domainId);
+                }
+            }
+            if (state != null) {
+                pstmt.setString(i, state.name());
+            }
+            try (ResultSet rs = pstmt.executeQuery()) {
+                while (rs.next()) {
+                    ids.add(rs.getLong(1));
+                }
+            }
+        } catch (SQLException e) {
+            throw new CloudRuntimeException("Unable to list user volume IDs", e);
+        }
+        return ids;
+    }
+
     @Override
     public List<VolumeVO> findIncludingRemovedByAccount(long accountId) {
         SearchCriteria<VolumeVO> sc = AllFieldsSearch.create();

engine/schema/src/main/java/com/cloud/vm/dao/UserVmDao.java

@@ -31,6 +31,8 @@
 public interface UserVmDao extends GenericDao<UserVmVO, Long> {
     List<UserVmVO> listByAccountId(long id);
 
+    List<Long> listIdsByAccountOrDomainsAndState(Long accountId, List<Long> domainIds, State state);
+
     List<UserVmVO> listByAccountAndPod(long accountId, long podId);
 
     List<UserVmVO> listByAccountAndDataCenter(long accountId, long dcId);

engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java

@@ -82,6 +82,7 @@ public class UserVmDaoImpl extends GenericDaoBase<UserVmVO, Long> implements Use
     protected SearchBuilder<UserVmVO> AccountDataCenterVirtualSearch;
     protected GenericSearchBuilder<UserVmVO, Long> CountByAccountPod;
     protected GenericSearchBuilder<UserVmVO, Long> CountByAccount;
+    protected GenericSearchBuilder<UserVmVO, Long> IdsByAccountOrDomainsAndStateSearch;
     protected GenericSearchBuilder<UserVmVO, Long> CountActiveAccount;
     protected GenericSearchBuilder<UserVmVO, Long> PodsHavingVmsForAccount;
 
@@ -143,6 +144,13 @@ void init() {
         AccountSearch.and("account", AccountSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
         AccountSearch.done();
 
+        IdsByAccountOrDomainsAndStateSearch = createSearchBuilder(Long.class);
+        IdsByAccountOrDomainsAndStateSearch.selectFields(IdsByAccountOrDomainsAndStateSearch.entity().getId());
+        IdsByAccountOrDomainsAndStateSearch.and("accountId", IdsByAccountOrDomainsAndStateSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
+        IdsByAccountOrDomainsAndStateSearch.and("domainIds", IdsByAccountOrDomainsAndStateSearch.entity().getDomainId(), SearchCriteria.Op.IN);
+        IdsByAccountOrDomainsAndStateSearch.and("state", IdsByAccountOrDomainsAndStateSearch.entity().getState(), SearchCriteria.Op.EQ);
+        IdsByAccountOrDomainsAndStateSearch.done();
+
         IdsSearch = createSearchBuilder();
         IdsSearch.and("ids", IdsSearch.entity().getId(), SearchCriteria.Op.IN);
         IdsSearch.done();
@@ -318,6 +326,21 @@ public List<UserVmVO> listByAccountId(long id) {
         return listBy(sc);
     }
 
+    @Override
+    public List<Long> listIdsByAccountOrDomainsAndState(Long accountId, List<Long> domainIds, State state) {
+        SearchCriteria<Long> sc = IdsByAccountOrDomainsAndStateSearch.create();
+        if (accountId != null) {
+            sc.setParameters("accountId", accountId);
+        }
+        if (domainIds != null) {
+            sc.setParameters("domainIds", domainIds.toArray());
+        }
+        if (state != null) {
+            sc.setParameters("state", state);
+        }
+        return customSearch(sc, null);
+    }
+
     @Override
     public List<UserVmVO> listByHostId(Long id) {
         SearchCriteria<UserVmVO> sc = HostSearch.create();

engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDao.java

@@ -41,6 +41,13 @@ public interface VmStatsDao extends GenericDao<VmStatsVO, Long> {
      */
     List<VmStatsVO> findByVmIdOrderByTimestampDesc(long vmId);
 
+    /**
+     * Finds the newest stats of a VM.
+     * @param vmId the VM ID.
+     * @return the newest stats, or null if there are none.
+     */
+    VmStatsVO findLatestByVmId(long vmId);
+
     /**
      * Finds stats by VM ID and timestamp >= a given time.
      * @param vmId the specific VM.

engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDaoImpl.java

@@ -85,6 +85,15 @@ public List<VmStatsVO> findByVmIdOrderByTimestampDesc(long vmId) {
         return search(sc, orderByFilter, null, false);
     }
 
+    @Override
+    public VmStatsVO findLatestByVmId(long vmId) {
+        SearchCriteria<VmStatsVO> sc = vmIdSearch.create();
+        sc.setParameters("vmId", vmId);
+        Filter orderByFilter = new Filter(VmStatsVO.class, "timestamp", false, 0L, 1L);
+        List<VmStatsVO> stats = search(sc, orderByFilter, null, false);
+        return stats.isEmpty() ? null : stats.get(0);
+    }
+
     @Override
     public List<VmStatsVO> findByVmIdAndTimestampGreaterThanEqual(long vmId, Date time) {
         SearchCriteria<VmStatsVO> sc = vmIdTimestampGreaterThanEqualSearch.create();

engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql

@@ -18,3 +18,64 @@
 --;
 -- Schema upgrade from 4.23.0.0 to 24.0.0
 --;
+
+-- resource_alert_rules: stores per-resource or generic metric threshold rules
+CREATE TABLE IF NOT EXISTS `cloud`.`resource_alert_rules` (
+    `id` bigint unsigned NOT NULL AUTO_INCREMENT,
+    `uuid` varchar(255) NOT NULL UNIQUE,
+    `name` varchar(255) NOT NULL,
+    `resource_type` varchar(64) NOT NULL COMMENT 'VirtualMachine, Volume, Host, StoragePool',
+    `resource_id` bigint unsigned DEFAULT NULL COMMENT 'null = applies to all resources of the type in scope',
+    `account_id` bigint unsigned NOT NULL,
+    `domain_id` bigint unsigned NOT NULL,
+    `metric` varchar(64) NOT NULL,
+    `condition_operator` varchar(8) NOT NULL COMMENT 'GT, GTE, LT, LTE, EQ',
+    `threshold` double NOT NULL,
+    `severity` varchar(32) NOT NULL COMMENT 'CRITICAL, HIGH, MEDIUM, LOW',
+    `message` varchar(4096) DEFAULT NULL,
+    `email` tinyint(1) NOT NULL DEFAULT 0,
+    `reset_interval` int unsigned NOT NULL DEFAULT 600 COMMENT 'minimum seconds between repeat firings of this rule',
+    `state` varchar(32) NOT NULL DEFAULT 'Enabled' COMMENT 'Enabled or Disabled; disabled rules are not checked',
+    `created` datetime DEFAULT NULL,
+    `updated` datetime DEFAULT NULL,
+    `removed` datetime DEFAULT NULL,
+    PRIMARY KEY (`id`),
+    INDEX `i_resource_alert_rules__account_id`(`account_id`),
+    INDEX `i_resource_alert_rules__domain_id`(`domain_id`),
+    INDEX `i_resource_alert_rules__resource_type__resource_id`(`resource_type`, `resource_id`),
+    CONSTRAINT `fk_resource_alert_rules__account_id` FOREIGN KEY (`account_id`) REFERENCES `account`(`id`) ON DELETE CASCADE,
+    CONSTRAINT `fk_resource_alert_rules__domain_id` FOREIGN KEY (`domain_id`) REFERENCES `domain`(`id`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- resource_alerts: immutable log of fired alerts
+CREATE TABLE IF NOT EXISTS `cloud`.`resource_alerts` (
+    `id` bigint unsigned NOT NULL AUTO_INCREMENT,
+    `uuid` varchar(255) NOT NULL UNIQUE,
+    `alert_rule_id` bigint unsigned NOT NULL,
+    `resource_id` bigint unsigned DEFAULT NULL COMMENT 'the specific resource that triggered the alert',
+    `metric_type` varchar(64) NOT NULL,
+    `metric_value` double NOT NULL,
+    `severity` varchar(32) NOT NULL,
+    `message` varchar(4096) DEFAULT NULL,
+    `alert_timestamp` datetime NOT NULL,
+    PRIMARY KEY (`id`),
+    INDEX `i_resource_alerts__alert_rule_id__resource_id__alert_timestamp`(`alert_rule_id`, `resource_id`, `alert_timestamp`),
+    INDEX `i_resource_alerts__resource_id__alert_timestamp`(`resource_id`, `alert_timestamp`),
+    INDEX `i_resource_alerts__alert_timestamp`(`alert_timestamp`),
+    CONSTRAINT `fk_resource_alerts__alert_rule_id` FOREIGN KEY (`alert_rule_id`) REFERENCES `resource_alert_rules`(`id`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- resource_alert_rules_webhook: webhooks a rule delivers its alerts to
+CREATE TABLE IF NOT EXISTS `cloud`.`resource_alert_rules_webhook` (
+    `id` bigint unsigned NOT NULL AUTO_INCREMENT,
+    `resource_alert_rule_id` bigint unsigned NOT NULL,
+    `webhook_id` bigint unsigned NOT NULL,
+    PRIMARY KEY (`id`),
+    UNIQUE KEY `uc_resource_alert_rules_webhook__rule_webhook`(`resource_alert_rule_id`, `webhook_id`),
+    CONSTRAINT `fk_resource_alert_rules_webhook__rule_id` FOREIGN KEY (`resource_alert_rule_id`) REFERENCES `resource_alert_rules`(`id`) ON DELETE CASCADE,
+    CONSTRAINT `fk_resource_alert_rules_webhook__webhook_id` FOREIGN KEY (`webhook_id`) REFERENCES `webhook`(`id`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- webhook_delivery: allow deliveries that are not tied to a stored event, such as resource alerts
+CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('cloud.webhook_delivery', 'event_id', 'event_id', 'bigint unsigned COMMENT "id of the event"');
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.webhook_delivery', 'event_type', 'varchar(255) COMMENT "type of the event when the delivery is not tied to a stored event"');

engine/schema/src/main/resources/META-INF/db/views/cloud.resource_alert_rule_view.sql

@@ -0,0 +1,52 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- VIEW `cloud`.`resource_alert_rule_view`;
+
+DROP VIEW IF EXISTS `cloud`.`resource_alert_rule_view`;
+CREATE VIEW `cloud`.`resource_alert_rule_view` AS
+    SELECT
+        r.id,
+        r.uuid,
+        r.name,
+        r.resource_type,
+        r.resource_id,
+        r.metric,
+        r.condition_operator,
+        r.threshold,
+        r.severity,
+        r.message,
+        r.email,
+        r.reset_interval,
+        r.state,
+        r.created,
+        r.updated,
+        r.removed,
+        a.id account_id,
+        a.uuid account_uuid,
+        a.account_name,
+        a.type account_type,
+        d.id domain_id,
+        d.uuid domain_uuid,
+        d.name domain_name,
+        d.path domain_path,
+        p.uuid project_uuid,
+        p.name project_name
+    FROM `cloud`.`resource_alert_rules` r
+    INNER JOIN `cloud`.`account` a ON r.account_id = a.id
+    INNER JOIN `cloud`.`domain` d ON r.domain_id = d.id
+    LEFT JOIN `cloud`.`projects` p ON p.project_account_id = a.id;

engine/schema/src/main/resources/META-INF/db/views/cloud.webhook_delivery_view.sql

@@ -28,9 +28,9 @@ CREATE VIEW `cloud`.`webhook_delivery_view` AS
         webhook_delivery.response,
         webhook_delivery.start_time,
         webhook_delivery.end_time,
-        event.id event_id,
+        webhook_delivery.event_id,
         event.uuid event_uuid,
-        event.type event_type,
+        IFNULL(event.type, webhook_delivery.event_type) event_type,
         webhook.id webhook_id,
         webhook.uuid webhook_uuid,
         webhook.name webhook_name,
@@ -40,7 +40,7 @@ CREATE VIEW `cloud`.`webhook_delivery_view` AS
         mshost.name mshost_name
     FROM
         `cloud`.`webhook_delivery`
-            INNER JOIN
+            LEFT JOIN
         `cloud`.`event` ON webhook_delivery.event_id = event.id
             INNER JOIN
         `cloud`.`webhook` ON webhook_delivery.webhook_id = webhook.id

engine/schema/src/test/java/com/cloud/storage/dao/VolumeDaoImplTest.java

@@ -114,6 +114,32 @@ public void testListPoolIdsByVolumeCount_without_cluster_details() throws SQLExc
         verify(preparedStatementMock, times(1)).executeQuery();
     }
 
+    @Test
+    public void listUserVolumeIdsByAccountOrDomainsAndStateSkipsSystemVmVolumes() throws SQLException {
+        final String expectedSql = "SELECT vol.id FROM `cloud`.`volumes` vol "
+                + "LEFT JOIN `cloud`.`vm_instance` vm ON vm.id = vol.instance_id "
+                + "WHERE vol.removed IS NULL AND (vol.instance_id IS NULL OR vm.type = 'User')"
+                + " AND vol.domain_id IN (?,?) AND vol.state = ?";
+        when(TransactionLegacy.currentTxn()).thenReturn(transactionMock);
+        when(transactionMock.prepareAutoCloseStatement(expectedSql)).thenReturn(preparedStatementMock);
+        ResultSet rs = Mockito.mock(ResultSet.class);
+        when(rs.next()).thenReturn(true, false);
+        when(rs.getLong(1)).thenReturn(7L);
+        when(preparedStatementMock.executeQuery()).thenReturn(rs);
+
+        List<Long> ids = volumeDao.listUserVolumeIdsByAccountOrDomainsAndState(null, List.of(1L, 2L), Volume.State.Ready);
+
+        Assert.assertEquals(List.of(7L), ids);
+        verify(preparedStatementMock).setLong(1, 1L);
+        verify(preparedStatementMock).setLong(2, 2L);
+        verify(preparedStatementMock).setString(3, "Ready");
+    }
+
+    @Test
+    public void listUserVolumeIdsByAccountOrDomainsAndStateWithNoDomains() {
+        Assert.assertTrue(volumeDao.listUserVolumeIdsByAccountOrDomainsAndState(null, List.of(), Volume.State.Ready).isEmpty());
+    }
+
     @Test
     public void findByInstanceAndNotState_queriesWithInstanceIdAndExcludedStates() {
         SearchBuilder<VolumeVO> sb = Mockito.mock(SearchBuilder.class);

plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookDelivery.java

@@ -27,7 +27,8 @@ public interface WebhookDelivery extends Identity, InternalIdentity {
     public static final String TEST_EVENT_TYPE = "TEST.WEBHOOK";
 
     long getId();
-    long getEventId();
+    Long getEventId();
+    String getEventType();
     long getWebhookId();
     long getManagementServerId();
     String getHeaders();