From 2b4070e2e80444e6bee28eb6aeaa625842f61d33 Mon Sep 17 00:00:00 2001 From: "Srivastava, Piyush" Date: Fri, 7 Aug 2026 11:30:04 +0530 Subject: [PATCH 1/2] feature/CSTACKEX-237: Pilot changes for scalebenmark framework --- private-cicd/benchmark/ontap/.gitignore | 8 + private-cicd/benchmark/ontap/README.md | 233 +++++++++++++ private-cicd/benchmark/ontap/USAGE.md | 179 ++++++++++ .../benchmark_storage_pool_concurrency.py | 208 ++++++++++++ .../benchmark_storage_pool_sequential.py | 198 +++++++++++ .../ontap/benchmark_vm_instance_combined.py | 123 +++++++ .../benchmark_vm_instance_concurrency.py | 220 ++++++++++++ .../ontap/benchmark_vm_instance_sequential.py | 211 ++++++++++++ .../benchmark/ontap/cloudstack_client.py | 159 +++++++++ .../benchmark/ontap/config.example.yaml | 124 +++++++ private-cicd/benchmark/ontap/render_report.py | 227 +++++++++++++ private-cicd/benchmark/ontap/requirements.txt | 2 + private-cicd/benchmark/ontap/results/.gitkeep | 0 .../benchmark/ontap/storage_pool_common.py | 218 ++++++++++++ .../benchmark/ontap/vm_instance_common.py | 319 ++++++++++++++++++ 15 files changed, 2429 insertions(+) create mode 100644 private-cicd/benchmark/ontap/.gitignore create mode 100644 private-cicd/benchmark/ontap/README.md create mode 100644 private-cicd/benchmark/ontap/USAGE.md create mode 100644 private-cicd/benchmark/ontap/benchmark_storage_pool_concurrency.py create mode 100644 private-cicd/benchmark/ontap/benchmark_storage_pool_sequential.py create mode 100644 private-cicd/benchmark/ontap/benchmark_vm_instance_combined.py create mode 100644 private-cicd/benchmark/ontap/benchmark_vm_instance_concurrency.py create mode 100644 private-cicd/benchmark/ontap/benchmark_vm_instance_sequential.py create mode 100644 private-cicd/benchmark/ontap/cloudstack_client.py create mode 100644 private-cicd/benchmark/ontap/config.example.yaml create mode 100644 private-cicd/benchmark/ontap/render_report.py create mode 100644 private-cicd/benchmark/ontap/requirements.txt create mode 100644 private-cicd/benchmark/ontap/results/.gitkeep create mode 100644 private-cicd/benchmark/ontap/storage_pool_common.py create mode 100644 private-cicd/benchmark/ontap/vm_instance_common.py diff --git a/private-cicd/benchmark/ontap/.gitignore b/private-cicd/benchmark/ontap/.gitignore new file mode 100644 index 000000000000..2e639cd15227 --- /dev/null +++ b/private-cicd/benchmark/ontap/.gitignore @@ -0,0 +1,8 @@ +config.yaml +config_sanity.yaml +.venv/ +__pycache__/ +*.pyc +results/*.csv +results/*.md +results/*.log diff --git a/private-cicd/benchmark/ontap/README.md b/private-cicd/benchmark/ontap/README.md new file mode 100644 index 000000000000..a0e3e0eb5265 --- /dev/null +++ b/private-cicd/benchmark/ontap/README.md @@ -0,0 +1,233 @@ + +# ONTAP plugin - storage pool benchmark (Step 1) + +> Looking for a quick "what does script X do / how do I run it" cheat +> sheet instead of the full design write-up below? See [USAGE.md](USAGE.md). + +Downstream-only benchmarking tool (not for Apache upstream) that drives +`createStoragePool` / `deleteStoragePool` over the CloudStack HTTP/REST API +to reproduce the **Storage Pool** rows of the Sequential Scale Matrix (5.1) +and Parallel/Concurrency Matrix (6.1) from the Confluence page +["ONTAP Plugin - CloudStack Operations, Scale & Parallel Test Matrix"](https://netapp.atlassian.net/wiki/spaces/OSSG/pages/608854350). + +This is step 1 of the automation follow-up (storage pool only). VM instance +benchmarks are now also covered (see below); volume/snapshot benchmarks will +be added as separate scripts later, reusing `cloudstack_client.py`. + +The storage-pool benchmark is split into **two independent scripts** (on +purpose — a sequential run is cheap/safe and worth reviewing before deciding +to launch a concurrency run, which is the one most likely to stress the +mgmt-server/plugin job queue): + +- `benchmark_storage_pool_sequential.py` — Section 5.1 (5.1.1 / 5.1.2) +- `benchmark_storage_pool_concurrency.py` — Section 6.1 (6.1.1 / 6.1.2) + +Both share the same createStoragePool/deleteStoragePool call shape, CSV +formats, and cleanup logic via `storage_pool_common.py`. Run them with the +**same `--run-id`** to accumulate both into a single +`results/summary_.csv` (and therefore a single `render_report.py` +output covering 5.1.x and 6.1.x together). + +The VM-instance benchmark is split into **three scripts** sharing +`vm_instance_common.py` (same call shape/CSV formats/cleanup logic): + +- `benchmark_vm_instance_sequential.py` — Section 5.2 (5.2.1 / 5.2.2) +- `benchmark_vm_instance_concurrency.py` — Section 6.2 (6.2.1 / 6.2.2) +- `benchmark_vm_instance_combined.py` — runs both of the above back to back + under one shared `--run-id`, for when you already trust the config/ + environment and just want the full 5.2.x + 6.2.x matrix in one invocation + +See "VM instance benchmark" below for prerequisites and usage. + +## What it does + +1. **Sequential scale test (5.1.1 / 5.1.2)** — for each protocol (NFS3, + iSCSI), creates storage pools one at a time up to N=30, logging every + single create call, and reports cumulative-total/avg-per-op timing at + checkpoints N = 1, 5, 10, 20, 30. Then deletes them one at a time and + reports the same at "N remaining" = 30, 20, 10, 5, 1. +2. **Concurrency test (6.1.1 / 6.1.2)** — for each concurrency level + C = 2, 5, 10, 20, 30 and each protocol, creates C pools in parallel + (`ThreadPoolExecutor`), records wall-clock time for the whole batch plus + success/failure/avg-per-op, then deletes the same C pools in parallel. +3. Writes **every** individual API call (start/end timestamp, duration, + success/failure, pool id, error) to `results/raw_ops_.csv`, and a + checkpoint-level roll-up to `results/summary_.csv`. +4. `render_report.py` turns those two CSVs into markdown tables that match + the Confluence page's table layout (including min/avg/p95/max stats for + the "Results Log" section), ready to paste back into the page. + +## Setup + +```bash +cd private-cicd/benchmark/ontap +python3 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +cp config.example.yaml config.yaml +``` + +Edit `config.yaml`: + +- `cloudstack.*` — API URL, admin username/password of your lab's + management server. +- `infrastructure.*` — zone/pod/cluster UUIDs to create pools under. +- `ontap.nfs3` / `ontap.iscsi` — connection details of the ONTAP SVM(s) you + want to benchmark against for each protocol. Delete a block if you only + have one protocol available. +- `benchmark.*` — checkpoints, concurrency levels, pool name prefix, output + directory. Defaults match the Confluence page (N/C = 1,5,10,20,30 / + 2,5,10,20,30). + +`config.yaml` and everything under `results/` are gitignored — never commit +real lab IPs/credentials or run output. + +## Running + +```bash +# Sanity-check either script/config without hitting a real management server: +python3 benchmark_storage_pool_sequential.py --config config.yaml --dry-run +python3 benchmark_storage_pool_concurrency.py --config config.yaml --dry-run + +# Sequential scale matrix, both protocols, up to config's sequential_checkpoints: +python3 benchmark_storage_pool_sequential.py --config config.yaml + +# Sequential scale matrix, NFS3 only: +python3 benchmark_storage_pool_sequential.py --config config.yaml --protocol nfs3 + +# Concurrency matrix, both protocols, up to config's concurrency_levels (e.g. 30): +python3 benchmark_storage_pool_concurrency.py --config config.yaml + +# Concurrency matrix, iSCSI only, overriding the levels to run: +python3 benchmark_storage_pool_concurrency.py --config config.yaml --protocol iscsi --levels 2,5,10,20,30 + +# Use the SAME --run-id across both scripts to combine 5.1.x + 6.1.x into one +# summary_.csv / report (run sequential first, review it, then decide +# whether to proceed with concurrency under the same run id): +python3 benchmark_storage_pool_sequential.py --config config.yaml --run-id RUN-0001 +python3 benchmark_storage_pool_concurrency.py --config config.yaml --run-id RUN-0001 +``` + +At the end of a real run each script automatically checks for and deletes any +storage pool whose name still contains the run id (safety net for pools that +were created but never got torn down because of a mid-run crash). Pass +`--skip-cleanup` to disable that and inspect the pools yourself. + +If a run crashes hard (script killed, network partition, etc.) and left +pools behind, recover with either script's `--cleanup-only` (they share the +same cleanup logic): + +```bash +# Deletes every pool whose name contains "bench_ontap" (the default prefix) +python3 benchmark_storage_pool_sequential.py --config config.yaml --cleanup-only + +# Or scope it to one specific run: +python3 benchmark_storage_pool_sequential.py --config config.yaml --cleanup-only RUN_20260722_101500 +``` + +## Rendering the Confluence-ready report + +```bash +python3 render_report.py --run-id RUN-20260722-101500 \ + --cloudstack-build 4.23.0.0-SNAPSHOT --ontap-version 9.15.1 +``` + +This prints markdown tables for 5.1.1, 5.1.2, 6.1.1, 6.1.2, and a set of +pre-filled rows for the page's Section 9 "Results Log" table (with Min/Avg/ +P95/Max computed from the raw per-op log), and also saves them to +`results/report_.md`. + +## VM instance benchmark + +Drives `deployVirtualMachine` / `destroyVirtualMachine` to reproduce the +**VM Instance** rows of the same Confluence page's Sequential Scale Matrix +(5.2) and Parallel/Concurrency Matrix (6.2). Every VM gets a root disk (from +the service offering) AND a data disk (from the disk offering), both landing +on the same tagged storage pool. + +### Prerequisites (one-time setup, per protocol) + +Unlike the storage-pool benchmark's transient pools, the VM-instance +benchmark needs a **persistent, pre-created** `bench_vm_` pool plus +matching offerings - it does not create/destroy the pool itself: + +1. A storage pool (e.g. `bench_vm_nfs3_pool`) with a distinct storage tag + (e.g. `bench_vm_nfs3`), sized comfortably below your aggregate's capacity. +2. A service offering (root disk) and a disk offering (data disk), both + tagged with that same storage tag, so root + data land on the same pool. + Set the service offering's `rootdisksize` (GB) to comfortably exceed the + template's actual `qemu-img` **virtual** size (not its sparse/download + file size) - too small silently "succeeds" on NFS but hard-fails on iSCSI + with `qemu-img: Cannot grow device files` (see Confluence Issue #2). +3. Fill in `vm_bench.templateid` / `networkid` / per-protocol + `serviceofferingid` / `diskofferingid` in `config.yaml` with the above. + +### Running + +```bash +# Sanity-check without hitting a real management server: +python3 benchmark_vm_instance_sequential.py --config config.yaml --dry-run +python3 benchmark_vm_instance_concurrency.py --config config.yaml --dry-run + +# Sequential scale matrix (5.2.1/5.2.2), both protocols: +python3 benchmark_vm_instance_sequential.py --config config.yaml + +# Concurrency matrix (6.2.1/6.2.2), iSCSI only, custom levels: +python3 benchmark_vm_instance_concurrency.py --config config.yaml --protocol iscsi --levels 1,2,5,10 + +# Both matrices in one invocation, combined into a single summary/report: +python3 benchmark_vm_instance_combined.py --config config.yaml + +# Or run the two standalone scripts under the SAME --run-id to combine them +# instead (lets you review the sequential results before committing to +# concurrency): +python3 benchmark_vm_instance_sequential.py --config config.yaml --run-id RUN-0001 +python3 benchmark_vm_instance_concurrency.py --config config.yaml --run-id RUN-0001 + +python3 render_report.py --run-id RUN-0001 --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm +``` + +All three scripts share the same `--skip-cleanup` / `--cleanup-only` safety +net as the storage-pool scripts (see above), via `vm_instance_common.py`. +Failed `deployVirtualMachine` calls are logged but their VM/disk are +intentionally left in place (not auto-destroyed) so you can inspect exactly +what got left behind - use `--cleanup-only` once you're done. + +## Notes / caveats + +- Auth is session-key based (`login` -> `sessionkey` + `JSESSIONID` cookie), + matching the approach documented on the Confluence API page. API + key/secret signed requests are not implemented (add to + `cloudstack_client.py` if your lab requires it). +- Timing is measured end-to-end from the caller's perspective, including any + async job polling (`queryAsyncJobResult`) — this is the number an operator + or automation would actually observe, not raw server-side processing time. +- The single `CloudStackClient`/`requests.Session` is shared across threads + during concurrency tests. We never mutate session state after login, so + this is safe; if you prefer full isolation, instantiate one + `CloudStackClient` per thread instead. +- `createStoragePool`/`deleteStoragePool`/`updateStoragePool` are + synchronous in this CloudStack version (no `jobid` in the response); + `enableStorageMaintenance`/`cancelStorageMaintenance` are async — the + client polls `queryAsyncJobResult` automatically for any response that + does contain a `jobid`, so the script works either way. +- This script intentionally covers **create/delete only** (per the current + ask). Maintenance mode and enable/disable pool are documented in + `cloudstack_client.py`'s API surface but not yet wired into a benchmark + phase — flag if you want those added to the matrix too. diff --git a/private-cicd/benchmark/ontap/USAGE.md b/private-cicd/benchmark/ontap/USAGE.md new file mode 100644 index 000000000000..f62dd3c89e01 --- /dev/null +++ b/private-cicd/benchmark/ontap/USAGE.md @@ -0,0 +1,179 @@ + +# Usage Reference + +Quick reference for **what each file does** and **how to run it**. For the +"why" behind the design (what each benchmark measures, how it maps to the +Confluence test matrix, known caveats) see [README.md](README.md) instead - +this file is deliberately just a lookup table / cheat sheet. + +## 1. Prerequisites + +| # | Requirement | Notes | +|---|---|---| +| 1 | Python 3.9+ | No other version-specific features used; anything 3.9-3.12 should work. | +| 2 | Network access to a CloudStack management server's API (`/client/api`) | Admin credentials with permission to create/delete storage pools, service/disk offerings, and deploy/destroy VMs. | +| 3 | Network access (HTTPS/443) from **your machine** to the ONTAP cluster's management LIF | Only needed if you run the storage-pool scripts' `--cleanup-only` helper functions that talk to ONTAP directly for orphan checks; the benchmark scripts themselves only ever call the CloudStack API, never ONTAP directly. | +| 4 | A CloudStack zone/pod/cluster with at least one KVM host, and the ONTAP plugin's SVM(s) reachable from the CloudStack management server | See README.md's "Test Environment" section on the Confluence page for the exact lab topology this was built/tested against. | +| 5 | For the VM-instance scripts only: a pre-created `bench_vm_` storage pool + matching service/disk offerings | One-time setup, see [Prerequisites (VM instance benchmark)](#4-vm-instance-benchmark-scripts) below. | + +### One-time environment setup + +```bash +cd private-cicd/benchmark/ontap +python3 -m venv .venv +source .venv/bin/activate # Windows: .venv\Scripts\activate +pip install -r requirements.txt +cp config.example.yaml config.yaml +``` + +Then edit `config.yaml` with your lab's real API URL, credentials, zone/pod/ +cluster IDs, and ONTAP connection details (see inline comments in +`config.example.yaml` for what each field means and why). + +> **About `.venv/`:** this is a completely standard, disposable Python +> virtual environment (`python3 -m venv .venv` + `pip install -r +> requirements.txt`, which just pulls in `requests` and `PyYAML` plus their +> small set of transitive dependencies). It is **never committed** (it's in +> `.gitignore`) because it is: +> - **Machine/OS-specific** - it stores an absolute path back to the Python +> interpreter that created it (`.venv/pyvenv.cfg`) and can contain +> platform-compiled binary wheels (e.g. a `.so`/`.pyd` file bundled with +> one of the dependencies) that won't load on a different OS/architecture. +> - **Python-version-specific** - the `lib/python3.9/...` directory layout +> is tied to whatever interpreter version created it. +> - **Fully reproducible** - anyone can regenerate an equivalent one in a +> few seconds with the two commands above; there is nothing hand-tuned or +> irreplaceable in it. +> +> If you ever see `.venv/` show up in `git status` as untracked, it means +> `.gitignore` isn't being picked up (e.g. it was created before the +> `.gitignore` entry was added) - just confirm `.venv/` is listed in +> `.gitignore` and it will stop appearing. + +## 2. Shared library modules (not run directly) + +| File | Purpose | +|---|---| +| `cloudstack_client.py` | Minimal CloudStack HTTP/REST client (session-key login, generic `call()`, automatic `queryAsyncJobResult` polling). Used by every benchmark script. | +| `storage_pool_common.py` | Shared helpers for the storage-pool scripts: `createStoragePool`/`deleteStoragePool` request building, CSV logging (`RawLogger`, `append_summary_csv`), cleanup-by-filter, config loading. | +| `vm_instance_common.py` | Shared helpers for the VM-instance scripts: `deployVirtualMachine`/`destroyVirtualMachine` request building (incl. force-purging data disks and `Destroy`-state volumes so ONTAP space is reclaimed immediately instead of waiting for CloudStack's 24h cleanup delay), CSV logging, cleanup-by-filter, config loading. | + +## 3. Storage-pool benchmark scripts + +Reproduces Confluence sections 5.1 (sequential) and 6.1 (concurrency) - +`createStoragePool` / `deleteStoragePool` timing at scale. + +| Script | What it does | Typical command | +|---|---|---| +| `benchmark_storage_pool_sequential.py` | Creates storage pools one at a time (per protocol) up to N=30, then deletes them one at a time, logging every call and reporting checkpoint totals at N = 1, 5, 10, 20, 30. | `python3 benchmark_storage_pool_sequential.py --config config.yaml` | +| `benchmark_storage_pool_concurrency.py` | For each concurrency level C (default 2, 5, 10, 20, 30), creates C pools in parallel via a thread pool, records wall-clock time for the whole batch, then deletes the same C pools in parallel. | `python3 benchmark_storage_pool_concurrency.py --config config.yaml` | + +Common flags (both scripts): + +| Flag | Meaning | +|---|---| +| `--config PATH` | Config YAML to use (default `config.yaml`). | +| `--protocol {nfs3,iscsi,both}` | Restrict the run to one protocol (default `both`). | +| `--run-id ID` | Reuse a specific run id (auto-generated otherwise) - use the **same id** across the sequential and concurrency scripts to merge both into one `summary_.csv`/report. | +| `--dry-run` | Simulate timings with no real API calls - use this first to sanity-check your config. | +| `--skip-cleanup` | Skip the automatic post-run sweep for leftover pools from this run id. | +| `--cleanup-only [FILTER]` | Don't run the benchmark - just delete every pool whose name contains `FILTER` (default: config's `pool_name_prefix`) and exit. Use this to recover after a crashed run. | +| `--levels 2,5,10` | *(concurrency script only)* Override the concurrency levels to run instead of the config's `concurrency_levels`. | + +## 4. VM-instance benchmark scripts + +Reproduces Confluence sections 5.2 (sequential) and 6.2 (concurrency) - +`deployVirtualMachine` / `destroyVirtualMachine` timing at scale, with each +VM getting both a root disk and a data disk on the same tagged pool. + +### Prerequisites (one-time, per protocol - not created/destroyed by the scripts) + +1. A persistent storage pool (e.g. `bench_vm_nfs3_pool`) with a distinct + storage tag (e.g. `bench_vm_nfs3`), sized comfortably below your + aggregate's capacity. +2. A service offering (root disk) and a disk offering (data disk), both + tagged with that same storage tag, so root + data land on the same pool. + Set the service offering's `rootdisksize` (GB) to comfortably exceed the + template's actual `qemu-img` **virtual** size - too small silently + "succeeds" on NFS but hard-fails on iSCSI with `qemu-img: Cannot grow + device files`. +3. Fill in `vm_bench.templateid` / `networkid` / per-protocol + `serviceofferingid` / `diskofferingid` in `config.yaml`. + +### Scripts + +| Script | What it does | Typical command | +|---|---|---| +| `benchmark_vm_instance_sequential.py` | Deploys VMs one at a time (per protocol) up to N=30, then destroys them one at a time, reporting checkpoint totals at N = 1, 2, 5, 10, 20, 30. | `python3 benchmark_vm_instance_sequential.py --config config.yaml` | +| `benchmark_vm_instance_concurrency.py` | For each concurrency level C, deploys C VMs in parallel, records wall-clock time, then destroys the same C VMs in parallel. | `python3 benchmark_vm_instance_concurrency.py --config config.yaml` | +| `benchmark_vm_instance_combined.py` | Runs the sequential matrix immediately followed by the concurrency matrix, under one shared run id, in a single invocation. Use once you already trust your config/environment. | `python3 benchmark_vm_instance_combined.py --config config.yaml` | + +Common flags: same as the storage-pool scripts above (`--config`, +`--protocol`, `--run-id`, `--dry-run`, `--skip-cleanup`, `--cleanup-only +[FILTER]`; `--levels` on the concurrency/combined scripts). Failed +`deployVirtualMachine` calls are intentionally **left in place** (not +auto-destroyed) so you can inspect what got left behind - clean them up +with `--cleanup-only` once done. + +## 5. Report rendering + +| Script | What it does | Typical command | +|---|---|---| +| `render_report.py` | Turns a run's `raw_ops_.csv` + `summary_.csv` into Confluence-ready markdown tables (matches the page's section layout, plus Min/Avg/P95/Max rows for the Results Log). Writes to stdout and to `results/report_.md`. | `python3 render_report.py --run-id RUN-0001 --cloudstack-build 4.23.0.0-SNAPSHOT --ontap-version 9.17.1` | + +For VM-instance runs (which write `raw_ops_vm_.csv` / +`summary_vm_.csv` instead of the storage-pool scripts' `raw_ops_`/ +`summary_` prefix), pass the matching prefixes: + +```bash +python3 render_report.py --run-id RUN-0001 \ + --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm +``` + +| Flag | Meaning | +|---|---| +| `--run-id ID` | **Required.** The run id used by the benchmark script(s). | +| `--output-dir DIR` | Where the CSVs live / the report gets written (default `results`). | +| `--cloudstack-build STR` | Label only, stamped into the Results Log rows. | +| `--ontap-version STR` | Label only, stamped into the Results Log rows. | +| `--run-date YYYY-MM-DD` | Defaults to today. | +| `--raw-prefix STR` | CSV filename prefix for raw per-op data (default `raw_ops`; use `raw_ops_vm` for VM-instance runs). | +| `--summary-prefix STR` | CSV filename prefix for checkpoint summaries (default `summary`; use `summary_vm` for VM-instance runs). | +| `--report-suffix STR` | Suffix appended to the output report filename (e.g. `_vm`) to avoid clobbering a storage-pool report with the same run id. | + +## 6. Config files at a glance + +| File | Committed? | Purpose | +|---|---|---| +| `config.example.yaml` | Yes | Sanitized template - copy to `config.yaml` and fill in your lab's real values. | +| `config.yaml` | **No** (gitignored) | Your real, lab-specific config (real IPs/credentials) - never commit this. | +| `config_sanity.yaml` | **No** (gitignored) | Optional scratch config some contributors keep locally for quick one-off dry-runs; not part of the checked-in tooling. | + +## 7. Output files at a glance + +All written under `results/` (gitignored - these are run artifacts, not +source): + +| Pattern | Produced by | Contents | +|---|---|---| +| `raw_ops_.csv` | storage-pool scripts | One row per individual API call (timestamps, duration, success/failure, error). | +| `raw_ops_vm_.csv` | VM-instance scripts | Same, for VM deploy/destroy calls. | +| `summary_.csv` / `summary_vm_.csv` | both | One row per checkpoint/concurrency level (totals, averages, success/failure counts). | +| `report_.md` / `report_vm_.md` | `render_report.py` | Confluence-ready markdown tables generated from the two CSVs above. | diff --git a/private-cicd/benchmark/ontap/benchmark_storage_pool_concurrency.py b/private-cicd/benchmark/ontap/benchmark_storage_pool_concurrency.py new file mode 100644 index 000000000000..f097dcd97330 --- /dev/null +++ b/private-cicd/benchmark/ontap/benchmark_storage_pool_concurrency.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Concurrency storage-pool benchmark for the NetApp ONTAP CloudStack plugin. + +Drives createStoragePool / deleteStoragePool in parallel (ThreadPoolExecutor) +over the CloudStack HTTP/REST API and reproduces Section 6.1 - Parallel/ +Concurrency Matrix (C = 2,5,10,20,30 workers) from the +"ONTAP Plugin - CloudStack Operations, Scale & Parallel Test Matrix" page: + + 6.1.1 Parallel pool creation + 6.1.2 Parallel pool deletion + +See benchmark_storage_pool_sequential.py for the sequential scale matrix +(kept as a separate script/entry point on purpose - concurrency runs are the +ones most likely to hit mgmt-server/plugin job-queue saturation, so they're +easy to run, review, and re-run independently of the sequential matrix). + +Every individual API call is logged to results/raw_ops_.csv, and a +per-checkpoint roll-up (matching the columns of the Confluence tables) is +appended to results/summary_.csv. Use render_report.py afterwards to +turn the summary CSV into paste-ready markdown for the Confluence page. Using +the same --run-id here and in benchmark_storage_pool_sequential.py combines +both into a single summary_.csv / report. + +Usage: + python3 benchmark_storage_pool_concurrency.py --config config.yaml --protocol both + python3 benchmark_storage_pool_concurrency.py --config config.yaml --levels 2,5,10 + python3 benchmark_storage_pool_concurrency.py --config config.yaml --dry-run + python3 benchmark_storage_pool_concurrency.py --config config.yaml --cleanup-only +""" + +import argparse +import os +import statistics +import time +from concurrent.futures import ThreadPoolExecutor, as_completed + +from storage_pool_common import ( + append_summary_csv, + cleanup_by_filter, + create_pool, + delete_pool, + fake_create, + fake_delete, + load_config, + make_cloudstack_client, + new_run_id, + RawLogger, + resolve_protocols, +) + + +def run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay, levels): + infra = cfg["infrastructure"] + ontap_cfg = cfg["ontap"][protocol_key] + prefix = cfg["benchmark"]["pool_name_prefix"] + + for level in levels: + print(f"\n=== [6.1.1] Parallel CREATE - protocol={protocol_key} C={level} ===") + names = [f"{prefix}_c{level}_{protocol_key}_{run_id}_{i:03d}" for i in range(1, level + 1)] + + t0 = time.perf_counter() + results = [] + if dry_run: + results = [fake_create(n) for n in names] + else: + with ThreadPoolExecutor(max_workers=level) as ex: + futures = {ex.submit(create_pool, client, n, infra, ontap_cfg): n for n in names} + for fut in as_completed(futures): + results.append(fut.result()) + wall = time.perf_counter() - t0 + + for idx, r in enumerate(results, start=1): + raw_logger.log(run_id, "concurrent_create", "6.1.1", protocol_key, level, idx, r) + + succ = [r for r in results if r.success] + fail = [r for r in results if not r.success] + avg = statistics.mean([r.duration_sec for r in succ]) if succ else 0 + notes = "Watch mgmt-server job-queue/thread-pool saturation as a confound" if level >= 30 else "" + summary_rows.append({ + "run_id": run_id, "phase": "concurrent_create", "test_id": "6.1.1", + "protocol": protocol_key, "checkpoint": level, + "total_time_sec": round(wall, 3), "avg_time_sec": round(avg, 3), + "success_count": len(succ), "failure_count": len(fail), "notes": notes, + }) + print(f" wall-clock={wall:.3f}s success={len(succ)} failure={len(fail)} avg/op={avg:.3f}s") + for r in fail: + print(f" !! {r.pool_name}: {r.error}") + + print(f"\n=== [6.1.2] Parallel DELETE - protocol={protocol_key} C={len(succ)} ===") + t0 = time.perf_counter() + del_results = [] + if dry_run: + del_results = [fake_delete(r.pool_id, r.pool_name) for r in succ] + elif succ: + with ThreadPoolExecutor(max_workers=len(succ)) as ex: + futures = {ex.submit(delete_pool, client, r.pool_id, r.pool_name): r for r in succ} + for fut in as_completed(futures): + del_results.append(fut.result()) + wall_del = time.perf_counter() - t0 + + for idx, r in enumerate(del_results, start=1): + raw_logger.log(run_id, "concurrent_delete", "6.1.2", protocol_key, level, idx, r) + + succ_d = [r for r in del_results if r.success] + fail_d = [r for r in del_results if not r.success] + avg_d = statistics.mean([r.duration_sec for r in succ_d]) if succ_d else 0 + summary_rows.append({ + "run_id": run_id, "phase": "concurrent_delete", "test_id": "6.1.2", + "protocol": protocol_key, "checkpoint": level, + "total_time_sec": round(wall_del, 3), "avg_time_sec": round(avg_d, 3), + "success_count": len(succ_d), "failure_count": len(fail_d), "notes": "", + }) + print(f" wall-clock={wall_del:.3f}s success={len(succ_d)} failure={len(fail_d)} avg/op={avg_d:.3f}s") + for r in fail_d: + print(f" !! {r.pool_name}: {r.error}") + if delay: + time.sleep(delay) + + +def parse_levels(raw, default_levels): + if not raw: + return sorted(default_levels) + return sorted(int(x.strip()) for x in raw.split(",") if x.strip()) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--protocol", default="both", help="nfs3 | iscsi | both (default: both)") + parser.add_argument("--run-id", default=None, help="Override auto-generated run id") + parser.add_argument( + "--levels", default=None, + help="Comma-separated concurrency levels to run, e.g. '2,5,10' " + "(default: config.benchmark.concurrency_levels, typically up to 30)", + ) + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover pools from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Delete all storage pools whose name contains FILTER (default: config pool_name_prefix) and exit", + ) + args = parser.parse_args() + + cfg = load_config(args.config) + os.makedirs(cfg["benchmark"].get("output_dir", "results"), exist_ok=True) + + if args.cleanup_only is not None: + client = make_cloudstack_client(cfg) + name_filter = args.cleanup_only + if name_filter == "__PREFIX__": + name_filter = cfg["benchmark"]["pool_name_prefix"] + cleanup_by_filter(client, name_filter) + return + + run_id = args.run_id or new_run_id() + protocols = resolve_protocols(cfg, args.protocol) + delay = cfg["benchmark"].get("inter_op_delay_sec", 0) + output_dir = cfg["benchmark"].get("output_dir", "results") + levels = parse_levels(args.levels, cfg["benchmark"]["concurrency_levels"]) + + print(f"Run ID: {run_id}") + print(f"Protocols: {protocols}") + print("Mode: concurrency") + print(f"Concurrency levels: {levels}") + print(f"Dry run: {args.dry_run}") + + client = None if args.dry_run else make_cloudstack_client(cfg) + + raw_logger = RawLogger(os.path.join(output_dir, f"raw_ops_{run_id}.csv")) + summary_rows = [] + + try: + for protocol_key in protocols: + run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, args.dry_run, delay, levels) + finally: + raw_logger.close() + + summary_path = os.path.join(output_dir, f"summary_{run_id}.csv") + append_summary_csv(summary_path, summary_rows) + + print(f"\nRaw per-operation log: {os.path.join(output_dir, f'raw_ops_{run_id}.csv')}") + print(f"Checkpoint summary: {summary_path}") + print("Next: python3 render_report.py --run-id " + run_id + f" --output-dir {output_dir}") + + if not args.dry_run and not args.skip_cleanup: + print(f"\nVerifying no orphaned pools remain for run {run_id}...") + cleanup_by_filter(client, run_id) + + +if __name__ == "__main__": + main() diff --git a/private-cicd/benchmark/ontap/benchmark_storage_pool_sequential.py b/private-cicd/benchmark/ontap/benchmark_storage_pool_sequential.py new file mode 100644 index 000000000000..b103596ab7d8 --- /dev/null +++ b/private-cicd/benchmark/ontap/benchmark_storage_pool_sequential.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Sequential storage-pool benchmark for the NetApp ONTAP CloudStack plugin. + +Drives createStoragePool / deleteStoragePool one at a time over the CloudStack +HTTP/REST API (see https://netapp.atlassian.net/wiki/spaces/OSSG/pages/608854350) +and reproduces Section 5.1 - Sequential Scale Matrix (N = 1,5,10,20,30) from the +"ONTAP Plugin - CloudStack Operations, Scale & Parallel Test Matrix" page: + + 5.1.1 Sequential create (1 pool at a time, cumulative to N) + 5.1.2 Sequential delete (1 pool at a time, from N remaining) + +See benchmark_storage_pool_concurrency.py for the parallel/concurrency matrix +(kept as a separate script/entry point on purpose, so a sequential run can be +kicked off and reviewed independently before committing to a concurrency run). + +Every individual API call is logged to results/raw_ops_.csv, and a +per-checkpoint roll-up (matching the columns of the Confluence tables) is +appended to results/summary_.csv. Use render_report.py afterwards to +turn the summary CSV into paste-ready markdown for the Confluence page. Using +the same --run-id here and in benchmark_storage_pool_concurrency.py combines +both into a single summary_.csv / report. + +Usage: + python3 benchmark_storage_pool_sequential.py --config config.yaml --protocol both + python3 benchmark_storage_pool_sequential.py --config config.yaml --dry-run + python3 benchmark_storage_pool_sequential.py --config config.yaml --cleanup-only +""" + +import argparse +import os +import time + +from storage_pool_common import ( + append_summary_csv, + cleanup_by_filter, + create_pool, + delete_pool, + fake_create, + fake_delete, + load_config, + make_cloudstack_client, + new_run_id, + RawLogger, + resolve_protocols, +) + + +def run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay): + infra = cfg["infrastructure"] + ontap_cfg = cfg["ontap"][protocol_key] + checkpoints = sorted(cfg["benchmark"]["sequential_checkpoints"]) + max_n = max(checkpoints) + prefix = cfg["benchmark"]["pool_name_prefix"] + + created = [] + create_durations = [] + print(f"\n=== [5.1.1] Sequential CREATE - protocol={protocol_key} up to N={max_n} ===") + for i in range(1, max_n + 1): + # ONTAP volume names only allow alphanumeric + underscore (no hyphens). + name = f"{prefix}_seq_{protocol_key}_{run_id}_{i:03d}" + result = fake_create(name) if dry_run else create_pool(client, name, infra, ontap_cfg) + raw_logger.log(run_id, "sequential_create", "5.1.1", protocol_key, max_n, i, result) + status = "OK" if result.success else "FAIL" + print(f" [{i:>3}/{max_n}] create {name} -> {status} ({result.duration_sec:.3f}s)") + if result.success: + created.append((name, result.pool_id)) + create_durations.append(result.duration_sec) + else: + print(f" !! {result.error}") + if i in checkpoints: + total = sum(create_durations) + avg = total / len(create_durations) if create_durations else 0 + summary_rows.append({ + "run_id": run_id, "phase": "sequential_create", "test_id": "5.1.1", + "protocol": protocol_key, "checkpoint": i, + "total_time_sec": round(total, 3), "avg_time_sec": round(avg, 3), + "success_count": len(created), "failure_count": i - len(created), "notes": "", + }) + print(f" >> checkpoint N={i}: total={total:.3f}s avg={avg:.3f}s/op " + f"success={len(created)} failure={i - len(created)}") + if delay: + time.sleep(delay) + + total_created = len(created) + print(f"\n=== [5.1.2] Sequential DELETE - protocol={protocol_key} from N={total_created} remaining ===") + if total_created in checkpoints: + summary_rows.append({ + "run_id": run_id, "phase": "sequential_delete", "test_id": "5.1.2", + "protocol": protocol_key, "checkpoint": total_created, + "total_time_sec": 0, "avg_time_sec": 0, "success_count": 0, "failure_count": 0, + "notes": "baseline - no deletes issued yet", + }) + delete_durations = [] + deleted_ok = 0 + for idx, (name, pool_id) in enumerate(created, start=1): + result = fake_delete(pool_id, name) if dry_run else delete_pool(client, pool_id, name) + remaining = total_created - idx + raw_logger.log(run_id, "sequential_delete", "5.1.2", protocol_key, total_created, idx, result) + status = "OK" if result.success else "FAIL" + print(f" [{idx:>3}/{total_created}] delete {name} -> {status} " + f"({result.duration_sec:.3f}s) remaining={remaining}") + if result.success: + delete_durations.append(result.duration_sec) + deleted_ok += 1 + else: + print(f" !! {result.error}") + if remaining in checkpoints: + total = sum(delete_durations) + avg = total / len(delete_durations) if delete_durations else 0 + summary_rows.append({ + "run_id": run_id, "phase": "sequential_delete", "test_id": "5.1.2", + "protocol": protocol_key, "checkpoint": remaining, + "total_time_sec": round(total, 3), "avg_time_sec": round(avg, 3), + "success_count": deleted_ok, "failure_count": idx - deleted_ok, "notes": "", + }) + print(f" >> checkpoint remaining={remaining}: total={total:.3f}s avg={avg:.3f}s/op " + f"success={deleted_ok} failure={idx - deleted_ok}") + if delay: + time.sleep(delay) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--protocol", default="both", help="nfs3 | iscsi | both (default: both)") + parser.add_argument("--run-id", default=None, help="Override auto-generated run id") + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover pools from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Delete all storage pools whose name contains FILTER (default: config pool_name_prefix) and exit", + ) + args = parser.parse_args() + + cfg = load_config(args.config) + os.makedirs(cfg["benchmark"].get("output_dir", "results"), exist_ok=True) + + if args.cleanup_only is not None: + client = make_cloudstack_client(cfg) + name_filter = args.cleanup_only + if name_filter == "__PREFIX__": + name_filter = cfg["benchmark"]["pool_name_prefix"] + cleanup_by_filter(client, name_filter) + return + + run_id = args.run_id or new_run_id() + protocols = resolve_protocols(cfg, args.protocol) + delay = cfg["benchmark"].get("inter_op_delay_sec", 0) + output_dir = cfg["benchmark"].get("output_dir", "results") + + print(f"Run ID: {run_id}") + print(f"Protocols: {protocols}") + print("Mode: sequential") + print(f"Dry run: {args.dry_run}") + + client = None if args.dry_run else make_cloudstack_client(cfg) + + raw_logger = RawLogger(os.path.join(output_dir, f"raw_ops_{run_id}.csv")) + summary_rows = [] + + try: + for protocol_key in protocols: + run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, args.dry_run, delay) + finally: + raw_logger.close() + + summary_path = os.path.join(output_dir, f"summary_{run_id}.csv") + append_summary_csv(summary_path, summary_rows) + + print(f"\nRaw per-operation log: {os.path.join(output_dir, f'raw_ops_{run_id}.csv')}") + print(f"Checkpoint summary: {summary_path}") + print("Next: python3 render_report.py --run-id " + run_id + f" --output-dir {output_dir}") + print(f" (or run benchmark_storage_pool_concurrency.py --run-id {run_id} first to add 6.1.x to the same report)") + + if not args.dry_run and not args.skip_cleanup: + print(f"\nVerifying no orphaned pools remain for run {run_id}...") + cleanup_by_filter(client, run_id) + + +if __name__ == "__main__": + main() diff --git a/private-cicd/benchmark/ontap/benchmark_vm_instance_combined.py b/private-cicd/benchmark/ontap/benchmark_vm_instance_combined.py new file mode 100644 index 000000000000..86e0430038c3 --- /dev/null +++ b/private-cicd/benchmark/ontap/benchmark_vm_instance_combined.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Combined (sequential + concurrency) VM-instance benchmark for the NetApp +ONTAP CloudStack plugin. + +Convenience wrapper that runs benchmark_vm_instance_sequential.py's Section +5.2 scale matrix immediately followed by benchmark_vm_instance_concurrency.py's +Section 6.2 parallel matrix, for each protocol, under a single shared +--run-id - equivalent to running both scripts back to back, but in one +invocation and one combined results/summary_vm_.csv. + +Prefer the two standalone scripts (benchmark_vm_instance_sequential.py / +benchmark_vm_instance_concurrency.py) when you want to review the sequential +results before deciding whether to proceed with a concurrency run - this +combined script is for when you already trust the config/environment and just +want the full 5.2.x + 6.2.x matrix in one go. + +Usage: + python3 benchmark_vm_instance_combined.py --config config.yaml --protocol both + python3 benchmark_vm_instance_combined.py --config config.yaml --dry-run + python3 benchmark_vm_instance_combined.py --config config.yaml --cleanup-only +""" + +import argparse +import os + +from benchmark_vm_instance_concurrency import parse_levels, run_concurrency +from benchmark_vm_instance_sequential import run_sequential +from vm_instance_common import ( + append_summary_csv, + cleanup_by_filter, + load_config, + make_cloudstack_client, + new_run_id, + RawLogger, + resolve_protocols, +) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--protocol", default="both", help="nfs3 | iscsi | both (default: both)") + parser.add_argument("--run-id", default=None, help="Override auto-generated run id") + parser.add_argument( + "--levels", default=None, + help="Comma-separated concurrency levels to run, e.g. '2,5,10' " + "(default: config.vm_bench.concurrency_levels, typically up to 30)", + ) + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover VMs from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Destroy all VMs whose name contains FILTER (default: config vm_name_prefix) and exit", + ) + args = parser.parse_args() + + cfg = load_config(args.config) + os.makedirs(cfg["vm_bench"].get("output_dir", "results"), exist_ok=True) + + if args.cleanup_only is not None: + client = make_cloudstack_client(cfg) + name_filter = args.cleanup_only + if name_filter == "__PREFIX__": + name_filter = cfg["vm_bench"]["vm_name_prefix"] + cleanup_by_filter(client, name_filter) + return + + run_id = args.run_id or new_run_id() + protocols = resolve_protocols(cfg, args.protocol) + delay = cfg["vm_bench"].get("inter_op_delay_sec", 0) + output_dir = cfg["vm_bench"].get("output_dir", "results") + levels = parse_levels(args.levels, cfg["vm_bench"]["concurrency_levels"]) + + print(f"Run ID: {run_id}") + print(f"Protocols: {protocols}") + print("Mode: sequential + concurrency (combined)") + print(f"Concurrency levels: {levels}") + print(f"Dry run: {args.dry_run}") + + client = None if args.dry_run else make_cloudstack_client(cfg) + + raw_logger = RawLogger(os.path.join(output_dir, f"raw_ops_vm_{run_id}.csv")) + summary_rows = [] + + try: + for protocol_key in protocols: + run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, args.dry_run, delay) + run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, args.dry_run, delay, levels) + finally: + raw_logger.close() + + summary_path = os.path.join(output_dir, f"summary_vm_{run_id}.csv") + append_summary_csv(summary_path, summary_rows) + + print(f"\nRaw per-operation log: {os.path.join(output_dir, f'raw_ops_vm_{run_id}.csv')}") + print(f"Checkpoint summary: {summary_path}") + print("Next: python3 render_report.py --run-id " + run_id + + f" --output-dir {output_dir} --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm") + + if not args.dry_run and not args.skip_cleanup: + print(f"\nVerifying no orphaned VMs remain for run {run_id}...") + cleanup_by_filter(client, run_id) + + +if __name__ == "__main__": + main() diff --git a/private-cicd/benchmark/ontap/benchmark_vm_instance_concurrency.py b/private-cicd/benchmark/ontap/benchmark_vm_instance_concurrency.py new file mode 100644 index 000000000000..c002e56b4d2b --- /dev/null +++ b/private-cicd/benchmark/ontap/benchmark_vm_instance_concurrency.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Concurrency VM-instance benchmark for the NetApp ONTAP CloudStack plugin. + +Drives deployVirtualMachine / destroyVirtualMachine in parallel +(ThreadPoolExecutor) over the CloudStack HTTP/REST API against pre- +provisioned bench_vm_nfs3 / bench_vm_iscsi storage pools (see README.md +"VM instance benchmark prerequisites") and reproduces Section 6.2 - Parallel/ +Concurrency Matrix from the "ONTAP Plugin - CloudStack Operations, Scale & +Parallel Test Matrix" page: + + 6.2.1 Parallel VM creation + 6.2.2 Parallel VM deletion + +Each deployed VM gets a root disk (from the service offering) AND one data +disk (from the disk offering) landing on the SAME tagged storage pool. + +See benchmark_vm_instance_sequential.py for the sequential scale matrix, or +benchmark_vm_instance_combined.py to run both in one invocation (kept as +separate scripts/entry points on purpose - concurrency runs are the ones most +likely to hit mgmt-server/plugin job-queue saturation or pool capacity limits, +so they're easy to run, review, and re-run independently of the sequential +matrix). + +Every individual API call is logged to results/raw_ops_vm_.csv, and a +per-checkpoint roll-up (matching the columns of the Confluence tables) is +appended to results/summary_vm_.csv. Use render_report.py afterwards +to turn the summary CSV into paste-ready markdown. Using the same --run-id +here and in benchmark_vm_instance_sequential.py combines both into a single +summary_vm_.csv / report. + +Usage: + python3 benchmark_vm_instance_concurrency.py --config config.yaml --protocol both + python3 benchmark_vm_instance_concurrency.py --config config.yaml --levels 2,5,10 + python3 benchmark_vm_instance_concurrency.py --config config.yaml --dry-run + python3 benchmark_vm_instance_concurrency.py --config config.yaml --cleanup-only +""" + +import argparse +import os +import statistics +import time +from concurrent.futures import ThreadPoolExecutor, as_completed + +from vm_instance_common import ( + append_summary_csv, + cleanup_by_filter, + deploy_vm, + destroy_vm, + fake_create, + fake_delete, + load_config, + make_cloudstack_client, + new_run_id, + RawLogger, + report_failed_creates, + resolve_protocols, +) + + +def run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay, levels): + infra = cfg["infrastructure"] + vm_cfg = cfg["vm_bench"] + proto_cfg = vm_cfg["protocols"][protocol_key] + prefix = vm_cfg["vm_name_prefix"] + + for level in levels: + print(f"\n=== [6.2.1] Parallel CREATE - protocol={protocol_key} C={level} ===") + names = [f"{prefix}-c{level}-{protocol_key}-{run_id}-{i:03d}".replace("_", "-") + for i in range(1, level + 1)] + + t0 = time.perf_counter() + results = [] + if dry_run: + results = [fake_create(n) for n in names] + else: + with ThreadPoolExecutor(max_workers=level) as ex: + futures = {ex.submit(deploy_vm, client, n, infra, vm_cfg, proto_cfg): n for n in names} + for fut in as_completed(futures): + results.append(fut.result()) + wall = time.perf_counter() - t0 + + for idx, r in enumerate(results, start=1): + raw_logger.log(run_id, "concurrent_create", "6.2.1", protocol_key, level, idx, r) + + succ = [r for r in results if r.success] + fail = [r for r in results if not r.success] + avg = statistics.mean([r.duration_sec for r in succ]) if succ else 0 + notes = "Watch mgmt-server job-queue/thread-pool + single-host KVM saturation as a confound" if level >= 20 else "" + summary_rows.append({ + "run_id": run_id, "phase": "concurrent_create", "test_id": "6.2.1", + "protocol": protocol_key, "checkpoint": level, + "total_time_sec": round(wall, 3), "avg_time_sec": round(avg, 3), + "success_count": len(succ), "failure_count": len(fail), "notes": notes, + }) + print(f" wall-clock={wall:.3f}s success={len(succ)} failure={len(fail)} avg/op={avg:.3f}s") + for r in fail: + print(f" !! {r.vm_name}: {r.error}") + report_failed_creates([(r.vm_name, r.vm_id) for r in fail]) + + print(f"\n=== [6.2.2] Parallel DELETE - protocol={protocol_key} C={len(succ)} ===") + t0 = time.perf_counter() + del_results = [] + if dry_run: + del_results = [fake_delete(r.vm_id, r.vm_name) for r in succ] + elif succ: + with ThreadPoolExecutor(max_workers=len(succ)) as ex: + futures = {ex.submit(destroy_vm, client, r.vm_id, r.vm_name): r for r in succ} + for fut in as_completed(futures): + del_results.append(fut.result()) + wall_del = time.perf_counter() - t0 + + for idx, r in enumerate(del_results, start=1): + raw_logger.log(run_id, "concurrent_delete", "6.2.2", protocol_key, level, idx, r) + + succ_d = [r for r in del_results if r.success] + fail_d = [r for r in del_results if not r.success] + avg_d = statistics.mean([r.duration_sec for r in succ_d]) if succ_d else 0 + summary_rows.append({ + "run_id": run_id, "phase": "concurrent_delete", "test_id": "6.2.2", + "protocol": protocol_key, "checkpoint": level, + "total_time_sec": round(wall_del, 3), "avg_time_sec": round(avg_d, 3), + "success_count": len(succ_d), "failure_count": len(fail_d), "notes": "", + }) + print(f" wall-clock={wall_del:.3f}s success={len(succ_d)} failure={len(fail_d)} avg/op={avg_d:.3f}s") + for r in fail_d: + print(f" !! {r.vm_name}: {r.error}") + if delay: + time.sleep(delay) + + +def parse_levels(raw, default_levels): + if not raw: + return sorted(default_levels) + return sorted(int(x.strip()) for x in raw.split(",") if x.strip()) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--protocol", default="both", help="nfs3 | iscsi | both (default: both)") + parser.add_argument("--run-id", default=None, help="Override auto-generated run id") + parser.add_argument( + "--levels", default=None, + help="Comma-separated concurrency levels to run, e.g. '2,5,10' " + "(default: config.vm_bench.concurrency_levels, typically up to 30)", + ) + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover VMs from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Destroy all VMs whose name contains FILTER (default: config vm_name_prefix) and exit", + ) + args = parser.parse_args() + + cfg = load_config(args.config) + os.makedirs(cfg["vm_bench"].get("output_dir", "results"), exist_ok=True) + + if args.cleanup_only is not None: + client = make_cloudstack_client(cfg) + name_filter = args.cleanup_only + if name_filter == "__PREFIX__": + name_filter = cfg["vm_bench"]["vm_name_prefix"] + cleanup_by_filter(client, name_filter) + return + + run_id = args.run_id or new_run_id() + protocols = resolve_protocols(cfg, args.protocol) + delay = cfg["vm_bench"].get("inter_op_delay_sec", 0) + output_dir = cfg["vm_bench"].get("output_dir", "results") + levels = parse_levels(args.levels, cfg["vm_bench"]["concurrency_levels"]) + + print(f"Run ID: {run_id}") + print(f"Protocols: {protocols}") + print("Mode: concurrency") + print(f"Concurrency levels: {levels}") + print(f"Dry run: {args.dry_run}") + + client = None if args.dry_run else make_cloudstack_client(cfg) + + raw_logger = RawLogger(os.path.join(output_dir, f"raw_ops_vm_{run_id}.csv")) + summary_rows = [] + + try: + for protocol_key in protocols: + run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, args.dry_run, delay, levels) + finally: + raw_logger.close() + + summary_path = os.path.join(output_dir, f"summary_vm_{run_id}.csv") + append_summary_csv(summary_path, summary_rows) + + print(f"\nRaw per-operation log: {os.path.join(output_dir, f'raw_ops_vm_{run_id}.csv')}") + print(f"Checkpoint summary: {summary_path}") + print("Next: python3 render_report.py --run-id " + run_id + + f" --output-dir {output_dir} --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm") + + if not args.dry_run and not args.skip_cleanup: + print(f"\nVerifying no orphaned VMs remain for run {run_id}...") + cleanup_by_filter(client, run_id) + + +if __name__ == "__main__": + main() diff --git a/private-cicd/benchmark/ontap/benchmark_vm_instance_sequential.py b/private-cicd/benchmark/ontap/benchmark_vm_instance_sequential.py new file mode 100644 index 000000000000..2890c99cfae7 --- /dev/null +++ b/private-cicd/benchmark/ontap/benchmark_vm_instance_sequential.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Sequential VM-instance benchmark for the NetApp ONTAP CloudStack plugin. + +Drives deployVirtualMachine / destroyVirtualMachine one at a time over the +CloudStack HTTP/REST API against pre-provisioned bench_vm_nfs3 / bench_vm_iscsi +storage pools (see README.md "VM instance benchmark prerequisites") and +reproduces Section 5.2 - Sequential Scale Matrix from the "ONTAP Plugin - +CloudStack Operations, Scale & Parallel Test Matrix" page: + + 5.2.1 Sequential create (1 VM at a time, cumulative to N) + 5.2.2 Sequential delete (1 VM at a time, from N remaining) + +Each deployed VM gets a root disk (from the service offering) AND one data +disk (from the disk offering) landing on the SAME tagged storage pool. + +See benchmark_vm_instance_concurrency.py for the parallel/concurrency matrix, +or benchmark_vm_instance_combined.py to run both in one invocation (kept as +separate scripts/entry points on purpose, so a sequential run can be kicked +off and reviewed independently before committing to a concurrency run). + +Every individual API call is logged to results/raw_ops_vm_.csv, and a +per-checkpoint roll-up (matching the columns of the Confluence tables) is +appended to results/summary_vm_.csv. Use render_report.py afterwards +to turn the summary CSV into paste-ready markdown. Using the same --run-id +here and in benchmark_vm_instance_concurrency.py combines both into a single +summary_vm_.csv / report. + +Usage: + python3 benchmark_vm_instance_sequential.py --config config.yaml --protocol both + python3 benchmark_vm_instance_sequential.py --config config.yaml --dry-run + python3 benchmark_vm_instance_sequential.py --config config.yaml --cleanup-only +""" + +import argparse +import os +import time + +from vm_instance_common import ( + append_summary_csv, + cleanup_by_filter, + deploy_vm, + destroy_vm, + fake_create, + fake_delete, + load_config, + make_cloudstack_client, + new_run_id, + RawLogger, + report_failed_creates, + resolve_protocols, +) + + +def run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay): + infra = cfg["infrastructure"] + vm_cfg = cfg["vm_bench"] + proto_cfg = vm_cfg["protocols"][protocol_key] + checkpoints = sorted(vm_cfg["sequential_checkpoints"]) + max_n = max(checkpoints) + prefix = vm_cfg["vm_name_prefix"] + + created = [] + failed = [] + create_durations = [] + print(f"\n=== [5.2.1] Sequential CREATE - protocol={protocol_key} up to N={max_n} ===") + for i in range(1, max_n + 1): + # CloudStack VM names are used as hostnames (RFC1123): letters, digits, + # and "-" only (no "_", the opposite of ONTAP's volume-name rule). + name = f"{prefix}-seq-{protocol_key}-{run_id}-{i:03d}".replace("_", "-") + result = fake_create(name) if dry_run else deploy_vm(client, name, infra, vm_cfg, proto_cfg) + raw_logger.log(run_id, "sequential_create", "5.2.1", protocol_key, max_n, i, result) + status = "OK" if result.success else "FAIL" + print(f" [{i:>3}/{max_n}] create {name} -> {status} ({result.duration_sec:.3f}s)") + if result.success: + created.append((name, result.vm_id)) + create_durations.append(result.duration_sec) + else: + print(f" !! {result.error}") + failed.append((name, result.vm_id)) + if i in checkpoints: + total = sum(create_durations) + avg = total / len(create_durations) if create_durations else 0 + summary_rows.append({ + "run_id": run_id, "phase": "sequential_create", "test_id": "5.2.1", + "protocol": protocol_key, "checkpoint": i, + "total_time_sec": round(total, 3), "avg_time_sec": round(avg, 3), + "success_count": len(created), "failure_count": i - len(created), "notes": "", + }) + print(f" >> checkpoint N={i}: total={total:.3f}s avg={avg:.3f}s/op " + f"success={len(created)} failure={i - len(created)}") + if delay: + time.sleep(delay) + + report_failed_creates(failed) + + total_created = len(created) + print(f"\n=== [5.2.2] Sequential DELETE - protocol={protocol_key} from N={total_created} remaining ===") + if total_created in checkpoints: + summary_rows.append({ + "run_id": run_id, "phase": "sequential_delete", "test_id": "5.2.2", + "protocol": protocol_key, "checkpoint": total_created, + "total_time_sec": 0, "avg_time_sec": 0, "success_count": 0, "failure_count": 0, + "notes": "baseline - no deletes issued yet", + }) + delete_durations = [] + deleted_ok = 0 + for idx, (name, vm_id) in enumerate(created, start=1): + result = fake_delete(vm_id, name) if dry_run else destroy_vm(client, vm_id, name) + remaining = total_created - idx + raw_logger.log(run_id, "sequential_delete", "5.2.2", protocol_key, total_created, idx, result) + status = "OK" if result.success else "FAIL" + print(f" [{idx:>3}/{total_created}] delete {name} -> {status} " + f"({result.duration_sec:.3f}s) remaining={remaining}") + if result.success: + delete_durations.append(result.duration_sec) + deleted_ok += 1 + else: + print(f" !! {result.error}") + if remaining in checkpoints: + total = sum(delete_durations) + avg = total / len(delete_durations) if delete_durations else 0 + summary_rows.append({ + "run_id": run_id, "phase": "sequential_delete", "test_id": "5.2.2", + "protocol": protocol_key, "checkpoint": remaining, + "total_time_sec": round(total, 3), "avg_time_sec": round(avg, 3), + "success_count": deleted_ok, "failure_count": idx - deleted_ok, "notes": "", + }) + print(f" >> checkpoint remaining={remaining}: total={total:.3f}s avg={avg:.3f}s/op " + f"success={deleted_ok} failure={idx - deleted_ok}") + if delay: + time.sleep(delay) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--protocol", default="both", help="nfs3 | iscsi | both (default: both)") + parser.add_argument("--run-id", default=None, help="Override auto-generated run id") + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover VMs from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Destroy all VMs whose name contains FILTER (default: config vm_name_prefix) and exit", + ) + args = parser.parse_args() + + cfg = load_config(args.config) + os.makedirs(cfg["vm_bench"].get("output_dir", "results"), exist_ok=True) + + if args.cleanup_only is not None: + client = make_cloudstack_client(cfg) + name_filter = args.cleanup_only + if name_filter == "__PREFIX__": + name_filter = cfg["vm_bench"]["vm_name_prefix"] + cleanup_by_filter(client, name_filter) + return + + run_id = args.run_id or new_run_id() + protocols = resolve_protocols(cfg, args.protocol) + delay = cfg["vm_bench"].get("inter_op_delay_sec", 0) + output_dir = cfg["vm_bench"].get("output_dir", "results") + + print(f"Run ID: {run_id}") + print(f"Protocols: {protocols}") + print("Mode: sequential") + print(f"Dry run: {args.dry_run}") + + client = None if args.dry_run else make_cloudstack_client(cfg) + + raw_logger = RawLogger(os.path.join(output_dir, f"raw_ops_vm_{run_id}.csv")) + summary_rows = [] + + try: + for protocol_key in protocols: + run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, args.dry_run, delay) + finally: + raw_logger.close() + + summary_path = os.path.join(output_dir, f"summary_vm_{run_id}.csv") + append_summary_csv(summary_path, summary_rows) + + print(f"\nRaw per-operation log: {os.path.join(output_dir, f'raw_ops_vm_{run_id}.csv')}") + print(f"Checkpoint summary: {summary_path}") + print("Next: python3 render_report.py --run-id " + run_id + + f" --output-dir {output_dir} --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm") + print(f" (or run benchmark_vm_instance_concurrency.py --run-id {run_id} first to add 6.2.x to the same report)") + + if not args.dry_run and not args.skip_cleanup: + print(f"\nVerifying no orphaned VMs remain for run {run_id}...") + cleanup_by_filter(client, run_id) + + +if __name__ == "__main__": + main() diff --git a/private-cicd/benchmark/ontap/cloudstack_client.py b/private-cicd/benchmark/ontap/cloudstack_client.py new file mode 100644 index 000000000000..7de56435fd8d --- /dev/null +++ b/private-cicd/benchmark/ontap/cloudstack_client.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Minimal CloudStack HTTP/REST API client used by the ONTAP plugin benchmark +scripts (private-cicd/benchmark/ontap/). + +Only session-key based auth (login -> sessionkey + JSESSIONID cookie) is +implemented, matching the approach documented in: +https://netapp.atlassian.net/wiki/spaces/OSSG/pages/608854350/CloudStack+API + +Every call is timed end-to-end (including async job polling, since that is +part of the wall-clock cost an operator/automation actually pays) and the +elapsed time is handed back to the caller so benchmark scripts can log it. +""" + +import logging +import time + +import requests + +log = logging.getLogger("cloudstack_client") + + +class CloudStackAPIError(Exception): + """Raised when CloudStack returns an errorcode/errortext, or the HTTP + call itself fails validation.""" + + def __init__(self, command, detail): + self.command = command + self.detail = detail + super().__init__(f"{command} failed: {detail}") + + +class CloudStackClient: + """Thin wrapper around the CloudStack query API. + + A single instance (and its underlying requests.Session) is safe to share + across threads for concurrency benchmarks: we never mutate session state + after login, and urllib3's connection pool is designed for concurrent + use. + """ + + def __init__( + self, + api_url, + username, + password, + verify_ssl=True, + http_timeout_sec=30, + job_timeout_sec=300, + job_poll_interval_sec=1.5, + ): + self.api_url = api_url.rstrip("/") + self.http_timeout_sec = http_timeout_sec + self.job_timeout_sec = job_timeout_sec + self.job_poll_interval_sec = job_poll_interval_sec + + self.session = requests.Session() + self.session.verify = verify_ssl + if not verify_ssl: + requests.packages.urllib3.disable_warnings( + requests.packages.urllib3.exceptions.InsecureRequestWarning + ) + + self.sessionkey = None + self._login(username, password) + + def _login(self, username, password): + params = { + "command": "login", + "username": username, + "password": password, + "response": "json", + } + resp = self.session.post(self.api_url, data=params, timeout=self.http_timeout_sec) + resp.raise_for_status() + data = resp.json() + login_resp = data.get("loginresponse") + if not login_resp or "sessionkey" not in login_resp: + raise CloudStackAPIError("login", data) + self.sessionkey = login_resp["sessionkey"] + log.info("Logged in as %s, session established", username) + + def call(self, command, params=None, poll_async=True): + """Issue one CloudStack API command. + + Returns a tuple (payload, elapsed_sec). `elapsed_sec` covers the + initial HTTP round trip AND any async job polling (i.e. the full + wall-clock time a caller would observe waiting for the operation to + finish). + """ + req_params = dict(params or {}) + req_params["command"] = command + req_params["response"] = "json" + req_params["sessionkey"] = self.sessionkey + + start = time.perf_counter() + resp = self.session.post(self.api_url, data=req_params, timeout=self.http_timeout_sec) + resp.raise_for_status() + data = resp.json() + + # Most commands wrap their payload as {"response": {...}}, but a + # few (e.g. enableStorageMaintenance -> prepareprimarystorageformaintenanceresponse) + # use a legacy internal command name instead. Since CloudStack always wraps + # the payload in exactly one top-level key, fall back to "the sole value" + # rather than assuming the key name matches the command. + resp_key = f"{command.lower()}response" + if resp_key in data: + payload = data[resp_key] + elif len(data) == 1: + payload = next(iter(data.values())) + else: + payload = data + + if isinstance(payload, dict) and "errorcode" in payload and "jobid" not in payload: + raise CloudStackAPIError(command, payload.get("errortext", payload)) + + if poll_async and isinstance(payload, dict) and "jobid" in payload: + payload = self._poll_job(payload["jobid"]) + + elapsed = time.perf_counter() - start + return payload, elapsed + + def _poll_job(self, jobid): + deadline = time.time() + self.job_timeout_sec + while time.time() < deadline: + params = { + "command": "queryAsyncJobResult", + "jobid": jobid, + "response": "json", + "sessionkey": self.sessionkey, + } + resp = self.session.post(self.api_url, data=params, timeout=self.http_timeout_sec) + resp.raise_for_status() + job = resp.json().get("queryasyncjobresultresponse", {}) + status = job.get("jobstatus", 0) + if status == 1: + return job.get("jobresult", job) + if status == 2: + raise CloudStackAPIError( + "queryAsyncJobResult", job.get("jobresult", job) + ) + time.sleep(self.job_poll_interval_sec) + raise TimeoutError(f"Async job {jobid} did not complete within {self.job_timeout_sec}s") diff --git a/private-cicd/benchmark/ontap/config.example.yaml b/private-cicd/benchmark/ontap/config.example.yaml new file mode 100644 index 000000000000..1c2ae3ff23ca --- /dev/null +++ b/private-cicd/benchmark/ontap/config.example.yaml @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Copy this file to config.yaml and fill in your lab's real values. +# config.yaml is gitignored - never commit real credentials/IPs. + +cloudstack: + api_url: "http://10.193.56.12:8080/client/api" + username: "admin" + password: "password" + verify_ssl: false # CloudStack mgmt server usually plain http/self-signed + # createStoragePool is synchronous from the client's POV and internally polls + # ONTAP jobs for up to 60s (StorageStrategy.jobPollForSuccess: 30 retries * + # 2000ms) before returning. Under C=20-30 concurrency, createStoragePool can + # legitimately take close to that full 60s. A 30s client timeout was firing + # "Read timed out" on calls that actually succeeded moments later on the + # server (confirmed via real CloudStack pool records found + deleted during + # cleanup) - so keep meaningful headroom above the plugin's poll budget. + http_timeout_sec: 90 # per HTTP request + job_timeout_sec: 300 # max wait for an async job (enable/cancel maintenance) + job_poll_interval_sec: 1.5 + +# Zone/pod/cluster the pools will be created under. scope=cluster needs +# podid+clusterid; scope=zone only needs zoneid. +infrastructure: + zoneid: "91659187-b00b-4997-8ae3-8955c05a7d48" + podid: "004494cc-8a24-4519-9cd4-5e68b2eff03a" + clusterid: "23788acb-1a63-45a9-8c61-5ed8f5374bc6" + scope: "cluster" + +# One entry per protocol you want to benchmark. Keys must be "nfs3"/"iscsi" +# to match --protocol on the CLI. Remove a block if you don't have that +# backend available. +# NOTE: the ONTAP plugin (StorageStrategy.java) builds its REST API base URL +# from storageIP for EVERY call (aggregates/volumes/svm/jobs/network/san/nas) - +# it never falls back to `url`. storageIP MUST answer HTTPS/443 with the ONTAP +# REST service (i.e. it needs the same management-https access as the mgmt +# LIF). A pure data-only LIF (NFS/iSCSI only, no REST) will fail with +# "errorcode":530 / "Connection refused" on createStoragePool. If your data +# LIFs don't have REST enabled, point storageIP at the mgmt LIF instead (as +# below) rather than the actual data LIF. +ontap: + nfs3: + provider: "NetApp ONTAP" + url: "https://10.0.0.10" # ONTAP mgmt LIF URL + storageIP: "10.0.0.10" # must answer HTTPS/443 (see NOTE above) + svmName: "vs0" + protocol: "NFS3" + username: "admin" + password_b64: "cGFzc3dvcmQ=" # base64-encoded ONTAP password (this is "password" - replace it) + capacitybytes: 10737418240 # 10 GiB per pool + tags: "bench_nfs3" + iscsi: + provider: "NetApp ONTAP" + url: "https://10.0.0.11" + storageIP: "10.0.0.11" # must answer HTTPS/443 (see NOTE above) + svmName: "vs0" + protocol: "ISCSI" + username: "admin" + password_b64: "cGFzc3dvcmQ=" + capacitybytes: 10737418240 + tags: "bench_iscsi" + +benchmark: + pool_name_prefix: "bench_ontap" # ONTAP volume names disallow "-"; use "_" only + sequential_checkpoints: [1, 5, 10, 20, 30] + concurrency_levels: [2, 5, 10, 20, 30] + # Pause between deleteStoragePool calls to give the mgmt server GC/ONTAP a + # beat; set to 0 to hammer as fast as possible. + inter_op_delay_sec: 0 + output_dir: "results" + +# VM instance benchmark (benchmark_vm_instance_sequential.py / +# benchmark_vm_instance_concurrency.py / benchmark_vm_instance_combined.py). Requires: +# - a template (templateid) +# - a network (networkid) +# - per protocol: a service offering (root disk) + disk offering (data disk) +# whose storage `tags` both match a persistent bench_vm_ pool, so +# root and data disk land on the same pool. These are one-time setup +# (createServiceOffering / createDiskOffering / createStoragePool) - not +# created/destroyed by the benchmark itself, unlike the storage-pool +# benchmark's transient pools. +# NOTE: watch aggregate capacity - StorageStrategy.java's aggregate-selection +# pre-check requires available space >= requested size even though volumes +# are created with guarantee=none (thin). On small/vsim aggregates this can +# reject valid requests; keep pool/disk sizes modest until that's addressed. +vm_bench: + templateid: "1af26eaa-8985-4b1c-ae8e-d39b7984b8f3" + networkid: "5e6f4c59-dc01-4245-8c0f-2c1c9639b37d" + # CloudStack VM names are used as hostnames (RFC1123): letters/digits/"-" only, + # no "_" (the opposite of ONTAP's volume-name rule used by pool_name_prefix above). + vm_name_prefix: "bench-vm" + sequential_checkpoints: [1, 5, 10, 20] + concurrency_levels: [2, 5, 10] + inter_op_delay_sec: 0 + output_dir: "results" + protocols: + nfs3: + # rootdisksize (GB, set on the offering) must be >= the template's + # actual qemu-img virtual size, not its download/file size - e.g. the + # stock Ubuntu 22.04 cloud image download is ~660MB on disk but has a + # 2.2 GiB virtual size. Undersizing this "works" on NFS (the backing + # file just grows past its nominal size) but hard-fails on iSCSI with + # "qemu-img: Cannot grow device files" since a LUN is a fixed-size + # block device that cannot be grown during the template copy. + serviceofferingid: "ac54f38c-5fd3-401b-b784-2c2163b08274" + diskofferingid: "e27813c0-6e8b-4375-8603-14c8ade75246" + iscsi: + serviceofferingid: "481feb00-16d8-4127-9937-cba3bec8a88b" + diskofferingid: "bf057b78-c5db-4ce4-820b-d7bd78f8ae69" diff --git a/private-cicd/benchmark/ontap/render_report.py b/private-cicd/benchmark/ontap/render_report.py new file mode 100644 index 000000000000..85935598df7a --- /dev/null +++ b/private-cicd/benchmark/ontap/render_report.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Turn benchmark_storage_pool_sequential.py / benchmark_storage_pool_concurrency.py's +raw_ops_.csv / summary_.csv into markdown tables that match the +layout of the storage-pool sections +(5.1.1, 5.1.2, 6.1.1, 6.1.2) and the Results Log (section 9) of the +"ONTAP Plugin - CloudStack Operations, Scale & Parallel Test Matrix" +Confluence page, ready to paste back in. + +Usage: + python3 render_report.py --run-id RUN-20260722-120000 \ + --cloudstack-build 4.23.0.0-SNAPSHOT --ontap-version 9.15.1 +""" + +import argparse +import csv +import math +import os +from collections import defaultdict + +PROTOCOL_ORDER = ["nfs3", "iscsi"] +PROTOCOL_LABEL = {"nfs3": "NFS3", "iscsi": "iSCSI"} + +# test_id -> (section title, sequential N-column label, category for section 9) +SECTION_DEFS = { + "5.1.1": ("5.1.1 Sequential create (1 pool at a time, cumulative to N)", "N (pools)", "Storage Pool"), + "5.1.2": ("5.1.2 Sequential delete (1 pool at a time, from N remaining)", "N (pools remaining)", "Storage Pool"), + "6.1.1": ("6.1.1 Parallel pool creation", None, "Storage Pool"), + "6.1.2": ("6.1.2 Parallel pool deletion", None, "Storage Pool"), + "5.2.1": ("5.2.1 Sequential create (1 VM at a time, cumulative to N)", "N (VMs)", "VM Instance"), + "5.2.2": ("5.2.2 Sequential delete (1 VM at a time, from N remaining)", "N (VMs remaining)", "VM Instance"), + "6.2.1": ("6.2.1 Parallel VM creation", None, "VM Instance"), + "6.2.2": ("6.2.2 Parallel VM deletion", None, "VM Instance"), +} + + +def percentile(sorted_values, pct): + if not sorted_values: + return None + if len(sorted_values) == 1: + return sorted_values[0] + k = (len(sorted_values) - 1) * (pct / 100.0) + f = math.floor(k) + c = math.ceil(k) + if f == c: + return sorted_values[int(k)] + d0 = sorted_values[f] * (c - k) + d1 = sorted_values[c] * (k - f) + return d0 + d1 + + +def load_csv(path): + if not os.path.exists(path): + return [] + with open(path, newline="") as f: + return list(csv.DictReader(f)) + + +def stats_for(durations): + if not durations: + return None, None, None, None + durations = sorted(durations) + return ( + round(durations[0], 3), + round(sum(durations) / len(durations), 3), + round(percentile(durations, 95), 3), + round(durations[-1], 3), + ) + + +def raw_durations(raw_rows, phase, protocol, predicate): + return [ + float(r["duration_sec"]) for r in raw_rows + if r["phase"] == phase and r["protocol"] == protocol + and r["success"] == "True" and predicate(r) + ] + + +def render_sequential_table(summary_rows, raw_rows, phase, test_id, n_label): + checkpoints = sorted({int(r["checkpoint"]) for r in summary_rows if r["test_id"] == test_id}) + by_key = {(r["protocol"], int(r["checkpoint"])): r for r in summary_rows if r["test_id"] == test_id} + + lines = [f"| {n_label} | NFS3 Total (s) | NFS3 Avg/op (s) | iSCSI Total (s) | iSCSI Avg/op (s) | Notes |", + "|---|---|---|---|---|---|"] + for cp in checkpoints: + cells = [str(cp)] + notes = [] + for proto in PROTOCOL_ORDER: + row = by_key.get((proto, cp)) + if row: + cells.append(row["total_time_sec"]) + cells.append(row["avg_time_sec"]) + if row.get("notes") and row["notes"] not in notes: + notes.append(row["notes"]) + else: + cells.append("") + cells.append("") + cells.append("; ".join(notes)) + lines.append("| " + " | ".join(cells) + " |") + return "\n".join(lines) + + +def render_concurrency_table(summary_rows, test_id): + rows = [r for r in summary_rows if r["test_id"] == test_id] + rows.sort(key=lambda r: (int(r["checkpoint"]), PROTOCOL_ORDER.index(r["protocol"]) + if r["protocol"] in PROTOCOL_ORDER else 99)) + lines = ["| Concurrency (C) | Protocol | Total Wall-clock (s) | Success | Failure | Avg Time/pool (s) | Notes |", + "|---|---|---|---|---|---|---|"] + for r in rows: + lines.append( + f"| {r['checkpoint']} | {PROTOCOL_LABEL.get(r['protocol'], r['protocol'])} | " + f"{r['total_time_sec']} | {r['success_count']} | {r['failure_count']} | " + f"{r['avg_time_sec']} | {r.get('notes', '')} |" + ) + return "\n".join(lines) + + +def build_results_log_rows(summary_rows, raw_rows, run_id, cs_build, ontap_version, run_date): + header = ("| Run Date | Run ID | CloudStack Build | ONTAP Version | Category | Test ID / Section | " + "Scale (N) / Concurrency (C) | Protocol | Min (s) | Avg (s) | P95 (s) | Max (s) | " + "Success Rate | Notes |") + sep = "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|" + lines = [header, sep] + + delete_total_created = {} + for r in raw_rows: + if r["phase"] == "sequential_delete": + delete_total_created[r["protocol"]] = int(r["scale_or_concurrency"]) + + for row in summary_rows: + phase, test_id, protocol, checkpoint = row["phase"], row["test_id"], row["protocol"], int(row["checkpoint"]) + if phase == "sequential_create": + durations = raw_durations(raw_rows, phase, protocol, lambda r: int(r["index"]) <= checkpoint) + scale_label = f"N={checkpoint}" + elif phase == "sequential_delete": + total_created = delete_total_created.get(protocol, checkpoint) + deleted_so_far = total_created - checkpoint + durations = raw_durations(raw_rows, phase, protocol, lambda r: int(r["index"]) <= deleted_so_far) + scale_label = f"N={checkpoint}" + else: # concurrent_create / concurrent_delete + durations = raw_durations(raw_rows, phase, protocol, lambda r: int(r["scale_or_concurrency"]) == checkpoint) + scale_label = f"C={checkpoint}" + + mn, avg, p95, mx = stats_for(durations) + total_ops = int(row["success_count"]) + int(row["failure_count"]) + success_rate = f"{(int(row['success_count']) / total_ops * 100):.0f}%" if total_ops else "-" + category = SECTION_DEFS.get(test_id, (None, None, "Storage Pool"))[2] + lines.append( + f"| {run_date} | {run_id} | {cs_build} | {ontap_version} | {category} | {test_id} | " + f"{scale_label} | {PROTOCOL_LABEL.get(protocol, protocol)} | " + f"{mn if mn is not None else '-'} | {avg if avg is not None else '-'} | " + f"{p95 if p95 is not None else '-'} | {mx if mx is not None else '-'} | " + f"{success_rate} | {row.get('notes', '')} |" + ) + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--run-id", required=True) + parser.add_argument("--output-dir", default="results") + parser.add_argument("--cloudstack-build", default="unknown") + parser.add_argument("--ontap-version", default="unknown") + parser.add_argument("--run-date", default=None, help="Defaults to today's date (YYYY-MM-DD)") + parser.add_argument("--raw-prefix", default="raw_ops", + help="Filename prefix for the raw ops CSV, e.g. raw_ops_vm for the benchmark_vm_instance_* scripts") + parser.add_argument("--summary-prefix", default="summary", + help="Filename prefix for the summary CSV, e.g. summary_vm for the benchmark_vm_instance_* scripts") + parser.add_argument("--report-suffix", default="", + help="Optional suffix to disambiguate the output report filename, e.g. _vm") + args = parser.parse_args() + + import datetime + run_date = args.run_date or datetime.date.today().isoformat() + + summary_path = os.path.join(args.output_dir, f"{args.summary_prefix}_{args.run_id}.csv") + raw_path = os.path.join(args.output_dir, f"{args.raw_prefix}_{args.run_id}.csv") + summary_rows = load_csv(summary_path) + raw_rows = load_csv(raw_path) + if not summary_rows: + raise SystemExit(f"No summary rows found at {summary_path}") + + # Render whichever sections actually have data in this run, in a stable order. + present_test_ids = sorted({r["test_id"] for r in summary_rows if r["test_id"] in SECTION_DEFS}) + sections = [] + for test_id in present_test_ids: + title, n_label, _category = SECTION_DEFS[test_id] + phase = { + "5.1.1": "sequential_create", "5.1.2": "sequential_delete", + "5.2.1": "sequential_create", "5.2.2": "sequential_delete", + }.get(test_id) + if n_label is not None: + sections.append(f"### {title}\n\n" + + render_sequential_table(summary_rows, raw_rows, phase, test_id, n_label)) + else: + sections.append(f"### {title}\n\n" + render_concurrency_table(summary_rows, test_id)) + sections.append("### 9. Results Log rows for this run\n\n" + + build_results_log_rows(summary_rows, raw_rows, args.run_id, args.cloudstack_build, + args.ontap_version, run_date)) + + report = f"# Benchmark report - {args.run_id}\n\n" + "\n\n".join(sections) + "\n" + print(report) + + report_path = os.path.join(args.output_dir, f"report{args.report_suffix}_{args.run_id}.md") + with open(report_path, "w") as f: + f.write(report) + print(f"\nSaved to {report_path}") + + +if __name__ == "__main__": + main() diff --git a/private-cicd/benchmark/ontap/requirements.txt b/private-cicd/benchmark/ontap/requirements.txt new file mode 100644 index 000000000000..34304aee700c --- /dev/null +++ b/private-cicd/benchmark/ontap/requirements.txt @@ -0,0 +1,2 @@ +requests>=2.31.0 +PyYAML>=6.0 diff --git a/private-cicd/benchmark/ontap/results/.gitkeep b/private-cicd/benchmark/ontap/results/.gitkeep new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/private-cicd/benchmark/ontap/storage_pool_common.py b/private-cicd/benchmark/ontap/storage_pool_common.py new file mode 100644 index 000000000000..e01c50c0e69d --- /dev/null +++ b/private-cicd/benchmark/ontap/storage_pool_common.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Shared helpers for the storage-pool benchmark scripts +(benchmark_storage_pool_sequential.py / benchmark_storage_pool_concurrency.py). + +Kept in one module (rather than duplicated per script) so the createStoragePool/ +deleteStoragePool call shape, CSV formats, and cleanup logic can't drift between +the two - only the sequential-vs-concurrent driving logic differs per script. +""" + +import csv +import dataclasses +import datetime +import os +import random +import sys +import time + +try: + import yaml +except ImportError: + sys.exit("PyYAML is required: pip install -r requirements.txt") + +from cloudstack_client import CloudStackAPIError, CloudStackClient + +RAW_FIELDNAMES = [ + "run_id", "phase", "test_id", "protocol", "scale_or_concurrency", + "index", "pool_name", "pool_id", "success", "duration_sec", + "start_ts", "end_ts", "error", +] + +SUMMARY_FIELDNAMES = [ + "run_id", "phase", "test_id", "protocol", "checkpoint", + "total_time_sec", "avg_time_sec", "success_count", "failure_count", "notes", +] + + +@dataclasses.dataclass +class OpResult: + success: bool + duration_sec: float + start_ts: str + end_ts: str + pool_name: str + pool_id: str = None + error: str = None + + +def now_iso(): + return datetime.datetime.utcnow().isoformat(timespec="milliseconds") + "Z" + + +def new_run_id(): + # ONTAP volume names only allow alphanumeric + underscore (no hyphens), and + # pool names flow straight through to the ONTAP volume name, so avoid "-" here. + return datetime.datetime.utcnow().strftime("RUN_%Y%m%d_%H%M%S") + + +def build_create_pool_params(name, infra_cfg, ontap_cfg): + params = { + "name": name, + "zoneid": infra_cfg["zoneid"], + "scope": infra_cfg.get("scope", "cluster"), + "provider": ontap_cfg["provider"], + "url": ontap_cfg["url"], + "managed": "true", + "details[0].username": ontap_cfg["username"], + "details[0].password": ontap_cfg["password_b64"], + "details[0].svmName": ontap_cfg["svmName"], + "details[0].protocol": ontap_cfg["protocol"], + "details[0].storageIP": ontap_cfg["storageIP"], + } + if infra_cfg.get("scope", "cluster") == "cluster": + params["podid"] = infra_cfg["podid"] + params["clusterid"] = infra_cfg["clusterid"] + if ontap_cfg.get("capacitybytes"): + params["capacitybytes"] = str(ontap_cfg["capacitybytes"]) + if ontap_cfg.get("tags"): + params["tags"] = ontap_cfg["tags"] + return params + + +def create_pool(client, name, infra_cfg, ontap_cfg): + start_ts = now_iso() + t0 = time.perf_counter() + try: + payload, elapsed = client.call( + "createStoragePool", build_create_pool_params(name, infra_cfg, ontap_cfg) + ) + pool = payload.get("storagepool", payload) if isinstance(payload, dict) else {} + pool_id = pool.get("id") if isinstance(pool, dict) else None + return OpResult(True, elapsed, start_ts, now_iso(), name, pool_id=pool_id) + except (CloudStackAPIError, TimeoutError, Exception) as exc: # noqa: BLE001 + elapsed = time.perf_counter() - t0 + return OpResult(False, elapsed, start_ts, now_iso(), name, error=str(exc)) + + +def delete_pool(client, pool_id, name, forced=True): + """deleteStoragePool requires the pool to already be in Maintenance state + (CloudStack error 431 otherwise), so this enables maintenance first and + waits for that async job before issuing the actual delete. Both steps are + included in the reported duration since that is the real wall-clock cost + of retiring a pool.""" + start_ts = now_iso() + t0 = time.perf_counter() + try: + client.call("enableStorageMaintenance", {"id": pool_id}) + payload, _ = client.call( + "deleteStoragePool", {"id": pool_id, "forced": "true" if forced else "false"} + ) + elapsed = time.perf_counter() - t0 + success = payload.get("success", True) if isinstance(payload, dict) else True + return OpResult(bool(success), elapsed, start_ts, now_iso(), name, pool_id=pool_id) + except (CloudStackAPIError, TimeoutError, Exception) as exc: # noqa: BLE001 + elapsed = time.perf_counter() - t0 + return OpResult(False, elapsed, start_ts, now_iso(), name, pool_id=pool_id, error=str(exc)) + + +def fake_create(name): + time.sleep(random.uniform(0.01, 0.05)) + return OpResult(True, random.uniform(0.5, 3.0), now_iso(), now_iso(), name, pool_id=f"dryrun-{name}") + + +def fake_delete(pool_id, name): + time.sleep(random.uniform(0.01, 0.05)) + return OpResult(True, random.uniform(0.3, 2.0), now_iso(), now_iso(), name, pool_id=pool_id) + + +class RawLogger: + def __init__(self, path): + self.file = open(path, "a", newline="") + self.writer = csv.DictWriter(self.file, fieldnames=RAW_FIELDNAMES) + if self.file.tell() == 0: + self.writer.writeheader() + + def log(self, run_id, phase, test_id, protocol, scale_or_concurrency, index, result: OpResult): + self.writer.writerow({ + "run_id": run_id, "phase": phase, "test_id": test_id, "protocol": protocol, + "scale_or_concurrency": scale_or_concurrency, "index": index, + "pool_name": result.pool_name, "pool_id": result.pool_id or "", + "success": result.success, "duration_sec": round(result.duration_sec, 4), + "start_ts": result.start_ts, "end_ts": result.end_ts, "error": result.error or "", + }) + self.file.flush() + + def close(self): + self.file.close() + + +def append_summary_csv(path, rows): + """Appends checkpoint rows to the run's summary CSV, writing the header only + if the file is new/empty. Append (not overwrite) so that running the + sequential and concurrency scripts separately under the *same* --run-id + accumulates into one combined summary_.csv that render_report.py + can render as a single report, instead of the second script clobbering the + first script's rows.""" + is_new = not os.path.exists(path) or os.path.getsize(path) == 0 + with open(path, "a", newline="") as f: + writer = csv.DictWriter(f, fieldnames=SUMMARY_FIELDNAMES) + if is_new: + writer.writeheader() + writer.writerows(rows) + + +def cleanup_by_filter(client, name_filter): + payload, _ = client.call("listStoragePools", {}, poll_async=False) + pools = payload.get("storagepool", []) if isinstance(payload, dict) else [] + matches = [p for p in pools if name_filter in p.get("name", "")] + print(f"Found {len(matches)} pool(s) whose name contains '{name_filter}'") + for p in matches: + try: + if p.get("state") != "Maintenance": + client.call("enableStorageMaintenance", {"id": p["id"]}) + client.call("deleteStoragePool", {"id": p["id"], "forced": "true"}) + print(f" deleted {p['name']} ({p['id']})") + except Exception as exc: # noqa: BLE001 + print(f" FAILED to delete {p['name']}: {exc}") + + +def load_config(path): + with open(path) as f: + return yaml.safe_load(f) + + +def resolve_protocols(cfg, requested): + available = list(cfg.get("ontap", {}).keys()) + if requested == "both": + return available + if requested not in available: + sys.exit(f"Protocol '{requested}' not found in config.ontap (available: {available})") + return [requested] + + +def make_cloudstack_client(cfg): + cs_cfg = cfg["cloudstack"] + return CloudStackClient( + cs_cfg["api_url"], cs_cfg["username"], cs_cfg["password"], + verify_ssl=cs_cfg.get("verify_ssl", True), + http_timeout_sec=cs_cfg.get("http_timeout_sec", 30), + job_timeout_sec=cs_cfg.get("job_timeout_sec", 300), + job_poll_interval_sec=cs_cfg.get("job_poll_interval_sec", 1.5), + ) diff --git a/private-cicd/benchmark/ontap/vm_instance_common.py b/private-cicd/benchmark/ontap/vm_instance_common.py new file mode 100644 index 000000000000..b671f3c576dc --- /dev/null +++ b/private-cicd/benchmark/ontap/vm_instance_common.py @@ -0,0 +1,319 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Shared helpers for the VM-instance benchmark scripts +(benchmark_vm_instance_sequential.py / benchmark_vm_instance_concurrency.py / +benchmark_vm_instance_combined.py). + +Kept in one module (rather than duplicated per script) so the +deployVirtualMachine/destroyVirtualMachine call shape, CSV formats, and +cleanup logic can't drift between the three - only the sequential-vs- +concurrent-vs-both driving logic differs per script. +""" + +import csv +import dataclasses +import datetime +import os +import random +import re +import sys +import time + +try: + import yaml +except ImportError: + sys.exit("PyYAML is required: pip install -r requirements.txt") + +from cloudstack_client import CloudStackAPIError, CloudStackClient + +RAW_FIELDNAMES = [ + "run_id", "phase", "test_id", "protocol", "scale_or_concurrency", + "index", "vm_name", "vm_id", "success", "duration_sec", + "start_ts", "end_ts", "error", +] + +SUMMARY_FIELDNAMES = [ + "run_id", "phase", "test_id", "protocol", "checkpoint", + "total_time_sec", "avg_time_sec", "success_count", "failure_count", "notes", +] + + +@dataclasses.dataclass +class OpResult: + success: bool + duration_sec: float + start_ts: str + end_ts: str + vm_name: str + vm_id: str = None + error: str = None + + +def now_iso(): + return datetime.datetime.utcnow().isoformat(timespec="milliseconds") + "Z" + + +def new_run_id(): + return datetime.datetime.utcnow().strftime("RUN_%Y%m%d_%H%M%S") + + +def build_deploy_vm_params(name, infra_cfg, vm_cfg, proto_cfg): + return { + "name": name, + "displayname": name, + "zoneid": infra_cfg["zoneid"], + "templateid": vm_cfg["templateid"], + "networkids": vm_cfg["networkid"], + "serviceofferingid": proto_cfg["serviceofferingid"], + "diskofferingid": proto_cfg["diskofferingid"], + "startvm": "true", + } + + +_UUID_RE = re.compile(r'"uuid"\s*:\s*"([0-9a-fA-F-]{36})"') + + +def _extract_vm_id_from_error(error_text): + """When deployVirtualMachine's async job fails deep in orchestration (e.g. the + ENOSPC-driven "Unable to orchestrate the start of VM instance" failures - see + Confluence Issue #10), CloudStack has ALREADY created the VM (and its ROOT/DATA + volume records) before the failure - it just never finished starting it. The + job's errortext embeds that VM's uuid (e.g. '...{"instanceName":"i-2-107-VM", + "uuid":"b8f4..."}.'), so scrape it out here so the leftover VM/volume can at + least be identified and reported for manual inspection (see + report_failed_creates()) instead of being silently invisible.""" + if not error_text: + return None + m = _UUID_RE.search(error_text) + return m.group(1) if m else None + + +def deploy_vm(client, name, infra_cfg, vm_cfg, proto_cfg): + start_ts = now_iso() + t0 = time.perf_counter() + try: + payload, elapsed = client.call( + "deployVirtualMachine", build_deploy_vm_params(name, infra_cfg, vm_cfg, proto_cfg) + ) + vm = payload.get("virtualmachine", payload) if isinstance(payload, dict) else {} + vm_id = vm.get("id") if isinstance(vm, dict) else None + return OpResult(True, elapsed, start_ts, now_iso(), name, vm_id=vm_id) + except (CloudStackAPIError, TimeoutError, Exception) as exc: # noqa: BLE001 + elapsed = time.perf_counter() - t0 + error = str(exc) + return OpResult(False, elapsed, start_ts, now_iso(), name, + vm_id=_extract_vm_id_from_error(error), error=error) + + +def _force_purge_destroy_state_volumes(client, volume_ids, max_wait_sec=10): + """destroyVirtualMachine(expunge=true) only guarantees the VM/volume DB records + move to "Destroy" state - actual physical removal from the backing storage + (ONTAP LUN/file deletion) is deferred to CloudStack's periodic storage-cleanup + background thread, which by default only *considers* volumes for real deletion + after `storage.cleanup.delay` (86400s / 24h) has elapsed, even though the + cleanup thread itself polls every `storage.cleanup.interval` (40s by default). + Confirmed directly against our management server via listConfigurations. + + That means every "successfully destroyed" VM's ROOT/DATA disks still + physically occupy space on the ONTAP pool for up to 24h unless something + calls deleteVolume() on them explicitly - which force-deletes a Destroy-state + volume immediately, bypassing the delay. Without this, repeated benchmark + runs progressively fill up the dedicated bench_vm_ pool with + "deleted" disks, compounding the ENOSPC failures in Confluence Issue #10. + """ + deadline = time.time() + max_wait_sec + remaining = set(volume_ids) + while remaining and time.time() < deadline: + still_pending = set() + for vol_id in remaining: + try: + vol_payload, _ = client.call("listVolumes", {"id": vol_id}, poll_async=False) + vols = vol_payload.get("volume", []) if isinstance(vol_payload, dict) else [] + if not vols: + continue # already fully gone + state = vols[0].get("state") + if state == "Destroy": + client.call("deleteVolume", {"id": vol_id}) + elif state not in ("Expunged",): + still_pending.add(vol_id) # not yet transitioned to Destroy - retry shortly + except (CloudStackAPIError, TimeoutError, Exception): # noqa: BLE001 + still_pending.add(vol_id) + remaining = still_pending + if remaining: + time.sleep(1) + + +def destroy_vm(client, vm_id, name, expunge=True): + """destroyVirtualMachine with expunge=true only reclaims the ROOT disk; + data disks are merely DETACHED (left behind as free-floating DATADISK + volumes) unless their ids are explicitly passed via `volumeids`. So this + looks up any attached data disks first and includes them, otherwise every + VM with a data disk leaks a LUN/file on the backing storage pool. + + It also force-purges any ROOT/DATA volumes still left in "Destroy" state + afterwards (see _force_purge_destroy_state_volumes) so this VM's disks are + truly gone from ONTAP before the next benchmark op runs, instead of + lingering for CloudStack's 24h background cleanup delay. + """ + start_ts = now_iso() + t0 = time.perf_counter() + try: + vol_payload, _ = client.call( + "listVolumes", {"virtualmachineid": vm_id}, poll_async=False + ) + all_vols = vol_payload.get("volume", []) if isinstance(vol_payload, dict) else [] + all_volume_ids = [v["id"] for v in all_vols] + data_disk_ids = [v["id"] for v in all_vols if v.get("type") == "DATADISK"] + + params = {"id": vm_id, "expunge": "true" if expunge else "false"} + if data_disk_ids: + params["volumeids"] = ",".join(data_disk_ids) + payload, _ = client.call("destroyVirtualMachine", params) + + if expunge and all_volume_ids: + _force_purge_destroy_state_volumes(client, all_volume_ids) + + elapsed = time.perf_counter() - t0 + success = payload.get("success", True) if isinstance(payload, dict) else True + return OpResult(bool(success), elapsed, start_ts, now_iso(), name, vm_id=vm_id) + except (CloudStackAPIError, TimeoutError, Exception) as exc: # noqa: BLE001 + elapsed = time.perf_counter() - t0 + return OpResult(False, elapsed, start_ts, now_iso(), name, vm_id=vm_id, error=str(exc)) + + +def fake_create(name): + time.sleep(random.uniform(0.01, 0.05)) + return OpResult(True, random.uniform(3.0, 15.0), now_iso(), now_iso(), name, vm_id=f"dryrun-{name}") + + +def fake_delete(vm_id, name): + time.sleep(random.uniform(0.01, 0.05)) + return OpResult(True, random.uniform(2.0, 8.0), now_iso(), now_iso(), name, vm_id=vm_id) + + +class RawLogger: + def __init__(self, path): + self.file = open(path, "a", newline="") + self.writer = csv.DictWriter(self.file, fieldnames=RAW_FIELDNAMES) + if self.file.tell() == 0: + self.writer.writeheader() + + def log(self, run_id, phase, test_id, protocol, scale_or_concurrency, index, result: OpResult): + self.writer.writerow({ + "run_id": run_id, "phase": phase, "test_id": test_id, "protocol": protocol, + "scale_or_concurrency": scale_or_concurrency, "index": index, + "vm_name": result.vm_name, "vm_id": result.vm_id or "", + "success": result.success, "duration_sec": round(result.duration_sec, 4), + "start_ts": result.start_ts, "end_ts": result.end_ts, "error": result.error or "", + }) + self.file.flush() + + def close(self): + self.file.close() + + +def append_summary_csv(path, rows): + """Appends checkpoint rows to the run's summary CSV, writing the header only + if the file is new/empty. Append (not overwrite) so that running the + sequential and concurrency scripts separately under the *same* --run-id + accumulates into one combined summary_vm_.csv that render_report.py + can render as a single report, instead of the second script clobbering the + first script's rows.""" + is_new = not os.path.exists(path) or os.path.getsize(path) == 0 + with open(path, "a", newline="") as f: + writer = csv.DictWriter(f, fieldnames=SUMMARY_FIELDNAMES) + if is_new: + writer.writeheader() + writer.writerows(rows) + + +def report_failed_creates(failed): + """Failed deployVirtualMachine calls (e.g. the ENOSPC "Unable to orchestrate + the start of VM instance" failures in Confluence Issue #10) still leave a real + VM + ROOT/DATA volume record behind in CloudStack - they just never finished + starting. These are intentionally NOT auto-destroyed here (unlike the + `created` list, which the benchmark itself deletes as part of 5.2.2/6.2.2) so + the leftover VM/volume artifacts stay inspectable afterwards - e.g. via + `listVolumes --state Destroy` on the bench_vm_ pool - to confirm + exactly what got left behind and why. Clean them up manually (or via + --cleanup-only) once you're done inspecting them.""" + failed_with_id = [(name, vm_id) for name, vm_id in failed if vm_id] + if not failed_with_id: + return + print(f"\n=== {len(failed_with_id)} failed-create VM(s) left in place for inspection (not cleaned up) ===") + for name, vm_id in failed_with_id: + print(f" {name} ({vm_id})") + + +def cleanup_by_filter(client, name_filter): + payload, _ = client.call("listVirtualMachines", {}, poll_async=False) + vms = payload.get("virtualmachine", []) if isinstance(payload, dict) else [] + matches = [v for v in vms if name_filter in v.get("name", "")] + print(f"Found {len(matches)} VM(s) whose name contains '{name_filter}'") + for v in matches: + try: + result = destroy_vm(client, v["id"], v["name"]) + if result.success: + print(f" destroyed {v['name']} ({v['id']}) [incl. any data disks]") + else: + print(f" FAILED to destroy {v['name']}: {result.error}") + except Exception as exc: # noqa: BLE001 + print(f" FAILED to destroy {v['name']}: {exc}") + + # Data disks can also end up orphaned (unattached) from prior runs whose + # VM was already destroyed without this volumeids fix. Volume names (e.g. + # "DATA-25") don't carry the run id/prefix, so instead of matching + # name_filter here, sweep any unattached data disk left on our dedicated + # bench_vm_* pools - nothing else legitimately lives there. + vol_payload, _ = client.call("listVolumes", {"type": "DATADISK"}, poll_async=False) + vols = vol_payload.get("volume", []) if isinstance(vol_payload, dict) else [] + orphan_vols = [v for v in vols if not v.get("virtualmachineid") and v.get("storage", "").startswith("bench_vm")] + if orphan_vols: + print(f"Found {len(orphan_vols)} orphaned unattached DATADISK volume(s) on matching pools") + for v in orphan_vols: + try: + client.call("deleteVolume", {"id": v["id"]}) + print(f" deleted orphan volume {v['name']} ({v['id']})") + except Exception as exc: # noqa: BLE001 + print(f" FAILED to delete orphan volume {v['name']}: {exc}") + + +def load_config(path): + with open(path) as f: + return yaml.safe_load(f) + + +def resolve_protocols(cfg, requested): + available = list(cfg.get("vm_bench", {}).get("protocols", {}).keys()) + if requested == "both": + return available + if requested not in available: + sys.exit(f"Protocol '{requested}' not found in config.vm_bench.protocols (available: {available})") + return [requested] + + +def make_cloudstack_client(cfg): + cs_cfg = cfg["cloudstack"] + return CloudStackClient( + cs_cfg["api_url"], cs_cfg["username"], cs_cfg["password"], + verify_ssl=cs_cfg.get("verify_ssl", True), + http_timeout_sec=cs_cfg.get("http_timeout_sec", 30), + job_timeout_sec=cs_cfg.get("job_timeout_sec", 300), + job_poll_interval_sec=cs_cfg.get("job_poll_interval_sec", 1.5), + ) From 8395e72eb81f4d770313ffa7e0b2221cd97cabb3 Mon Sep 17 00:00:00 2001 From: sp16743 Date: Wed, 30 Sep 2026 22:39:29 +0530 Subject: [PATCH 2/2] Address ONTAP benchmark review comments --- private-cicd/benchmark/ontap/AGENTS.md | 25 +++ private-cicd/benchmark/ontap/README.md | 85 +++++----- private-cicd/benchmark/ontap/USAGE.md | 36 +++-- .../benchmark_storage_pool_concurrency.py | 146 +++++++++-------- .../benchmark_storage_pool_sequential.py | 142 ++++++++++------- .../benchmark/ontap/benchmark_support.py | 111 +++++++++++++ .../ontap/benchmark_vm_instance_combined.py | 105 ++++++------ .../benchmark_vm_instance_concurrency.py | 149 ++++++++++-------- .../ontap/benchmark_vm_instance_sequential.py | 145 +++++++++-------- .../benchmark/ontap/cloudstack_client.py | 5 +- .../benchmark/ontap/config.example.yaml | 10 +- private-cicd/benchmark/ontap/render_report.py | 13 +- private-cicd/benchmark/ontap/run.py | 77 +++++++++ .../benchmark/ontap/storage_pool_common.py | 73 +++++++-- .../benchmark/ontap/vm_instance_common.py | 136 ++++++++++++---- 15 files changed, 831 insertions(+), 427 deletions(-) create mode 100644 private-cicd/benchmark/ontap/AGENTS.md create mode 100644 private-cicd/benchmark/ontap/benchmark_support.py create mode 100644 private-cicd/benchmark/ontap/run.py diff --git a/private-cicd/benchmark/ontap/AGENTS.md b/private-cicd/benchmark/ontap/AGENTS.md new file mode 100644 index 000000000000..6066360ba3a3 --- /dev/null +++ b/private-cicd/benchmark/ontap/AGENTS.md @@ -0,0 +1,25 @@ + +# ONTAP scale benchmark + +- Refer to the scale matrix from `README.md` and `USAGE.md` only. +- Protocols, checkpoints, and concurrency levels live in `config.yaml`. `run.py` is the entry point. Do not add a CLI flag for a field that already exists in config. +- Use `benchmark_support` for run ids, resource names, `mean_seconds`, and logging. Sequential and concurrency scripts must use the same helpers and the same log format. Do not hand-roll a second average or a second logger. +- Run ids are letters, digits, and underscores. Storage pool names disallow `-`. VM hostnames disallow `_`; `format_vm_name` / `vm_name_token` are the only rewrite, and cleanup must use them so created VMs are the ones deleted. +- Cleanup must stay inside the configured name prefix, zone, cluster (pools), and storage provider. Do not delete a resource just because its name contains a short substring. diff --git a/private-cicd/benchmark/ontap/README.md b/private-cicd/benchmark/ontap/README.md index a0e3e0eb5265..fa07a63bc0f0 100644 --- a/private-cicd/benchmark/ontap/README.md +++ b/private-cicd/benchmark/ontap/README.md @@ -100,51 +100,52 @@ real lab IPs/credentials or run output. ## Running -```bash -# Sanity-check either script/config without hitting a real management server: -python3 benchmark_storage_pool_sequential.py --config config.yaml --dry-run -python3 benchmark_storage_pool_concurrency.py --config config.yaml --dry-run +`run.py` is the entry point. Protocols, checkpoints, and concurrency levels +come from `config.yaml` (`ontap` / `benchmark` for pools, `vm_bench` for +VMs). Drop a protocol block to skip it, and edit `sequential_checkpoints` or +`concurrency_levels` to change the matrix. -# Sequential scale matrix, both protocols, up to config's sequential_checkpoints: -python3 benchmark_storage_pool_sequential.py --config config.yaml +Run ids are generated as `RUN__<4 hex chars>` (for example +`RUN_20260927_081500_a1b2`). Letters, digits, and underscores only: a hyphen +is rewritten to `_` because it is embedded in the storage pool name. Reuse +one id across sequential and concurrency when you want a single report. -# Sequential scale matrix, NFS3 only: -python3 benchmark_storage_pool_sequential.py --config config.yaml --protocol nfs3 +```bash +# Sanity-check without hitting a real management server: +python3 run.py --test storage-pool-sequential --config config.yaml --dry-run +python3 run.py --test storage-pool-concurrency --config config.yaml --dry-run -# Concurrency matrix, both protocols, up to config's concurrency_levels (e.g. 30): -python3 benchmark_storage_pool_concurrency.py --config config.yaml +# Sequential scale matrix, using the protocols and checkpoints in config.yaml: +python3 run.py --test storage-pool-sequential --config config.yaml -# Concurrency matrix, iSCSI only, overriding the levels to run: -python3 benchmark_storage_pool_concurrency.py --config config.yaml --protocol iscsi --levels 2,5,10,20,30 +# Concurrency matrix, using config.yaml's concurrency_levels: +python3 run.py --test storage-pool-concurrency --config config.yaml -# Use the SAME --run-id across both scripts to combine 5.1.x + 6.1.x into one -# summary_.csv / report (run sequential first, review it, then decide -# whether to proceed with concurrency under the same run id): -python3 benchmark_storage_pool_sequential.py --config config.yaml --run-id RUN-0001 -python3 benchmark_storage_pool_concurrency.py --config config.yaml --run-id RUN-0001 +# Same --run-id across both tests to combine 5.1.x + 6.1.x into one report. +# Run sequential first, review it, then decide on concurrency: +python3 run.py --test storage-pool-sequential --config config.yaml --run-id RUN_20260927_081500_a1b2 +python3 run.py --test storage-pool-concurrency --config config.yaml --run-id RUN_20260927_081500_a1b2 ``` -At the end of a real run each script automatically checks for and deletes any -storage pool whose name still contains the run id (safety net for pools that -were created but never got torn down because of a mid-run crash). Pass -`--skip-cleanup` to disable that and inspect the pools yourself. +At the end of a real run each script deletes leftover pools from that run id. +The sweep only deletes pools that match this config's name prefix, name +pattern, zone, cluster, and storage provider, so other pools in a shared zone +are left alone. Pass `--skip-cleanup` to inspect the pools yourself. -If a run crashes hard (script killed, network partition, etc.) and left -pools behind, recover with either script's `--cleanup-only` (they share the -same cleanup logic): +If a run crashes and leaves pools behind: ```bash -# Deletes every pool whose name contains "bench_ontap" (the default prefix) -python3 benchmark_storage_pool_sequential.py --config config.yaml --cleanup-only +# Deletes benchmark pools for this config's pool_name_prefix (still provider/zone/cluster scoped): +python3 run.py --test storage-pool-sequential --config config.yaml --cleanup-only -# Or scope it to one specific run: -python3 benchmark_storage_pool_sequential.py --config config.yaml --cleanup-only RUN_20260722_101500 +# Or scope it to one specific run (underscores, not hyphens): +python3 run.py --test storage-pool-sequential --config config.yaml --cleanup-only RUN_20260927_081500_a1b2 ``` ## Rendering the Confluence-ready report ```bash -python3 render_report.py --run-id RUN-20260722-101500 \ +python3 render_report.py --run-id RUN_20260722_101500_a1b2 \ --cloudstack-build 4.23.0.0-SNAPSHOT --ontap-version 9.15.1 ``` @@ -182,25 +183,23 @@ matching offerings - it does not create/destroy the pool itself: ```bash # Sanity-check without hitting a real management server: -python3 benchmark_vm_instance_sequential.py --config config.yaml --dry-run -python3 benchmark_vm_instance_concurrency.py --config config.yaml --dry-run +python3 run.py --test vm-instance-sequential --config config.yaml --dry-run +python3 run.py --test vm-instance-concurrency --config config.yaml --dry-run -# Sequential scale matrix (5.2.1/5.2.2), both protocols: -python3 benchmark_vm_instance_sequential.py --config config.yaml +# Sequential scale matrix (5.2.1/5.2.2), protocols from config.yaml: +python3 run.py --test vm-instance-sequential --config config.yaml -# Concurrency matrix (6.2.1/6.2.2), iSCSI only, custom levels: -python3 benchmark_vm_instance_concurrency.py --config config.yaml --protocol iscsi --levels 1,2,5,10 +# Concurrency matrix (6.2.1/6.2.2). Levels come from vm_bench.concurrency_levels: +python3 run.py --test vm-instance-concurrency --config config.yaml -# Both matrices in one invocation, combined into a single summary/report: -python3 benchmark_vm_instance_combined.py --config config.yaml +# Both matrices in one invocation: +python3 run.py --test vm-instance-combined --config config.yaml -# Or run the two standalone scripts under the SAME --run-id to combine them -# instead (lets you review the sequential results before committing to -# concurrency): -python3 benchmark_vm_instance_sequential.py --config config.yaml --run-id RUN-0001 -python3 benchmark_vm_instance_concurrency.py --config config.yaml --run-id RUN-0001 +# Or run the two tests under the SAME --run-id (underscores only): +python3 run.py --test vm-instance-sequential --config config.yaml --run-id RUN_20260927_081500_a1b2 +python3 run.py --test vm-instance-concurrency --config config.yaml --run-id RUN_20260927_081500_a1b2 -python3 render_report.py --run-id RUN-0001 --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm +python3 render_report.py --run-id RUN_20260927_081500_a1b2 --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm ``` All three scripts share the same `--skip-cleanup` / `--cleanup-only` safety diff --git a/private-cicd/benchmark/ontap/USAGE.md b/private-cicd/benchmark/ontap/USAGE.md index f62dd3c89e01..9082a095a49f 100644 --- a/private-cicd/benchmark/ontap/USAGE.md +++ b/private-cicd/benchmark/ontap/USAGE.md @@ -71,6 +71,7 @@ cluster IDs, and ONTAP connection details (see inline comments in | File | Purpose | |---|---| +| `benchmark_support.py` | Shared run ids, resource names, `mean_seconds`, and the one log format used by every script. | | `cloudstack_client.py` | Minimal CloudStack HTTP/REST client (session-key login, generic `call()`, automatic `queryAsyncJobResult` polling). Used by every benchmark script. | | `storage_pool_common.py` | Shared helpers for the storage-pool scripts: `createStoragePool`/`deleteStoragePool` request building, CSV logging (`RawLogger`, `append_summary_csv`), cleanup-by-filter, config loading. | | `vm_instance_common.py` | Shared helpers for the VM-instance scripts: `deployVirtualMachine`/`destroyVirtualMachine` request building (incl. force-purging data disks and `Destroy`-state volumes so ONTAP space is reclaimed immediately instead of waiting for CloudStack's 24h cleanup delay), CSV logging, cleanup-by-filter, config loading. | @@ -82,20 +83,25 @@ Reproduces Confluence sections 5.1 (sequential) and 6.1 (concurrency) - | Script | What it does | Typical command | |---|---|---| -| `benchmark_storage_pool_sequential.py` | Creates storage pools one at a time (per protocol) up to N=30, then deletes them one at a time, logging every call and reporting checkpoint totals at N = 1, 5, 10, 20, 30. | `python3 benchmark_storage_pool_sequential.py --config config.yaml` | -| `benchmark_storage_pool_concurrency.py` | For each concurrency level C (default 2, 5, 10, 20, 30), creates C pools in parallel via a thread pool, records wall-clock time for the whole batch, then deletes the same C pools in parallel. | `python3 benchmark_storage_pool_concurrency.py --config config.yaml` | +| `run.py` | Entry point. Loads `config.yaml` and runs the test named by `--test`. | `python3 run.py --test storage-pool-sequential --config config.yaml` | +| `benchmark_storage_pool_sequential.py` | Creates storage pools one at a time (per protocol) up to N=30, then deletes them one at a time, logging every call and reporting checkpoint totals at N = 1, 5, 10, 20, 30. | `python3 run.py --test storage-pool-sequential --config config.yaml` | +| `benchmark_storage_pool_concurrency.py` | For each concurrency level C (default 2, 5, 10, 20, 30), creates C pools in parallel via a thread pool, records wall-clock time for the whole batch, then deletes the same C pools in parallel. | `python3 run.py --test storage-pool-concurrency --config config.yaml` | Common flags (both scripts): | Flag | Meaning | |---|---| -| `--config PATH` | Config YAML to use (default `config.yaml`). | -| `--protocol {nfs3,iscsi,both}` | Restrict the run to one protocol (default `both`). | -| `--run-id ID` | Reuse a specific run id (auto-generated otherwise) - use the **same id** across the sequential and concurrency scripts to merge both into one `summary_.csv`/report. | +| `--config PATH` | Config YAML to use (default `config.yaml`). Protocols, checkpoints, and concurrency levels are read from here. | +| `--run-id ID` | Reuse a run id. Auto-generated otherwise as `RUN__<4 hex chars>`. Letters, digits, and underscores only — a hyphen is rewritten to `_` because storage-pool names (and the ONTAP volume behind them) disallow `-`. Use the **same id** across sequential and concurrency to merge one report. | | `--dry-run` | Simulate timings with no real API calls - use this first to sanity-check your config. | | `--skip-cleanup` | Skip the automatic post-run sweep for leftover pools from this run id. | -| `--cleanup-only [FILTER]` | Don't run the benchmark - just delete every pool whose name contains `FILTER` (default: config's `pool_name_prefix`) and exit. Use this to recover after a crashed run. | -| `--levels 2,5,10` | *(concurrency script only)* Override the concurrency levels to run instead of the config's `concurrency_levels`. | +| `--cleanup-only [FILTER]` | Don't run the benchmark. Delete pools that match `FILTER` (default: config's `pool_name_prefix`) **and** this config's name pattern, zone, cluster, and storage provider. | + +`run.py` also requires `--test`. Protocols, checkpoints, and concurrency levels are not flags; change them in `config.yaml`. + +| Flag | Meaning | +|---|---| +| `--test NAME` | Required. `storage-pool-sequential`, `storage-pool-concurrency`, `vm-instance-sequential`, `vm-instance-concurrency`, or `vm-instance-combined`. | ## 4. VM-instance benchmark scripts @@ -121,13 +127,15 @@ VM getting both a root disk and a data disk on the same tagged pool. | Script | What it does | Typical command | |---|---|---| -| `benchmark_vm_instance_sequential.py` | Deploys VMs one at a time (per protocol) up to N=30, then destroys them one at a time, reporting checkpoint totals at N = 1, 2, 5, 10, 20, 30. | `python3 benchmark_vm_instance_sequential.py --config config.yaml` | -| `benchmark_vm_instance_concurrency.py` | For each concurrency level C, deploys C VMs in parallel, records wall-clock time, then destroys the same C VMs in parallel. | `python3 benchmark_vm_instance_concurrency.py --config config.yaml` | -| `benchmark_vm_instance_combined.py` | Runs the sequential matrix immediately followed by the concurrency matrix, under one shared run id, in a single invocation. Use once you already trust your config/environment. | `python3 benchmark_vm_instance_combined.py --config config.yaml` | +| `benchmark_vm_instance_sequential.py` | Deploys VMs one at a time (per protocol) up to N=30, then destroys them one at a time, reporting checkpoint totals at N = 1, 2, 5, 10, 20, 30. | `python3 run.py --test vm-instance-sequential --config config.yaml` | +| `benchmark_vm_instance_concurrency.py` | For each concurrency level C, deploys C VMs in parallel, records wall-clock time, then destroys the same C VMs in parallel. | `python3 run.py --test vm-instance-concurrency --config config.yaml` | +| `benchmark_vm_instance_combined.py` | Runs the sequential matrix immediately followed by the concurrency matrix, under one shared run id, in a single invocation. Use once you already trust your config/environment. | `python3 run.py --test vm-instance-combined --config config.yaml` | Common flags: same as the storage-pool scripts above (`--config`, -`--protocol`, `--run-id`, `--dry-run`, `--skip-cleanup`, `--cleanup-only -[FILTER]`; `--levels` on the concurrency/combined scripts). Failed +`--run-id`, `--dry-run`, `--skip-cleanup`, `--cleanup-only [FILTER]`). +VM cleanup rewrites `_` in the filter to `-` so it matches the hostname +form of the run id, and it only destroys VMs in the configured zone whose +names match the benchmark prefix and pattern. Failed `deployVirtualMachine` calls are intentionally **left in place** (not auto-destroyed) so you can inspect what got left behind - clean them up with `--cleanup-only` once done. @@ -136,14 +144,14 @@ with `--cleanup-only` once done. | Script | What it does | Typical command | |---|---|---| -| `render_report.py` | Turns a run's `raw_ops_.csv` + `summary_.csv` into Confluence-ready markdown tables (matches the page's section layout, plus Min/Avg/P95/Max rows for the Results Log). Writes to stdout and to `results/report_.md`. | `python3 render_report.py --run-id RUN-0001 --cloudstack-build 4.23.0.0-SNAPSHOT --ontap-version 9.17.1` | +| `render_report.py` | Turns a run's `raw_ops_.csv` + `summary_.csv` into markdown tables (section layout plus Min/Avg/P95/Max rows for the Results Log). Writes to stdout and to `results/report_.md`. | `python3 render_report.py --run-id RUN_20260927_081500_a1b2 --cloudstack-build 4.23.0.0-SNAPSHOT --ontap-version 9.17.1` | For VM-instance runs (which write `raw_ops_vm_.csv` / `summary_vm_.csv` instead of the storage-pool scripts' `raw_ops_`/ `summary_` prefix), pass the matching prefixes: ```bash -python3 render_report.py --run-id RUN-0001 \ +python3 render_report.py --run-id RUN_20260927_081500_a1b2 \ --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm ``` diff --git a/private-cicd/benchmark/ontap/benchmark_storage_pool_concurrency.py b/private-cicd/benchmark/ontap/benchmark_storage_pool_concurrency.py index f097dcd97330..470338ac0c5a 100644 --- a/private-cicd/benchmark/ontap/benchmark_storage_pool_concurrency.py +++ b/private-cicd/benchmark/ontap/benchmark_storage_pool_concurrency.py @@ -20,8 +20,7 @@ Drives createStoragePool / deleteStoragePool in parallel (ThreadPoolExecutor) over the CloudStack HTTP/REST API and reproduces Section 6.1 - Parallel/ -Concurrency Matrix (C = 2,5,10,20,30 workers) from the -"ONTAP Plugin - CloudStack Operations, Scale & Parallel Test Matrix" page: +Concurrency Matrix (C = 2,5,10,20,30 workers): 6.1.1 Parallel pool creation 6.1.2 Parallel pool deletion @@ -32,25 +31,32 @@ easy to run, review, and re-run independently of the sequential matrix). Every individual API call is logged to results/raw_ops_.csv, and a -per-checkpoint roll-up (matching the columns of the Confluence tables) is -appended to results/summary_.csv. Use render_report.py afterwards to -turn the summary CSV into paste-ready markdown for the Confluence page. Using -the same --run-id here and in benchmark_storage_pool_sequential.py combines -both into a single summary_.csv / report. +per-checkpoint roll-up is appended to results/summary_.csv. Use +render_report.py afterwards to turn the summary CSV into paste-ready markdown. +Using the same --run-id here and in benchmark_storage_pool_sequential.py +combines both into a single summary_.csv / report. + +Protocols and concurrency levels come from config.yaml. Prefer run.py. Usage: - python3 benchmark_storage_pool_concurrency.py --config config.yaml --protocol both - python3 benchmark_storage_pool_concurrency.py --config config.yaml --levels 2,5,10 + python3 run.py --test storage-pool-concurrency --config config.yaml python3 benchmark_storage_pool_concurrency.py --config config.yaml --dry-run python3 benchmark_storage_pool_concurrency.py --config config.yaml --cleanup-only """ import argparse import os -import statistics import time from concurrent.futures import ThreadPoolExecutor, as_completed +from benchmark_support import ( + assert_pool_name_prefix, + configure_logging, + format_pool_name, + get_logger, + mean_seconds, + normalize_run_id, +) from storage_pool_common import ( append_summary_csv, cleanup_by_filter, @@ -60,11 +66,12 @@ fake_delete, load_config, make_cloudstack_client, - new_run_id, RawLogger, resolve_protocols, ) +log = get_logger() + def run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay, levels): infra = cfg["infrastructure"] @@ -72,8 +79,9 @@ def run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, prefix = cfg["benchmark"]["pool_name_prefix"] for level in levels: - print(f"\n=== [6.1.1] Parallel CREATE - protocol={protocol_key} C={level} ===") - names = [f"{prefix}_c{level}_{protocol_key}_{run_id}_{i:03d}" for i in range(1, level + 1)] + log.info("[6.1.1] Parallel CREATE protocol=%s C=%s", protocol_key, level) + names = [format_pool_name(prefix, protocol_key, run_id, i, concurrency_level=level) + for i in range(1, level + 1)] t0 = time.perf_counter() results = [] @@ -91,7 +99,7 @@ def run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, succ = [r for r in results if r.success] fail = [r for r in results if not r.success] - avg = statistics.mean([r.duration_sec for r in succ]) if succ else 0 + avg = mean_seconds([r.duration_sec for r in succ]) notes = "Watch mgmt-server job-queue/thread-pool saturation as a confound" if level >= 30 else "" summary_rows.append({ "run_id": run_id, "phase": "concurrent_create", "test_id": "6.1.1", @@ -99,11 +107,14 @@ def run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, "total_time_sec": round(wall, 3), "avg_time_sec": round(avg, 3), "success_count": len(succ), "failure_count": len(fail), "notes": notes, }) - print(f" wall-clock={wall:.3f}s success={len(succ)} failure={len(fail)} avg/op={avg:.3f}s") + log.info( + "checkpoint C=%s protocol=%s total=%.3fs avg=%.3fs/op success=%s failure=%s", + level, protocol_key, wall, avg, len(succ), len(fail), + ) for r in fail: - print(f" !! {r.pool_name}: {r.error}") + log.error("%s: %s", r.pool_name, r.error) - print(f"\n=== [6.1.2] Parallel DELETE - protocol={protocol_key} C={len(succ)} ===") + log.info("[6.1.2] Parallel DELETE protocol=%s C=%s", protocol_key, len(succ)) t0 = time.perf_counter() del_results = [] if dry_run: @@ -120,88 +131,87 @@ def run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, succ_d = [r for r in del_results if r.success] fail_d = [r for r in del_results if not r.success] - avg_d = statistics.mean([r.duration_sec for r in succ_d]) if succ_d else 0 + avg_d = mean_seconds([r.duration_sec for r in succ_d]) summary_rows.append({ "run_id": run_id, "phase": "concurrent_delete", "test_id": "6.1.2", "protocol": protocol_key, "checkpoint": level, "total_time_sec": round(wall_del, 3), "avg_time_sec": round(avg_d, 3), "success_count": len(succ_d), "failure_count": len(fail_d), "notes": "", }) - print(f" wall-clock={wall_del:.3f}s success={len(succ_d)} failure={len(fail_d)} avg/op={avg_d:.3f}s") + log.info( + "checkpoint C=%s protocol=%s total=%.3fs avg=%.3fs/op success=%s failure=%s", + level, protocol_key, wall_del, avg_d, len(succ_d), len(fail_d), + ) for r in fail_d: - print(f" !! {r.pool_name}: {r.error}") + log.error("%s: %s", r.pool_name, r.error) if delay: time.sleep(delay) -def parse_levels(raw, default_levels): - if not raw: - return sorted(default_levels) - return sorted(int(x.strip()) for x in raw.split(",") if x.strip()) - - -def main(): - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") - parser.add_argument("--protocol", default="both", help="nfs3 | iscsi | both (default: both)") - parser.add_argument("--run-id", default=None, help="Override auto-generated run id") - parser.add_argument( - "--levels", default=None, - help="Comma-separated concurrency levels to run, e.g. '2,5,10' " - "(default: config.benchmark.concurrency_levels, typically up to 30)", - ) - parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") - parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover pools from this run in place") - parser.add_argument( - "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", - help="Delete all storage pools whose name contains FILTER (default: config pool_name_prefix) and exit", - ) - args = parser.parse_args() - - cfg = load_config(args.config) - os.makedirs(cfg["benchmark"].get("output_dir", "results"), exist_ok=True) +def execute(cfg, dry_run=False, skip_cleanup=False, cleanup_only=None, run_id=None): + configure_logging() + assert_pool_name_prefix(cfg["benchmark"]["pool_name_prefix"]) + output_dir = cfg["benchmark"].get("output_dir", "results") + os.makedirs(output_dir, exist_ok=True) - if args.cleanup_only is not None: + if cleanup_only is not None: client = make_cloudstack_client(cfg) - name_filter = args.cleanup_only - if name_filter == "__PREFIX__": - name_filter = cfg["benchmark"]["pool_name_prefix"] - cleanup_by_filter(client, name_filter) + name_filter = cfg["benchmark"]["pool_name_prefix"] if cleanup_only == "__PREFIX__" else cleanup_only + cleanup_by_filter(client, name_filter, cfg) return - run_id = args.run_id or new_run_id() - protocols = resolve_protocols(cfg, args.protocol) + run_id = normalize_run_id(run_id) + protocols = resolve_protocols(cfg, "both") delay = cfg["benchmark"].get("inter_op_delay_sec", 0) - output_dir = cfg["benchmark"].get("output_dir", "results") - levels = parse_levels(args.levels, cfg["benchmark"]["concurrency_levels"]) + levels = sorted(cfg["benchmark"]["concurrency_levels"]) - print(f"Run ID: {run_id}") - print(f"Protocols: {protocols}") - print("Mode: concurrency") - print(f"Concurrency levels: {levels}") - print(f"Dry run: {args.dry_run}") + log.info("Run ID: %s", run_id) + log.info("Protocols: %s", protocols) + log.info("Mode: concurrency") + log.info("Concurrency levels: %s", levels) + log.info("Dry run: %s", dry_run) - client = None if args.dry_run else make_cloudstack_client(cfg) + client = None if dry_run else make_cloudstack_client(cfg) raw_logger = RawLogger(os.path.join(output_dir, f"raw_ops_{run_id}.csv")) summary_rows = [] try: for protocol_key in protocols: - run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, args.dry_run, delay, levels) + run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay, levels) finally: raw_logger.close() summary_path = os.path.join(output_dir, f"summary_{run_id}.csv") append_summary_csv(summary_path, summary_rows) - print(f"\nRaw per-operation log: {os.path.join(output_dir, f'raw_ops_{run_id}.csv')}") - print(f"Checkpoint summary: {summary_path}") - print("Next: python3 render_report.py --run-id " + run_id + f" --output-dir {output_dir}") + log.info("Raw per-operation log: %s", os.path.join(output_dir, f"raw_ops_{run_id}.csv")) + log.info("Checkpoint summary: %s", summary_path) + log.info("Next: python3 render_report.py --run-id %s --output-dir %s", run_id, output_dir) + + if not dry_run and not skip_cleanup: + log.info("Verifying no orphaned pools remain for run %s", run_id) + cleanup_by_filter(client, run_id, cfg) - if not args.dry_run and not args.skip_cleanup: - print(f"\nVerifying no orphaned pools remain for run {run_id}...") - cleanup_by_filter(client, run_id) + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--run-id", default=None, help="Override auto-generated run id") + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover pools from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Delete benchmark pools whose name contains FILTER (default: config pool_name_prefix) and exit", + ) + args = parser.parse_args() + execute( + load_config(args.config), + dry_run=args.dry_run, + skip_cleanup=args.skip_cleanup, + cleanup_only=args.cleanup_only, + run_id=args.run_id, + ) if __name__ == "__main__": diff --git a/private-cicd/benchmark/ontap/benchmark_storage_pool_sequential.py b/private-cicd/benchmark/ontap/benchmark_storage_pool_sequential.py index b103596ab7d8..13cfd95ab439 100644 --- a/private-cicd/benchmark/ontap/benchmark_storage_pool_sequential.py +++ b/private-cicd/benchmark/ontap/benchmark_storage_pool_sequential.py @@ -19,9 +19,8 @@ Sequential storage-pool benchmark for the NetApp ONTAP CloudStack plugin. Drives createStoragePool / deleteStoragePool one at a time over the CloudStack -HTTP/REST API (see https://netapp.atlassian.net/wiki/spaces/OSSG/pages/608854350) -and reproduces Section 5.1 - Sequential Scale Matrix (N = 1,5,10,20,30) from the -"ONTAP Plugin - CloudStack Operations, Scale & Parallel Test Matrix" page: +HTTP/REST API and reproduces Section 5.1 - Sequential Scale Matrix +(N = 1,5,10,20,30): 5.1.1 Sequential create (1 pool at a time, cumulative to N) 5.1.2 Sequential delete (1 pool at a time, from N remaining) @@ -31,14 +30,15 @@ kicked off and reviewed independently before committing to a concurrency run). Every individual API call is logged to results/raw_ops_.csv, and a -per-checkpoint roll-up (matching the columns of the Confluence tables) is -appended to results/summary_.csv. Use render_report.py afterwards to -turn the summary CSV into paste-ready markdown for the Confluence page. Using -the same --run-id here and in benchmark_storage_pool_concurrency.py combines -both into a single summary_.csv / report. +per-checkpoint roll-up is appended to results/summary_.csv. Use +render_report.py afterwards to turn the summary CSV into paste-ready markdown. +Using the same --run-id here and in benchmark_storage_pool_concurrency.py +combines both into a single summary_.csv / report. + +Protocols and checkpoints come from config.yaml. Prefer run.py. Usage: - python3 benchmark_storage_pool_sequential.py --config config.yaml --protocol both + python3 run.py --test storage-pool-sequential --config config.yaml python3 benchmark_storage_pool_sequential.py --config config.yaml --dry-run python3 benchmark_storage_pool_sequential.py --config config.yaml --cleanup-only """ @@ -47,6 +47,14 @@ import os import time +from benchmark_support import ( + assert_pool_name_prefix, + configure_logging, + format_pool_name, + get_logger, + mean_seconds, + normalize_run_id, +) from storage_pool_common import ( append_summary_csv, cleanup_by_filter, @@ -56,11 +64,12 @@ fake_delete, load_config, make_cloudstack_client, - new_run_id, RawLogger, resolve_protocols, ) +log = get_logger() + def run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay): infra = cfg["infrastructure"] @@ -71,35 +80,36 @@ def run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, created = [] create_durations = [] - print(f"\n=== [5.1.1] Sequential CREATE - protocol={protocol_key} up to N={max_n} ===") + log.info("[5.1.1] Sequential CREATE protocol=%s up to N=%s", protocol_key, max_n) for i in range(1, max_n + 1): - # ONTAP volume names only allow alphanumeric + underscore (no hyphens). - name = f"{prefix}_seq_{protocol_key}_{run_id}_{i:03d}" + name = format_pool_name(prefix, protocol_key, run_id, i) result = fake_create(name) if dry_run else create_pool(client, name, infra, ontap_cfg) raw_logger.log(run_id, "sequential_create", "5.1.1", protocol_key, max_n, i, result) status = "OK" if result.success else "FAIL" - print(f" [{i:>3}/{max_n}] create {name} -> {status} ({result.duration_sec:.3f}s)") + log.info("[%s/%s] create %s -> %s (%.3fs)", i, max_n, name, status, result.duration_sec) if result.success: created.append((name, result.pool_id)) create_durations.append(result.duration_sec) else: - print(f" !! {result.error}") + log.error("%s", result.error) if i in checkpoints: total = sum(create_durations) - avg = total / len(create_durations) if create_durations else 0 + avg = mean_seconds(create_durations) summary_rows.append({ "run_id": run_id, "phase": "sequential_create", "test_id": "5.1.1", "protocol": protocol_key, "checkpoint": i, "total_time_sec": round(total, 3), "avg_time_sec": round(avg, 3), "success_count": len(created), "failure_count": i - len(created), "notes": "", }) - print(f" >> checkpoint N={i}: total={total:.3f}s avg={avg:.3f}s/op " - f"success={len(created)} failure={i - len(created)}") + log.info( + "checkpoint N=%s protocol=%s total=%.3fs avg=%.3fs/op success=%s failure=%s", + i, protocol_key, total, avg, len(created), i - len(created), + ) if delay: time.sleep(delay) total_created = len(created) - print(f"\n=== [5.1.2] Sequential DELETE - protocol={protocol_key} from N={total_created} remaining ===") + log.info("[5.1.2] Sequential DELETE protocol=%s from N=%s remaining", protocol_key, total_created) if total_created in checkpoints: summary_rows.append({ "run_id": run_id, "phase": "sequential_delete", "test_id": "5.1.2", @@ -114,84 +124,94 @@ def run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, remaining = total_created - idx raw_logger.log(run_id, "sequential_delete", "5.1.2", protocol_key, total_created, idx, result) status = "OK" if result.success else "FAIL" - print(f" [{idx:>3}/{total_created}] delete {name} -> {status} " - f"({result.duration_sec:.3f}s) remaining={remaining}") + log.info( + "[%s/%s] delete %s -> %s (%.3fs) remaining=%s", + idx, total_created, name, status, result.duration_sec, remaining, + ) if result.success: delete_durations.append(result.duration_sec) deleted_ok += 1 else: - print(f" !! {result.error}") + log.error("%s", result.error) if remaining in checkpoints: total = sum(delete_durations) - avg = total / len(delete_durations) if delete_durations else 0 + avg = mean_seconds(delete_durations) summary_rows.append({ "run_id": run_id, "phase": "sequential_delete", "test_id": "5.1.2", "protocol": protocol_key, "checkpoint": remaining, "total_time_sec": round(total, 3), "avg_time_sec": round(avg, 3), "success_count": deleted_ok, "failure_count": idx - deleted_ok, "notes": "", }) - print(f" >> checkpoint remaining={remaining}: total={total:.3f}s avg={avg:.3f}s/op " - f"success={deleted_ok} failure={idx - deleted_ok}") + log.info( + "checkpoint remaining=%s protocol=%s total=%.3fs avg=%.3fs/op success=%s failure=%s", + remaining, protocol_key, total, avg, deleted_ok, idx - deleted_ok, + ) if delay: time.sleep(delay) -def main(): - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") - parser.add_argument("--protocol", default="both", help="nfs3 | iscsi | both (default: both)") - parser.add_argument("--run-id", default=None, help="Override auto-generated run id") - parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") - parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover pools from this run in place") - parser.add_argument( - "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", - help="Delete all storage pools whose name contains FILTER (default: config pool_name_prefix) and exit", - ) - args = parser.parse_args() - - cfg = load_config(args.config) - os.makedirs(cfg["benchmark"].get("output_dir", "results"), exist_ok=True) +def execute(cfg, dry_run=False, skip_cleanup=False, cleanup_only=None, run_id=None): + configure_logging() + assert_pool_name_prefix(cfg["benchmark"]["pool_name_prefix"]) + output_dir = cfg["benchmark"].get("output_dir", "results") + os.makedirs(output_dir, exist_ok=True) - if args.cleanup_only is not None: + if cleanup_only is not None: client = make_cloudstack_client(cfg) - name_filter = args.cleanup_only - if name_filter == "__PREFIX__": - name_filter = cfg["benchmark"]["pool_name_prefix"] - cleanup_by_filter(client, name_filter) + name_filter = cfg["benchmark"]["pool_name_prefix"] if cleanup_only == "__PREFIX__" else cleanup_only + cleanup_by_filter(client, name_filter, cfg) return - run_id = args.run_id or new_run_id() - protocols = resolve_protocols(cfg, args.protocol) + run_id = normalize_run_id(run_id) + protocols = resolve_protocols(cfg, "both") delay = cfg["benchmark"].get("inter_op_delay_sec", 0) - output_dir = cfg["benchmark"].get("output_dir", "results") - print(f"Run ID: {run_id}") - print(f"Protocols: {protocols}") - print("Mode: sequential") - print(f"Dry run: {args.dry_run}") + log.info("Run ID: %s", run_id) + log.info("Protocols: %s", protocols) + log.info("Mode: sequential") + log.info("Dry run: %s", dry_run) - client = None if args.dry_run else make_cloudstack_client(cfg) + client = None if dry_run else make_cloudstack_client(cfg) raw_logger = RawLogger(os.path.join(output_dir, f"raw_ops_{run_id}.csv")) summary_rows = [] try: for protocol_key in protocols: - run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, args.dry_run, delay) + run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay) finally: raw_logger.close() summary_path = os.path.join(output_dir, f"summary_{run_id}.csv") append_summary_csv(summary_path, summary_rows) - print(f"\nRaw per-operation log: {os.path.join(output_dir, f'raw_ops_{run_id}.csv')}") - print(f"Checkpoint summary: {summary_path}") - print("Next: python3 render_report.py --run-id " + run_id + f" --output-dir {output_dir}") - print(f" (or run benchmark_storage_pool_concurrency.py --run-id {run_id} first to add 6.1.x to the same report)") + log.info("Raw per-operation log: %s", os.path.join(output_dir, f"raw_ops_{run_id}.csv")) + log.info("Checkpoint summary: %s", summary_path) + log.info("Next: python3 render_report.py --run-id %s --output-dir %s", run_id, output_dir) + + if not dry_run and not skip_cleanup: + log.info("Verifying no orphaned pools remain for run %s", run_id) + cleanup_by_filter(client, run_id, cfg) - if not args.dry_run and not args.skip_cleanup: - print(f"\nVerifying no orphaned pools remain for run {run_id}...") - cleanup_by_filter(client, run_id) + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--run-id", default=None, help="Override auto-generated run id") + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover pools from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Delete benchmark pools whose name contains FILTER (default: config pool_name_prefix) and exit", + ) + args = parser.parse_args() + execute( + load_config(args.config), + dry_run=args.dry_run, + skip_cleanup=args.skip_cleanup, + cleanup_only=args.cleanup_only, + run_id=args.run_id, + ) if __name__ == "__main__": diff --git a/private-cicd/benchmark/ontap/benchmark_support.py b/private-cicd/benchmark/ontap/benchmark_support.py new file mode 100644 index 000000000000..bdafda215027 --- /dev/null +++ b/private-cicd/benchmark/ontap/benchmark_support.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Shared run-id, naming, statistics, and logging helpers for every benchmark script.""" + +import datetime +import logging +import re +import secrets +import statistics +import sys + +LOG_NAME = "ontap.benchmark" + + +def configure_logging(): + """One format for every benchmark script and the API client.""" + root = logging.getLogger(LOG_NAME) + if not root.handlers: + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter( + "%(asctime)s %(levelname)s %(message)s", + "%Y-%m-%dT%H:%M:%S", + )) + root.addHandler(handler) + root.setLevel(logging.INFO) + root.propagate = False + return root + + +def get_logger(): + return logging.getLogger(LOG_NAME) + + +def mean_seconds(values): + """Average duration. Empty input is 0 so checkpoint rows stay numeric.""" + if not values: + return 0.0 + return statistics.mean(values) + + +def new_run_id(): + """Timestamp plus a short random suffix. Underscores only: pool names reject hyphens.""" + stamp = datetime.datetime.utcnow().strftime("%Y%m%d_%H%M%S") + return f"RUN_{stamp}_{secrets.token_hex(2)}" + + +def normalize_run_id(run_id): + """Accept a caller-supplied id, or generate one. Hyphens are rewritten to underscores.""" + if not run_id: + return new_run_id() + normalized = run_id.strip().replace("-", "_") + if not re.fullmatch(r"[A-Za-z0-9_]+", normalized): + sys.exit( + f"Run id {run_id!r} is not valid. Use letters, digits, and underscores " + "(hyphens are rewritten to underscores because storage pool names disallow '-')." + ) + if normalized != run_id.strip(): + get_logger().warning( + "Rewrote run id %r to %r so it can be embedded in a storage pool name", + run_id, normalized, + ) + return normalized + + +def format_pool_name(prefix, protocol_key, run_id, index, concurrency_level=None): + if concurrency_level is None: + return f"{prefix}_seq_{protocol_key}_{run_id}_{index:03d}" + return f"{prefix}_c{concurrency_level}_{protocol_key}_{run_id}_{index:03d}" + + +def format_vm_name(prefix, protocol_key, run_id, index, concurrency_level=None): + """VM names are hostnames: underscores in the run id become hyphens.""" + kind = "seq" if concurrency_level is None else f"c{concurrency_level}" + raw = f"{prefix}-{kind}-{protocol_key}-{run_id}-{index:03d}" + return raw.replace("_", "-") + + +def vm_name_token(value): + """Same underscore-to-hyphen rewrite used by format_vm_name, for cleanup filters.""" + return (value or "").replace("_", "-") + + +def assert_pool_name_prefix(prefix): + if not prefix or not re.fullmatch(r"[A-Za-z0-9_]+", prefix): + sys.exit( + f"benchmark.pool_name_prefix {prefix!r} must be letters, digits, and underscores " + "(storage pool names disallow '-')." + ) + + +def assert_vm_name_prefix(prefix): + if not prefix or "_" in prefix or not re.fullmatch(r"[A-Za-z0-9-]+", prefix): + sys.exit( + f"vm_bench.vm_name_prefix {prefix!r} must be letters, digits, and hyphens " + "(VM names are hostnames and disallow '_')." + ) diff --git a/private-cicd/benchmark/ontap/benchmark_vm_instance_combined.py b/private-cicd/benchmark/ontap/benchmark_vm_instance_combined.py index 86e0430038c3..95da56dd496b 100644 --- a/private-cicd/benchmark/ontap/benchmark_vm_instance_combined.py +++ b/private-cicd/benchmark/ontap/benchmark_vm_instance_combined.py @@ -31,8 +31,10 @@ combined script is for when you already trust the config/environment and just want the full 5.2.x + 6.2.x matrix in one go. +Protocols and concurrency levels come from config.yaml. Prefer run.py. + Usage: - python3 benchmark_vm_instance_combined.py --config config.yaml --protocol both + python3 run.py --test vm-instance-combined --config config.yaml python3 benchmark_vm_instance_combined.py --config config.yaml --dry-run python3 benchmark_vm_instance_combined.py --config config.yaml --cleanup-only """ @@ -40,83 +42,94 @@ import argparse import os -from benchmark_vm_instance_concurrency import parse_levels, run_concurrency +from benchmark_support import ( + assert_vm_name_prefix, + configure_logging, + get_logger, + normalize_run_id, +) +from benchmark_vm_instance_concurrency import run_concurrency from benchmark_vm_instance_sequential import run_sequential from vm_instance_common import ( append_summary_csv, cleanup_by_filter, load_config, make_cloudstack_client, - new_run_id, RawLogger, resolve_protocols, ) +log = get_logger() -def main(): - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") - parser.add_argument("--protocol", default="both", help="nfs3 | iscsi | both (default: both)") - parser.add_argument("--run-id", default=None, help="Override auto-generated run id") - parser.add_argument( - "--levels", default=None, - help="Comma-separated concurrency levels to run, e.g. '2,5,10' " - "(default: config.vm_bench.concurrency_levels, typically up to 30)", - ) - parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") - parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover VMs from this run in place") - parser.add_argument( - "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", - help="Destroy all VMs whose name contains FILTER (default: config vm_name_prefix) and exit", - ) - args = parser.parse_args() - cfg = load_config(args.config) - os.makedirs(cfg["vm_bench"].get("output_dir", "results"), exist_ok=True) +def execute(cfg, dry_run=False, skip_cleanup=False, cleanup_only=None, run_id=None): + configure_logging() + assert_vm_name_prefix(cfg["vm_bench"]["vm_name_prefix"]) + output_dir = cfg["vm_bench"].get("output_dir", "results") + os.makedirs(output_dir, exist_ok=True) - if args.cleanup_only is not None: + if cleanup_only is not None: client = make_cloudstack_client(cfg) - name_filter = args.cleanup_only - if name_filter == "__PREFIX__": - name_filter = cfg["vm_bench"]["vm_name_prefix"] - cleanup_by_filter(client, name_filter) + name_filter = cfg["vm_bench"]["vm_name_prefix"] if cleanup_only == "__PREFIX__" else cleanup_only + cleanup_by_filter(client, name_filter, cfg) return - run_id = args.run_id or new_run_id() - protocols = resolve_protocols(cfg, args.protocol) + run_id = normalize_run_id(run_id) + protocols = resolve_protocols(cfg, "both") delay = cfg["vm_bench"].get("inter_op_delay_sec", 0) - output_dir = cfg["vm_bench"].get("output_dir", "results") - levels = parse_levels(args.levels, cfg["vm_bench"]["concurrency_levels"]) + levels = sorted(cfg["vm_bench"]["concurrency_levels"]) - print(f"Run ID: {run_id}") - print(f"Protocols: {protocols}") - print("Mode: sequential + concurrency (combined)") - print(f"Concurrency levels: {levels}") - print(f"Dry run: {args.dry_run}") + log.info("Run ID: %s", run_id) + log.info("Protocols: %s", protocols) + log.info("Mode: sequential + concurrency (combined)") + log.info("Concurrency levels: %s", levels) + log.info("Dry run: %s", dry_run) - client = None if args.dry_run else make_cloudstack_client(cfg) + client = None if dry_run else make_cloudstack_client(cfg) raw_logger = RawLogger(os.path.join(output_dir, f"raw_ops_vm_{run_id}.csv")) summary_rows = [] try: for protocol_key in protocols: - run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, args.dry_run, delay) - run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, args.dry_run, delay, levels) + run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay) + run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay, levels) finally: raw_logger.close() summary_path = os.path.join(output_dir, f"summary_vm_{run_id}.csv") append_summary_csv(summary_path, summary_rows) - print(f"\nRaw per-operation log: {os.path.join(output_dir, f'raw_ops_vm_{run_id}.csv')}") - print(f"Checkpoint summary: {summary_path}") - print("Next: python3 render_report.py --run-id " + run_id + - f" --output-dir {output_dir} --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm") + log.info("Raw per-operation log: %s", os.path.join(output_dir, f"raw_ops_vm_{run_id}.csv")) + log.info("Checkpoint summary: %s", summary_path) + log.info( + "Next: python3 render_report.py --run-id %s --output-dir %s --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm", + run_id, output_dir, + ) + + if not dry_run and not skip_cleanup: + log.info("Verifying no orphaned VMs remain for run %s", run_id) + cleanup_by_filter(client, run_id, cfg) + - if not args.dry_run and not args.skip_cleanup: - print(f"\nVerifying no orphaned VMs remain for run {run_id}...") - cleanup_by_filter(client, run_id) +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--run-id", default=None, help="Override auto-generated run id") + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover VMs from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Destroy benchmark VMs whose name contains FILTER (default: config vm_name_prefix) and exit", + ) + args = parser.parse_args() + execute( + load_config(args.config), + dry_run=args.dry_run, + skip_cleanup=args.skip_cleanup, + cleanup_only=args.cleanup_only, + run_id=args.run_id, + ) if __name__ == "__main__": diff --git a/private-cicd/benchmark/ontap/benchmark_vm_instance_concurrency.py b/private-cicd/benchmark/ontap/benchmark_vm_instance_concurrency.py index c002e56b4d2b..d4d7bf349145 100644 --- a/private-cicd/benchmark/ontap/benchmark_vm_instance_concurrency.py +++ b/private-cicd/benchmark/ontap/benchmark_vm_instance_concurrency.py @@ -22,8 +22,7 @@ (ThreadPoolExecutor) over the CloudStack HTTP/REST API against pre- provisioned bench_vm_nfs3 / bench_vm_iscsi storage pools (see README.md "VM instance benchmark prerequisites") and reproduces Section 6.2 - Parallel/ -Concurrency Matrix from the "ONTAP Plugin - CloudStack Operations, Scale & -Parallel Test Matrix" page: +Concurrency Matrix: 6.2.1 Parallel VM creation 6.2.2 Parallel VM deletion @@ -39,25 +38,32 @@ matrix). Every individual API call is logged to results/raw_ops_vm_.csv, and a -per-checkpoint roll-up (matching the columns of the Confluence tables) is -appended to results/summary_vm_.csv. Use render_report.py afterwards -to turn the summary CSV into paste-ready markdown. Using the same --run-id -here and in benchmark_vm_instance_sequential.py combines both into a single -summary_vm_.csv / report. +per-checkpoint roll-up is appended to results/summary_vm_.csv. Use +render_report.py afterwards to turn the summary CSV into paste-ready markdown. +Using the same --run-id here and in benchmark_vm_instance_sequential.py +combines both into a single summary_vm_.csv / report. + +Protocols and concurrency levels come from config.yaml. Prefer run.py. Usage: - python3 benchmark_vm_instance_concurrency.py --config config.yaml --protocol both - python3 benchmark_vm_instance_concurrency.py --config config.yaml --levels 2,5,10 + python3 run.py --test vm-instance-concurrency --config config.yaml python3 benchmark_vm_instance_concurrency.py --config config.yaml --dry-run python3 benchmark_vm_instance_concurrency.py --config config.yaml --cleanup-only """ import argparse import os -import statistics import time from concurrent.futures import ThreadPoolExecutor, as_completed +from benchmark_support import ( + assert_vm_name_prefix, + configure_logging, + format_vm_name, + get_logger, + mean_seconds, + normalize_run_id, +) from vm_instance_common import ( append_summary_csv, cleanup_by_filter, @@ -67,12 +73,13 @@ fake_delete, load_config, make_cloudstack_client, - new_run_id, RawLogger, report_failed_creates, resolve_protocols, ) +log = get_logger() + def run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay, levels): infra = cfg["infrastructure"] @@ -81,8 +88,8 @@ def run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, prefix = vm_cfg["vm_name_prefix"] for level in levels: - print(f"\n=== [6.2.1] Parallel CREATE - protocol={protocol_key} C={level} ===") - names = [f"{prefix}-c{level}-{protocol_key}-{run_id}-{i:03d}".replace("_", "-") + log.info("[6.2.1] Parallel CREATE protocol=%s C=%s", protocol_key, level) + names = [format_vm_name(prefix, protocol_key, run_id, i, concurrency_level=level) for i in range(1, level + 1)] t0 = time.perf_counter() @@ -101,7 +108,7 @@ def run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, succ = [r for r in results if r.success] fail = [r for r in results if not r.success] - avg = statistics.mean([r.duration_sec for r in succ]) if succ else 0 + avg = mean_seconds([r.duration_sec for r in succ]) notes = "Watch mgmt-server job-queue/thread-pool + single-host KVM saturation as a confound" if level >= 20 else "" summary_rows.append({ "run_id": run_id, "phase": "concurrent_create", "test_id": "6.2.1", @@ -109,12 +116,15 @@ def run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, "total_time_sec": round(wall, 3), "avg_time_sec": round(avg, 3), "success_count": len(succ), "failure_count": len(fail), "notes": notes, }) - print(f" wall-clock={wall:.3f}s success={len(succ)} failure={len(fail)} avg/op={avg:.3f}s") + log.info( + "checkpoint C=%s protocol=%s total=%.3fs avg=%.3fs/op success=%s failure=%s", + level, protocol_key, wall, avg, len(succ), len(fail), + ) for r in fail: - print(f" !! {r.vm_name}: {r.error}") + log.error("%s: %s", r.vm_name, r.error) report_failed_creates([(r.vm_name, r.vm_id) for r in fail]) - print(f"\n=== [6.2.2] Parallel DELETE - protocol={protocol_key} C={len(succ)} ===") + log.info("[6.2.2] Parallel DELETE protocol=%s C=%s", protocol_key, len(succ)) t0 = time.perf_counter() del_results = [] if dry_run: @@ -131,89 +141,90 @@ def run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, succ_d = [r for r in del_results if r.success] fail_d = [r for r in del_results if not r.success] - avg_d = statistics.mean([r.duration_sec for r in succ_d]) if succ_d else 0 + avg_d = mean_seconds([r.duration_sec for r in succ_d]) summary_rows.append({ "run_id": run_id, "phase": "concurrent_delete", "test_id": "6.2.2", "protocol": protocol_key, "checkpoint": level, "total_time_sec": round(wall_del, 3), "avg_time_sec": round(avg_d, 3), "success_count": len(succ_d), "failure_count": len(fail_d), "notes": "", }) - print(f" wall-clock={wall_del:.3f}s success={len(succ_d)} failure={len(fail_d)} avg/op={avg_d:.3f}s") + log.info( + "checkpoint C=%s protocol=%s total=%.3fs avg=%.3fs/op success=%s failure=%s", + level, protocol_key, wall_del, avg_d, len(succ_d), len(fail_d), + ) for r in fail_d: - print(f" !! {r.vm_name}: {r.error}") + log.error("%s: %s", r.vm_name, r.error) if delay: time.sleep(delay) -def parse_levels(raw, default_levels): - if not raw: - return sorted(default_levels) - return sorted(int(x.strip()) for x in raw.split(",") if x.strip()) - - -def main(): - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") - parser.add_argument("--protocol", default="both", help="nfs3 | iscsi | both (default: both)") - parser.add_argument("--run-id", default=None, help="Override auto-generated run id") - parser.add_argument( - "--levels", default=None, - help="Comma-separated concurrency levels to run, e.g. '2,5,10' " - "(default: config.vm_bench.concurrency_levels, typically up to 30)", - ) - parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") - parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover VMs from this run in place") - parser.add_argument( - "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", - help="Destroy all VMs whose name contains FILTER (default: config vm_name_prefix) and exit", - ) - args = parser.parse_args() - - cfg = load_config(args.config) - os.makedirs(cfg["vm_bench"].get("output_dir", "results"), exist_ok=True) +def execute(cfg, dry_run=False, skip_cleanup=False, cleanup_only=None, run_id=None): + configure_logging() + assert_vm_name_prefix(cfg["vm_bench"]["vm_name_prefix"]) + output_dir = cfg["vm_bench"].get("output_dir", "results") + os.makedirs(output_dir, exist_ok=True) - if args.cleanup_only is not None: + if cleanup_only is not None: client = make_cloudstack_client(cfg) - name_filter = args.cleanup_only - if name_filter == "__PREFIX__": - name_filter = cfg["vm_bench"]["vm_name_prefix"] - cleanup_by_filter(client, name_filter) + name_filter = cfg["vm_bench"]["vm_name_prefix"] if cleanup_only == "__PREFIX__" else cleanup_only + cleanup_by_filter(client, name_filter, cfg) return - run_id = args.run_id or new_run_id() - protocols = resolve_protocols(cfg, args.protocol) + run_id = normalize_run_id(run_id) + protocols = resolve_protocols(cfg, "both") delay = cfg["vm_bench"].get("inter_op_delay_sec", 0) - output_dir = cfg["vm_bench"].get("output_dir", "results") - levels = parse_levels(args.levels, cfg["vm_bench"]["concurrency_levels"]) + levels = sorted(cfg["vm_bench"]["concurrency_levels"]) - print(f"Run ID: {run_id}") - print(f"Protocols: {protocols}") - print("Mode: concurrency") - print(f"Concurrency levels: {levels}") - print(f"Dry run: {args.dry_run}") + log.info("Run ID: %s", run_id) + log.info("Protocols: %s", protocols) + log.info("Mode: concurrency") + log.info("Concurrency levels: %s", levels) + log.info("Dry run: %s", dry_run) - client = None if args.dry_run else make_cloudstack_client(cfg) + client = None if dry_run else make_cloudstack_client(cfg) raw_logger = RawLogger(os.path.join(output_dir, f"raw_ops_vm_{run_id}.csv")) summary_rows = [] try: for protocol_key in protocols: - run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, args.dry_run, delay, levels) + run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay, levels) finally: raw_logger.close() summary_path = os.path.join(output_dir, f"summary_vm_{run_id}.csv") append_summary_csv(summary_path, summary_rows) - print(f"\nRaw per-operation log: {os.path.join(output_dir, f'raw_ops_vm_{run_id}.csv')}") - print(f"Checkpoint summary: {summary_path}") - print("Next: python3 render_report.py --run-id " + run_id + - f" --output-dir {output_dir} --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm") + log.info("Raw per-operation log: %s", os.path.join(output_dir, f"raw_ops_vm_{run_id}.csv")) + log.info("Checkpoint summary: %s", summary_path) + log.info( + "Next: python3 render_report.py --run-id %s --output-dir %s --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm", + run_id, output_dir, + ) + + if not dry_run and not skip_cleanup: + log.info("Verifying no orphaned VMs remain for run %s", run_id) + cleanup_by_filter(client, run_id, cfg) - if not args.dry_run and not args.skip_cleanup: - print(f"\nVerifying no orphaned VMs remain for run {run_id}...") - cleanup_by_filter(client, run_id) + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--run-id", default=None, help="Override auto-generated run id") + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover VMs from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Destroy benchmark VMs whose name contains FILTER (default: config vm_name_prefix) and exit", + ) + args = parser.parse_args() + execute( + load_config(args.config), + dry_run=args.dry_run, + skip_cleanup=args.skip_cleanup, + cleanup_only=args.cleanup_only, + run_id=args.run_id, + ) if __name__ == "__main__": diff --git a/private-cicd/benchmark/ontap/benchmark_vm_instance_sequential.py b/private-cicd/benchmark/ontap/benchmark_vm_instance_sequential.py index 2890c99cfae7..a583eba3ba5e 100644 --- a/private-cicd/benchmark/ontap/benchmark_vm_instance_sequential.py +++ b/private-cicd/benchmark/ontap/benchmark_vm_instance_sequential.py @@ -21,8 +21,7 @@ Drives deployVirtualMachine / destroyVirtualMachine one at a time over the CloudStack HTTP/REST API against pre-provisioned bench_vm_nfs3 / bench_vm_iscsi storage pools (see README.md "VM instance benchmark prerequisites") and -reproduces Section 5.2 - Sequential Scale Matrix from the "ONTAP Plugin - -CloudStack Operations, Scale & Parallel Test Matrix" page: +reproduces Section 5.2 - Sequential Scale Matrix: 5.2.1 Sequential create (1 VM at a time, cumulative to N) 5.2.2 Sequential delete (1 VM at a time, from N remaining) @@ -36,14 +35,15 @@ off and reviewed independently before committing to a concurrency run). Every individual API call is logged to results/raw_ops_vm_.csv, and a -per-checkpoint roll-up (matching the columns of the Confluence tables) is -appended to results/summary_vm_.csv. Use render_report.py afterwards -to turn the summary CSV into paste-ready markdown. Using the same --run-id -here and in benchmark_vm_instance_concurrency.py combines both into a single -summary_vm_.csv / report. +per-checkpoint roll-up is appended to results/summary_vm_.csv. Use +render_report.py afterwards to turn the summary CSV into paste-ready markdown. +Using the same --run-id here and in benchmark_vm_instance_concurrency.py +combines both into a single summary_vm_.csv / report. + +Protocols and checkpoints come from config.yaml. Prefer run.py. Usage: - python3 benchmark_vm_instance_sequential.py --config config.yaml --protocol both + python3 run.py --test vm-instance-sequential --config config.yaml python3 benchmark_vm_instance_sequential.py --config config.yaml --dry-run python3 benchmark_vm_instance_sequential.py --config config.yaml --cleanup-only """ @@ -52,6 +52,14 @@ import os import time +from benchmark_support import ( + assert_vm_name_prefix, + configure_logging, + format_vm_name, + get_logger, + mean_seconds, + normalize_run_id, +) from vm_instance_common import ( append_summary_csv, cleanup_by_filter, @@ -61,12 +69,13 @@ fake_delete, load_config, make_cloudstack_client, - new_run_id, RawLogger, report_failed_creates, resolve_protocols, ) +log = get_logger() + def run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay): infra = cfg["infrastructure"] @@ -79,39 +88,39 @@ def run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, created = [] failed = [] create_durations = [] - print(f"\n=== [5.2.1] Sequential CREATE - protocol={protocol_key} up to N={max_n} ===") + log.info("[5.2.1] Sequential CREATE protocol=%s up to N=%s", protocol_key, max_n) for i in range(1, max_n + 1): - # CloudStack VM names are used as hostnames (RFC1123): letters, digits, - # and "-" only (no "_", the opposite of ONTAP's volume-name rule). - name = f"{prefix}-seq-{protocol_key}-{run_id}-{i:03d}".replace("_", "-") + name = format_vm_name(prefix, protocol_key, run_id, i) result = fake_create(name) if dry_run else deploy_vm(client, name, infra, vm_cfg, proto_cfg) raw_logger.log(run_id, "sequential_create", "5.2.1", protocol_key, max_n, i, result) status = "OK" if result.success else "FAIL" - print(f" [{i:>3}/{max_n}] create {name} -> {status} ({result.duration_sec:.3f}s)") + log.info("[%s/%s] create %s -> %s (%.3fs)", i, max_n, name, status, result.duration_sec) if result.success: created.append((name, result.vm_id)) create_durations.append(result.duration_sec) else: - print(f" !! {result.error}") + log.error("%s", result.error) failed.append((name, result.vm_id)) if i in checkpoints: total = sum(create_durations) - avg = total / len(create_durations) if create_durations else 0 + avg = mean_seconds(create_durations) summary_rows.append({ "run_id": run_id, "phase": "sequential_create", "test_id": "5.2.1", "protocol": protocol_key, "checkpoint": i, "total_time_sec": round(total, 3), "avg_time_sec": round(avg, 3), "success_count": len(created), "failure_count": i - len(created), "notes": "", }) - print(f" >> checkpoint N={i}: total={total:.3f}s avg={avg:.3f}s/op " - f"success={len(created)} failure={i - len(created)}") + log.info( + "checkpoint N=%s protocol=%s total=%.3fs avg=%.3fs/op success=%s failure=%s", + i, protocol_key, total, avg, len(created), i - len(created), + ) if delay: time.sleep(delay) report_failed_creates(failed) total_created = len(created) - print(f"\n=== [5.2.2] Sequential DELETE - protocol={protocol_key} from N={total_created} remaining ===") + log.info("[5.2.2] Sequential DELETE protocol=%s from N=%s remaining", protocol_key, total_created) if total_created in checkpoints: summary_rows.append({ "run_id": run_id, "phase": "sequential_delete", "test_id": "5.2.2", @@ -126,85 +135,97 @@ def run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, remaining = total_created - idx raw_logger.log(run_id, "sequential_delete", "5.2.2", protocol_key, total_created, idx, result) status = "OK" if result.success else "FAIL" - print(f" [{idx:>3}/{total_created}] delete {name} -> {status} " - f"({result.duration_sec:.3f}s) remaining={remaining}") + log.info( + "[%s/%s] delete %s -> %s (%.3fs) remaining=%s", + idx, total_created, name, status, result.duration_sec, remaining, + ) if result.success: delete_durations.append(result.duration_sec) deleted_ok += 1 else: - print(f" !! {result.error}") + log.error("%s", result.error) if remaining in checkpoints: total = sum(delete_durations) - avg = total / len(delete_durations) if delete_durations else 0 + avg = mean_seconds(delete_durations) summary_rows.append({ "run_id": run_id, "phase": "sequential_delete", "test_id": "5.2.2", "protocol": protocol_key, "checkpoint": remaining, "total_time_sec": round(total, 3), "avg_time_sec": round(avg, 3), "success_count": deleted_ok, "failure_count": idx - deleted_ok, "notes": "", }) - print(f" >> checkpoint remaining={remaining}: total={total:.3f}s avg={avg:.3f}s/op " - f"success={deleted_ok} failure={idx - deleted_ok}") + log.info( + "checkpoint remaining=%s protocol=%s total=%.3fs avg=%.3fs/op success=%s failure=%s", + remaining, protocol_key, total, avg, deleted_ok, idx - deleted_ok, + ) if delay: time.sleep(delay) -def main(): - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") - parser.add_argument("--protocol", default="both", help="nfs3 | iscsi | both (default: both)") - parser.add_argument("--run-id", default=None, help="Override auto-generated run id") - parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") - parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover VMs from this run in place") - parser.add_argument( - "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", - help="Destroy all VMs whose name contains FILTER (default: config vm_name_prefix) and exit", - ) - args = parser.parse_args() - - cfg = load_config(args.config) - os.makedirs(cfg["vm_bench"].get("output_dir", "results"), exist_ok=True) +def execute(cfg, dry_run=False, skip_cleanup=False, cleanup_only=None, run_id=None): + configure_logging() + assert_vm_name_prefix(cfg["vm_bench"]["vm_name_prefix"]) + output_dir = cfg["vm_bench"].get("output_dir", "results") + os.makedirs(output_dir, exist_ok=True) - if args.cleanup_only is not None: + if cleanup_only is not None: client = make_cloudstack_client(cfg) - name_filter = args.cleanup_only - if name_filter == "__PREFIX__": - name_filter = cfg["vm_bench"]["vm_name_prefix"] - cleanup_by_filter(client, name_filter) + name_filter = cfg["vm_bench"]["vm_name_prefix"] if cleanup_only == "__PREFIX__" else cleanup_only + cleanup_by_filter(client, name_filter, cfg) return - run_id = args.run_id or new_run_id() - protocols = resolve_protocols(cfg, args.protocol) + run_id = normalize_run_id(run_id) + protocols = resolve_protocols(cfg, "both") delay = cfg["vm_bench"].get("inter_op_delay_sec", 0) - output_dir = cfg["vm_bench"].get("output_dir", "results") - print(f"Run ID: {run_id}") - print(f"Protocols: {protocols}") - print("Mode: sequential") - print(f"Dry run: {args.dry_run}") + log.info("Run ID: %s", run_id) + log.info("Protocols: %s", protocols) + log.info("Mode: sequential") + log.info("Dry run: %s", dry_run) - client = None if args.dry_run else make_cloudstack_client(cfg) + client = None if dry_run else make_cloudstack_client(cfg) raw_logger = RawLogger(os.path.join(output_dir, f"raw_ops_vm_{run_id}.csv")) summary_rows = [] try: for protocol_key in protocols: - run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, args.dry_run, delay) + run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows, dry_run, delay) finally: raw_logger.close() summary_path = os.path.join(output_dir, f"summary_vm_{run_id}.csv") append_summary_csv(summary_path, summary_rows) - print(f"\nRaw per-operation log: {os.path.join(output_dir, f'raw_ops_vm_{run_id}.csv')}") - print(f"Checkpoint summary: {summary_path}") - print("Next: python3 render_report.py --run-id " + run_id + - f" --output-dir {output_dir} --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm") - print(f" (or run benchmark_vm_instance_concurrency.py --run-id {run_id} first to add 6.2.x to the same report)") + log.info("Raw per-operation log: %s", os.path.join(output_dir, f"raw_ops_vm_{run_id}.csv")) + log.info("Checkpoint summary: %s", summary_path) + log.info( + "Next: python3 render_report.py --run-id %s --output-dir %s --raw-prefix raw_ops_vm --summary-prefix summary_vm --report-suffix _vm", + run_id, output_dir, + ) + + if not dry_run and not skip_cleanup: + log.info("Verifying no orphaned VMs remain for run %s", run_id) + cleanup_by_filter(client, run_id, cfg) + - if not args.dry_run and not args.skip_cleanup: - print(f"\nVerifying no orphaned VMs remain for run {run_id}...") - cleanup_by_filter(client, run_id) +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--run-id", default=None, help="Override auto-generated run id") + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave any leftover VMs from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Destroy benchmark VMs whose name contains FILTER (default: config vm_name_prefix) and exit", + ) + args = parser.parse_args() + execute( + load_config(args.config), + dry_run=args.dry_run, + skip_cleanup=args.skip_cleanup, + cleanup_only=args.cleanup_only, + run_id=args.run_id, + ) if __name__ == "__main__": diff --git a/private-cicd/benchmark/ontap/cloudstack_client.py b/private-cicd/benchmark/ontap/cloudstack_client.py index 7de56435fd8d..0baed80b9c1e 100644 --- a/private-cicd/benchmark/ontap/cloudstack_client.py +++ b/private-cicd/benchmark/ontap/cloudstack_client.py @@ -20,8 +20,7 @@ scripts (private-cicd/benchmark/ontap/). Only session-key based auth (login -> sessionkey + JSESSIONID cookie) is -implemented, matching the approach documented in: -https://netapp.atlassian.net/wiki/spaces/OSSG/pages/608854350/CloudStack+API +implemented. API key/secret signed requests are not. Every call is timed end-to-end (including async job polling, since that is part of the wall-clock cost an operator/automation actually pays) and the @@ -33,7 +32,7 @@ import requests -log = logging.getLogger("cloudstack_client") +log = logging.getLogger("ontap.benchmark.client") class CloudStackAPIError(Exception): diff --git a/private-cicd/benchmark/ontap/config.example.yaml b/private-cicd/benchmark/ontap/config.example.yaml index 1c2ae3ff23ca..606298a06ca9 100644 --- a/private-cicd/benchmark/ontap/config.example.yaml +++ b/private-cicd/benchmark/ontap/config.example.yaml @@ -42,9 +42,8 @@ infrastructure: clusterid: "23788acb-1a63-45a9-8c61-5ed8f5374bc6" scope: "cluster" -# One entry per protocol you want to benchmark. Keys must be "nfs3"/"iscsi" -# to match --protocol on the CLI. Remove a block if you don't have that -# backend available. +# One entry per protocol you want to benchmark. Keys must be "nfs3"/"iscsi". +# Remove a block to skip that protocol. # NOTE: the ONTAP plugin (StorageStrategy.java) builds its REST API base URL # from storageIP for EVERY call (aggregates/volumes/svm/jobs/network/san/nas) - # it never falls back to `url`. storageIP MUST answer HTTPS/443 with the ONTAP @@ -77,6 +76,8 @@ ontap: benchmark: pool_name_prefix: "bench_ontap" # ONTAP volume names disallow "-"; use "_" only + # Checkpoints and concurrency levels are the combinations for a run. + # Change them here. sequential_checkpoints: [1, 5, 10, 20, 30] concurrency_levels: [2, 5, 10, 20, 30] # Pause between deleteStoragePool calls to give the mgmt server GC/ONTAP a @@ -104,6 +105,9 @@ vm_bench: # CloudStack VM names are used as hostnames (RFC1123): letters/digits/"-" only, # no "_" (the opposite of ONTAP's volume-name rule used by pool_name_prefix above). vm_name_prefix: "bench-vm" + # Unattached data disks are deleted only from pools with this name prefix, + # and only when that pool's provider and zone match this config. + cleanup_pool_name_prefix: "bench_vm" sequential_checkpoints: [1, 5, 10, 20] concurrency_levels: [2, 5, 10] inter_op_delay_sec: 0 diff --git a/private-cicd/benchmark/ontap/render_report.py b/private-cicd/benchmark/ontap/render_report.py index 85935598df7a..ee0d1946964d 100644 --- a/private-cicd/benchmark/ontap/render_report.py +++ b/private-cicd/benchmark/ontap/render_report.py @@ -16,15 +16,12 @@ # specific language governing permissions and limitations # under the License. """ -Turn benchmark_storage_pool_sequential.py / benchmark_storage_pool_concurrency.py's -raw_ops_.csv / summary_.csv into markdown tables that match the -layout of the storage-pool sections -(5.1.1, 5.1.2, 6.1.1, 6.1.2) and the Results Log (section 9) of the -"ONTAP Plugin - CloudStack Operations, Scale & Parallel Test Matrix" -Confluence page, ready to paste back in. +Turn a benchmark run's raw and summary CSVs into markdown tables for sections +5.1.1, 5.1.2, 6.1.1, 6.1.2 (storage pools) or 5.2.x / 6.2.x (VM instances), +plus Results Log rows. Usage: - python3 render_report.py --run-id RUN-20260722-120000 \ + python3 render_report.py --run-id RUN_20260722_120000_ab12 \ --cloudstack-build 4.23.0.0-SNAPSHOT --ontap-version 9.15.1 """ @@ -120,7 +117,7 @@ def render_concurrency_table(summary_rows, test_id): rows = [r for r in summary_rows if r["test_id"] == test_id] rows.sort(key=lambda r: (int(r["checkpoint"]), PROTOCOL_ORDER.index(r["protocol"]) if r["protocol"] in PROTOCOL_ORDER else 99)) - lines = ["| Concurrency (C) | Protocol | Total Wall-clock (s) | Success | Failure | Avg Time/pool (s) | Notes |", + lines = ["| Concurrency (C) | Protocol | Total Wall-clock (s) | Success | Failure | Avg Time/op (s) | Notes |", "|---|---|---|---|---|---|---|"] for r in rows: lines.append( diff --git a/private-cicd/benchmark/ontap/run.py b/private-cicd/benchmark/ontap/run.py new file mode 100644 index 000000000000..691150da5b6c --- /dev/null +++ b/private-cicd/benchmark/ontap/run.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Single entry point for the ONTAP scale benchmarks. + +Protocols, checkpoints, and concurrency levels come from config.yaml. Drop a +protocol block to skip it, and edit sequential_checkpoints or concurrency_levels +to change the matrix. The selected test reads those fields as written. + +Usage: + python3 run.py --test storage-pool-sequential --config config.yaml + python3 run.py --test storage-pool-concurrency --config config.yaml --dry-run + python3 run.py --test vm-instance-combined --config config.yaml + python3 run.py --test vm-instance-sequential --config config.yaml --cleanup-only +""" + +import argparse + +from benchmark_storage_pool_concurrency import execute as run_storage_pool_concurrency +from benchmark_storage_pool_sequential import execute as run_storage_pool_sequential +from benchmark_support import configure_logging +from benchmark_vm_instance_combined import execute as run_vm_instance_combined +from benchmark_vm_instance_concurrency import execute as run_vm_instance_concurrency +from benchmark_vm_instance_sequential import execute as run_vm_instance_sequential +from storage_pool_common import load_config + +TESTS = { + "storage-pool-sequential": run_storage_pool_sequential, + "storage-pool-concurrency": run_storage_pool_concurrency, + "vm-instance-sequential": run_vm_instance_sequential, + "vm-instance-concurrency": run_vm_instance_concurrency, + "vm-instance-combined": run_vm_instance_combined, +} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--test", required=True, choices=sorted(TESTS), help="Benchmark to run") + parser.add_argument("--config", default="config.yaml", help="Path to config YAML (default: config.yaml)") + parser.add_argument("--run-id", default=None, help="Reuse a run id (auto-generated timestamp + random suffix otherwise)") + parser.add_argument("--dry-run", action="store_true", help="Simulate timings, no real API calls") + parser.add_argument("--skip-cleanup", action="store_true", help="Leave leftovers from this run in place") + parser.add_argument( + "--cleanup-only", nargs="?", const="__PREFIX__", default=None, metavar="FILTER", + help="Delete benchmark resources whose name contains FILTER (default: configured name prefix) and exit", + ) + args = parser.parse_args() + + configure_logging() + cfg = load_config(args.config) + runner = TESTS[args.test] + runner( + cfg, + dry_run=args.dry_run, + skip_cleanup=args.skip_cleanup, + cleanup_only=args.cleanup_only, + run_id=args.run_id, + ) + + +if __name__ == "__main__": + main() diff --git a/private-cicd/benchmark/ontap/storage_pool_common.py b/private-cicd/benchmark/ontap/storage_pool_common.py index e01c50c0e69d..496d69804334 100644 --- a/private-cicd/benchmark/ontap/storage_pool_common.py +++ b/private-cicd/benchmark/ontap/storage_pool_common.py @@ -29,6 +29,7 @@ import datetime import os import random +import re import sys import time @@ -37,6 +38,7 @@ except ImportError: sys.exit("PyYAML is required: pip install -r requirements.txt") +from benchmark_support import get_logger from cloudstack_client import CloudStackAPIError, CloudStackClient RAW_FIELDNAMES = [ @@ -66,12 +68,6 @@ def now_iso(): return datetime.datetime.utcnow().isoformat(timespec="milliseconds") + "Z" -def new_run_id(): - # ONTAP volume names only allow alphanumeric + underscore (no hyphens), and - # pool names flow straight through to the ONTAP volume name, so avoid "-" here. - return datetime.datetime.utcnow().strftime("RUN_%Y%m%d_%H%M%S") - - def build_create_pool_params(name, infra_cfg, ontap_cfg): params = { "name": name, @@ -178,19 +174,66 @@ def append_summary_csv(path, rows): writer.writerows(rows) -def cleanup_by_filter(client, name_filter): +def _pool_name_pattern(prefix): + return re.compile( + rf"^{re.escape(prefix)}_(?:seq|c\d+)_[A-Za-z0-9]+_[A-Za-z0-9_]+_\d{{3}}$" + ) + + +def cleanup_by_filter(client, name_filter, cfg): + """Delete benchmark pools only. + + A shared zone can contain pools this run did not create. A match has to use + this config's name prefix and naming pattern, sit in the configured zone and + cluster, and use one of the configured storage providers. + """ + log = get_logger() + if not name_filter or len(name_filter) < 4: + sys.exit("Refusing cleanup: name filter is missing or shorter than 4 characters") + + prefix = cfg["benchmark"]["pool_name_prefix"] + infra = cfg.get("infrastructure", {}) + zoneid = infra.get("zoneid") + clusterid = infra.get("clusterid") if infra.get("scope", "cluster") == "cluster" else None + providers = { + str(entry.get("provider", "")).strip().lower() + for entry in cfg.get("ontap", {}).values() + if entry.get("provider") + } + pattern = _pool_name_pattern(prefix) + payload, _ = client.call("listStoragePools", {}, poll_async=False) pools = payload.get("storagepool", []) if isinstance(payload, dict) else [] - matches = [p for p in pools if name_filter in p.get("name", "")] - print(f"Found {len(matches)} pool(s) whose name contains '{name_filter}'") - for p in matches: + name_hits = [p for p in pools if name_filter in (p.get("name") or "")] + matches = [] + for pool in name_hits: + name = pool.get("name") or "" + provider = str(pool.get("provider") or "").strip().lower() + if not pattern.match(name): + continue + if providers and provider not in providers: + continue + if zoneid and pool.get("zoneid") and pool.get("zoneid") != zoneid: + continue + if clusterid and pool.get("clusterid") and pool.get("clusterid") != clusterid: + continue + matches.append(pool) + + skipped = len(name_hits) - len(matches) + log.info("Found %s pool(s) eligible for cleanup (filter=%r)", len(matches), name_filter) + if skipped: + log.warning( + "Skipped %s pool(s) whose name contains %r but failed the prefix, zone, cluster, or provider check", + skipped, name_filter, + ) + for pool in matches: try: - if p.get("state") != "Maintenance": - client.call("enableStorageMaintenance", {"id": p["id"]}) - client.call("deleteStoragePool", {"id": p["id"], "forced": "true"}) - print(f" deleted {p['name']} ({p['id']})") + if pool.get("state") != "Maintenance": + client.call("enableStorageMaintenance", {"id": pool["id"]}) + client.call("deleteStoragePool", {"id": pool["id"], "forced": "true"}) + log.info("deleted %s (%s)", pool["name"], pool["id"]) except Exception as exc: # noqa: BLE001 - print(f" FAILED to delete {p['name']}: {exc}") + log.error("FAILED to delete %s: %s", pool["name"], exc) def load_config(path): diff --git a/private-cicd/benchmark/ontap/vm_instance_common.py b/private-cicd/benchmark/ontap/vm_instance_common.py index b671f3c576dc..8824c9ae625d 100644 --- a/private-cicd/benchmark/ontap/vm_instance_common.py +++ b/private-cicd/benchmark/ontap/vm_instance_common.py @@ -40,6 +40,7 @@ except ImportError: sys.exit("PyYAML is required: pip install -r requirements.txt") +from benchmark_support import get_logger, vm_name_token from cloudstack_client import CloudStackAPIError, CloudStackClient RAW_FIELDNAMES = [ @@ -69,10 +70,6 @@ def now_iso(): return datetime.datetime.utcnow().isoformat(timespec="milliseconds") + "Z" -def new_run_id(): - return datetime.datetime.utcnow().strftime("RUN_%Y%m%d_%H%M%S") - - def build_deploy_vm_params(name, infra_cfg, vm_cfg, proto_cfg): return { "name": name, @@ -90,14 +87,14 @@ def build_deploy_vm_params(name, infra_cfg, vm_cfg, proto_cfg): def _extract_vm_id_from_error(error_text): - """When deployVirtualMachine's async job fails deep in orchestration (e.g. the - ENOSPC-driven "Unable to orchestrate the start of VM instance" failures - see - Confluence Issue #10), CloudStack has ALREADY created the VM (and its ROOT/DATA - volume records) before the failure - it just never finished starting it. The - job's errortext embeds that VM's uuid (e.g. '...{"instanceName":"i-2-107-VM", - "uuid":"b8f4..."}.'), so scrape it out here so the leftover VM/volume can at - least be identified and reported for manual inspection (see - report_failed_creates()) instead of being silently invisible.""" + """When deployVirtualMachine's async job fails deep in orchestration (for example + an ENOSPC-driven "Unable to orchestrate the start of VM instance" failure), + CloudStack has ALREADY created the VM (and its ROOT/DATA volume records) before + the failure - it just never finished starting it. The job's errortext embeds + that VM's uuid (e.g. '...{"instanceName":"i-2-107-VM", "uuid":"b8f4..."}.'), + so scrape it out here so the leftover VM/volume can at least be identified and + reported for manual inspection (see report_failed_creates()) instead of being + silently invisible.""" if not error_text: return None m = _UUID_RE.search(error_text) @@ -135,7 +132,7 @@ def _force_purge_destroy_state_volumes(client, volume_ids, max_wait_sec=10): calls deleteVolume() on them explicitly - which force-deletes a Destroy-state volume immediately, bypassing the delay. Without this, repeated benchmark runs progressively fill up the dedicated bench_vm_ pool with - "deleted" disks, compounding the ENOSPC failures in Confluence Issue #10. + "deleted" disks, compounding ENOSPC failures on the dedicated bench pool. """ deadline = time.time() + max_wait_sec remaining = set(volume_ids) @@ -244,8 +241,8 @@ def append_summary_csv(path, rows): def report_failed_creates(failed): - """Failed deployVirtualMachine calls (e.g. the ENOSPC "Unable to orchestrate - the start of VM instance" failures in Confluence Issue #10) still leave a real + """Failed deployVirtualMachine calls (for example ENOSPC "Unable to orchestrate + the start of VM instance" failures) still leave a real VM + ROOT/DATA volume record behind in CloudStack - they just never finished starting. These are intentionally NOT auto-destroyed here (unlike the `created` list, which the benchmark itself deletes as part of 5.2.2/6.2.2) so @@ -253,45 +250,114 @@ def report_failed_creates(failed): `listVolumes --state Destroy` on the bench_vm_ pool - to confirm exactly what got left behind and why. Clean them up manually (or via --cleanup-only) once you're done inspecting them.""" + log = get_logger() failed_with_id = [(name, vm_id) for name, vm_id in failed if vm_id] if not failed_with_id: return - print(f"\n=== {len(failed_with_id)} failed-create VM(s) left in place for inspection (not cleaned up) ===") + log.info("%s failed-create VM(s) left in place for inspection (not cleaned up)", len(failed_with_id)) for name, vm_id in failed_with_id: - print(f" {name} ({vm_id})") + log.info("left in place: %s (%s)", name, vm_id) + + +def _vm_name_pattern(prefix): + return re.compile( + rf"^{re.escape(prefix)}-(?:seq|c\d+)-[A-Za-z0-9]+-[A-Za-z0-9-]+-\d{{3}}$" + ) -def cleanup_by_filter(client, name_filter): +def _allowed_orphan_pool_names(client, cfg): + """Pools this config is allowed to reclaim unattached data disks from.""" + vm_cfg = cfg.get("vm_bench", {}) + pool_prefix = vm_cfg.get("cleanup_pool_name_prefix", "bench_vm") + infra = cfg.get("infrastructure", {}) + zoneid = infra.get("zoneid") + providers = { + str(entry.get("provider", "")).strip().lower() + for entry in cfg.get("ontap", {}).values() + if entry.get("provider") + } + payload, _ = client.call("listStoragePools", {}, poll_async=False) + pools = payload.get("storagepool", []) if isinstance(payload, dict) else [] + allowed = set() + for pool in pools: + name = pool.get("name") or "" + provider = str(pool.get("provider") or "").strip().lower() + if not name.startswith(pool_prefix): + continue + if providers and provider not in providers: + continue + if zoneid and pool.get("zoneid") and pool.get("zoneid") != zoneid: + continue + allowed.add(name) + return allowed + + +def cleanup_by_filter(client, name_filter, cfg): + """Destroy benchmark VMs only. + + VM names rewrite '_' to '-' (hostnames). The filter is rewritten the same + way, otherwise a run id such as RUN_20260927_081500 never matches the VM + that was actually created. A shared zone is protected by the name prefix, + the benchmark name pattern, and the configured zone. + """ + log = get_logger() + token = vm_name_token(name_filter) + if not token or len(token) < 4: + sys.exit("Refusing cleanup: name filter is missing or shorter than 4 characters") + + prefix = cfg["vm_bench"]["vm_name_prefix"] + zoneid = cfg.get("infrastructure", {}).get("zoneid") + pattern = _vm_name_pattern(prefix) + payload, _ = client.call("listVirtualMachines", {}, poll_async=False) vms = payload.get("virtualmachine", []) if isinstance(payload, dict) else [] - matches = [v for v in vms if name_filter in v.get("name", "")] - print(f"Found {len(matches)} VM(s) whose name contains '{name_filter}'") - for v in matches: + name_hits = [v for v in vms if token in (v.get("name") or "")] + matches = [] + for vm in name_hits: + name = vm.get("name") or "" + if not pattern.match(name): + continue + if zoneid and vm.get("zoneid") and vm.get("zoneid") != zoneid: + continue + matches.append(vm) + + skipped = len(name_hits) - len(matches) + log.info("Found %s VM(s) eligible for cleanup (filter=%r, match token=%r)", len(matches), name_filter, token) + if skipped: + log.warning( + "Skipped %s VM(s) whose name contains %r but failed the prefix, name pattern, or zone check", + skipped, token, + ) + for vm in matches: try: - result = destroy_vm(client, v["id"], v["name"]) + result = destroy_vm(client, vm["id"], vm["name"]) if result.success: - print(f" destroyed {v['name']} ({v['id']}) [incl. any data disks]") + log.info("destroyed %s (%s) including any data disks", vm["name"], vm["id"]) else: - print(f" FAILED to destroy {v['name']}: {result.error}") + log.error("FAILED to destroy %s: %s", vm["name"], result.error) except Exception as exc: # noqa: BLE001 - print(f" FAILED to destroy {v['name']}: {exc}") + log.error("FAILED to destroy %s: %s", vm["name"], exc) # Data disks can also end up orphaned (unattached) from prior runs whose - # VM was already destroyed without this volumeids fix. Volume names (e.g. - # "DATA-25") don't carry the run id/prefix, so instead of matching - # name_filter here, sweep any unattached data disk left on our dedicated - # bench_vm_* pools - nothing else legitimately lives there. + # VM was already destroyed without the volumeids argument. Volume names + # (e.g. "DATA-25") don't carry the run id, so only unattached data disks + # on pools that match this config's provider, zone, and pool-name prefix + # are deleted. + allowed_pools = _allowed_orphan_pool_names(client, cfg) vol_payload, _ = client.call("listVolumes", {"type": "DATADISK"}, poll_async=False) vols = vol_payload.get("volume", []) if isinstance(vol_payload, dict) else [] - orphan_vols = [v for v in vols if not v.get("virtualmachineid") and v.get("storage", "").startswith("bench_vm")] + orphan_vols = [ + v for v in vols + if not v.get("virtualmachineid") and (v.get("storage") or "") in allowed_pools + ] if orphan_vols: - print(f"Found {len(orphan_vols)} orphaned unattached DATADISK volume(s) on matching pools") - for v in orphan_vols: + log.info("Found %s orphaned unattached DATADISK volume(s) on configured pools", len(orphan_vols)) + for vol in orphan_vols: try: - client.call("deleteVolume", {"id": v["id"]}) - print(f" deleted orphan volume {v['name']} ({v['id']})") + client.call("deleteVolume", {"id": vol["id"]}) + log.info("deleted orphan volume %s (%s)", vol["name"], vol["id"]) except Exception as exc: # noqa: BLE001 - print(f" FAILED to delete orphan volume {v['name']}: {exc}") + log.error("FAILED to delete orphan volume %s: %s", vol["name"], exc) def load_config(path):