diff --git a/Dockerfile.dev b/Dockerfile.dev index 1c747f8d8..83e99ab8b 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -38,7 +38,8 @@ RUN --mount=type=cache,target=/root/.cache/pip /opt/venv/bin/pip install -r requ WORKDIR /app RUN mkdir -p /etc/cron.d && mv /app/setup_utils/cronjob.txt /etc/cron.d/cronjob -RUN chmod +x /app/setup_utils/start.sh /app/setup_utils/init_db.sh /app/setup_utils/generate_config_ini.sh +# CLUSTOX: trigger_sync.sh added so cron can send the internal API token. +RUN chmod +x /app/setup_utils/start.sh /app/setup_utils/init_db.sh /app/setup_utils/generate_config_ini.sh /app/setup_utils/trigger_sync.sh RUN mv /app/setup_utils/supervisord.conf /etc/supervisord.conf RUN mv /app/database-docker/db/ /app/ && rm -rf /app/database-docker/ RUN echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/15/main/pg_hba.conf diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..d1529adf0 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,301 @@ +pipeline { + agent any + + environment { + APP_DIR = '/opt/app/middleware' + // CLUSTOX: the production compose file, not the default one. See the + // Build & Deploy stage for why that distinction is the whole point. + COMPOSE_FILE = 'docker-compose.prod.yml' + } + + stages { + stage('Checkout') { + steps { + checkout scm + } + } + + stage('Update Deployment Copy') { + steps { + sh ''' + cd ${APP_DIR} + git fetch origin main + git reset --hard origin/main + ''' + } + } + + // CLUSTOX: .env is gitignored, so a release that needs a new variable + // cannot deliver it. Without this stage the app builds, starts, and + // then fails at runtime in ways the health check cannot see -- an unset + // INTERNAL_API_TOKEN makes the Flask servers reject every request, and + // an unset SUPERADMIN_PASSWORD means no account is ever created. + // + // Failing here, before anything is torn down, leaves the running + // deployment alive through a misconfigured release. + stage('Verify Environment') { + steps { + sh ''' + cd ${APP_DIR} + + if [ ! -f .env ]; then + echo "FATAL: ${APP_DIR}/.env does not exist." + exit 1 + fi + + MISSING="" + for VAR in NEXTAUTH_URL NEXTAUTH_SECRET INTERNAL_API_TOKEN \ + SUPERADMIN_EMAIL SUPERADMIN_PASSWORD; do + # Present AND non-empty. `grep -q "^VAR="` alone would + # pass on a variable set to nothing. + if ! grep -qE "^${VAR}=.+" .env; then + MISSING="${MISSING} ${VAR}" + fi + done + + if [ -n "${MISSING}" ]; then + echo "FATAL: required variables missing or empty in ${APP_DIR}/.env:" + for VAR in ${MISSING}; do echo " - ${VAR}"; done + echo "" + echo "See env.example. Nothing has been changed." + exit 1 + fi + + echo "All required environment variables are present." + ''' + } + } + + stage('SonarQube Analysis') { + steps { + // CLUSTOX: placed before Build/Deploy on purpose -- this is a deploy + // pipeline, so the gate below has to be able to stop a release. It sits + // after Verify Environment so a missing .env variable still fails fast + // and cheaply, before spending a scan. + // + // The agent has Docker but no sonar-scanner CLI and no SonarQube Scanner + // tool installation, so the official scanner image is used. + // sonar.working.directory is redirected into the bind-mounted workspace: + // the image's baked-in /tmp/.scannerwork is owned by uid 1000 and + // unwritable under -u, and the override also lands report-task.txt where + // waitForQualityGate looks for it. + withSonarQubeEnv('MySonarQube') { + sh ''' + docker run --rm \ + -u "$(id -u):$(id -g)" \ + -e SONAR_HOST_URL="$SONAR_HOST_URL" \ + -e SONAR_TOKEN="$SONAR_AUTH_TOKEN" \ + -e SONAR_USER_HOME=/tmp/.sonar \ + -v "$WORKSPACE:/usr/src" \ + -w /usr/src \ + sonarsource/sonar-scanner-cli:latest \ + -Dsonar.working.directory=/usr/src/.scannerwork \ + -Dsonar.projectVersion="${GIT_COMMIT:-$BUILD_NUMBER}" + ''' + } + } + } + + stage('Quality Gate') { + steps { + // CLUSTOX: bounded wait. waitForQualityGate depends on SonarQube calling + // back to /sonarqube-webhook/; without the timeout a missed webhook would + // hang this stage until the build is killed, holding the deploy hostage. + timeout(time: 5, unit: 'MINUTES') { + waitForQualityGate abortPipeline: true + } + } + } + + stage('Fetch Sonar Issues') { + steps { + // CLUSTOX: reporting only -- catchError keeps a SonarQube outage or an + // expired token from blocking a deploy that already passed the gate. Uses + // the read-only token, not the scan credential, and sends it as an + // Authorization header so it cannot land in proxy or access logs. + catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + withCredentials([string(credentialsId: 'sonarqube-readonly-token', variable: 'SONAR_RO_TOKEN')]) { + sh ''' + set -eu + PS=500 + TMP=$(mktemp -d) + trap 'rm -rf "$TMP"' EXIT + page=1 + while : ; do + curl --fail --silent --show-error \ + -H "Authorization: Bearer $SONAR_RO_TOKEN" \ + -o "$TMP/page-$page.json" \ + "https://sonar.theclustox.com/api/issues/search?componentKeys=middleware&resolved=false&ps=$PS&p=$page" + total=$(jq -r '.paging.total' "$TMP/page-$page.json") + fetched=$(( page * PS )) + echo "fetched page $page (up to $fetched of $total open issues)" + [ "$fetched" -ge "$total" ] && break + [ "$fetched" -ge 10000 ] && { echo "WARNING: capped at 10000 issues"; break; } + page=$(( page + 1 )) + done + # The warnings-ng SonarQube parser sniffs the response format from + # the top-level keys, so the merged document has to look like one + # big api/issues/search page -- dropping total/p/ps makes it + # silently parse to zero issues. + jq -s ' + (map(.issues) | add) as $iss | + (.[0].paging.total) as $tot | + { + total: $tot, + p: 1, + ps: ($iss | length), + paging: {pageIndex: 1, pageSize: ($iss | length), total: $tot}, + effortTotal: (map(.effortTotal // 0) | add), + issues: $iss, + components: (map(.components // []) | add | unique_by(.key)) + }' "$TMP"/page-*.json > sonar-issues.json + echo "merged $(jq '.issues | length' sonar-issues.json) issues into sonar-issues.json" + ''' + } + } + } + } + + stage('Publish Issue Report') { + steps { + catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + recordIssues(tools: [sonarQube(pattern: 'sonar-issues.json')]) + } + } + } + + stage('Build') { + steps { + // CLUSTOX: builds docker-compose.prod.yml, which uses the + // production Dockerfile and runs `yarn build`. The default + // docker-compose.yml is upstream's *local development* file: it + // builds Dockerfile.dev, which never builds the frontend, so the + // container serves the app through `next dev` and compiles every + // route inside the first request that asks for it. Measured on a + // cold container: /login 7.5s, /workspaces 5.7s, /users 21.8s, + // against 0.02s once warm. + // + // The build is deliberately its own stage. `docker compose build + // && docker compose up -d` in one step is a trap: when the build + // fails, `up -d` happily recreates the container from the + // PREVIOUS image, so a broken release presents as a healthy app + // serving stale code and the pipeline reports success. That is + // not hypothetical -- it happened during development when a + // dependency pin was wrong for the image's Python version. + sh ''' + cd ${APP_DIR} + docker compose -f ${COMPOSE_FILE} build + ''' + } + } + + stage('Deploy') { + steps { + sh ''' + cd ${APP_DIR} + docker compose -f ${COMPOSE_FILE} up -d + ''' + } + } + + stage('Health Check') { + steps { + // CLUSTOX: the previous check was `sleep 10` then a curl of `/`. + // Both halves were wrong. `/` answers 307 to /login for a + // signed-out request and curl treats a redirect as success, so + // it passed against an app whose API was entirely broken. And + // 10s is short for a cold start that also runs migrations, so it + // produced false failures too. + // + // Instead: poll until the app answers, then assert the API + // returns 401. Reaching 401 proves Next.js is serving, the + // database is reachable, and auth is wired up. A misconfigured + // instance returns 500 or refuses the connection. + sh ''' + PORT_NUM=${PORT:-3333} + BASE="http://127.0.0.1:${PORT_NUM}" + + echo "Waiting for the app to accept connections..." + READY=0 + for i in $(seq 1 60); do + if curl -sf -o /dev/null "${BASE}/login"; then + READY=1 + echo "App responded after $((i * 5))s." + break + fi + sleep 5 + done + + if [ "${READY}" != "1" ]; then + echo "FATAL: app did not respond within 300s." + cd ${APP_DIR} && docker compose -f ${COMPOSE_FILE} logs --tail=80 + exit 1 + fi + + echo "Checking that the API is up and enforcing auth..." + CODE=$(curl -s -o /dev/null -w '%{http_code}' "${BASE}/api/clustox/me") + if [ "${CODE}" != "401" ]; then + echo "FATAL: /api/clustox/me returned ${CODE}, expected 401." + echo "The app is serving pages but the API is not healthy." + cd ${APP_DIR} && docker compose -f ${COMPOSE_FILE} logs --tail=80 + exit 1 + fi + + echo "Health check passed." + ''' + } + } + + // CLUSTOX: proves the deploy is actually serving a compiled bundle + // rather than silently falling back to the dev server. A production + // build answers a cold page request in well under a second; `next dev` + // takes seconds to tens of seconds because it compiles on demand. + // + // This is a warning rather than a failure: a slow disk or a loaded host + // could trip it without the deploy being wrong, and failing a healthy + // release over a timing measurement would be worse than the problem. + stage('Verify Production Build') { + steps { + sh ''' + BASE="http://127.0.0.1:${PORT:-3333}" + + ELAPSED=$(curl -s -o /dev/null -w '%{time_total}' "${BASE}/login") + echo "Cold /login response: ${ELAPSED}s" + + # Integer compare; `sh` has no float arithmetic. + WHOLE=$(echo "${ELAPSED}" | cut -d. -f1) + if [ "${WHOLE}" -ge 3 ]; then + echo "WARNING: ${ELAPSED}s is slow for a compiled build." + echo "The container may be running 'next dev' -- check that" + echo "ENVIRONMENT=prod reached it and that ${COMPOSE_FILE} was used." + else + echo "Response time is consistent with a production build." + fi + ''' + } + } + } + + post { + success { + echo 'Deployment succeeded.' + withCredentials([string(credentialsId: 'slack-webhook-ci-build-alerts', variable: 'SLACK_WEBHOOK_URL')]) { + sh ''' + curl -X POST -H "Content-type: application/json" \ + --data "{\\"text\\":\\"✅ *middleware* deployment succeeded — build #${BUILD_NUMBER} — ${BUILD_URL}\\"}" \ + "$SLACK_WEBHOOK_URL" + ''' + } + } + failure { + echo 'Deployment failed - check console output.' + withCredentials([string(credentialsId: 'slack-webhook-ci-build-alerts', variable: 'SLACK_WEBHOOK_URL')]) { + sh ''' + curl -X POST -H "Content-type: application/json" \ + --data "{\\"text\\":\\"❌ *middleware* deployment failed — build #${BUILD_NUMBER} — ${BUILD_URL}\\"}" \ + "$SLACK_WEBHOOK_URL" + ''' + } + } + } +} diff --git a/NOTICE b/NOTICE new file mode 100644 index 000000000..db447077b --- /dev/null +++ b/NOTICE @@ -0,0 +1,54 @@ +Middleware +========== + +This product includes software developed by Middleware HQ and the Middleware +open-source contributors. + + Project: Middleware — open-source engineering management / DORA metrics + Source: https://github.com/middlewarehq/middleware + Homepage: https://www.middlewarehq.com/ + Licence: Apache License, Version 2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. The +full licence text is included in the LICENSE file at the root of this +repository. + + +Modifications +------------- + +This repository is a fork of middlewarehq/middleware maintained by Clustox. + + Fork: https://github.com/Clustox/middleware + Fork point: commit 844eb42 on branch main + +Modifications made by Clustox are marked in source files with `CLUSTOX:` +sentinel comments. The complete set of modifications can be listed with: + + grep -rn "CLUSTOX:" --include='*.py' --include='*.ts' --include='*.tsx' \ + --include='*.sql' . | grep -v node_modules + +and the full diff from the fork point with: + + git diff 844eb42..main + +See docs/FORK_STRATEGY.md for how this fork is maintained. + + +Trademarks +---------- + +The Apache License, Version 2.0 does not grant permission to use the trade +names, trademarks, service marks, or product names of the licensor. "Middleware" +and "Middleware HQ", together with associated logos and brand assets (including +those under media_files/), remain the property of their respective owners and +are used here solely to attribute the origin of this software. diff --git a/backend/analytics_server/app.py b/backend/analytics_server/app.py index c246a9c87..2cee73ab2 100644 --- a/backend/analytics_server/app.py +++ b/backend/analytics_server/app.py @@ -19,6 +19,9 @@ from mhq.store.initialise_db import initialize_database +# CLUSTOX: restrict this API to the BFF. See mhq/clustox_auth/internal_token.py +from mhq.clustox_auth.internal_token import register_internal_token_guard + ANALYTICS_SERVER_PORT = getenv("ANALYTICS_SERVER_PORT") app = Flask(__name__) @@ -33,6 +36,9 @@ app.register_blueprint(bookmark_api) app.register_blueprint(ai_api) +# CLUSTOX: reject any request that did not come from the BFF. +register_internal_token_guard(app) + configure_db_with_app(app) initialize_database(app) diff --git a/backend/analytics_server/mhq/api/integrations.py b/backend/analytics_server/mhq/api/integrations.py index 35ac97d0b..9b9118b31 100644 --- a/backend/analytics_server/mhq/api/integrations.py +++ b/backend/analytics_server/mhq/api/integrations.py @@ -1,13 +1,42 @@ -from typing import List +# CLUSTOX: no typing.Optional here on purpose -- voluptuous.Optional is imported +# below and shadows it, and an annotation that reads Optional[...] resolves to +# voluptuous's marker class, which is not subscriptable. Under Python 3.9, +# where annotations are evaluated at def time, that is a TypeError on import of +# this module, taking every route in it down. Default arguments say "optional" +# here instead. +from typing import Dict, List, Tuple from flask import Blueprint, jsonify from github import GithubException -from voluptuous import Schema, Optional, Coerce, Range, All +from sqlalchemy.exc import IntegrityError + +# CLUSTOX: Required is used by the Jenkins mapping request schemas below. Any +# is used by the JiraConnection routes' generated_by field, which is either a +# uuid string or None. +from voluptuous import Schema, Optional, Required, Coerce, Range, All, Any from mhq.exapi.models.gitlab import GitlabRepo -from mhq.api.request_utils import queryschema + +# CLUSTOX: the Jenkins mapping routes take a JSON body rather than query +# params, and resolve repos and workflow rows directly -- hence dataschema, +# uuid_validator, the RepoWorkflow model and the two repo services. +from mhq.api.request_utils import dataschema, queryschema, uuid_validator from mhq.service.external_integrations_service import get_external_integrations_service from mhq.service.query_validator import get_query_validator from mhq.store.models import UserIdentityProvider +from mhq.store.models.code import RepoWorkflowProviders, RepoWorkflowType +from mhq.store.models.code.enums import TeamReposDeploymentType +from mhq.store.models.code.workflows.workflows import RepoWorkflow +from mhq.store.repos.code import CodeRepoService +from mhq.store.repos.workflows import WorkflowRepoService + +# CLUSTOX: JiraConnection routes -- see docs/JIRA_MULTI_ACCOUNT_PLAN.md Task 3. +from mhq.store.repos.jira_connection import ( + JiraConnectionInUseError, + JiraConnectionNotFoundError, + JiraConnectionRepoService, +) + +# END CLUSTOX from mhq.utils.github import github_org_data_multi_thread_worker app = Blueprint("integrations", __name__) @@ -15,6 +44,124 @@ STATUS_TOO_MANY_REQUESTS = 429 +# CLUSTOX: key under RepoWorkflow.meta on the Jenkins row, holding the ids of +# the deployment workflows that this mapping switched off. Written on mapping, +# read on unmapping. Without it an unmapping has no way to tell a workflow it +# displaced from one that was already inactive for an unrelated reason. +DISPLACED_WORKFLOW_IDS_KEY = "jenkins_displaced_workflow_ids" + +# CLUSTOX: second key on the same blob, holding the TeamRepos.deployment_type +# values this mapping overwrote, keyed by team id. Mapping a job is useless +# without the switch it records: DeploymentsService splits a team's repos on +# that column and only consults RepoWorkflowRuns for the WORKFLOW side, so a +# repo left on PR_MERGE ingests Jenkins builds and counts none of them -- +# mapping appears to work, changes nothing, and reports no error. Recorded for +# the same reason the displaced workflow ids are: unmapping has to put back +# exactly what mapping changed, and "everything is PR_MERGE now" would demote a +# repo the admin had deliberately configured as WORKFLOW. +PREVIOUS_DEPLOYMENT_TYPES_KEY = "jenkins_previous_deployment_types" + + +# CLUSTOX: JSONB columns on this model default to the string "{}" rather than a +# dict, and a row written before DISPLACED_WORKFLOW_IDS_KEY existed has no +# record at all. Both read as "displaced nothing". +def _workflow_meta(workflow) -> dict: + meta = getattr(workflow, "meta", None) + return meta if isinstance(meta, dict) else {} + + +# CLUSTOX: one active deployment source per repo, whatever the provider. A repo +# tracked through both GitHub Actions and Jenkins -- or through two Jenkins jobs, +# which is two clicks away if an admin maps the wrong job and then the right one +# -- would count every deploy twice, doubling Deployment Frequency with nothing +# visibly wrong. Deactivation is reversible: the rows survive, and +# reactivate_github_actions_workflows_for_repo restores them when the Jenkins +# mapping is removed. Returns the rows it actually switched off, because those +# -- and only those -- are what the unmapping has to switch back on. +def deactivate_deployment_workflows_for_repo(workflows: List) -> List: + deactivated = [] + for workflow in workflows: + if workflow.is_active: + workflow.is_active = False + deactivated.append(workflow) + return deactivated + + +# CLUSTOX: the other half of the invariant above. docs/JENKINS_INTEGRATION.md and +# the mapping dialog both promise the admin can undo a mapping, which means the +# GitHub Actions rows the mapping switched off have to come back on. +# +# Restricted to displaced_workflow_ids, which is the whole point. Mapping only +# ever deactivates *active* rows, but a repo can hold inactive GitHub Actions +# deployment workflows that Jenkins never touched: teams/v2.ts deactivates all +# of a repo's deployment workflows and re-enables only the ones the admin +# selected, so every deselected workflow sits there inactive. Reactivating +# every inactive row -- as this did -- turns a repo with one selected and four +# deselected workflows into a repo with five active ones after a single +# map/unmap round trip, inflating Deployment Frequency through the GitHub path +# instead of the Jenkins one. +def reactivate_github_actions_workflows_for_repo( + workflows: List, displaced_workflow_ids: List[str] +) -> List: + displaced = {str(workflow_id) for workflow_id in displaced_workflow_ids or []} + reactivated = [] + for workflow in workflows: + if ( + workflow.provider == RepoWorkflowProviders.GITHUB_ACTIONS + and not workflow.is_active + and str(workflow.id) in displaced + ): + workflow.is_active = True + reactivated.append(workflow) + return reactivated + + +# CLUSTOX: points every team that tracks this repo at its workflow runs, which +# is what actually makes a Jenkins build count as a deployment. Returns the rows +# it changed together with the values it overwrote; rows already on WORKFLOW are +# neither changed nor recorded, so unmapping leaves them alone rather than +# demoting a repo the admin configured that way himself. +def switch_team_repos_to_workflow_deployments( + team_repos: List, +) -> Tuple[List, Dict[str, str]]: + switched = [] + previous_deployment_types: Dict[str, str] = {} + for team_repo in team_repos: + if team_repo.deployment_type == TeamReposDeploymentType.WORKFLOW: + continue + # A null deployment_type is recorded as PR_MERGE -- the column default, + # and the only value that restores to something DeploymentsService can + # read, since it dereferences .value without a null check. + previous_deployment_types[str(team_repo.team_id)] = ( + team_repo.deployment_type.value + if team_repo.deployment_type + else TeamReposDeploymentType.PR_MERGE.value + ) + team_repo.deployment_type = TeamReposDeploymentType.WORKFLOW + switched.append(team_repo) + return switched, previous_deployment_types + + +# CLUSTOX: the other half of the switch above, and the same restraint as +# reactivate_github_actions_workflows_for_repo: restore only the rows this +# mapping recorded, and only while they still hold the value the mapping wrote. +# A row the admin has since changed by hand is his decision, not ours to undo. +def restore_team_repo_deployment_types( + team_repos: List, previous_deployment_types: Dict[str, str] = None +) -> List: + recorded = previous_deployment_types or {} + restored = [] + for team_repo in team_repos: + previous = recorded.get(str(team_repo.team_id)) + if not previous: + continue + if team_repo.deployment_type != TeamReposDeploymentType.WORKFLOW: + continue + team_repo.deployment_type = TeamReposDeploymentType(previous) + restored.append(team_repo) + return restored + + @app.route("/orgs//integrations/github/orgs", methods={"GET"}) def get_github_orgs(org_id: str): query_validator = get_query_validator() @@ -210,6 +357,274 @@ def get_gitlab_projects(org_id: str, group_id: str, page_size: int, page: int): ] +@app.route("/orgs//integrations/jenkins/jobs", methods={"GET"}) +def get_jenkins_jobs(org_id: str): + # CLUSTOX: Jenkins-specific imports stay local to the handler, mirroring + # the lazy-import precedent for provider modules in + # mhq/service/workflows/sync/etl_jenkins_handler.py. + import requests + + from mhq.exapi.jenkins import JenkinsAddressNotAllowed, JenkinsApiService + from mhq.store.repos.core import CoreRepoService + from mhq.utils.jenkins import get_jenkins_config + + query_validator = get_query_validator() + query_validator.org_validator(org_id) + + api_token = CoreRepoService().get_access_token(org_id, UserIdentityProvider.JENKINS) + base_url, username = get_jenkins_config(org_id) + if not (api_token and base_url and username): + return {"error": "Jenkins is not configured for this workspace"}, 400 + + try: + return JenkinsApiService(base_url, username, api_token).get_jobs() + except JenkinsAddressNotAllowed as e: + # CLUSTOX: caught ahead of RequestException, which it subclasses. The + # configured URL points somewhere this server refuses to fetch, which is + # a 400 the admin can fix -- reported as "could not reach Jenkins" it + # reads as a network problem and sends him to look at a firewall. + return jsonify({"error": str(e)}), 400 + except requests.RequestException as e: + # An unreachable, slow or erroring Jenkins is not a bug in this server. + # Surfaced as a 500 it reached the setup form as "check your + # credentials", which sends the admin looking in the wrong place. + return ( + jsonify({"error": f"Could not reach Jenkins: {str(e)}"}), + 502, + ) + + +# CLUSTOX: the workspace's live Jenkins mappings. Without this the mapping table +# reloads showing "Select a Jenkins job" for a repo that is mapped and ingesting +# deployments -- an admin's configuration looks lost -- and nothing can supply +# the repo_workflow_id the DELETE route needs, so a mapping cannot be undone +# from the UI at all. +@app.route("/orgs//integrations/jenkins/mappings", methods={"GET"}) +def get_jenkins_mappings(org_id: str): + query_validator = get_query_validator() + query_validator.org_validator(org_id) + + # Active Jenkins deployment workflows only: an inactive row is a mapping the + # admin removed, and offering it back would misreport where a repo's + # deployments come from. Scoping is the OrgRepo join inside the query. + mappings = WorkflowRepoService().get_active_repo_workflows_by_org_id_and_provider( + org_id, RepoWorkflowProviders.JENKINS, RepoWorkflowType.DEPLOYMENT + ) + + return [ + { + "repo_workflow_id": str(repo_workflow.id), + "org_repo_id": str(repo_workflow.org_repo_id), + "job_full_name": repo_workflow.provider_workflow_id, + "repo_name": org_repo.name, + } + for repo_workflow, org_repo in mappings + ] + + +# CLUSTOX: maps a Jenkins job to a repo as its deployment source. Enforces the +# one-active-deployment-source-per-repo invariant and reuses the row for an +# already-known (org_repo_id, provider_workflow_id) pair. +@app.route("/orgs//integrations/jenkins/mappings", methods={"POST"}) +@dataschema( + Schema( + { + Required("org_repo_id"): All(str, Coerce(uuid_validator)), + Required("job_full_name"): str, + } + ), +) +def create_jenkins_mapping(org_id: str, org_repo_id: str, job_full_name: str): + query_validator = get_query_validator() + query_validator.org_validator(org_id) + + code_repo_service = CodeRepoService() + org_repo = code_repo_service.get_repo_by_id(org_repo_id) + if org_repo is None or str(org_repo.org_id) != str(org_id): + return jsonify({"error": f"Repo {org_repo_id} not found in org {org_id}"}), 404 + + workflow_repo_service = WorkflowRepoService() + + # A row may already exist for this (org_repo_id, provider_workflow_id) -- + # left behind inactive by a previous unmapping. The pair is uniquely + # indexed, so inserting a second one raises IntegrityError; reuse it. + existing_workflow = ( + workflow_repo_service.get_repo_workflow_by_repo_id_and_provider_workflow_id( + org_repo_id, job_full_name + ) + ) + # The lookup matches the index, which is not scoped by provider, so the row + # occupying this pair may belong to another provider -- a Jenkins job whose + # full name happens to equal a GitHub Actions workflow id. Reusing it would + # silently convert someone else's deployment workflow into a Jenkins + # mapping, taking its runs and its history with it. Refuse instead: an + # admin who sees this can rename the job, and one who does not would never + # have found the takeover. + if ( + existing_workflow is not None + and existing_workflow.provider != RepoWorkflowProviders.JENKINS + ): + return ( + jsonify( + { + "error": ( + f"Repo {org_repo_id} already has a " + f"{existing_workflow.provider.value} workflow with the " + f"id {job_full_name}" + ) + } + ), + 409, + ) + existing_jenkins_workflow = existing_workflow + + active_workflows = workflow_repo_service.get_repo_workflow_by_repo_ids( + [org_repo_id], RepoWorkflowType.DEPLOYMENT + ) + # Every active deployment workflow on this repo except the one being mapped, + # regardless of provider: an already-active Jenkins job for a different + # pipeline has to go too, or the repo ends up with two live sources. + workflows_to_deactivate = [ + workflow + for workflow in active_workflows + if existing_jenkins_workflow is None + or str(workflow.id) != str(existing_jenkins_workflow.id) + ] + # Mutates the rows in place; the actual write happens together with the + # Jenkins row below, in one commit. + deactivated = deactivate_deployment_workflows_for_repo(workflows_to_deactivate) + + # Mutated in place like the workflows above, and written in the same commit. + team_repos = code_repo_service.get_active_team_repos_by_repo_id(org_repo_id) + switched_team_repos, previous_deployment_types = ( + switch_team_repos_to_workflow_deployments(team_repos) + ) + # A Jenkins mapping this one displaces already switched these rows, so this + # mapping records nothing for them and the values it is overwriting sit on + # the displaced row's meta. Carrying them forward is what keeps the last + # unmapping in a chain of remappings able to restore the repo; dropping them + # strands it on WORKFLOW with no workflow behind it -- zero deployments, + # silently. + for workflow in deactivated: + if workflow.provider != RepoWorkflowProviders.JENKINS: + continue + carried = _workflow_meta(workflow).get(PREVIOUS_DEPLOYMENT_TYPES_KEY) or {} + for team_id, deployment_type in carried.items(): + previous_deployment_types.setdefault(team_id, deployment_type) + + if existing_jenkins_workflow is not None: + existing_jenkins_workflow.is_active = True + existing_jenkins_workflow.name = job_full_name + jenkins_workflow = existing_jenkins_workflow + else: + jenkins_workflow = RepoWorkflow( + org_repo_id=org_repo_id, + type=RepoWorkflowType.DEPLOYMENT, + provider=RepoWorkflowProviders.JENKINS, + provider_workflow_id=job_full_name, + is_active=True, + name=job_full_name, + ) + # CLUSTOX: the record of what this mapping displaced. Replaced rather than + # merged: a remapping recomputes the set from scratch, and anything left + # over from a previous mapping was already restored when that one was + # removed. A new dict, not a mutation, so SQLAlchemy sees the change. + jenkins_workflow.meta = { + **_workflow_meta(jenkins_workflow), + DISPLACED_WORKFLOW_IDS_KEY: [str(workflow.id) for workflow in deactivated], + PREVIOUS_DEPLOYMENT_TYPES_KEY: previous_deployment_types, + } + workflow_repo_service.create_jenkins_repo_workflow( + jenkins_workflow, workflows_to_deactivate, switched_team_repos + ) + + return { + "ok": True, + "deactivated_workflows": len(deactivated), + "switched_team_repos": len(switched_team_repos), + } + + +# CLUSTOX: removes a Jenkins mapping and restores the GitHub Actions workflows +# the mapping displaced. Jenkins rows only -- see the provider check below. +@app.route("/orgs//integrations/jenkins/mappings", methods={"DELETE"}) +@dataschema( + Schema( + { + Required("repo_workflow_id"): All(str, Coerce(uuid_validator)), + } + ), +) +def delete_jenkins_mapping(org_id: str, repo_workflow_id: str): + query_validator = get_query_validator() + query_validator.org_validator(org_id) + + workflow_repo_service = WorkflowRepoService() + repo_workflow = workflow_repo_service.get_repo_workflow_by_id(repo_workflow_id) + if repo_workflow is None: + return jsonify({"error": f"Workflow {repo_workflow_id} not found"}), 404 + + code_repo_service = CodeRepoService() + org_repo = code_repo_service.get_repo_by_id(repo_workflow.org_repo_id) + if org_repo is None or str(org_repo.org_id) != str(org_id): + return ( + jsonify( + {"error": f"Workflow {repo_workflow_id} not found in org {org_id}"} + ), + 404, + ) + + # get_repo_workflow_by_id is not filtered by provider, so without this an + # admin could pass any workflow id from their own workspace and silently + # disable a GitHub Actions deployment workflow through the Jenkins route. + if repo_workflow.provider != RepoWorkflowProviders.JENKINS: + return ( + jsonify({"error": f"Jenkins mapping {repo_workflow_id} not found"}), + 404, + ) + + # Removing the mapping gives the repo its deployment source back, which is + # what both the docs and the mapping dialog promise -- but only the rows + # the mapping itself displaced. A mapping written before this record + # existed restores nothing, which is the safe direction: a repo with no + # active deployment source reads as zero deployments, a repo with several + # reads as several times too many, and only the second one is silent. + meta = _workflow_meta(repo_workflow) + github_workflows = workflow_repo_service.get_repo_workflows_by_repo_id_and_provider( + str(repo_workflow.org_repo_id), + RepoWorkflowProviders.GITHUB_ACTIONS, + RepoWorkflowType.DEPLOYMENT, + ) + reactivated = reactivate_github_actions_workflows_for_repo( + github_workflows, meta.get(DISPLACED_WORKFLOW_IDS_KEY) + ) + # The deployment_type switch is undone on the same terms: only the team rows + # this mapping recorded. A repo already on WORKFLOW when it was mapped was + # never recorded, so it stays on WORKFLOW here. + team_repos = code_repo_service.get_active_team_repos_by_repo_id( + str(repo_workflow.org_repo_id) + ) + restored_team_repos = restore_team_repo_deployment_types( + team_repos, meta.get(PREVIOUS_DEPLOYMENT_TYPES_KEY) + ) + # Drop the record in the same commit that acts on it, so a second unmapping + # -- or one that follows a team-config edit that deselected these same + # workflows -- does not restore them a second time. + dropped_keys = {DISPLACED_WORKFLOW_IDS_KEY, PREVIOUS_DEPLOYMENT_TYPES_KEY} + repo_workflow.meta = { + key: value for key, value in meta.items() if key not in dropped_keys + } + workflow_repo_service.deactivate_repo_workflow( + repo_workflow, reactivated, restored_team_repos + ) + + return { + "ok": True, + "reactivated_github_workflows": len(reactivated), + "restored_team_repos": len(restored_team_repos), + } + + @app.route("/orgs//integrations/gitlab/user/repos", methods={"GET"}) @queryschema( Schema( @@ -247,3 +662,118 @@ def get_gitlab_user_projects(org_id: str, page_size: int, page: int): } for project in projects ] + + +# CLUSTOX: JiraConnection routes -- Task 3 of docs/JIRA_MULTI_ACCOUNT_PLAN.md. +# The web-server's jira-connections/* BFF routes proxy straight through to +# these, the same way the Jenkins mapping routes above do, rather than writing +# JiraConnection/OrgProjectConnection from knex directly: create/delete/ +# set-default all carry business rules (encrypt-on-write, delete-blocked-by- +# reference, default-switch atomicity) that belong in one place, enforced +# for every caller, not re-implemented in TypeScript. +def _serialize_jira_connection(connection) -> Dict: + # access_token_enc_chunks is deliberately never included -- there is no + # caller of this serializer that should ever see even the encrypted form. + return { + "id": str(connection.id), + "site_url": connection.site_url, + "email": connection.email, + "is_default": connection.is_default, + "provider_meta": connection.provider_meta or {}, + "created_at": ( + connection.created_at.isoformat() if connection.created_at else None + ), + } + + +@app.route("/orgs//integrations/jira-connections", methods={"GET"}) +def list_jira_connections(org_id: str): + query_validator = get_query_validator() + query_validator.org_validator(org_id) + + service = JiraConnectionRepoService() + connections = service.list_jira_connections(org_id) + return jsonify([_serialize_jira_connection(c) for c in connections]) + + +@app.route("/orgs//integrations/jira-connections", methods={"POST"}) +@dataschema( + Schema( + { + Required("site_url"): str, + Required("email"): str, + Required("access_token"): str, + Optional("provider_meta", default={}): dict, + Optional("generated_by", default=None): Any( + All(str, Coerce(uuid_validator)), None + ), + } + ), +) +def create_jira_connection( + org_id: str, + site_url: str, + email: str, + access_token: str, + provider_meta: dict, + generated_by: str = None, +): + query_validator = get_query_validator() + query_validator.org_validator(org_id) + + service = JiraConnectionRepoService() + try: + connection = service.create_jira_connection( + org_id, site_url, email, access_token, provider_meta, generated_by + ) + except IntegrityError: + # jira_connection_unique_account: this (org_id, site_url, email) is + # already connected. Named explicitly rather than surfacing the raw + # constraint violation -- see delete/set-default below for the same + # rule applied to their own DB-level invariants. + return ( + jsonify( + { + "error": ( + f"{email} is already connected to {site_url} for this " + "workspace" + ) + } + ), + 409, + ) + return jsonify(_serialize_jira_connection(connection)), 201 + + +@app.route( + "/orgs//integrations/jira-connections/", + methods={"DELETE"}, +) +def delete_jira_connection(org_id: str, connection_id: str): + query_validator = get_query_validator() + query_validator.org_validator(org_id) + + service = JiraConnectionRepoService() + try: + service.delete_jira_connection(org_id, connection_id) + except JiraConnectionNotFoundError as e: + return jsonify({"error": str(e)}), 404 + except JiraConnectionInUseError as e: + return jsonify({"error": str(e)}), 409 + return jsonify({"ok": True}) + + +@app.route( + "/orgs//integrations/jira-connections/", + methods={"PATCH"}, +) +def set_default_jira_connection(org_id: str, connection_id: str): + query_validator = get_query_validator() + query_validator.org_validator(org_id) + + service = JiraConnectionRepoService() + try: + service.set_default_jira_connection(org_id, connection_id) + except JiraConnectionNotFoundError as e: + return jsonify({"error": str(e)}), 404 + return jsonify({"ok": True}) diff --git a/backend/analytics_server/mhq/api/pull_requests.py b/backend/analytics_server/mhq/api/pull_requests.py index 302387462..16eb2a49a 100644 --- a/backend/analytics_server/mhq/api/pull_requests.py +++ b/backend/analytics_server/mhq/api/pull_requests.py @@ -7,6 +7,7 @@ from voluptuous import Required, Schema, Coerce, All, Optional from mhq.service.code.models.lead_time import LeadTimeMetrics from mhq.service.code.lead_time import get_lead_time_service +from mhq.service.code.loc import LOCMetrics, get_loc_service from mhq.service.code.pr_filter import apply_pr_filter from mhq.store.models.code import PRFilter @@ -16,10 +17,12 @@ from mhq.api.request_utils import queryschema from mhq.api.resources.code_resouces import ( adapt_lead_time_metrics, + adapt_loc_metrics, adapt_pull_request, get_non_paginated_pr_response, ) from mhq.store.models.code.pull_requests import PullRequest +from mhq.store.repos.code import CodeRepoService from mhq.service.code.pr_analytics import get_pr_analytics_service from mhq.service.settings.models import ExcludedPRsSetting @@ -30,7 +33,6 @@ from mhq.store.models import SettingType, EntityType - app = Blueprint("pull_requests", __name__) @@ -167,3 +169,103 @@ def get_team_lead_time_trends( week.isoformat(): adapt_lead_time_metrics(average_lead_time_metrics) for week, average_lead_time_metrics in weekly_lead_time_metrics_avg_map.items() } + + +# CLUSTOX: lines of code is a fifth metric on the same dashboard as the four +# DORA metrics, so it goes through the same filter path -- the contributor +# filter, branch mode and the excluded-PRs setting then apply to it for free. +@app.route("/teams//loc", methods={"GET"}) +@queryschema( + Schema( + { + Required("from_time"): All(str, Coerce(datetime.fromisoformat)), + Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + Optional("pr_filter"): All(str, Coerce(json.loads)), + } + ), +) +def get_team_loc( + team_id: str, + from_time: datetime, + to_time: datetime, + pr_filter: Dict = None, +): + + query_validator = get_query_validator() + + interval: Interval = query_validator.interval_validator(from_time, to_time) + team: Team = query_validator.team_validator(team_id) + + pr_filter: PRFilter = apply_pr_filter( + pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] + ) + + loc_metrics: LOCMetrics = get_loc_service().get_team_loc_metrics( + str(team.id), interval, pr_filter + ) + + return adapt_loc_metrics(loc_metrics) + + +@app.route("/teams//loc/trends", methods={"GET"}) +@queryschema( + Schema( + { + Required("from_time"): All(str, Coerce(datetime.fromisoformat)), + Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + Optional("pr_filter"): All(str, Coerce(json.loads)), + } + ), +) +def get_team_loc_trends( + team_id: str, + from_time: datetime, + to_time: datetime, + pr_filter: Dict = None, +): + + query_validator = get_query_validator() + + interval: Interval = query_validator.interval_validator(from_time, to_time) + team: Team = query_validator.team_validator(team_id) + + pr_filter: PRFilter = apply_pr_filter( + pr_filter, EntityType.TEAM, team_id, [SettingType.EXCLUDED_PRS_SETTING] + ) + + weekly_loc_metrics_map: Dict[datetime, LOCMetrics] = ( + get_loc_service().get_team_loc_trends(str(team.id), interval, pr_filter) + ) + + return { + week.isoformat(): adapt_loc_metrics(loc_metrics) + for week, loc_metrics in weekly_loc_metrics_map.items() + } + + +# CLUSTOX: contributor list backing the DORA metrics filter. Scoped to the +# team's repos and the window on screen, so the dropdown cannot offer someone +# with no data in view. +@app.route("/teams//contributors", methods={"GET"}) +@queryschema( + Schema( + { + Required("from_time"): All(str, Coerce(datetime.fromisoformat)), + Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + } + ), +) +def get_team_contributors(team_id: str, from_time: datetime, to_time: datetime): + query_validator = get_query_validator() + team: Team = query_validator.team_validator(team_id) + + repos = CodeRepoService().get_team_repos(str(team.id)) + repo_ids = [str(repo.id) for repo in repos] + + contributors = CodeRepoService().get_contributors_for_repos( + repo_ids, from_time, to_time + ) + return [ + {"username": username, "pr_count": pr_count} + for username, pr_count in contributors + ] diff --git a/backend/analytics_server/mhq/api/request_utils.py b/backend/analytics_server/mhq/api/request_utils.py index 0ffc819a3..93a5ea1f4 100644 --- a/backend/analytics_server/mhq/api/request_utils.py +++ b/backend/analytics_server/mhq/api/request_utils.py @@ -9,6 +9,7 @@ from werkzeug.exceptions import BadRequest from mhq.store.models.code.repository import TeamRepos from mhq.service.code.models.org_repo import RawTeamOrgRepo +from mhq.service.project.models.org_project import RawTeamOrgProject from mhq.store.models.code import WorkflowFilter, CodeProvider from mhq.service.workflows.workflow_filter import get_workflow_filter_processor @@ -116,3 +117,18 @@ def coerce_team_repo(team_repo: Dict[str, str]) -> TeamRepos: def coerce_team_repos(repos: List[Dict[str, str]]) -> List[TeamRepos]: return [coerce_team_repo(repo) for repo in repos] + + +def coerce_org_project(project: Dict[str, str]) -> RawTeamOrgProject: + return RawTeamOrgProject( + team_id=project.get("team_id"), + provider=project.get("provider"), + key=project.get("key"), + name=project.get("name"), + idempotency_key=project.get("idempotency_key"), + connection_id=project.get("connection_id"), + ) + + +def coerce_org_projects(projects: List[Dict[str, str]]) -> List[RawTeamOrgProject]: + return [coerce_org_project(project) for project in projects] diff --git a/backend/analytics_server/mhq/api/resources/code_resouces.py b/backend/analytics_server/mhq/api/resources/code_resouces.py index c709b5112..e300b4bde 100644 --- a/backend/analytics_server/mhq/api/resources/code_resouces.py +++ b/backend/analytics_server/mhq/api/resources/code_resouces.py @@ -1,4 +1,5 @@ from typing import Dict, List +from mhq.service.code.loc import LOCMetrics from mhq.service.code.models.lead_time import LeadTimeMetrics from mhq.api.resources.core_resources import adapt_user_info from mhq.store.models.code import PullRequest, OrgRepo, TeamRepos @@ -121,6 +122,16 @@ def adapt_lead_time_metrics(lead_time_metric: LeadTimeMetrics) -> Dict[str, any] } +def adapt_loc_metrics(loc_metrics: LOCMetrics) -> Dict[str, any]: + return { + "additions": loc_metrics.additions, + "deletions": loc_metrics.deletions, + "total": loc_metrics.additions + loc_metrics.deletions, + "avg_pr_size": loc_metrics.avg_pr_size, + "avg_daily": loc_metrics.avg_daily, + } + + def adapt_org_repo(org_repo: OrgRepo) -> Dict[str, any]: return { "id": str(org_repo.id), diff --git a/backend/analytics_server/mhq/api/resources/project_resources.py b/backend/analytics_server/mhq/api/resources/project_resources.py new file mode 100644 index 000000000..d09ac807c --- /dev/null +++ b/backend/analytics_server/mhq/api/resources/project_resources.py @@ -0,0 +1,21 @@ +from typing import Dict, List + +from mhq.store.models.projects import OrgProject + + +def adapt_org_project(org_project: OrgProject) -> Dict[str, any]: + return { + "id": str(org_project.id), + "org_id": str(org_project.org_id), + "key": org_project.key, + "name": org_project.name, + "provider": org_project.provider, + "is_active": org_project.is_active, + "idempotency_key": org_project.idempotency_key, + "created_at": org_project.created_at.isoformat(), + "updated_at": org_project.updated_at.isoformat(), + } + + +def adapt_org_projects(org_projects: List[OrgProject]) -> List[Dict[str, any]]: + return [adapt_org_project(project) for project in org_projects] diff --git a/backend/analytics_server/mhq/api/resources/settings_resource.py b/backend/analytics_server/mhq/api/resources/settings_resource.py index 9e95e693d..088466a5c 100644 --- a/backend/analytics_server/mhq/api/resources/settings_resource.py +++ b/backend/analytics_server/mhq/api/resources/settings_resource.py @@ -6,6 +6,8 @@ IncidentTypesSetting, IncidentSourcesSetting, IncidentPRsSetting, + JiraIncidentIssueTypesSetting, + BenchmarkSetting, ) from mhq.store.models import EntityType @@ -22,6 +24,13 @@ def _add_entity(config_settings: ConfigurationSettings, response): if config_settings.entity_type == EntityType.ORG: response["org_id"] = str(config_settings.entity_id) + # CLUSTOX: the global row's entity_id is a fixed sentinel, not a real + # entity -- echoing it back as an id invites a caller to treat it as + # one (e.g. look it up as a team). A "scope" marker says what the row + # is without implying it resolves to anything. + if config_settings.entity_type == EntityType.GLOBAL: + response["scope"] = "global" + return response def _add_setting_data(config_settings: ConfigurationSettings, response): @@ -62,6 +71,24 @@ def _add_setting_data(config_settings: ConfigurationSettings, response): "filters": config_settings.specific_settings.filters, } + if isinstance(config_settings.specific_settings, JiraIncidentIssueTypesSetting): + response["setting"] = { + "issue_types": config_settings.specific_settings.issue_types + } + + if isinstance(config_settings.specific_settings, BenchmarkSetting): + # CLUSTOX: None means "not set at this scope, inherit" and must + # round-trip as null, not be dropped or coerced to 0 -- the + # config form distinguishes "no target", "target of 0" and + # "target inherited from the global baseline". + response["setting"] = { + "lead_time": config_settings.specific_settings.lead_time, + "deployment_frequency": config_settings.specific_settings.deployment_frequency, + "change_failure_rate": config_settings.specific_settings.change_failure_rate, + "mean_time_to_recovery": config_settings.specific_settings.mean_time_to_recovery, + "lines_of_code": config_settings.specific_settings.lines_of_code, + } + # ADD NEW API ADAPTER HERE return response diff --git a/backend/analytics_server/mhq/api/resources/sprint_resources.py b/backend/analytics_server/mhq/api/resources/sprint_resources.py new file mode 100644 index 000000000..fb4b17a4a --- /dev/null +++ b/backend/analytics_server/mhq/api/resources/sprint_resources.py @@ -0,0 +1,17 @@ +from typing import Dict, List + +from mhq.store.models.projects import Sprint + + +def adapt_sprints(sprints: List[Sprint]) -> List[Dict]: + return [ + { + "name": sprint.name, + "state": sprint.state, + "start_date": sprint.start_date.isoformat() if sprint.start_date else None, + "end_date": sprint.end_date.isoformat() if sprint.end_date else None, + "planned_count": sprint.planned_count, + "completed_count": sprint.completed_count, + } + for sprint in sprints + ] diff --git a/backend/analytics_server/mhq/api/resources/ticket_insights_resources.py b/backend/analytics_server/mhq/api/resources/ticket_insights_resources.py new file mode 100644 index 000000000..d701fb727 --- /dev/null +++ b/backend/analytics_server/mhq/api/resources/ticket_insights_resources.py @@ -0,0 +1,36 @@ +from typing import Dict, List + +from mhq.store.models.code import PullRequest + + +def adapt_unlinked_prs(prs: List[PullRequest]) -> List[Dict]: + return [ + { + "id": str(pr.id), + "title": pr.title, + "url": pr.url, + "head_branch": pr.head_branch, + "author": pr.author, + "merged_at": pr.state_changed_at.isoformat(), + } + for pr in prs + ] + + +def adapt_ticket_insights(insights: Dict) -> Dict: + return { + "cycle_time_by_project": [ + { + "project_key": project.project_key, + "project_name": project.project_name, + "ticket_count": project.ticket_count, + "avg_total_seconds": round(project.avg_total_seconds), + "avg_seconds_by_category": { + category: round(seconds) + for category, seconds in project.avg_seconds_by_category.items() + }, + } + for project in insights["cycle_time_by_project"] + ], + "prs_without_ticket_count": insights["prs_without_ticket_count"], + } diff --git a/backend/analytics_server/mhq/api/resources/ticket_lead_time_resources.py b/backend/analytics_server/mhq/api/resources/ticket_lead_time_resources.py new file mode 100644 index 000000000..37598389b --- /dev/null +++ b/backend/analytics_server/mhq/api/resources/ticket_lead_time_resources.py @@ -0,0 +1,14 @@ +from mhq.service.code.models.ticket_lead_time import TicketLeadTimeMetrics + + +def adapt_ticket_lead_time_metrics(metrics: TicketLeadTimeMetrics) -> dict: + return { + "matched_pr_count": metrics.matched_pr_count, + "avg_ticket_to_first_commit_seconds": round( + metrics.avg_ticket_to_first_commit_seconds + ), + "avg_commit_only_lead_time_seconds": round( + metrics.avg_commit_only_lead_time_seconds + ), + "avg_extended_lead_time_seconds": round(metrics.avg_extended_lead_time_seconds), + } diff --git a/backend/analytics_server/mhq/api/settings.py b/backend/analytics_server/mhq/api/settings.py index b0015f788..c945bd601 100644 --- a/backend/analytics_server/mhq/api/settings.py +++ b/backend/analytics_server/mhq/api/settings.py @@ -8,6 +8,12 @@ from mhq.api.resources.settings_resource import adapt_configuration_settings_response from mhq.service.query_validator import get_query_validator from mhq.service.settings import get_settings_service, settings_type_validator +from mhq.service.settings.benchmarks import ( + GLOBAL_BENCHMARK_ENTITY_ID, + empty_benchmark_settings, + get_resolved_benchmarks_for_team, + validate_benchmark_payload, +) from mhq.store.models import SettingType, EntityType app = Blueprint("settings", __name__) @@ -44,16 +50,47 @@ def get_team_settings(team_id: str, setting_type: SettingType, setter_id: str = ) if not settings: - settings = settings_service.save_settings( - setting_type=setting_type, - entity_type=EntityType.TEAM, - entity_id=team_id, - setter=setter, - ) + # CLUSTOX: benchmarks opt out of the auto-create below. Every other + # setting type has a meaningful shipped default and relies on this + # row being written on first read; a benchmark has no default that is + # safe to invent, because writing one at TEAM scope silently ends + # that team's per-metric inheritance from the global baseline and + # makes the card claim "your team's benchmark" for a target the team + # never set. Gated on the setting type rather than removed, so + # INCIDENT_SETTING, EXCLUDED_PRS_SETTING and the rest are untouched. + if setting_type == SettingType.BENCHMARK_SETTING: + settings = empty_benchmark_settings(EntityType.TEAM, team_id) + else: + settings = settings_service.save_settings( + setting_type=setting_type, + entity_type=EntityType.TEAM, + entity_id=team_id, + setter=setter, + ) return adapt_configuration_settings_response(settings) +# CLUSTOX: the resolved benchmarks for a team -- the team row falling back to +# the global baseline, per metric, with the source of each. +# +# This exists because the four DORA cards read the targets out of +# `metrics_summary.benchmarks`, which is written only by the dora_metrics BFF +# route. The resolved values used to be attached to +# /teams//deployment_analytics, whose response lands in a different redux +# slice that no card reads, so the entire user-visible half of the feature +# rendered nothing. Serving them here lets the dora_metrics BFF fold them into +# the response the cards actually consume, inside the Promise.all it already +# runs, without adding a round trip to the page. +@app.route("/teams//benchmarks", methods={"GET"}) +def get_team_benchmarks(team_id: str): + + query_validator = get_query_validator() + query_validator.team_validator(team_id) + + return get_resolved_benchmarks_for_team(team_id) + + @app.route("/teams//settings", methods={"PUT"}) @dataschema( Schema( @@ -83,6 +120,17 @@ def put_team_settings( if setter and str(setter.org_id) != str(team.org_id): raise BadRequest(f"User {setter_id} does not belong to team {team_id}") + # CLUSTOX: this is the route every workspace admin's benchmark form + # writes to, and it used to store setting_data verbatim -- a lead_time of + # -1, a change_failure_rate of 5000, or a typo'd `leadtime` key that + # _adapt_benchmark_setting_from_json drops on the floor while the admin + # gets a success toast. The client-side mirror in BenchmarkSettingsForm is + # not a trust boundary. Validation also normalises the payload to all four + # keys, which is what makes clearing a field mean "inherit" rather than + # "write the defaults" (see validate_benchmark_payload). + if setting_type == SettingType.BENCHMARK_SETTING: + setting_data = validate_benchmark_payload(setting_data) + settings_service = get_settings_service() settings = settings_service.save_settings( setting_type=setting_type, @@ -170,3 +218,78 @@ def put_org_settings( setting_data=setting_data, ) return adapt_configuration_settings_response(settings) + + +# CLUSTOX: GLOBAL scope exists for exactly one setting type. These routes are +# reachable by anything holding the internal token, and every other setting +# type is meaningless without an entity to scope it to -- an unguarded +# setting_type would let a caller create a GLOBAL row under the benchmark +# sentinel id for, say, EXCLUDED_PRS_SETTING, and would run the benchmark +# validator's unknown-keys check against a payload it knows nothing about, +# rejecting it with a confusing message. Reject it plainly instead. +def _assert_global_scope_is_benchmarks(setting_type: SettingType): + if setting_type != SettingType.BENCHMARK_SETTING: + raise BadRequest( + f"Global scope is only supported for " + f"{SettingType.BENCHMARK_SETTING.value}, got {setting_type.value}" + ) + + +# CLUSTOX: the superadmin's baseline belongs to no team and no workspace, so +# it cannot use the team- or org-scoped routes above. Authorisation is enforced +# at the BFF -- this layer only knows the internal token, not who is calling. +@app.route("/settings/global", methods={"GET"}) +@queryschema( + Schema( + { + Required("setting_type"): All(str, Coerce(settings_type_validator)), + } + ), +) +def get_global_settings(setting_type: SettingType): + + _assert_global_scope_is_benchmarks(setting_type) + + settings_service = get_settings_service() + settings = settings_service.get_settings( + setting_type=setting_type, + entity_type=EntityType.GLOBAL, + entity_id=GLOBAL_BENCHMARK_ENTITY_ID, + ) + + # CLUSTOX: read-only. Creating the row here meant a single admin opening a + # single team form (which fetches this to build its placeholders) + # materialised the installation-wide baseline, so "no benchmark anywhere" + # -- the state the spec requires every card to render unchanged in -- was + # unreachable in practice. A superadmin creates the row by saving one. + if not settings: + settings = empty_benchmark_settings( + EntityType.GLOBAL, GLOBAL_BENCHMARK_ENTITY_ID + ) + + return adapt_configuration_settings_response(settings) + + +@app.route("/settings/global", methods={"PUT"}) +@dataschema( + Schema( + { + Required("setting_type"): All(str, Coerce(settings_type_validator)), + Required("setting_data"): dict, + } + ), +) +def put_global_settings(setting_type: SettingType, setting_data: Dict = None): + + _assert_global_scope_is_benchmarks(setting_type) + + setting_data = validate_benchmark_payload(setting_data) + + settings_service = get_settings_service() + settings = settings_service.save_settings( + setting_type=setting_type, + entity_type=EntityType.GLOBAL, + entity_id=GLOBAL_BENCHMARK_ENTITY_ID, + setting_data=setting_data, + ) + return adapt_configuration_settings_response(settings) diff --git a/backend/analytics_server/mhq/api/sync.py b/backend/analytics_server/mhq/api/sync.py index e626e7df9..5898f5640 100644 --- a/backend/analytics_server/mhq/api/sync.py +++ b/backend/analytics_server/mhq/api/sync.py @@ -1,7 +1,15 @@ -from flask import Blueprint, jsonify +from typing import List -from mhq.service.query_validator import get_query_validator +from flask import Blueprint + +from mhq.clustox_auth.sync_run import ( + SyncRunStatus, + as_dict, + get_sync_run_service, +) from mhq.service.sync_data import trigger_data_sync +from mhq.store.models.core import Organization +from mhq.store.repos.core import CoreRepoService from mhq.utils.lock import get_redis_lock_service from mhq.utils.log import LOG from mhq.utils.time import time_now @@ -11,14 +19,62 @@ @app.route("/sync", methods=["POST"]) def sync(): - default_org = get_query_validator().get_default_org() - if not default_org: - return jsonify({"message": "Default org not found"}), 404 - org_id = str(default_org.id) - with get_redis_lock_service().acquire_lock("{org}:" + f"{str(org_id)}:data_sync"): + """ + CLUSTOX: sync every workspace, not just the default one. + + Upstream synced a single organisation, because only one ever existed. Each + workspace now brings its own integration and its own repositories, so each + needs its own sync -- against its own provider token and therefore its own + rate limit. + + Workspaces are isolated from one another: one failing must not stop the + rest, which is the same reasoning upstream applies per repository inside a + sync. The difference is that the outcome is recorded per workspace instead + of being lost to a rotating log file. + """ + orgs: List[Organization] = CoreRepoService().get_all_orgs() + + if not orgs: + return {"message": "no workspaces to sync", "results": []} + + sync_run_service = get_sync_run_service() + lock_service = get_redis_lock_service() + results = [] + + for org in orgs: + org_id = str(org.id) + run = sync_run_service.start(org_id) + try: - trigger_data_sync(org_id) + # Per-workspace lock: a long sync in one workspace must not block + # another, and a repeat trigger must not run the same workspace + # twice concurrently. + with lock_service.acquire_lock("{org}:" + f"{org_id}:data_sync"): + trigger_data_sync(org_id) + + sync_run_service.finish(run, SyncRunStatus.SUCCESS) + LOG.info(f"[CLUSTOX] Synced workspace {org.name} ({org_id})") + except Exception as e: - LOG.error(f"Error syncing data for org {org_id}: {str(e)}") - return {"message": "sync failed", "time": time_now().isoformat()}, 500 - return {"message": "sync started", "time": time_now().isoformat()} + detail = f"{type(e).__name__}: {str(e)}" + sync_run_service.finish(run, SyncRunStatus.FAILED, detail) + LOG.error(f"[CLUSTOX] Sync failed for workspace {org_id}: {detail}") + + latest = sync_run_service.latest_for_org(org_id) + if latest: + results.append(as_dict(latest)) + + failed = [r for r in results if r["status"] == SyncRunStatus.FAILED] + + return { + "message": f"synced {len(results) - len(failed)}/{len(results)} workspaces", + "time": time_now().isoformat(), + "results": results, + } + + +@app.route("/sync/status", methods=["GET"]) +def sync_status(): + """Latest sync outcome per workspace.""" + runs = get_sync_run_service().latest_per_org() + return {"results": [as_dict(r) for r in runs]} diff --git a/backend/analytics_server/mhq/api/teams.py b/backend/analytics_server/mhq/api/teams.py index 08b0f32d2..485fe54b7 100644 --- a/backend/analytics_server/mhq/api/teams.py +++ b/backend/analytics_server/mhq/api/teams.py @@ -1,20 +1,45 @@ +from datetime import datetime + from flask import Blueprint from typing import Any, Dict, List from voluptuous import Required, Schema, Optional, All, Coerce from werkzeug.exceptions import BadRequest from mhq.store.models.code.repository import OrgRepo, TeamRepos from mhq.service.code.models.org_repo import RawTeamOrgRepo +from mhq.service.project.models.org_project import RawTeamOrgProject from mhq.api.resources.code_resouces import ( adapt_team_repos, adapt_team_repo_and_org_repo, ) +from mhq.api.resources.project_resources import adapt_org_projects +from mhq.api.resources.ticket_insights_resources import ( + adapt_ticket_insights, + adapt_unlinked_prs, +) +from mhq.api.resources.ticket_lead_time_resources import ( + adapt_ticket_lead_time_metrics, +) +from mhq.api.resources.sprint_resources import adapt_sprints from mhq.service.code.repository_service import get_repository_service +from mhq.service.code.ticket_lead_time import get_ticket_lead_time_service +from mhq.service.project.repository_service import get_project_service +from mhq.service.sprints import get_sprint_service +from mhq.service.ticket_insights import get_ticket_insights_service from mhq.api.resources.core_resources import adapt_team from mhq.store.models.core.teams import Team +from mhq.store.models.projects import OrgProject from mhq.service.core.teams import get_team_service -from mhq.api.request_utils import coerce_org_repos, coerce_team_repos, dataschema +from mhq.api.request_utils import ( + coerce_org_repos, + coerce_org_projects, + coerce_team_repos, + dataschema, + queryschema, + uuid_validator, +) from mhq.service.query_validator import get_query_validator +from mhq.store.repos.team_repo_project_mapping import TeamRepoProjectMappingRepoService app = Blueprint("teams", __name__) @@ -156,3 +181,206 @@ def patch_team_repos_mapping(team_id: str, team_repos_data: List[TeamRepos]): team_repos_service = get_repository_service() team_repos = team_repos_service.patch_team_repos_mapping(team, team_repos_data) return adapt_team_repos(team_repos) + + +# CLUSTOX: Jira integration, Phase 2 (project selection) -- a team's chosen +# Jira project(s), same "GET the current set / PUT the full replacement set" +# shape as /teams//repos above. See +# docs/JIRA_INTEGRATION_PROPOSAL.md. +@app.route("/teams//projects", methods={"GET"}) +def fetch_team_projects(team_id: str): + + query_validator = get_query_validator() + team: Team = query_validator.team_validator(team_id) + + project_service = get_project_service() + team_org_projects: List[OrgProject] = project_service.get_team_projects(team) + + return adapt_org_projects(team_org_projects) + + +@app.route("/teams//projects", methods={"PUT"}) +@dataschema( + Schema( + { + Required("projects"): All(list, Coerce(coerce_org_projects)), + } + ), +) +def update_team_projects(team_id: str, projects: List[RawTeamOrgProject]): + + query_validator = get_query_validator() + team: Team = query_validator.team_validator(team_id) + + project_service = get_project_service() + updated_org_projects = project_service.update_team_projects(team, projects) + + return adapt_org_projects(updated_org_projects) + + +# CLUSTOX: explicit, informational repo<->Jira-project pairing -- see +# docs/JIRA_MULTI_ACCOUNT_PLAN.md's follow-up on Jira<->repo relationships +# and TeamRepoProjectMapping's own docstring for why this never drives +# PR<->ticket matching. Same "GET the current set / PUT the full +# replacement set" shape as /teams//projects above. +@app.route("/teams//repo_project_mappings", methods={"GET"}) +def fetch_team_repo_project_mappings(team_id: str): + query_validator = get_query_validator() + team: Team = query_validator.team_validator(team_id) + + mappings = TeamRepoProjectMappingRepoService().get_mappings_for_team(team.id) + return [ + { + "org_repo_id": str(mapping.org_repo_id), + "org_project_id": str(mapping.org_project_id), + } + for mapping in mappings + ] + + +@app.route("/teams//repo_project_mappings", methods={"PUT"}) +@dataschema( + Schema( + { + Required("mappings"): [ + { + Required("org_repo_id"): All(str, Coerce(uuid_validator)), + Required("org_project_id"): All(str, Coerce(uuid_validator)), + } + ], + } + ), +) +def update_team_repo_project_mappings(team_id: str, mappings: List[Dict[str, str]]): + query_validator = get_query_validator() + team: Team = query_validator.team_validator(team_id) + + # Informational only, but a pair naming a repo or project this team + # hasn't actually selected would be a mapping to nothing meaningful -- + # rejected rather than silently stored, same "fail loud at the + # boundary" convention the rest of this API follows. + team_repo_ids = { + str(repo.id) for repo in get_repository_service().get_team_repos(team) + } + team_project_ids = { + str(project.id) for project in get_project_service().get_team_projects(team) + } + for mapping in mappings: + if mapping["org_repo_id"] not in team_repo_ids: + raise BadRequest( + f"Repo {mapping['org_repo_id']} is not tracked by team {team_id}." + ) + if mapping["org_project_id"] not in team_project_ids: + raise BadRequest( + f"Project {mapping['org_project_id']} is not tracked by team {team_id}." # noqa E501 + ) + + saved = TeamRepoProjectMappingRepoService().set_mappings_for_team( + team.id, + [(mapping["org_repo_id"], mapping["org_project_id"]) for mapping in mappings], + ) + return [ + { + "org_repo_id": str(mapping.org_repo_id), + "org_project_id": str(mapping.org_project_id), + } + for mapping in saved + ] + + +# CLUSTOX: Jira integration, Phase 4 (§6C/§6E) -- the DORA Metrics page's +# ticket-cycle-time widget and "N PRs merged with no linked ticket" +# callout. Deliberately its own read-only endpoint, not folded into the +# existing dora_metrics/deployment_analytics ones -- keeps this additive +# to the 4 existing DORA cards rather than touching their code. See +# docs/JIRA_INTEGRATION_PROPOSAL.md. +@app.route("/teams//ticket_insights", methods={"GET"}) +@queryschema( + Schema( + { + Required("from_time"): All(str, Coerce(datetime.fromisoformat)), + Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + } + ), +) +def get_team_ticket_insights(team_id: str, from_time: datetime, to_time: datetime): + + query_validator = get_query_validator() + team: Team = query_validator.team_validator(team_id) + interval = query_validator.interval_validator(from_time, to_time) + + ticket_insights_service = get_ticket_insights_service() + insights = ticket_insights_service.get_team_ticket_insights(team, interval) + + return adapt_ticket_insights(insights) + + +# CLUSTOX: Jira integration, Phase 4 (§6E) -- the Data Hygiene card's +# drill-down. Its own route/fetch, not folded into ticket_insights above: +# that one loads on every DORA Metrics page view, this one only when +# someone actually opens the unmatched-PRs list. See +# docs/JIRA_INTEGRATION_PROPOSAL.md. +@app.route("/teams//unlinked_prs", methods={"GET"}) +@queryschema( + Schema( + { + Required("from_time"): All(str, Coerce(datetime.fromisoformat)), + Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + } + ), +) +def get_team_unlinked_prs(team_id: str, from_time: datetime, to_time: datetime): + + query_validator = get_query_validator() + team: Team = query_validator.team_validator(team_id) + interval = query_validator.interval_validator(from_time, to_time) + + ticket_insights_service = get_ticket_insights_service() + prs = ticket_insights_service.get_team_unlinked_prs(team, interval) + + return adapt_unlinked_prs(prs) + + +# CLUSTOX: Jira integration -- the extended Lead Time breakdown's +# "ticket created -> first commit" leading phase (docs/ +# JIRA_INTEGRATION_PROPOSAL.md §6A). Its own route, not a change to +# /teams//lead_time above: that one's response and the plain +# DORA "Lead Time for Changes" card that depends on it stay exactly as +# they are. See TicketLeadTimeMetrics for why this is a separate, +# matched-subset-only average rather than a new field mixed into the +# org-wide one. +@app.route("/teams//ticket_lead_time", methods={"GET"}) +@queryschema( + Schema( + { + Required("from_time"): All(str, Coerce(datetime.fromisoformat)), + Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + } + ), +) +def get_team_ticket_lead_time(team_id: str, from_time: datetime, to_time: datetime): + + query_validator = get_query_validator() + team: Team = query_validator.team_validator(team_id) + interval = query_validator.interval_validator(from_time, to_time) + + ticket_lead_time_service = get_ticket_lead_time_service() + metrics = ticket_lead_time_service.get_team_ticket_lead_time_metrics(team, interval) + + return adapt_ticket_lead_time_metrics(metrics) + + +# CLUSTOX: Jira integration -- the Sprint rollup chart (docs/ +# JIRA_INTEGRATION_PROPOSAL.md §6D). No date-range params -- a sprint's +# own start/end dates are its natural window, unlike the DORA Metrics +# page's selected period. +@app.route("/teams//sprints", methods={"GET"}) +def get_team_sprints(team_id: str): + + query_validator = get_query_validator() + team: Team = query_validator.team_validator(team_id) + + sprint_service = get_sprint_service() + sprints = sprint_service.get_team_sprints(team) + + return adapt_sprints(sprints) diff --git a/backend/analytics_server/mhq/clustox_auth/__init__.py b/backend/analytics_server/mhq/clustox_auth/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/mhq/clustox_auth/internal_token.py b/backend/analytics_server/mhq/clustox_auth/internal_token.py new file mode 100644 index 000000000..d14e76a80 --- /dev/null +++ b/backend/analytics_server/mhq/clustox_auth/internal_token.py @@ -0,0 +1,37 @@ +""" +CLUSTOX: restrict the Flask servers to traffic from the Next.js BFF. + +Upstream ships these APIs unauthenticated, relying on the loopback port binding +in docker-compose. Once authorization lives in the BFF, a direct call to :9696 +would bypass every check, so every request must carry a shared secret. + +Fails closed: if INTERNAL_API_TOKEN is unset, all requests are rejected. +""" + +import hmac +from os import getenv + +from flask import Flask, jsonify, request + +HEADER = "X-Internal-Token" + +# Health checks must stay reachable for container and UI status probes. +EXEMPT_PATHS = {"/"} + + +def register_internal_token_guard(app: Flask) -> None: + @app.before_request + def _verify_internal_token(): + if request.path in EXEMPT_PATHS: + return None + + expected = getenv("INTERNAL_API_TOKEN") + if not expected: + return jsonify({"error": "internal token not configured"}), 403 + + provided = request.headers.get(HEADER, "") + # compare_digest avoids leaking the token through response timing. + if not hmac.compare_digest(provided, expected): + return jsonify({"error": "forbidden"}), 403 + + return None diff --git a/backend/analytics_server/mhq/clustox_auth/sync_run.py b/backend/analytics_server/mhq/clustox_auth/sync_run.py new file mode 100644 index 000000000..c05768c03 --- /dev/null +++ b/backend/analytics_server/mhq/clustox_auth/sync_run.py @@ -0,0 +1,113 @@ +""" +CLUSTOX: per-workspace sync outcomes. + +Upstream syncs a single organisation and reports success regardless of what +happened inside it. With one sync per workspace that is untenable: a failing +workspace would be invisible among the ones that worked. + +Every workspace sync records a row here, so "did last night's sync work?" is a +query rather than an exercise in reading log files before they rotate. +""" + +from datetime import datetime +from typing import List, Optional + +from sqlalchemy import Column, DateTime, String, Text, func +from sqlalchemy.dialects.postgresql import UUID + +from mhq.store import db, rollback_on_exc +from mhq.utils.time import time_now + + +class SyncRunStatus: + RUNNING = "RUNNING" + SUCCESS = "SUCCESS" + FAILED = "FAILED" + SKIPPED = "SKIPPED" + + +class ClustoxSyncRun(db.Model): + __tablename__ = "ClustoxSyncRun" + + id = Column(UUID(as_uuid=True), primary_key=True) + org_id = Column(UUID(as_uuid=True), nullable=False) + started_at = Column(DateTime(timezone=True), server_default=func.now()) + finished_at = Column(DateTime(timezone=True)) + status = Column(String, nullable=False) + detail = Column(Text) + + +class SyncRunService: + def __init__(self): + self._db = db + + @rollback_on_exc + def start(self, org_id: str) -> ClustoxSyncRun: + from mhq.utils.string import uuid4_str + + run = ClustoxSyncRun( + id=uuid4_str(), + org_id=org_id, + started_at=time_now(), + status=SyncRunStatus.RUNNING, + ) + self._db.session.add(run) + self._db.session.commit() + return run + + @rollback_on_exc + def finish( + self, run: ClustoxSyncRun, status: str, detail: Optional[str] = None + ) -> None: + run.status = status + run.finished_at = time_now() + # Truncated: a stack trace helps, an unbounded one bloats the row. + run.detail = detail[:2000] if detail else None + self._db.session.commit() + + @rollback_on_exc + def latest_per_org(self) -> List[ClustoxSyncRun]: + """Most recent run for each workspace.""" + subq = ( + self._db.session.query( + ClustoxSyncRun.org_id, + func.max(ClustoxSyncRun.started_at).label("latest"), + ) + .group_by(ClustoxSyncRun.org_id) + .subquery() + ) + return ( + self._db.session.query(ClustoxSyncRun) + .join( + subq, + (ClustoxSyncRun.org_id == subq.c.org_id) + & (ClustoxSyncRun.started_at == subq.c.latest), + ) + .all() + ) + + @rollback_on_exc + def latest_for_org(self, org_id: str) -> Optional[ClustoxSyncRun]: + return ( + self._db.session.query(ClustoxSyncRun) + .filter(ClustoxSyncRun.org_id == org_id) + .order_by(ClustoxSyncRun.started_at.desc()) + .first() + ) + + +def get_sync_run_service() -> SyncRunService: + return SyncRunService() + + +def as_dict(run: ClustoxSyncRun) -> dict: + def iso(value: Optional[datetime]) -> Optional[str]: + return value.isoformat() if value else None + + return { + "org_id": str(run.org_id), + "started_at": iso(run.started_at), + "finished_at": iso(run.finished_at), + "status": run.status, + "detail": run.detail, + } diff --git a/backend/analytics_server/mhq/exapi/bitbucket.py b/backend/analytics_server/mhq/exapi/bitbucket.py new file mode 100644 index 000000000..a19d1af4e --- /dev/null +++ b/backend/analytics_server/mhq/exapi/bitbucket.py @@ -0,0 +1,129 @@ +from datetime import datetime +from typing import Dict, List, Optional + +import requests +from requests.exceptions import HTTPError + +PAGE_SIZE = 50 + + +class BitbucketRateLimitExceeded(Exception): + """Raised on HTTP 429. The client only reports; the ETL handler owns the + pause-and-resume policy, because only it knows what has been safely + stored so far.""" + + def __init__(self, message: str, retry_after_seconds: Optional[int] = None): + super().__init__(message) + self.retry_after_seconds = retry_after_seconds + + +class BitbucketApiService: + """Bitbucket Cloud v2 REST client. + + CLUSTOX: auth is an Atlassian API token via HTTP Basic auth + (email, token) -- app passwords are being phased out by Atlassian, and + OAuth would be a different shape from every other integration here. The + pair rides on the session, never in a header string and never in a URL. + """ + + def __init__(self, email: str, api_token: str): + self.base_url = "https://api.bitbucket.org/2.0" + self._session = requests.Session() + self._session.auth = (email, api_token) + + def check_pat(self) -> bool: + url = f"{self.base_url}/user" + try: + response = self._session.get(url) + except Exception as e: + raise Exception(f"Error in Bitbucket token validation, Error: {e}") + # CLUSTOX: a 429 here must surface as a rate limit, not as False -- + # False becomes "token is invalid, revoked or expired" at sync start, + # and the pause-and-resume window after a 429 is exactly when the next + # sync's validity check runs. Telling a paused org its token died + # would send an admin to rotate a perfectly good credential. + if response.status_code == 429: + self._handle_error(response) + return response.status_code == 200 + + def _handle_error(self, response): + if response.status_code == 429: + retry_after = response.headers.get("Retry-After") + raise BitbucketRateLimitExceeded( + "Bitbucket rate limit exceeded", + retry_after_seconds=int(retry_after) if retry_after else None, + ) + if response.status_code != 200: + try: + error = (response.json().get("error") or {}).get("message", "") + except ValueError: + error = "" + raise HTTPError( + f"Request failed with status {response.status_code}: {error}" + ) + + def _get_paginated(self, url: str, params: Optional[Dict] = None) -> List[Dict]: + """Bitbucket paginates by `next` link, not page numbers: each page is + `{"values": [...], "next": ""}` and the last page has no + `next`. The `next` URL already carries every query param, so params go + only on the first request.""" + values: List[Dict] = [] + request_params = {**(params or {}), "pagelen": PAGE_SIZE} + + while url: + response = self._session.get(url, params=request_params) + self._handle_error(response) + body = response.json() + values.extend(body.get("values") or []) + url = body.get("next") + request_params = None + + return values + + def get_workspaces(self) -> List[Dict]: + """CLUSTOX: /2.0/workspaces is dead -- it returns 410 "CHANGE-2770 - + Functionality has been deprecated" (discovered live; every fixture + test passed happily against it). The supported listing is the user's + workspace permissions, whose entries wrap the workspace object.""" + permissions = self._get_paginated( + f"{self.base_url}/user/permissions/workspaces" + ) + return [entry["workspace"] for entry in permissions if entry.get("workspace")] + + def get_workspace_repos(self, workspace: str) -> List[Dict]: + return self._get_paginated(f"{self.base_url}/repositories/{workspace}") + + def get_repo_pull_requests( + self, workspace: str, repo_slug: str, updated_since: datetime + ) -> List[Dict]: + url = f"{self.base_url}/repositories/{workspace}/{repo_slug}/pullrequests" + # CLUSTOX: `state` must list every state explicitly -- without it the + # API returns ONLY open PRs, and every merged PR silently vanishes + # from lead time. Fetched "successfully", wrong, and invisible. + params = { + "q": f'updated_on > "{updated_since.isoformat()}"', + "sort": "updated_on", + "state": ["MERGED", "OPEN", "DECLINED", "SUPERSEDED"], + } + return self._get_paginated(url, params) + + def get_pr_activity(self, workspace: str, repo_slug: str, pr_id: int) -> List[Dict]: + url = ( + f"{self.base_url}/repositories/{workspace}/{repo_slug}" + f"/pullrequests/{pr_id}/activity" + ) + return self._get_paginated(url) + + def get_pr_commits(self, workspace: str, repo_slug: str, pr_id: int) -> List[Dict]: + url = ( + f"{self.base_url}/repositories/{workspace}/{repo_slug}" + f"/pullrequests/{pr_id}/commits" + ) + return self._get_paginated(url) + + def get_pr_diffstat(self, workspace: str, repo_slug: str, pr_id: int) -> List[Dict]: + url = ( + f"{self.base_url}/repositories/{workspace}/{repo_slug}" + f"/pullrequests/{pr_id}/diffstat" + ) + return self._get_paginated(url) diff --git a/backend/analytics_server/mhq/exapi/jenkins.py b/backend/analytics_server/mhq/exapi/jenkins.py new file mode 100644 index 000000000..267bdaf16 --- /dev/null +++ b/backend/analytics_server/mhq/exapi/jenkins.py @@ -0,0 +1,319 @@ +# CLUSTOX: Jenkins REST client. Knows Jenkins' API shape and nothing about our +# models -- adaptation to RepoWorkflowRuns lives in the ETL handler. +import ipaddress +import json +import socket +from datetime import datetime +from time import monotonic +from typing import Dict, List, Optional, Tuple, Union +from urllib.parse import urlsplit + +import pytz +import requests + +# Connect and read timeouts. The workspace sync loop is sequential, so an +# untimed request against a hung Jenkins stalls every workspace behind it. +DEFAULT_TIMEOUT: Tuple[int, int] = (5, 30) + +# Total wall-clock ceiling for a single Jenkins request. requests' read timeout +# is between bytes, not total: a Jenkins dribbling one byte every 29 seconds +# satisfies a 30s read timeout forever and holds the sequential sync loop open +# indefinitely. This is the ceiling the design doc promised. +DEFAULT_MAX_SECONDS: int = 60 + +# Upper bound on a single read, not a demand for that many bytes -- see _get. +RESPONSE_CHUNK_SIZE = 64 * 1024 + +# Fetched per build. Explicit rather than a wildcard so a Jenkins with many +# plugins does not return megabytes of action data per build. +BUILD_TREE = ( + "builds[number,result,timestamp,duration,url,building," + "actions[causes[userId,userName,shortDescription]," + "lastBuiltRevision[SHA1,branch[name]]]]" +) + +JOB_FIELDS = "name,fullName,url,_class" + +# Folders and multibranch projects nest, and a non-recursive tree offers the +# admin the container instead of the job. Mapping a container yields a URL with +# no "builds" key, so the sync reports zero deployments forever without an +# error. Three levels covers folder/multibranch layouts without asking a large +# Jenkins for its whole tree. +JOB_TREE_DEPTH = 3 + +# Substrings of the _class Jenkins reports for things that hold jobs rather than +# being one. Matched loosely because the plugin class names are long and +# version-dependent, and because a container sitting exactly at JOB_TREE_DEPTH +# has no nested "jobs" key to give it away. +CONTAINER_CLASS_MARKERS = ("Folder", "MultiBranchProject") + + +def _build_job_tree(depth: int) -> str: + tree = f"jobs[{JOB_FIELDS}]" + for _ in range(max(depth - 1, 0)): + tree = f"jobs[{JOB_FIELDS},{tree}]" + return tree + + +JOB_TREE = _build_job_tree(JOB_TREE_DEPTH) + + +def job_path(job_full_name: str) -> str: + """Jenkins addresses nested jobs by repeating /job/ for each segment.""" + return "/".join(f"job/{segment}" for segment in job_full_name.split("/")) + + +# CLUSTOX: base_url arrives from an admin through the setup form and the server +# then fetches it. The browser validated it; the server did not, so an +# authenticated admin could point the workspace at http://169.254.169.254/latest +# or any address inside our network and read the response back out of the jobs +# endpoint -- cloud instance credentials included. Everything below exists to +# stop that. +ALLOWED_SCHEMES = ("http", "https") + + +class JenkinsAddressNotAllowed(requests.RequestException): + """ + Raised before any request leaves the process. A RequestException so the + routes that already handle an unreachable Jenkins keep working, but its own + type so they can say "this address is not allowed" rather than "Jenkins is + unreachable" -- an admin who typed an internal hostname needs to be told + which of the two happened. + """ + + +def _resolved_addresses(hostname: str, port: Optional[int]) -> List[str]: + try: + infos = socket.getaddrinfo( + hostname, port or 0, type=socket.SOCK_STREAM, proto=socket.IPPROTO_TCP + ) + except socket.gaierror as e: + # A name that does not resolve is an unreachable Jenkins, not an + # attempt to reach somewhere it should not -- different message, and + # the existing 502 path is the right home for it. + raise requests.ConnectionError( + f"Could not resolve Jenkins host {hostname}: {e}" + ) + return [info[4][0] for info in infos] + + +def _is_disallowed(ip: Union[ipaddress.IPv4Address, ipaddress.IPv6Address]) -> bool: + # An IPv4 address written as ::ffff:10.0.0.5 is an IPv4 address, and + # is_private on the IPv6 form does not say so. + mapped = getattr(ip, "ipv4_mapped", None) + if mapped is not None: + ip = mapped + return ( + ip.is_private # RFC1918, and loopback and link-local with it + or ip.is_loopback + or ip.is_link_local # 169.254.0.0/16, the cloud metadata range + or ip.is_reserved + or ip.is_multicast + or ip.is_unspecified + ) + + +def assert_url_is_allowed(url: str) -> None: + """ + Rejects anything that is not an http(s) URL pointing at a public address. + Called before the request rather than after, because a request to + 169.254.169.254 has already leaked whatever it was going to leak by the + time a response comes back. + """ + parts = urlsplit(url) + if parts.scheme not in ALLOWED_SCHEMES: + raise JenkinsAddressNotAllowed( + f"Jenkins URL must start with http:// or https://, got '{url}'" + ) + + try: + hostname, port = parts.hostname, parts.port + except ValueError as e: + raise JenkinsAddressNotAllowed(f"Jenkins URL '{url}' is not valid: {e}") + if not hostname: + raise JenkinsAddressNotAllowed(f"Jenkins URL '{url}' has no host") + + # Every address the name resolves to, not the first: a hostname an admin + # controls can resolve to a public address and an internal one at once, and + # which of them is connected to is not ours to predict. + for address in _resolved_addresses(hostname, port): + try: + ip = ipaddress.ip_address(address) + except ValueError: + raise JenkinsAddressNotAllowed( + f"Jenkins host {hostname} resolved to something that is not an " + f"IP address: {address}" + ) + if _is_disallowed(ip): + raise JenkinsAddressNotAllowed( + f"Jenkins host {hostname} resolves to {address}, which is a " + f"private, loopback or link-local address. Middleware will not " + f"fetch internal addresses." + ) + + +class JenkinsResponse: + """ + The slice of requests.Response this client uses, with the body already read + under a wall-clock deadline. Exists so _get can enforce that deadline while + reading, which a plain Response cannot do. + """ + + def __init__(self, status_code: int, body: bytes, url: str, location: str = None): + self.status_code = status_code + self._body = body + self._url = url + self._location = location + + def json(self): + return json.loads(self._body or b"{}") + + def raise_for_status(self) -> None: + # CLUSTOX: redirects are no longer followed, so a 3xx arrives here with + # an empty body. Left to fall through it parses as {} and reports "no + # jobs" -- a Jenkins behind a login redirect or a moved URL would look + # like an empty one. Named instead. + if 300 <= self.status_code < 400: + raise requests.HTTPError( + f"Jenkins redirected {self._url} to " + f"'{self._location or 'an unnamed location'}'. Redirects are " + f"not followed; check the base URL is the one Jenkins serves." + ) + if self.status_code >= 400: + raise requests.HTTPError( + f"Jenkins returned HTTP {self.status_code} for {self._url}" + ) + + +class JenkinsApiService: + def __init__( + self, + base_url: str, + username: str, + api_token: str, + timeout: Tuple[int, int] = DEFAULT_TIMEOUT, + max_seconds: int = DEFAULT_MAX_SECONDS, + ): + self._base_url = base_url.rstrip("/") + self._auth = (username, api_token) + self._timeout = timeout + self._max_seconds = max_seconds + + def _get(self, path: str) -> JenkinsResponse: + url = f"{self._base_url}/{path}" + # CLUSTOX: checked per request, not once in __init__. A service object + # outlives a DNS record, and the check is worth nothing if the address + # it approved is not the one being fetched now. + assert_url_is_allowed(url) + deadline = monotonic() + self._max_seconds + connect_timeout, read_timeout = self._timeout + # CLUSTOX: a read timeout longer than the total ceiling can never be + # reached without the ceiling firing first, and it is the only thing + # standing between us and a server that stops sending entirely. Capping + # it keeps the worst case at ceiling + one gap instead of + # ceiling + read_timeout. + read_timeout = min(read_timeout, self._max_seconds) + # TLS verification is deliberately not configurable. A skip-verify + # option is easy to add, hard to remove, and this connection carries an + # API token. + response = requests.get( + url, + auth=self._auth, + timeout=(connect_timeout, read_timeout), + verify=True, + stream=True, + # CLUSTOX: the address check above covers the URL we ask for, and a + # followed redirect is a second URL nobody checked. A public host -- + # the admin's own, or one that answers it -- redirecting to + # 169.254.169.254 walks straight past the check. Not following + # means the check and the request are about the same address. + allow_redirects=False, + ) + body = bytearray() + try: + # CLUSTOX: read1(), not iter_content(). iter_content() asks + # urllib3.stream() for a whole chunk, and on a Content-Length + # response urllib3 blocks until it has that many bytes -- so a + # deadline checked between chunks is not consulted until 64 KiB has + # accumulated. Measured against a server dribbling 100 bytes every + # 50ms, the first chunk arrived after 36 seconds with the ceiling + # set well below that. read1() returns whatever bytes are already + # available, so the deadline is re-checked at the pace the server + # actually sends and the ceiling genuinely bounds the call. + # Chunked-transfer responses happen to yield per HTTP chunk, but + # nothing guarantees a Jenkins sends chunked. + raw = response.raw + while True: + if monotonic() > deadline: + raise requests.exceptions.Timeout( + f"Jenkins request to {url} exceeded the " + f"{self._max_seconds}s ceiling" + ) + # decode_content=True so a gzipped body -- requests advertises + # gzip on every request -- is still decoded, which reading + # response.raw directly would otherwise skip. + chunk = raw.read1(RESPONSE_CHUNK_SIZE, decode_content=True) + if not chunk: + break + body.extend(chunk) + finally: + response.close() + return JenkinsResponse( + response.status_code, bytes(body), url, response.headers.get("Location") + ) + + def check_pat(self) -> bool: + response = self._get("api/json") + return response.status_code == 200 + + def get_jobs(self) -> List[Dict]: + response = self._get(f"api/json?tree={JOB_TREE}") + response.raise_for_status() + return self._collect_jobs(response.json().get("jobs", [])) + + @classmethod + def _collect_jobs(cls, nodes: Optional[List[Dict]]) -> List[Dict]: + """Flattens the nested tree down to the entries that can actually run.""" + jobs: List[Dict] = [] + for node in nodes or []: + if not node: + continue + children = node.get("jobs") + if isinstance(children, list): + jobs.extend(cls._collect_jobs(children)) + continue + if cls._is_container(node): + # A container deeper than the tree we asked for. Offering it + # would let an admin map something that reports zero + # deployments forever, so leave it out. + continue + jobs.append( + { + "name": node.get("name"), + "full_name": node.get("fullName") or node.get("name"), + "url": node.get("url"), + } + ) + return jobs + + @staticmethod + def _is_container(node: Dict) -> bool: + job_class = node.get("_class") or "" + return any(marker in job_class for marker in CONTAINER_CLASS_MARKERS) + + def get_builds(self, job_full_name: str, bookmark: datetime) -> List[Dict]: + response = self._get(f"{job_path(job_full_name)}/api/json?tree={BUILD_TREE}") + response.raise_for_status() + builds = response.json().get("builds", []) + return [ + build + for build in builds + if self._build_time(build) and self._build_time(build) > bookmark + ] + + @staticmethod + def _build_time(build: Dict): + timestamp = build.get("timestamp") + if not timestamp: + return None + return datetime.fromtimestamp(timestamp / 1000, tz=pytz.UTC) diff --git a/backend/analytics_server/mhq/exapi/jira.py b/backend/analytics_server/mhq/exapi/jira.py new file mode 100644 index 000000000..0c36d728e --- /dev/null +++ b/backend/analytics_server/mhq/exapi/jira.py @@ -0,0 +1,184 @@ +from typing import Dict, List, Optional, Tuple + +import requests +from requests.auth import HTTPBasicAuth + +from mhq.exapi.models.jira import JiraBoard, JiraIssue, JiraSprint + +# Jira retired GET/POST /rest/api/3/search in favor of /rest/api/3/search/jql +# (the old endpoint now returns 410 Gone) -- pagination there is cursor-based +# via nextPageToken, not startAt/total. +_SEARCH_PAGE_SIZE = 100 +# Deliberately narrow: only what Ticket/JiraIssue actually use. Story +# points/sprint are intentionally not requested -- Jira exposes those as +# instance-specific customfield_* ids that need a separate /field lookup +# to resolve by name, and nothing downstream reads them yet. Add that +# lookup if/when a consumer needs them, rather than paying for it now. +_ISSUE_FIELDS = ["summary", "status", "issuetype", "created", "updated"] + + +class JiraApiError(Exception): + def __init__(self, message: str, status_code: Optional[int] = None): + super().__init__(message) + self.status_code = status_code + + +class JiraRateLimitExceeded(JiraApiError): + def __init__(self, retry_after: Optional[str] = None): + super().__init__("Jira rate limit exceeded", status_code=429) + self.retry_after = retry_after + + +class JiraApiService: + def __init__(self, email: str, api_token: str, site_url: str): + self.base_url = f"https://{site_url}/rest/api/3" + # Sprints/boards live on a different API surface than issues -- + # /rest/agile/1.0, not /rest/api/3 -- same site, same auth. + self.agile_base_url = f"https://{site_url}/rest/agile/1.0" + self._auth = HTTPBasicAuth(email, api_token) + + def check_pat(self) -> bool: + response = requests.get(f"{self.base_url}/myself", auth=self._auth, timeout=8) + return response.status_code == 200 + + def search_issues( + self, + jql: str, + next_page_token: Optional[str] = None, + max_results: int = _SEARCH_PAGE_SIZE, + ) -> Dict: + """ + One page of /rest/api/3/search/jql, with the changelog expanded + so status-transition history comes back in the same call -- + avoids a separate per-issue changelog request (a real N+1 this + way, not just a theoretical one, since a project can easily have + hundreds of issues). + """ + body = { + "jql": jql, + "fields": _ISSUE_FIELDS, + "expand": "changelog", + "maxResults": max_results, + } + if next_page_token: + body["nextPageToken"] = next_page_token + + response = requests.post( + f"{self.base_url}/search/jql", json=body, auth=self._auth, timeout=15 + ) + self._raise_for_error(response) + return response.json() + + def get_all_issues(self, jql: str) -> List[JiraIssue]: + issues: List[JiraIssue] = [] + next_page_token = None + + while True: + page = self.search_issues(jql, next_page_token=next_page_token) + page_issues = page.get("issues") or [] + issues.extend(JiraIssue(issue) for issue in page_issues) + + next_page_token = page.get("nextPageToken") + # Defensive on both signals: this endpoint dropped the old + # /search's explicit isLast/total fields in favor of a bare + # cursor, so stop on either "no cursor for a next page" or "the + # page came back empty" -- either alone is a safe stop + # condition, so checking both can't loop forever even if one + # of them doesn't behave exactly as expected for some query. + if not next_page_token or not page_issues: + break + + return issues + + def get_boards_for_project(self, project_key: str) -> List[JiraBoard]: + """ + CLUSTOX: Jira integration -- the Sprint rollup chart (docs/ + JIRA_INTEGRATION_PROPOSAL.md §6D). Only scrum boards have + sprints -- callers filter on board_type before ever calling the + sprint endpoints below, so a Kanban-only project cleanly syncs + zero sprints rather than erroring. + """ + boards: List[JiraBoard] = [] + start_at = 0 + + while True: + response = requests.get( + f"{self.agile_base_url}/board", + params={"projectKeyOrId": project_key, "startAt": start_at}, + auth=self._auth, + timeout=8, + ) + self._raise_for_error(response) + page = response.json() + page_values = page.get("values") or [] + boards.extend(JiraBoard(board) for board in page_values) + + if page.get("isLast", True) or not page_values: + break + start_at += len(page_values) + + return boards + + def get_sprints_for_board(self, board_id: int) -> List[JiraSprint]: + sprints: List[JiraSprint] = [] + start_at = 0 + + while True: + response = requests.get( + f"{self.agile_base_url}/board/{board_id}/sprint", + params={"startAt": start_at}, + auth=self._auth, + timeout=8, + ) + self._raise_for_error(response) + page = response.json() + page_values = page.get("values") or [] + sprints.extend(JiraSprint(sprint) for sprint in page_values) + + if page.get("isLast", True) or not page_values: + break + start_at += len(page_values) + + return sprints + + def get_sprint_issue_counts(self, sprint_id: int) -> Tuple[int, int]: + """ + (planned_count, completed_count) for one sprint -- two + maxResults=0 calls that read only each response's own `total` + metadata field, never the issue bodies themselves. A sprint can + carry hundreds of issues on a real site (confirmed live against + this org's own data), so fetching full issue lists just to count + them would be needless -- Jira's sprint-issue endpoint already + reports `total` without paginating through the issues at all. + """ + planned = self._sprint_issue_total(sprint_id) + completed = self._sprint_issue_total(sprint_id, jql="statusCategory = Done") + return planned, completed + + def _sprint_issue_total(self, sprint_id: int, jql: Optional[str] = None) -> int: + params = {"maxResults": 0} + if jql: + params["jql"] = jql + + response = requests.get( + f"{self.agile_base_url}/sprint/{sprint_id}/issue", + params=params, + auth=self._auth, + timeout=8, + ) + self._raise_for_error(response) + return response.json().get("total", 0) + + def _raise_for_error(self, response: requests.Response) -> None: + if response.status_code == 429: + raise JiraRateLimitExceeded(retry_after=response.headers.get("Retry-After")) + if response.status_code in (401, 403): + raise JiraApiError( + "Jira credentials were rejected", status_code=response.status_code + ) + if response.status_code >= 400: + raise JiraApiError( + f"Jira request failed with status {response.status_code}: " + f"{response.text}", + status_code=response.status_code, + ) diff --git a/backend/analytics_server/mhq/exapi/models/bitbucket.py b/backend/analytics_server/mhq/exapi/models/bitbucket.py new file mode 100644 index 000000000..4627eb5cd --- /dev/null +++ b/backend/analytics_server/mhq/exapi/models/bitbucket.py @@ -0,0 +1,72 @@ +from dataclasses import dataclass +from datetime import datetime +from typing import Dict, List, Optional + +from mhq.utils.time import dt_from_iso_time_string + + +@dataclass +class BitbucketRepo: + name: str + org_name: str + default_branch: str + idempotency_key: str + slug: str + description: str + web_url: str + + def __init__(self, repo: Dict): + self.name = repo.get("name") + # CLUSTOX: the workspace slug is Bitbucket's org-level container -- + # the counterpart of a GitHub org / GitLab namespace. + self.org_name = (repo.get("workspace") or {}).get("slug") + self.default_branch = (repo.get("mainbranch") or {}).get("name") + # The uuid, not the slug: slugs can be renamed, uuids cannot, and the + # idempotency key is what sync dedupes on across renames. + self.idempotency_key = repo.get("uuid") + self.slug = repo.get("slug") + self.description = repo.get("description") + self.web_url = ((repo.get("links") or {}).get("html") or {}).get("href") + + +@dataclass +class BitbucketPR: + number: int + title: str + state: str + author_uuid: Optional[str] + author_nickname: Optional[str] + head_branch: Optional[str] + base_branch: Optional[str] + created_on: Optional[datetime] + updated_on: Optional[datetime] + merge_commit_sha: Optional[str] + url: Optional[str] + participants: List[Dict] + raw_data: Dict + + def __init__(self, pr: Dict): + self.number = pr.get("id") + self.title = pr.get("title") or "" + self.state = pr.get("state") + author = pr.get("author") or {} + self.author_uuid = author.get("uuid") + self.author_nickname = author.get("nickname") + self.head_branch = ((pr.get("source") or {}).get("branch") or {}).get("name") + self.base_branch = ((pr.get("destination") or {}).get("branch") or {}).get( + "name" + ) + self.created_on = _dt_or_none(pr.get("created_on")) + self.updated_on = _dt_or_none(pr.get("updated_on")) + # CLUSTOX: null on every unmerged PR -- the common case, not an edge + # case. `or {}` keeps a None from exploding the chain. + self.merge_commit_sha = (pr.get("merge_commit") or {}).get("hash") + self.url = ((pr.get("links") or {}).get("html") or {}).get("href") + self.participants = pr.get("participants") or [] + self.raw_data = pr + + +def _dt_or_none(value: Optional[str]) -> Optional[datetime]: + if not value: + return None + return dt_from_iso_time_string(value) diff --git a/backend/analytics_server/mhq/exapi/models/jira.py b/backend/analytics_server/mhq/exapi/models/jira.py new file mode 100644 index 000000000..8652798a2 --- /dev/null +++ b/backend/analytics_server/mhq/exapi/models/jira.py @@ -0,0 +1,140 @@ +from dataclasses import dataclass, field +from datetime import datetime +from typing import Dict, List, Optional + +from mhq.utils.time import dt_from_iso_time_string + + +@dataclass +class JiraChangelogEntry: + """ + One Jira changelog "history" entry that includes a status change. + A single history entry can bundle several field changes at once (e.g. + status + assignee edited together) -- this only exists for entries + that actually changed `status`; entries with no status item are + filtered out by the caller (see JiraIssue.__init__) rather than kept + around with a meaningless to_status. + """ + + idempotency_key: str + from_status: Optional[str] + to_status: Optional[str] + changed_at: datetime + data: dict + + def __init__(self, history: Dict): + self.data = history + self.idempotency_key = str(history.get("id")) + self.changed_at = dt_from_iso_time_string(history.get("created")) + status_item = next( + ( + item + for item in history.get("items", []) + if item.get("field") == "status" + ), + None, + ) + self.from_status = status_item.get("fromString") if status_item else None + self.to_status = status_item.get("toString") if status_item else None + + +@dataclass +class JiraIssue: + """ + One Jira issue from a /rest/api/3/search/jql response (with + expand=changelog). Deliberately keeps only the fields this + integration actually uses (status, status_category, plus a handful + exposed via `data` for display) -- Jira's full issue payload has far + more fields than that, most of them provider-specific (custom fields + whose ids vary per Jira instance) and unused here. + """ + + id: str + key: str + status: Optional[str] + status_category: Optional[str] + created: datetime + updated: datetime + changelog: List[JiraChangelogEntry] = field(default_factory=list) + data: dict = field(default_factory=dict) + + def __init__(self, issue: Dict): + fields = issue.get("fields", {}) or {} + self.id = str(issue.get("id")) + self.key = issue.get("key") + + status = fields.get("status") or {} + self.status = status.get("name") + self.status_category = (status.get("statusCategory") or {}).get("name") + + self.created = dt_from_iso_time_string(fields.get("created")) + self.updated = dt_from_iso_time_string(fields.get("updated")) + + self.data = { + "summary": fields.get("summary"), + "issue_type": (fields.get("issuetype") or {}).get("name"), + "assignee": (fields.get("assignee") or {}).get("displayName"), + "reporter": (fields.get("reporter") or {}).get("displayName"), + } + + histories = (issue.get("changelog") or {}).get("histories") or [] + # Only entries that actually changed `status` -- most changelog + # history entries are for other fields (comments, priority, + # assignee, ...) and would otherwise become TicketState rows with + # a meaningless null to_status. + self.changelog = [ + entry + for entry in (JiraChangelogEntry(h) for h in histories) + if entry.to_status is not None + ] + + +@dataclass +class JiraBoard: + """ + One board from /rest/agile/1.0/board. `board_type` is "scrum", + "kanban", or "simple" -- only scrum boards have sprints, so + JiraETLHandler filters on this before ever calling the sprint + endpoints. See docs/JIRA_INTEGRATION_PROPOSAL.md §6D. + """ + + id: int + name: str + board_type: str + + def __init__(self, board: Dict): + self.id = board.get("id") + self.name = board.get("name") + self.board_type = board.get("type") + + +@dataclass +class JiraSprint: + """ + One sprint from /rest/agile/1.0/board/{boardId}/sprint. Deliberately + doesn't carry per-issue detail -- planned_count/completed_count are + fetched separately (2 lightweight maxResults=0 calls, not a full + issue list) and attached by the caller, since the Sprint rollup + chart only ever needs the counts. + """ + + id: int + name: str + state: str + start_date: Optional[datetime] + end_date: Optional[datetime] + + def __init__(self, sprint: Dict): + self.id = sprint.get("id") + self.name = sprint.get("name") + self.state = sprint.get("state") + self.start_date = ( + dt_from_iso_time_string(sprint["startDate"]) + if sprint.get("startDate") + else None + ) + self.end_date = ( + dt_from_iso_time_string(sprint["endDate"]) + if sprint.get("endDate") + else None + ) diff --git a/backend/analytics_server/mhq/service/bookmark/bookmark.py b/backend/analytics_server/mhq/service/bookmark/bookmark.py index 11c53410c..60a9ef758 100644 --- a/backend/analytics_server/mhq/service/bookmark/bookmark.py +++ b/backend/analytics_server/mhq/service/bookmark/bookmark.py @@ -19,6 +19,8 @@ IncidentProvider, ) from mhq.store.repos.incidents import IncidentsRepoService +from mhq.store.models.projects import ProjectIssuesBookmark +from mhq.store.repos.projects import ProjectRepoService class BookmarkService: @@ -32,10 +34,12 @@ def __init__( code_repo_service: CodeRepoService, workflow_repo_service: WorkflowRepoService, incident_repo_service: IncidentsRepoService, + project_repo_service: ProjectRepoService, ): self._code_repo_service = code_repo_service self._workflow_repo_service = workflow_repo_service self._incident_repo_service = incident_repo_service + self._project_repo_service = project_repo_service def get_bookmark( self, @@ -70,6 +74,13 @@ def get_bookmark( else mtd_broker_bookmark ) + if bookmark_type == BookmarkType.PROJECT_ISSUES_BOOKMARK: + return datetime.fromisoformat( + self._get_project_issues_bookmark( + entity_id, provider, default_sync_days + ).bookmark + ) + raise ValueError(f"Unsupported BookmarkType: {bookmark_type}.") def update_bookmark( @@ -96,6 +107,11 @@ def update_bookmark( entity_id, bookmark_timestamp ) + if bookmark_type == BookmarkType.PROJECT_ISSUES_BOOKMARK: + return self._update_project_issues_bookmark( + entity_id, provider, bookmark_timestamp + ) + raise ValueError(f"Unsupported BookmarkType: {bookmark_type}.") def reset_org_bookmarks(self, org_id: str, bookmark_timestamp: datetime): @@ -103,6 +119,7 @@ def reset_org_bookmarks(self, org_id: str, bookmark_timestamp: datetime): self._reset_incident_bookmarks(org_id, bookmark_timestamp) self._reset_workflow_bookmarks(org_id, bookmark_timestamp) self._reset_merge_to_deploy_broker_bookmarks(org_id, bookmark_timestamp) + self._reset_project_issues_bookmarks(org_id, bookmark_timestamp) def _get_org_repo_bookmark( self, repo_id: str, default_sync_days: int = DEFAULT_SYNC_DAYS @@ -258,9 +275,53 @@ def _reset_merge_to_deploy_broker_bookmarks( merge_to_deploy_broker_bookmarks ) + def _get_project_issues_bookmark( + self, + org_project_id: str, + provider: str, + default_sync_days: int = DEFAULT_SYNC_DAYS, + ) -> ProjectIssuesBookmark: + bookmark = self._project_repo_service.get_project_issues_bookmark( + org_project_id, provider + ) + if not bookmark: + default_bookmark = time_now() - timedelta(days=default_sync_days) + bookmark = ProjectIssuesBookmark( + org_project_id=org_project_id, + provider=provider, + bookmark=default_bookmark.isoformat(), + ) + return bookmark + + def _update_project_issues_bookmark( + self, org_project_id: str, provider: str, bookmark_time_stamp: datetime + ): + bookmark = self._get_project_issues_bookmark(org_project_id, provider) + bookmark.bookmark = bookmark_time_stamp.isoformat() + bookmark.updated_at = time_now() + self._project_repo_service.update_project_issues_bookmark(bookmark) + + def _reset_project_issues_bookmarks( + self, org_id: str, bookmark_timestamp: datetime + ): + project_issues_bookmarks = ( + self._project_repo_service.get_all_org_project_issues_bookmarks(org_id) + ) + + for bookmark in project_issues_bookmarks: + bookmark.bookmark = bookmark_timestamp.isoformat() + bookmark.updated_at = time_now() + + self._project_repo_service.update_project_issues_bookmarks( + project_issues_bookmarks + ) + def get_bookmark_service(): return BookmarkService( - CodeRepoService(), WorkflowRepoService(), IncidentsRepoService() + CodeRepoService(), + WorkflowRepoService(), + IncidentsRepoService(), + ProjectRepoService(), ) diff --git a/backend/analytics_server/mhq/service/bookmark/bookmark_types.py b/backend/analytics_server/mhq/service/bookmark/bookmark_types.py index 13724a519..e3e6d84ba 100644 --- a/backend/analytics_server/mhq/service/bookmark/bookmark_types.py +++ b/backend/analytics_server/mhq/service/bookmark/bookmark_types.py @@ -7,3 +7,6 @@ class BookmarkType(Enum): INCIDENT_SERVICE_BOOKMARK = "INCIDENT_SERVICE_BOOKMARK" REPO_WORKFLOW_BOOKMARK = "REPO_WORKFLOW_BOOKMARK" MERGE_TO_DEPLOY_BOOKMARK = "MERGE_TO_DEPLOY_BOOKMARK" + # CLUSTOX: Jira integration, Phase 2 (issue sync). See + # docs/JIRA_INTEGRATION_PROPOSAL.md. + PROJECT_ISSUES_BOOKMARK = "PROJECT_ISSUES_BOOKMARK" diff --git a/backend/analytics_server/mhq/service/code/integration.py b/backend/analytics_server/mhq/service/code/integration.py index e79a1792c..35d47e8e2 100644 --- a/backend/analytics_server/mhq/service/code/integration.py +++ b/backend/analytics_server/mhq/service/code/integration.py @@ -3,9 +3,15 @@ from mhq.store.models import UserIdentityProvider, Integration from mhq.store.repos.core import CoreRepoService +# CLUSTOX: every code provider must be listed here as well as in +# CodeETLFactory. This bucket decides WHICH orgs the scheduler syncs at all -- +# a provider registered in the factory but absent here links successfully in +# the UI and then never syncs, with no error anywhere. (The Jenkins +# integration shipped exactly this class of gap.) CODE_INTEGRATION_BUCKET = [ UserIdentityProvider.GITHUB.value, UserIdentityProvider.GITLAB.value, + UserIdentityProvider.BITBUCKET.value, ] diff --git a/backend/analytics_server/mhq/service/code/lead_time.py b/backend/analytics_server/mhq/service/code/lead_time.py index 3122bf780..2e240e078 100644 --- a/backend/analytics_server/mhq/service/code/lead_time.py +++ b/backend/analytics_server/mhq/service/code/lead_time.py @@ -41,6 +41,25 @@ def get_team_lead_time_metrics( self._get_team_repos_lead_time_metrics(team_repos, interval, pr_filter) ) + def get_team_pr_lead_time_metrics( + self, + team: Team, + interval: Interval, + pr_filter: Dict[str, PRFilter] = None, + ) -> List[LeadTimeMetrics]: + """ + The same per-PR LeadTimeMetrics get_team_lead_time_metrics + already computes internally before averaging them together -- + exposed as-is, with no change to that averaging logic, so a + caller needing per-PR granularity (CLUSTOX: the extended Lead + Time breakdown's ticket-matched subset, docs/ + JIRA_INTEGRATION_PROPOSAL.md §6A) doesn't have to duplicate this + method's deployment-config/workflow-vs-PR branching to get it. + """ + team_repos = self._code_repo_service.get_active_team_repos_by_team_id(team.id) + + return self._get_team_repos_lead_time_metrics(team_repos, interval, pr_filter) + def get_team_lead_time_metrics_trends( self, team: Team, diff --git a/backend/analytics_server/mhq/service/code/loc.py b/backend/analytics_server/mhq/service/code/loc.py new file mode 100644 index 000000000..c2fbdc616 --- /dev/null +++ b/backend/analytics_server/mhq/service/code/loc.py @@ -0,0 +1,134 @@ +from dataclasses import dataclass +from datetime import datetime +from typing import Dict, List + +from mhq.store.models.code import PRFilter, PullRequest +from mhq.store.models.code.repository import TeamRepos +from mhq.store.repos.code import CodeRepoService +from mhq.utils.time import ( + Interval, + fill_missing_week_buckets, + generate_expanded_buckets, +) + + +@dataclass +class LOCMetrics: + additions: int = 0 + deletions: int = 0 + # CLUSTOX: gross lines per merged PR, not net. A 2,000-line refactor that + # nets to zero is still 2,000 lines to review, and PR size is benchmarked + # precisely because it predicts review latency. + avg_pr_size: int = 0 + # CLUSTOX: gross lines per CALENDAR day across the selected range. The + # headline it replaces was a period total, which is not comparable between + # two ranges of different lengths -- a month always looks four times + # "better" than a week. A rate is. + # + # Calendar days, not working days: the range the user picked is calendar + # days, and a working-day divisor would silently disagree with the dates on + # screen. It does mean a team that works five days a week reads ~29% lower + # than its working-day rate; that is a consistent scaling, not a bias + # between teams. + avg_daily: int = 0 + + +def aggregate_loc(prs: List[PullRequest], days: int = 0) -> LOCMetrics: + """`days` is the calendar span the PRs were drawn from, used only for + `avg_daily`. Zero (the default) leaves `avg_daily` at 0 rather than + guessing a divisor -- a caller that has no interval has no rate.""" + if not prs: + # CLUSTOX: zero, never None -- the card would otherwise need a third + # empty state on top of "no data" and "no target". + return LOCMetrics() + + additions = sum(pr.additions for pr in prs) + deletions = sum(pr.deletions for pr in prs) + total = additions + deletions + return LOCMetrics( + additions=additions, + deletions=deletions, + avg_pr_size=round(total / len(prs)), + # `days > 0` guards the divisor. A range shorter than a day rounds up + # to 1 rather than dividing by zero -- see interval_days(). + avg_daily=round(total / days) if days > 0 else 0, + ) + + +def interval_days(interval: Interval) -> int: + """Calendar days in an interval, never less than 1. + + CLUSTOX: a same-day range has a duration of 0 and would divide by zero. + Treating it as one day is also the honest reading -- a single day's work + over a single day is that day's rate. + """ + return max(1, round(interval.duration.total_seconds() / 86400)) + + +class LOCService: + def __init__(self, code_repo_service: CodeRepoService): + self._code_repo_service = code_repo_service + + def get_team_loc_metrics( + self, + team_id: str, + interval: Interval, + pr_filter: PRFilter = None, + ) -> LOCMetrics: + return aggregate_loc( + self._get_team_merged_prs(team_id, interval, pr_filter), + days=interval_days(interval), + ) + + def get_team_loc_trends( + self, + team_id: str, + interval: Interval, + pr_filter: PRFilter = None, + ) -> Dict[datetime, LOCMetrics]: + + prs: List[PullRequest] = self._get_team_merged_prs(team_id, interval, pr_filter) + + weekly_prs_map: Dict[datetime, List[PullRequest]] = generate_expanded_buckets( + prs, interval, "state_changed_at", "weekly" + ) + + weekly_loc_metrics_map: Dict[datetime, LOCMetrics] = { + # CLUSTOX: 7, not the whole range. Dividing each weekly bucket by + # the full interval would make every bar shrink as the user widens + # the date picker, while the same weeks' work stayed identical. + week: aggregate_loc(week_prs, days=7) + for week, week_prs in weekly_prs_map.items() + } + + # CLUSTOX: a week with no merges has to come back as a zero bucket, not + # be missing -- the trend line is plotted per week and a hole in the map + # would shift every later point left. + return fill_missing_week_buckets(weekly_loc_metrics_map, interval, LOCMetrics) + + def _get_team_merged_prs( + self, + team_id: str, + interval: Interval, + pr_filter: PRFilter = None, + ) -> List[PullRequest]: + + team_repos: List[TeamRepos] = ( + self._code_repo_service.get_active_team_repos_by_team_id(team_id) + ) + + repo_ids = [str(team_repo.org_repo_id) for team_repo in team_repos] + if not repo_ids: + return [] + + # CLUSTOX: go through the shared filtered query rather than a LOC-specific + # one. It is what makes the contributor filter, branch mode and the + # excluded-PRs setting apply here for free, and it already restricts to + # MERGED PRs -- an abandoned 5,000-line PR is not delivered work. + return self._code_repo_service.get_prs_merged_in_interval( + repo_ids, interval, pr_filter + ) + + +def get_loc_service() -> LOCService: + return LOCService(CodeRepoService()) diff --git a/backend/analytics_server/mhq/service/code/models/ticket_lead_time.py b/backend/analytics_server/mhq/service/code/models/ticket_lead_time.py new file mode 100644 index 000000000..932936658 --- /dev/null +++ b/backend/analytics_server/mhq/service/code/models/ticket_lead_time.py @@ -0,0 +1,34 @@ +from dataclasses import dataclass + + +@dataclass +class TicketLeadTimeMetrics: + """ + CLUSTOX: Jira integration -- the extended Lead Time breakdown's + "ticket created -> first commit" leading phase (docs/ + JIRA_INTEGRATION_PROPOSAL.md §6A). + + Deliberately its own dataclass, not a new field bolted onto + LeadTimeMetrics: that one is averaged across *every* merged PR + (matched to a ticket or not), and mixing a field that's only + meaningful for a subset into that same weighted average would + silently understate it -- exactly the population-mismatch bug this + integration already had to fix once for Ticket Cycle Time. Every + field here is computed over the *same* matched_pr_count PRs, so the + two averages are always a fair, like-for-like comparison. + """ + + matched_pr_count: int = 0 + avg_ticket_to_first_commit_seconds: float = 0 + # This is NOT the org-wide lead_time average -- it's the plain + # (existing, unmodified) lead_time averaged over just this same + # matched subset, so "extended vs commit-only" compares one + # population against itself, not two different ones. + avg_commit_only_lead_time_seconds: float = 0 + + @property + def avg_extended_lead_time_seconds(self) -> float: + return ( + self.avg_ticket_to_first_commit_seconds + + self.avg_commit_only_lead_time_seconds + ) diff --git a/backend/analytics_server/mhq/service/code/repository_service.py b/backend/analytics_server/mhq/service/code/repository_service.py index bda48bbea..75fd3ac59 100644 --- a/backend/analytics_server/mhq/service/code/repository_service.py +++ b/backend/analytics_server/mhq/service/code/repository_service.py @@ -122,7 +122,7 @@ def update_org_repos( idempotency_keys = [repo.idempotency_key for repo in raw_org_repos] existing_org_repos = self._code_repo_service.get_repos_by_idempotency_keys( - idempotency_keys + org_id, idempotency_keys ) updated_org_repos = [] @@ -136,13 +136,11 @@ def update_org_repos( raw_org_repo.idempotency_key ) if existing_org_repo: - - # ToDo update idempotency key to idempotency_key, provider, org. - if str(existing_org_repo.org_id) != str(org_id): - raise Exception( - f"Data integrity error, matching idempotency key across orgs. Team OrgId: {str(org_id)}. Existing Repo OrgID: {str(existing_org_repo.org_id)}. idempotency_key: {raw_org_repo.idempotency_key}" - ) - + # CLUSTOX: the lookup above is org-scoped, so a row here always + # belongs to this org. The cross-org "Data integrity error" + # raise that used to sit here -- a permanent 500 whenever + # another workspace already tracked the same repo -- is gone + # with the global lookup that made it reachable. existing_org_repo.is_active = True existing_org_repo.slug = raw_org_repo.slug existing_org_repo.name = raw_org_repo.name diff --git a/backend/analytics_server/mhq/service/code/sync/etl_bitbucket_handler.py b/backend/analytics_server/mhq/service/code/sync/etl_bitbucket_handler.py new file mode 100644 index 000000000..48b671dbc --- /dev/null +++ b/backend/analytics_server/mhq/service/code/sync/etl_bitbucket_handler.py @@ -0,0 +1,456 @@ +from datetime import datetime +from typing import Dict, List, Optional, Set, Tuple +from uuid import uuid4 + +from mhq.exapi.bitbucket import BitbucketApiService, BitbucketRateLimitExceeded +from mhq.exapi.models.bitbucket import BitbucketPR, BitbucketRepo +from mhq.service.code.sync.etl_code_analytics import CodeETLAnalyticsService +from mhq.service.code.sync.etl_provider_handler import CodeProviderETLHandler +from mhq.store.models import UserIdentityProvider +from mhq.store.models.code import ( + CodeProvider, + OrgRepo, + PullRequest, + PullRequestCommit, + PullRequestEvent, + PullRequestEventState, + PullRequestEventType, + PullRequestRevertPRMapping, + PullRequestState, +) +from mhq.store.repos.code import CodeRepoService +from mhq.store.repos.core import CoreRepoService +from mhq.utils.log import LOG +from mhq.utils.string import uuid4_str +from mhq.utils.time import dt_from_iso_time_string, time_now + +# CLUSTOX: our model has three PR states, Bitbucket four. DECLINED (rejected) +# and SUPERSEDED (replaced by another PR) are both terminal-without-merge, so +# both fold into CLOSED. An unmapped state skips that PR rather than guessing: +# a guessed state would flow into lead time as a plausible wrong number. +BITBUCKET_STATE_MAP = { + "MERGED": PullRequestState.MERGED, + "OPEN": PullRequestState.OPEN, + "DECLINED": PullRequestState.CLOSED, + "SUPERSEDED": PullRequestState.CLOSED, +} + + +class BitbucketETLHandler(CodeProviderETLHandler): + def __init__( + self, + org_id: str, + bitbucket_api_service: BitbucketApiService, + code_repo_service: CodeRepoService, + code_etl_analytics_service: CodeETLAnalyticsService, + bitbucket_revert_pr_sync_handler, + ): + self.org_id: str = org_id + self._api: BitbucketApiService = bitbucket_api_service + self.code_repo_service: CodeRepoService = code_repo_service + self.code_etl_analytics_service: CodeETLAnalyticsService = ( + code_etl_analytics_service + ) + self.bitbucket_revert_pr_sync_handler = bitbucket_revert_pr_sync_handler + self.provider: str = CodeProvider.BITBUCKET.value + + def check_pat_validity(self) -> bool: + is_valid = self._api.check_pat() + if not is_valid: + # The API cannot tell a revoked token from an expired one. + raise Exception("Bitbucket API token is invalid, revoked or expired") + return is_valid + + def get_org_repos(self, org_repos: List[OrgRepo]) -> List[OrgRepo]: + """CLUSTOX: one workspace listing per workspace rather than one call + per repo -- repos are matched back by uuid. Against a ~1,000 req/hr + ceiling, N repos in one workspace cost 1 request instead of N.""" + repos_by_workspace: Dict[str, List[OrgRepo]] = {} + for org_repo in org_repos: + repos_by_workspace.setdefault(org_repo.org_name, []).append(org_repo) + + processed: List[OrgRepo] = [] + for workspace, workspace_org_repos in repos_by_workspace.items(): + try: + bitbucket_repos = [ + BitbucketRepo(repo) + for repo in self._api.get_workspace_repos(workspace) + ] + except Exception as e: + LOG.error(f"Error listing Bitbucket workspace {workspace}: {str(e)}") + continue + + by_key = {repo.idempotency_key: repo for repo in bitbucket_repos} + for org_repo in workspace_org_repos: + bitbucket_repo = by_key.get(str(org_repo.idempotency_key)) + if not bitbucket_repo: + LOG.error( + f"Bitbucket repo not found in workspace {workspace}: " + f"{org_repo.slug}" + ) + continue + processed.append(self._process_bitbucket_repo(org_repo, bitbucket_repo)) + + return processed + + def _process_bitbucket_repo( + self, org_repo: OrgRepo, bitbucket_repo: BitbucketRepo + ) -> OrgRepo: + return OrgRepo( + id=org_repo.id, + org_id=self.org_id, + name=bitbucket_repo.name, + provider=self.provider, + org_name=bitbucket_repo.org_name, + default_branch=bitbucket_repo.default_branch, + language=None, + # CLUSTOX: Bitbucket has no contributors endpoint. Empty is safe: + # the contributor filter reads PR authors from our own tables, not + # this map. + contributors={"contributions": []}, + idempotency_key=str(bitbucket_repo.idempotency_key), + slug=bitbucket_repo.slug, + updated_at=time_now(), + ) + + def get_revert_prs_mapping( + self, prs: List[PullRequest] + ) -> List[PullRequestRevertPRMapping]: + return self.bitbucket_revert_pr_sync_handler(prs) + + def get_repo_pull_requests_data( + self, org_repo: OrgRepo, bookmark: datetime + ) -> Tuple[List[PullRequest], List[PullRequestCommit], List[PullRequestEvent]]: + workspace = org_repo.org_name + repo_slug = org_repo.slug + + prs_to_process = self._api.get_repo_pull_requests( + workspace, repo_slug, bookmark + ) + if not prs_to_process: + return [], [], [] + + pull_requests: List[PullRequest] = [] + pr_commits: List[PullRequestCommit] = [] + pr_events: List[PullRequestEvent] = [] + prs_added: Set[int] = set() + + for raw_pr in prs_to_process: + bitbucket_pr = BitbucketPR(raw_pr) + if bitbucket_pr.number in prs_added: + continue + if bitbucket_pr.state not in BITBUCKET_STATE_MAP: + LOG.warning( + f"Skipping Bitbucket PR {bitbucket_pr.number} with " + f"unknown state {bitbucket_pr.state}" + ) + continue + + try: + pr_model, event_models, pr_commit_models = self.process_pr( + str(org_repo.id), workspace, repo_slug, bitbucket_pr + ) + except BitbucketRateLimitExceeded as e: + # CLUSTOX: pause, not fail. Everything fully processed so far + # is returned and stored; the bookmark then rests at the last + # stored PR, so the next scheduled sync resumes exactly there. + # Failing instead would throw away this batch; continuing + # would hammer a locked-out API. + LOG.warning( + f"Bitbucket rate limit hit at PR {bitbucket_pr.number} of " + f"{workspace}/{repo_slug}; resuming next sync" + + ( + f" (retry-after {e.retry_after_seconds}s)" + if e.retry_after_seconds + else "" + ) + ) + break + + pull_requests.append(pr_model) + pr_events += event_models + pr_commits += pr_commit_models + prs_added.add(bitbucket_pr.number) + + return pull_requests, pr_commits, pr_events + + def process_pr( + self, repo_id: str, workspace: str, repo_slug: str, pr: BitbucketPR + ) -> Tuple[PullRequest, List[PullRequestEvent], List[PullRequestCommit]]: + pr_model: Optional[PullRequest] = self.code_repo_service.get_repo_pr_by_number( + repo_id, pr.number + ) + # CLUSTOX: kept before the model is rebuilt. A merged PR re-syncs + # whenever anything bumps its updated_on (a late comment, say); the + # rebuilt meta starts empty, and if THIS run's diffstat fails, the + # merge would overwrite the row and erase code_stats it already had -- + # LOC silently dropping for a PR whose lines never changed. + previous_code_stats = ( + (pr_model.meta or {}).get("code_stats") if pr_model else None + ) + pr_event_model_list: List[PullRequestEvent] = ( + self.code_repo_service.get_pr_events(pr_model) + ) + + activity = self._api.get_pr_activity(workspace, repo_slug, pr.number) + + pr_model = self._to_pr_model(pr, pr_model, repo_id) + pr_events_models = self._to_pr_events(activity, pr_model, pr_event_model_list) + + pr_commits_model_list: List[PullRequestCommit] = [] + if pr_model.state == PullRequestState.MERGED: + commits = self._api.get_pr_commits(workspace, repo_slug, pr.number) + pr_commits_model_list = self._to_pr_commits(commits, pr_model) + + additions, deletions, files_changed = self.process_pr_code_stats( + workspace, repo_slug, pr.number + ) + meta = {"user_profile": dict(username=pr_model.author)} + # CLUSTOX: a failed diffstat leaves code_stats out entirely rather + # than writing zeros -- the PullRequest properties then read 0 via + # their own defaults, and LOC undercounts honestly while the PR + # stays present for lead time and the contributor filter. Writing + # zeros here would be indistinguishable from a genuinely empty + # diff. + if additions is not None: + meta["code_stats"] = dict( + commits=len(pr_commits_model_list), + additions=additions, + deletions=deletions, + changed_files=files_changed, + comments=None, + ) + elif previous_code_stats: + meta["code_stats"] = previous_code_stats + pr_model.meta = meta + + pr_model = self.code_etl_analytics_service.create_pr_metrics( + pr_model, pr_events_models, pr_commits_model_list + ) + + return pr_model, pr_events_models, pr_commits_model_list + + def process_pr_code_stats( + self, workspace: str, repo_slug: str, pr_number: int + ) -> Tuple[Optional[int], Optional[int], Optional[int]]: + try: + diffstat = self._api.get_pr_diffstat(workspace, repo_slug, pr_number) + except BitbucketRateLimitExceeded: + raise + except Exception as e: + LOG.warning( + f"Diffstat failed for PR {pr_number} of {workspace}/{repo_slug}: " + f"{str(e)}" + ) + return None, None, None + + additions = sum(entry.get("lines_added") or 0 for entry in diffstat) + deletions = sum(entry.get("lines_removed") or 0 for entry in diffstat) + return additions, deletions, len(diffstat) + + def _to_pr_model( + self, + pr: BitbucketPR, + pr_model: Optional[PullRequest], + repo_id: str, + ) -> PullRequest: + pr_id = pr_model.id if pr_model else uuid4() + state = BITBUCKET_STATE_MAP[pr.state] + reviewers = [ + (participant.get("user") or {}).get("nickname") + for participant in pr.participants + if participant.get("role") == "REVIEWER" + and (participant.get("user") or {}).get("nickname") + ] + + return PullRequest( + id=pr_id, + number=pr.number, + title=pr.title, + url=pr.url, + # CLUSTOX: nickname, not uuid. The contributor dropdown lists + # `author` verbatim, and "{a1b2c3d4-...}" is not a name anyone can + # pick from a list. The uuid stays in `data` for a future identity + # layer. Nickname renames make history split under two handles -- + # the same accepted cost GitHub logins already carry here. + author=pr.author_nickname or pr.author_uuid, + state=state, + base_branch=pr.base_branch, + head_branch=pr.head_branch, + data=pr.raw_data, + created_at=pr.created_on, + updated_at=pr.updated_on, + # CLUSTOX: Bitbucket has no merged_at anywhere in the PR object. + # updated_on at the moment of merge is the closest truth, and it + # is what lead time keys on. None for OPEN PRs, matching the + # GitHub handler -- an open PR's state has not changed, and a + # timestamp there would be a silent cross-provider inconsistency + # in a load-bearing column. + state_changed_at=( + pr.updated_on if state != PullRequestState.OPEN else None + ), + repo_id=repo_id, + requested_reviews=[], + meta=dict(), + reviewers=reviewers, + provider=UserIdentityProvider.BITBUCKET.value, + merge_commit_sha=pr.merge_commit_sha, + ) + + @staticmethod + def _to_pr_commits( + commits: List[Dict], pr_model: PullRequest + ) -> List[PullRequestCommit]: + pr_commits: List[PullRequestCommit] = [] + for commit in commits: + if not commit.get("hash"): + LOG.warning("Skipping Bitbucket commit without a hash") + continue + pr_commits.append( + PullRequestCommit( + hash=commit.get("hash"), + pull_request_id=pr_model.id, + message=commit.get("message") or "", + url=((commit.get("links") or {}).get("html") or {}).get("href"), + data=commit, + author=(commit.get("author") or {}).get("raw"), + created_at=_dt_or_none(commit.get("date")), + org_repo_id=pr_model.repo_id, + ) + ) + return pr_commits + + def _to_pr_events( + self, + activity: List[Dict], + pr_model: PullRequest, + pr_events_model: List[PullRequestEvent], + ) -> List[PullRequestEvent]: + """CLUSTOX: Bitbucket's activity feed mixes approvals, comments, + changes-requested and plain updates in one stream, each under a + different top-level key. Anything unrecognised is skipped + individually -- one malformed entry must never cost the batch.""" + pr_events: List[PullRequestEvent] = [] + pr_review_id_map = { + event.idempotency_key: event.id for event in pr_events_model + } + + for entry in activity: + adapted = self._adapt_activity_entry(entry) + if not adapted: + continue + event_state, actor, created_at, idempotency_key, data = adapted + data["state"] = event_state + pr_events.append( + PullRequestEvent( + id=pr_review_id_map.get(idempotency_key, uuid4_str()), + pull_request_id=str(pr_model.id), + type=PullRequestEventType.REVIEW.value, + data=data, + created_at=created_at, + idempotency_key=idempotency_key, + org_repo_id=pr_model.repo_id, + actor_username=actor, + ) + ) + + return pr_events + + @staticmethod + def _adapt_activity_entry(entry: Dict): + try: + if "approval" in entry: + approval = entry["approval"] or {} + created_at = _dt_or_none(approval.get("date")) + user = approval.get("user") or {} + actor = user.get("nickname") + if not created_at: + return None + # CLUSTOX: keyed on the account uuid, not the nickname -- a + # nickname rename would change the key and duplicate this + # event on the next re-sync of the PR. + key = f"approval-{user.get('uuid') or actor}-{approval.get('date')}" + return ( + PullRequestEventState.APPROVED.value, + actor, + created_at, + key, + dict(approval), + ) + if "changes_requested" in entry: + changes = entry["changes_requested"] or {} + created_at = _dt_or_none(changes.get("date")) + user = changes.get("user") or {} + actor = user.get("nickname") + if not created_at: + return None + key = f"changes-{user.get('uuid') or actor}-{changes.get('date')}" + return ( + PullRequestEventState.CHANGES_REQUESTED.value, + actor, + created_at, + key, + dict(changes), + ) + if "comment" in entry: + comment = entry["comment"] or {} + created_at = _dt_or_none(comment.get("created_on")) + actor = (comment.get("user") or {}).get("nickname") + if not created_at: + return None + key = f"comment-{comment.get('id')}" + return ( + PullRequestEventState.COMMENTED.value, + actor, + created_at, + key, + dict(comment, content=None), + ) + except Exception as e: + LOG.warning(f"Skipping malformed Bitbucket activity entry: {str(e)}") + return None + + +def _dt_or_none(value: Optional[str]) -> Optional[datetime]: + if not value: + return None + return dt_from_iso_time_string(value) + + +def get_bitbucket_etl_handler(org_id: str) -> BitbucketETLHandler: + from mhq.service.code.sync.revert_prs_bitbucket_sync import ( + get_revert_prs_bitbucket_sync_handler, + ) + + def _get_credentials() -> Tuple[Optional[str], Optional[str]]: + core_repo_service = CoreRepoService() + access_token = core_repo_service.get_access_token( + org_id, UserIdentityProvider.BITBUCKET + ) + integrations = core_repo_service.get_org_integrations_for_names( + org_id, [UserIdentityProvider.BITBUCKET.value] + ) + # CLUSTOX: the email is the Basic-auth username. It lives in + # provider_meta -- the same home GitLab uses for custom_domain -- + # because it is not a secret; only the token is encrypted. + email = ( + integrations[0].provider_meta.get("email") + if integrations and integrations[0].provider_meta + else None + ) + if not access_token or not email: + LOG.error( + f"Bitbucket credentials incomplete for org {org_id}: " + f"token {'present' if access_token else 'missing'}, " + f"email {'present' if email else 'missing'}" + ) + return email, access_token + + email, access_token = _get_credentials() + return BitbucketETLHandler( + org_id, + BitbucketApiService(email, access_token), + CodeRepoService(), + CodeETLAnalyticsService(), + get_revert_prs_bitbucket_sync_handler(), + ) diff --git a/backend/analytics_server/mhq/service/code/sync/etl_code_factory.py b/backend/analytics_server/mhq/service/code/sync/etl_code_factory.py index 327bb92c8..89823c93d 100644 --- a/backend/analytics_server/mhq/service/code/sync/etl_code_factory.py +++ b/backend/analytics_server/mhq/service/code/sync/etl_code_factory.py @@ -1,3 +1,4 @@ +from mhq.service.code.sync.etl_bitbucket_handler import get_bitbucket_etl_handler from mhq.service.code.sync.etl_gitlab_handler import get_gitlab_etl_handler from mhq.service.code.sync.etl_github_handler import get_github_etl_handler from mhq.service.code.sync.etl_provider_handler import CodeProviderETLHandler @@ -15,4 +16,7 @@ def __call__(self, provider: str) -> CodeProviderETLHandler: if provider == CodeProvider.GITLAB.value: return get_gitlab_etl_handler(self.org_id) + if provider == CodeProvider.BITBUCKET.value: + return get_bitbucket_etl_handler(self.org_id) + raise NotImplementedError(f"Unknown provider - {provider}") diff --git a/backend/analytics_server/mhq/service/code/sync/revert_prs_bitbucket_sync.py b/backend/analytics_server/mhq/service/code/sync/revert_prs_bitbucket_sync.py new file mode 100644 index 000000000..80689e3a3 --- /dev/null +++ b/backend/analytics_server/mhq/service/code/sync/revert_prs_bitbucket_sync.py @@ -0,0 +1,134 @@ +import re +from typing import Dict, List, Optional, Set + +from mhq.store.models.code.enums import ( + PullRequestRevertPRMappingActorType, + PullRequestState, +) +from mhq.store.models.code.pull_requests import ( + PullRequest, + PullRequestRevertPRMapping, +) +from mhq.store.repos.code import CodeRepoService +from mhq.utils.time import time_now + +# CLUSTOX: Bitbucket's own "Revert" button creates the branch +# `revert-pr-` -- a structured link to the original PR's number in the +# same repo. This is a STRONGER signal than the title-prefix heuristic the +# spec originally proposed: a title alone identifies a PR as "a revert" but +# names no target, and a mapping without both ends is useless to CFR. A +# manual revert from a hand-named branch goes undetected -- accepted, and +# recorded in the spec. +REVERT_BRANCH_PATTERN = re.compile(r"^revert-pr-(\d+)$") + + +class RevertPRsBitbucketSyncHandler: + def __init__(self, code_repo_service: CodeRepoService): + self.code_repo_service = code_repo_service + + def __call__(self, *args, **kwargs): + return self.process_revert_prs(*args, **kwargs) + + def process_revert_prs( + self, prs: List[PullRequest] + ) -> List[PullRequestRevertPRMapping]: + revert_prs: List[PullRequest] = [] + original_prs: List[PullRequest] = [] + + for pr in prs: + if pr.head_branch and REVERT_BRANCH_PATTERN.match(pr.head_branch): + revert_prs.append(pr) + else: + original_prs.append(pr) + + mappings = self._mappings_for_revert_prs( + revert_prs + ) + self._mappings_for_original_prs(original_prs) + return list(set(mappings)) + + def _mappings_for_revert_prs( + self, prs: List[PullRequest] + ) -> List[PullRequestRevertPRMapping]: + """A revert PR in this batch names its original by number; the + original may have synced in any earlier batch, so it is looked up in + the store rather than in the batch.""" + mappings: List[PullRequestRevertPRMapping] = [] + + for pr in prs: + reverted_number = self.get_reverted_pr_number(pr.head_branch) + if reverted_number is None: + continue + + original_pr: Optional[PullRequest] = ( + self.code_repo_service.get_repo_pr_by_number( + str(pr.repo_id), reverted_number + ) + ) + if original_pr is None: + continue + + mappings.append( + PullRequestRevertPRMapping( + pr_id=pr.id, + actor_type=PullRequestRevertPRMappingActorType.SYSTEM, + actor=None, + reverted_pr=original_pr.id, + updated_at=time_now(), + ) + ) + + return mappings + + def _mappings_for_original_prs( + self, prs: List[PullRequest] + ) -> List[PullRequestRevertPRMapping]: + """The mirror direction: an original PR syncing now may already have + been reverted by a PR from an earlier batch.""" + merged_prs = [pr for pr in prs if pr.state == PullRequestState.MERGED] + if not merged_prs: + return [] + + repo_ids: Set[str] = {str(pr.repo_id) for pr in merged_prs} + match_strings = [f"revert-pr-{pr.number}" for pr in merged_prs] + number_by_repo: Dict[str, Dict[str, str]] = {} + for pr in merged_prs: + number_by_repo.setdefault(str(pr.repo_id), {})[str(pr.number)] = pr.id + + revert_prs: List[PullRequest] = ( + self.code_repo_service.get_prs_by_head_branch_match_strings( + list(repo_ids), match_strings + ) + ) + + mappings: List[PullRequestRevertPRMapping] = [] + for revert_pr in revert_prs: + reverted_number = self.get_reverted_pr_number(revert_pr.head_branch) + if reverted_number is None: + continue + original_pr_id = number_by_repo.get(str(revert_pr.repo_id), {}).get( + str(reverted_number) + ) + if original_pr_id is None: + continue + mappings.append( + PullRequestRevertPRMapping( + pr_id=revert_pr.id, + actor_type=PullRequestRevertPRMappingActorType.SYSTEM, + actor=None, + reverted_pr=original_pr_id, + updated_at=time_now(), + ) + ) + + return mappings + + @staticmethod + def get_reverted_pr_number(head_branch: Optional[str]) -> Optional[int]: + if not head_branch: + return None + match = REVERT_BRANCH_PATTERN.match(head_branch) + return int(match.group(1)) if match else None + + +def get_revert_prs_bitbucket_sync_handler() -> RevertPRsBitbucketSyncHandler: + return RevertPRsBitbucketSyncHandler(CodeRepoService()) diff --git a/backend/analytics_server/mhq/service/code/ticket_lead_time.py b/backend/analytics_server/mhq/service/code/ticket_lead_time.py new file mode 100644 index 000000000..d8776ef2e --- /dev/null +++ b/backend/analytics_server/mhq/service/code/ticket_lead_time.py @@ -0,0 +1,114 @@ +from typing import List + +from mhq.service.code.lead_time import LeadTimeService, get_lead_time_service +from mhq.service.code.models.lead_time import LeadTimeMetrics +from mhq.service.code.models.ticket_lead_time import TicketLeadTimeMetrics +from mhq.store.models.core import Team +from mhq.store.repos.code import CodeRepoService +from mhq.store.repos.ticket_matching import TicketMatchingRepoService +from mhq.utils.time import Interval + + +class TicketLeadTimeService: + """ + CLUSTOX: Jira integration -- the extended Lead Time breakdown's + "ticket created -> first commit" leading phase (docs/ + JIRA_INTEGRATION_PROPOSAL.md §6A). + + Deliberately additive alongside LeadTimeService, not a change to it: + every existing method there (and the plain, org-wide "lead_time" DORA + card that depends on them) is untouched. This composes + get_team_pr_lead_time_metrics' per-PR list with the ticket-matching + and commit stores to build a second, separate average scoped to only + the PRs that actually have a matched ticket -- see + TicketLeadTimeMetrics for why that scoping matters. + """ + + def __init__( + self, + lead_time_service: LeadTimeService, + code_repo_service: CodeRepoService, + ticket_matching_repo_service: TicketMatchingRepoService, + ): + self._lead_time_service = lead_time_service + self._code_repo_service = code_repo_service + self._ticket_matching_repo_service = ticket_matching_repo_service + + def get_team_ticket_lead_time_metrics( + self, team: Team, interval: Interval + ) -> TicketLeadTimeMetrics: + pr_metrics = self._lead_time_service.get_team_pr_lead_time_metrics( + team, interval + ) + if not pr_metrics: + return TicketLeadTimeMetrics() + + pr_ids = [str(metric.pr_id) for metric in pr_metrics] + ticket_created_at_by_pr = ( + self._ticket_matching_repo_service.get_ticket_created_at_by_pr_ids(pr_ids) + ) + matched_metrics = [ + metric + for metric in pr_metrics + if str(metric.pr_id) in ticket_created_at_by_pr + ] + if not matched_metrics: + return TicketLeadTimeMetrics() + + first_commit_at_by_pr = self._code_repo_service.get_first_commit_at_by_pr_ids( + [str(metric.pr_id) for metric in matched_metrics] + ) + + return self._average_over_matched_prs( + matched_metrics, ticket_created_at_by_pr, first_commit_at_by_pr + ) + + def _average_over_matched_prs( + self, + matched_metrics: List[LeadTimeMetrics], + ticket_created_at_by_pr: dict, + first_commit_at_by_pr: dict, + ) -> TicketLeadTimeMetrics: + ticket_to_commit_durations = [] + commit_only_lead_times = [] + + for metric in matched_metrics: + pr_id = str(metric.pr_id) + first_commit_at = first_commit_at_by_pr.get(pr_id) + ticket_created_at = ticket_created_at_by_pr.get(pr_id) + # A PR matched to a ticket but with no PullRequestCommit row + # synced -- can't anchor a real "first commit" timestamp, so + # this PR is excluded rather than guessed. + if not first_commit_at or not ticket_created_at: + continue + + duration = (first_commit_at - ticket_created_at).total_seconds() + # A ticket created *after* the first commit (filed + # retroactively, or the regex matched a ticket key that was + # only created later) isn't a real "idea to commit" span -- + # excluded rather than shown as a negative duration. + if duration < 0: + continue + + ticket_to_commit_durations.append(duration) + commit_only_lead_times.append(metric.lead_time) + + matched_pr_count = len(ticket_to_commit_durations) + if not matched_pr_count: + return TicketLeadTimeMetrics() + + return TicketLeadTimeMetrics( + matched_pr_count=matched_pr_count, + avg_ticket_to_first_commit_seconds=( + sum(ticket_to_commit_durations) / matched_pr_count + ), + avg_commit_only_lead_time_seconds=( + sum(commit_only_lead_times) / matched_pr_count + ), + ) + + +def get_ticket_lead_time_service() -> TicketLeadTimeService: + return TicketLeadTimeService( + get_lead_time_service(), CodeRepoService(), TicketMatchingRepoService() + ) diff --git a/backend/analytics_server/mhq/service/incidents/integration.py b/backend/analytics_server/mhq/service/incidents/integration.py index fd10e09a6..b892375fd 100644 --- a/backend/analytics_server/mhq/service/incidents/integration.py +++ b/backend/analytics_server/mhq/service/incidents/integration.py @@ -7,6 +7,7 @@ from mhq.store.repos.core import CoreRepoService GIT_INCIDENT_INTEGRATION_BUCKET = [IncidentProvider.GITHUB.value] +JIRA_INCIDENT_INTEGRATION_BUCKET = [IncidentProvider.JIRA.value] class IncidentsIntegrationService: @@ -37,6 +38,9 @@ def _get_possible_incident_providers(self, org_id: str) -> List[str]: if IncidentSource.GIT_REPO in incident_source_setting.incident_sources: valid_integration_types += GIT_INCIDENT_INTEGRATION_BUCKET + if IncidentSource.JIRA_ISSUE in incident_source_setting.incident_sources: + valid_integration_types += JIRA_INCIDENT_INTEGRATION_BUCKET + return valid_integration_types def _get_or_create_incident_source_setting( diff --git a/backend/analytics_server/mhq/service/incidents/sync/etl_incidents_factory.py b/backend/analytics_server/mhq/service/incidents/sync/etl_incidents_factory.py index 571178196..7169d0dd6 100644 --- a/backend/analytics_server/mhq/service/incidents/sync/etl_incidents_factory.py +++ b/backend/analytics_server/mhq/service/incidents/sync/etl_incidents_factory.py @@ -1,6 +1,9 @@ from mhq.service.incidents.sync.etl_git_incidents_handler import ( get_incidents_sync_etl_handler, ) +from mhq.service.incidents.sync.etl_jira_incidents_handler import ( + get_jira_incidents_etl_handler, +) from mhq.service.incidents.sync.etl_provider_handler import IncidentsProviderETLHandler from mhq.store.models.incidents import IncidentProvider @@ -16,4 +19,7 @@ def __call__(self, provider: str) -> IncidentsProviderETLHandler: if provider == IncidentProvider.GITLAB.value: return get_incidents_sync_etl_handler(self.org_id) + if provider == IncidentProvider.JIRA.value: + return get_jira_incidents_etl_handler(self.org_id) + raise NotImplementedError(f"Unknown provider - {provider}") diff --git a/backend/analytics_server/mhq/service/incidents/sync/etl_jira_incidents_handler.py b/backend/analytics_server/mhq/service/incidents/sync/etl_jira_incidents_handler.py new file mode 100644 index 000000000..7e5647f01 --- /dev/null +++ b/backend/analytics_server/mhq/service/incidents/sync/etl_jira_incidents_handler.py @@ -0,0 +1,286 @@ +from datetime import datetime +from typing import Dict, List, Optional, Tuple + +from mhq.exapi.jira import JiraApiService +from mhq.service.incidents.sync.etl_provider_handler import IncidentsProviderETLHandler +from mhq.service.settings import SettingsService, get_settings_service +from mhq.service.settings.models import JiraIncidentIssueTypesSetting +from mhq.store.models import UserIdentityProvider +from mhq.store.models.settings import EntityType, SettingType +from mhq.store.models.incidents import ( + Incident, + IncidentOrgIncidentServiceMap, + IncidentProvider, + IncidentSource, + IncidentStatus, + IncidentType, + OrgIncidentService, + TeamIncidentService, +) +from mhq.store.models.projects import OrgProject, Ticket, TicketState +from mhq.store.repos.core import CoreRepoService +from mhq.store.repos.incidents import IncidentsRepoService +from mhq.store.repos.projects import ProjectRepoService +from mhq.utils.log import LOG +from mhq.utils.string import uuid4_str +from mhq.utils.time import time_now + + +class JiraIncidentsETLHandler(IncidentsProviderETLHandler): + """ + Treats Ticket rows of a configured Jira issue type (default: "Bug", + see JiraIncidentIssueTypesSetting) as incidents for Change Failure + Rate / MTTR -- see docs/JIRA_INTEGRATION_PROPOSAL.md. Mirrors + GitIncidentsETLHandler's shape: one OrgIncidentService per active + Jira project (that file's equivalent of "one per repo"), upserted + Incidents per ticket. + + Unlike revert-PR incidents (created already-resolved, single-shot), + a Jira-ticket incident can be open when first synced and resolved on + a later sync -- status/resolved_date are recomputed from the + ticket's current state every sync, not fixed at creation. + """ + + def __init__( + self, + org_id: str, + jira_api_service: JiraApiService, + project_repo_service: ProjectRepoService, + incidents_repo_service: IncidentsRepoService, + settings_service: SettingsService, + site_url: Optional[str], + ): + self.org_id = org_id + self._api = jira_api_service + self._project_repo_service = project_repo_service + self._incidents_repo_service = incidents_repo_service + self._settings_service = settings_service + self._site_url = site_url + + def check_pat_validity(self) -> bool: + return self._api.check_pat() + + def get_updated_incident_services( + self, incident_services: List[OrgIncidentService] + ) -> List[OrgIncidentService]: + jira_services = [ + service + for service in incident_services + if service.source_type == IncidentSource.JIRA_ISSUE + ] + key_to_service_map: Dict[str, OrgIncidentService] = { + service.key: service for service in jira_services + } + + active_projects: List[OrgProject] = ( + self._project_repo_service.get_active_org_projects_for_provider( + self.org_id, UserIdentityProvider.JIRA.value + ) + ) + + return [ + self._adapt_org_incident_service( + project, key_to_service_map.get(str(project.id)) + ) + for project in active_projects + ] + + def process_service_incidents( + self, incident_service: OrgIncidentService, bookmark: datetime + ) -> Tuple[List[Incident], List[IncidentOrgIncidentServiceMap], datetime]: + # incident_service.key is the org_project id (see + # _adapt_org_incident_service) -- link any team that has this + # project actively selected (MID-3) before syncing its incidents, + # so a team never has to configure "which Jira project" twice. + self._ensure_team_links(incident_service) + + issue_types = self._get_issue_types() + if not issue_types: + return [], [], bookmark + + from_time = bookmark + to_time = time_now() + + tickets, ticket_states = ( + self._project_repo_service.get_tickets_by_issue_types_for_project( + incident_service.key, issue_types, from_time, to_time + ) + ) + if not tickets: + LOG.info( + f"[Jira Incidents Sync] No qualifying tickets for service " + f"{str(incident_service.id)} in org {self.org_id} since " + f"{from_time.isoformat()}" + ) + return [], [], bookmark + + states_by_ticket: Dict[str, List[TicketState]] = {} + for state in ticket_states: + states_by_ticket.setdefault(str(state.ticket_id), []).append(state) + + incidents: List[Incident] = [] + incident_service_maps: List[IncidentOrgIncidentServiceMap] = [] + for ticket in tickets: + incident = self._to_incident( + ticket, states_by_ticket.get(str(ticket.id), []) + ) + incidents.append(incident) + incident_service_maps.append( + IncidentOrgIncidentServiceMap( + incident_id=incident.id, service_id=incident_service.id + ) + ) + + bookmark = max(bookmark, max(ticket.updated_at for ticket in tickets)) + return incidents, incident_service_maps, bookmark + + def _ensure_team_links(self, incident_service: OrgIncidentService) -> None: + team_ids = self._project_repo_service.get_team_ids_for_org_project( + incident_service.key + ) + if not team_ids: + return + + existing_links = { + (str(link.team_id), str(link.service_id)) + for team_id in team_ids + for link in self._incidents_repo_service.get_team_incident_services( + _TeamIdOnly(team_id) + ) + } + new_links = [ + TeamIncidentService(team_id=team_id, service_id=incident_service.id) + for team_id in team_ids + if (team_id, str(incident_service.id)) not in existing_links + ] + if new_links: + self._incidents_repo_service.add_team_incident_services(new_links) + + def _get_issue_types(self) -> List[str]: + setting: JiraIncidentIssueTypesSetting = ( + self._settings_service.get_or_set_default_settings( + setting_type=SettingType.JIRA_INCIDENT_ISSUE_TYPES_SETTING, + entity_type=EntityType.ORG, + entity_id=self.org_id, + ).specific_settings + ) + return setting.issue_types + + def _to_incident(self, ticket: Ticket, states: List[TicketState]) -> Incident: + # CLUSTOX: org-scoped lookup -- ticket.key ("PROJ-123") is + # site-local, not globally unique, so two orgs' Jira sites can + # land on the same key. See + # get_incident_by_key_type_provider_and_org's own docstring. + existing_incident: Optional[Incident] = ( + self._incidents_repo_service.get_incident_by_key_type_provider_and_org( + self.org_id, ticket.key, IncidentType.JIRA_ISSUE, IncidentProvider.JIRA + ) + ) + incident_id = existing_incident.id if existing_incident else uuid4_str() + + is_resolved = ticket.status_category == "Done" + resolved_date = self._resolved_date(ticket, states) if is_resolved else None + + return Incident( + id=incident_id, + provider=IncidentProvider.JIRA.value, + key=ticket.key, + title=ticket.summary, + status=( + IncidentStatus.RESOLVED.value + if is_resolved + else IncidentStatus.TRIGGERED.value + ), + creation_date=ticket.created_at, + resolved_date=resolved_date, + assigned_to=ticket.assignee, + assignees=[ticket.assignee] if ticket.assignee else [], + url=self._ticket_url(ticket), + meta={"issue_type": ticket.issue_type, "status": ticket.status}, + created_at=( + existing_incident.created_at if existing_incident else time_now() + ), + updated_at=time_now(), + incident_type=IncidentType.JIRA_ISSUE, + ) + + @staticmethod + def _resolved_date(ticket: Ticket, states: List[TicketState]) -> datetime: + # The ticket's current status is Done-category, so whichever + # transition most recently landed it there (its own last recorded + # transition, since states are the ticket's complete history) is + # when it actually resolved. No states recorded at all (never + # transitioned since sync started tracking it) falls back to the + # ticket's own last-updated timestamp -- the same fallback + # ticket_insights/cycle_time.py's _status_segments uses. + if not states: + return ticket.updated_at + return max(states, key=lambda s: s.changed_at).changed_at + + def _ticket_url(self, ticket: Ticket) -> Optional[str]: + if not self._site_url: + return None + return f"{self._site_url.rstrip('/')}/browse/{ticket.key}" + + @staticmethod + def _adapt_org_incident_service( + org_project: OrgProject, + org_incident_service: Optional[OrgIncidentService], + ) -> OrgIncidentService: + return OrgIncidentService( + id=org_incident_service.id if org_incident_service else uuid4_str(), + org_id=org_project.org_id, + provider=IncidentProvider.JIRA.value, + name=org_project.name, + # The org_project id, not the Jira project key -- keys aren't + # guaranteed unique across an org's multiple Jira sites/boards + # the way this app's own ids are. Also doubles as the + # org_project_id the ticket-fetch query needs directly. + key=str(org_project.id), + meta={}, + created_at=( + org_incident_service.created_at if org_incident_service else time_now() + ), + updated_at=time_now(), + source_type=IncidentSource.JIRA_ISSUE, + ) + + +class _TeamIdOnly: + """Minimal stand-in for Team -- IncidentsRepoService.get_team_incident_services + only ever reads `.id` off what it's given.""" + + def __init__(self, team_id: str): + self.id = team_id + + +def get_jira_incidents_etl_handler(org_id: str) -> JiraIncidentsETLHandler: + core_repo_service = CoreRepoService() + site_url, email = _get_jira_site_and_email(core_repo_service, org_id) + api_token = core_repo_service.get_access_token(org_id, UserIdentityProvider.JIRA) + + if not (site_url and email and api_token): + LOG.error(f"Jira credentials incomplete for org {org_id}") + + return JiraIncidentsETLHandler( + org_id, + JiraApiService(email, api_token, site_url), + ProjectRepoService(), + IncidentsRepoService(), + get_settings_service(), + site_url, + ) + + +def _get_jira_site_and_email( + core_repo_service: CoreRepoService, org_id: str +) -> Tuple[Optional[str], Optional[str]]: + # CLUSTOX: mirrors etl_jira_handler.py's private helper of the same + # name -- small and self-contained enough that duplicating it here + # was preferable to reaching into that (separately-owned, already + # in-review) module for a shared import. + integrations = core_repo_service.get_org_integrations_for_names( + org_id, [UserIdentityProvider.JIRA.value] + ) + provider_meta: Dict = (integrations[0].provider_meta or {}) if integrations else {} + return provider_meta.get("site_url"), provider_meta.get("email") diff --git a/backend/analytics_server/mhq/service/project/__init__.py b/backend/analytics_server/mhq/service/project/__init__.py new file mode 100644 index 000000000..8601efc12 --- /dev/null +++ b/backend/analytics_server/mhq/service/project/__init__.py @@ -0,0 +1,3 @@ +from .repository_service import get_project_service, ProjectService +from .integration import get_project_integration_service +from .sync import sync_project_issues diff --git a/backend/analytics_server/mhq/service/project/integration.py b/backend/analytics_server/mhq/service/project/integration.py new file mode 100644 index 000000000..de1bb0907 --- /dev/null +++ b/backend/analytics_server/mhq/service/project/integration.py @@ -0,0 +1,50 @@ +from typing import List + +from mhq.store.models import Integration, UserIdentityProvider +from mhq.store.repos.core import CoreRepoService +from mhq.store.repos.jira_connection import JiraConnectionRepoService + +# CLUSTOX: Jira integration, Phase 2 (issue sync). Mirrors +# mhq/service/code/integration.py's CODE_INTEGRATION_BUCKET -- one entry +# today, but kept as a bucket (not a single value) for the same reason +# that one is: room for another project-tracking tool later without +# reshaping this service. +PROJECT_INTEGRATION_BUCKET = [ + UserIdentityProvider.JIRA.value, +] + + +class ProjectIntegrationService: + def __init__( + self, + core_repo_service: CoreRepoService, + jira_connection_repo_service: JiraConnectionRepoService, + ): + self.core_repo_service = core_repo_service + self.jira_connection_repo_service = jira_connection_repo_service + + def get_org_providers(self, org_id: str) -> List[str]: + integrations: List[Integration] = ( + self.core_repo_service.get_org_integrations_for_names( + org_id, PROJECT_INTEGRATION_BUCKET + ) + ) + providers = {integration.name for integration in integrations} + + # CLUSTOX: an org can go straight to JiraConnection without ever + # linking the legacy Integration(name='jira') row -- see + # docs/JIRA_MULTI_ACCOUNT_PLAN.md. Checking only Integration here + # left such an org with an empty provider list, and + # sync_project_issues returns before ever calling the Jira ETL + # factory -- a JiraConnection-only org would silently never sync. + if self.jira_connection_repo_service.list_jira_connections(org_id): + providers.add(UserIdentityProvider.JIRA.value) + + return list(providers) + + +def get_project_integration_service(): + return ProjectIntegrationService( + core_repo_service=CoreRepoService(), + jira_connection_repo_service=JiraConnectionRepoService(), + ) diff --git a/backend/analytics_server/mhq/service/project/models/org_project.py b/backend/analytics_server/mhq/service/project/models/org_project.py new file mode 100644 index 000000000..86b5f9abc --- /dev/null +++ b/backend/analytics_server/mhq/service/project/models/org_project.py @@ -0,0 +1,15 @@ +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class RawTeamOrgProject: + team_id: str + provider: str + key: str + name: str + idempotency_key: str + # CLUSTOX: which JiraConnection this project was picked under, if any -- + # None means either a non-Jira provider or the legacy single-account + # Integration flow. See docs/JIRA_MULTI_ACCOUNT_PLAN.md Task 6 part 2. + connection_id: Optional[str] = None diff --git a/backend/analytics_server/mhq/service/project/repository_service.py b/backend/analytics_server/mhq/service/project/repository_service.py new file mode 100644 index 000000000..dd916d1e3 --- /dev/null +++ b/backend/analytics_server/mhq/service/project/repository_service.py @@ -0,0 +1,182 @@ +from typing import Dict, List + +from mhq.service.project.models.org_project import RawTeamOrgProject +from mhq.store.models.core import Team +from mhq.store.models.projects import OrgProject, OrgProjectConnection +from mhq.store.repos.projects import ProjectRepoService, TeamProjects +from mhq.utils.string import uuid4_str + + +class ProjectService: + """ + Business layer for a team's selected project-tracking-tool projects + (Jira, to start). Mirrors RepositoryService's repo/team methods, minus + the incident-service coupling that's specific to code repos -- see + docs/JIRA_INTEGRATION_PROPOSAL.md for the phase this belongs to. + """ + + def __init__(self, project_repo_service: ProjectRepoService): + self._project_repo_service = project_repo_service + + def get_team_projects(self, team: Team) -> List[OrgProject]: + return self._project_repo_service.get_team_projects(team_id=str(team.id)) + + def get_project_id_team_projects_map( + self, team: Team, org_projects: List[OrgProject] + ) -> Dict[str, TeamProjects]: + project_ids = [str(project.id) for project in org_projects] + team_projects: List[TeamProjects] = ( + self._project_repo_service.get_team_projects_by_project_id_for_team( + team.id, project_ids + ) + ) + return {str(tp.org_project_id): tp for tp in team_projects} + + def update_team_projects( + self, team: Team, raw_org_projects: List[RawTeamOrgProject] + ) -> List[OrgProject]: + updated_projects = self._update_org_projects(team.org_id, raw_org_projects) + self._update_team_projects(team, updated_projects, raw_org_projects) + self._set_unused_projects_as_inactive(team.org_id) + + return updated_projects + + def _update_org_projects( + self, org_id: str, raw_org_projects: List[RawTeamOrgProject] + ) -> List[OrgProject]: + idempotency_keys = [project.idempotency_key for project in raw_org_projects] + + # One batch lookup for all incoming projects, not one query per + # project -- avoids the per-item DB round trip this would otherwise + # cost (see docs/JIRA_INTEGRATION_PROPOSAL.md's review notes). + existing_org_projects = ( + self._project_repo_service.get_projects_by_idempotency_keys( + idempotency_keys + ) + ) + idempotency_key_to_project_map = { + project.idempotency_key: project for project in existing_org_projects + } + + updated_org_projects = [] + # CLUSTOX: which connection each project (new or existing) came from, + # keyed by the project's own id -- collected alongside the loop below + # since that's the only place both the project id (freshly minted for + # a new row) and the raw request's connection_id are in scope + # together. Written only after updated_org_projects is committed + # (see the OrgProjectConnection FK), so this stays a plan, not a + # write, until after the return below. + connection_id_by_project_id: Dict[str, str] = {} + for raw_project in raw_org_projects: + existing_project = idempotency_key_to_project_map.get( + raw_project.idempotency_key + ) + if existing_project: + if str(existing_project.org_id) != str(org_id): + raise Exception( + "Data integrity error, matching idempotency key across " + f"orgs. Team OrgId: {str(org_id)}. Existing Project " + f"OrgID: {str(existing_project.org_id)}. " + f"idempotency_key: {raw_project.idempotency_key}" + ) + + existing_project.is_active = True + existing_project.key = raw_project.key + existing_project.name = raw_project.name + project = existing_project + else: + project = OrgProject( + id=uuid4_str(), + org_id=org_id, + key=raw_project.key, + name=raw_project.name, + provider=raw_project.provider, + idempotency_key=raw_project.idempotency_key, + # Explicit rather than relying on the column's + # SQLAlchemy-level default -- that default only + # materializes on this instance once it's actually + # flushed through a real DB session, which makes + # the object momentarily wrong (is_active=None) to + # anything inspecting it beforehand. + is_active=True, + ) + updated_org_projects.append(project) + # A project saved without a connection_id (legacy flow, or a + # non-Jira provider) is left alone here -- not de-associated -- + # see save_org_project_connections's own docstring. + if raw_project.connection_id: + connection_id_by_project_id[str(project.id)] = raw_project.connection_id + + saved_projects = self._project_repo_service.update_org_projects( + updated_org_projects + ) + if connection_id_by_project_id: + self._project_repo_service.save_org_project_connections( + [ + OrgProjectConnection( + org_project_id=project_id, + jira_connection_id=connection_id, + ) + for project_id, connection_id in connection_id_by_project_id.items() + ] + ) + return saved_projects + + def _update_team_projects( + self, + team: Team, + updated_org_projects: List[OrgProject], + raw_projects_data: List[RawTeamOrgProject], + ): + existing_team_projects = self._project_repo_service.get_existing_team_projects( + team + ) + for team_project in existing_team_projects: + team_project.is_active = False + + project_id_to_team_project_map = { + str(tp.org_project_id): tp for tp in existing_team_projects + } + idempotency_key_raw_project_map = { + raw.idempotency_key: raw for raw in raw_projects_data + } + + updated_team_projects = [] + for project in updated_org_projects: + team_project = project_id_to_team_project_map.get(str(project.id)) + # Every project in updated_org_projects came from raw_projects_data + # (see _update_org_projects), so this lookup should never miss -- + # asserting rather than silently defaulting surfaces a broken + # idempotency_key match instead of writing a mystery row. + assert idempotency_key_raw_project_map.get(project.idempotency_key) + + if team_project: + team_project.is_active = True + else: + team_project = TeamProjects( + team_id=team.id, + org_project_id=str(project.id), + ) + + updated_team_projects.append(team_project) + + self._project_repo_service.update_team_projects(updated_team_projects) + + def _set_unused_projects_as_inactive(self, org_id: str) -> List[OrgProject]: + active_projects = self._project_repo_service.get_active_org_projects(org_id) + active_projects_used_across_teams = ( + self._project_repo_service.get_org_projects_used_across_teams(org_id) + ) + active_used_ids = { + str(project.id) for project in active_projects_used_across_teams + } + + for project in active_projects: + if str(project.id) not in active_used_ids: + project.is_active = False + + return self._project_repo_service.update_org_projects(active_projects) + + +def get_project_service() -> ProjectService: + return ProjectService(ProjectRepoService()) diff --git a/backend/analytics_server/mhq/service/project/sync/__init__.py b/backend/analytics_server/mhq/service/project/sync/__init__.py new file mode 100644 index 000000000..b50dd389e --- /dev/null +++ b/backend/analytics_server/mhq/service/project/sync/__init__.py @@ -0,0 +1 @@ +from .etl_handler import sync_project_issues diff --git a/backend/analytics_server/mhq/service/project/sync/etl_handler.py b/backend/analytics_server/mhq/service/project/sync/etl_handler.py new file mode 100644 index 000000000..350d3588f --- /dev/null +++ b/backend/analytics_server/mhq/service/project/sync/etl_handler.py @@ -0,0 +1,138 @@ +from datetime import datetime +from typing import List + +import pytz + +from mhq.service.bookmark import BookmarkService, BookmarkType, get_bookmark_service +from mhq.service.project.integration import get_project_integration_service +from mhq.service.project.sync.etl_project_factory import ProjectETLFactory +from mhq.service.project.sync.etl_provider_handler import ProjectProviderETLHandler +from mhq.store.models.projects import OrgProject +from mhq.store.repos.projects import ProjectRepoService +from mhq.utils.log import LOG + + +class ProjectETLHandler: + """Provider-agnostic orchestrator -- mirrors CodeETLHandler.""" + + def __init__( + self, + project_repo_service: ProjectRepoService, + etl_service: ProjectProviderETLHandler, + bookmark_service: BookmarkService, + ): + self.project_repo_service = project_repo_service + self.etl_service = etl_service + self.bookmark_service = bookmark_service + + def sync_org_projects(self, org_id: str, provider: str) -> None: + if not self.etl_service.check_pat_validity(): + LOG.error("Invalid PAT for project provider") + return + + # CLUSTOX: which projects to sync is now the handler's own call, not + # a blanket "every active project for org+provider" query here -- a + # connection-scoped JiraETLHandler must only touch its connection's + # own projects. See docs/JIRA_MULTI_ACCOUNT_PLAN.md Task 4. + org_projects: List[OrgProject] = self.etl_service.get_org_projects_to_sync( + org_id + ) + for org_project in org_projects: + try: + self._sync_project_issues(org_project, provider) + except Exception as e: + LOG.error( + f"Error syncing issues for project {org_project.key}: {str(e)}" + ) + continue + + try: + self._sync_project_sprints(org_project) + except Exception as e: + LOG.error( + f"Error syncing sprints for project {org_project.key}: {str(e)}" + ) + continue + + def _sync_project_issues(self, org_project: OrgProject, provider: str) -> None: + bookmark: datetime = self.bookmark_service.get_bookmark( + str(org_project.id), + BookmarkType.PROJECT_ISSUES_BOOKMARK, + provider, + ) + tickets, ticket_states = self.etl_service.get_project_issues_data( + org_project, bookmark + ) + self.project_repo_service.save_tickets_data(tickets, ticket_states) + + if not tickets: + # Nothing changed since the bookmark -- re-save it as-is so + # the row exists (first sync for this project) rather than + # leaving it perpetually unset. + self.bookmark_service.update_bookmark( + str(org_project.id), + BookmarkType.PROJECT_ISSUES_BOOKMARK, + provider, + bookmark, + ) + return + + tickets.sort(key=lambda ticket: ticket.updated_at) + new_bookmark = tickets[-1].updated_at.astimezone(tz=pytz.UTC) + self.bookmark_service.update_bookmark( + str(org_project.id), + BookmarkType.PROJECT_ISSUES_BOOKMARK, + provider, + new_bookmark, + ) + + def _sync_project_sprints(self, org_project: OrgProject) -> None: + # No bookmark -- see Sprint's own docstring for why re-fetching + # everything each cycle is the right call here, unlike issues. + sprints = self.etl_service.get_project_sprints_data(org_project) + if not sprints: + return + self.project_repo_service.save_sprints(sprints) + + +def sync_project_issues(org_id: str) -> None: + providers: List[str] = get_project_integration_service().get_org_providers(org_id) + if not providers: + LOG.info(f"No project-tracking integrations found for org {org_id}") + return + + etl_factory = ProjectETLFactory(org_id) + + for provider in providers: + try: + # CLUSTOX: one provider can now resolve to several handlers -- + # one per JiraConnection, see docs/JIRA_MULTI_ACCOUNT_PLAN.md + # Task 4. Sequential on purpose: N connections make N sync + # passes, same as N providers already did, and the whole call + # already runs inside the per-org "{org_id}:data_sync" Redis + # lock (mhq/api/sync.py) -- nothing here runs concurrently, so + # no new lock or connection-scoped isolation primitive is + # needed for this. + etl_services = etl_factory(provider) + except Exception as e: + LOG.error(f"Error building ETL handlers for provider {provider}: {str(e)}") + continue + + for etl_service in etl_services: + try: + project_etl_handler = ProjectETLHandler( + ProjectRepoService(), + etl_service, + get_bookmark_service(), + ) + project_etl_handler.sync_org_projects(org_id, provider) + except Exception as e: + # One connection's failure must not stop its siblings' -- + # the same reasoning as the per-project try/except above, + # one level up. + LOG.error( + f"Error syncing org project issues for provider {provider}: {str(e)}" + ) + continue + LOG.info(f"Synced org project issues for provider {provider}") + LOG.info(f"Synced all org project issues for org {org_id}") diff --git a/backend/analytics_server/mhq/service/project/sync/etl_jira_handler.py b/backend/analytics_server/mhq/service/project/sync/etl_jira_handler.py new file mode 100644 index 000000000..11fa87bec --- /dev/null +++ b/backend/analytics_server/mhq/service/project/sync/etl_jira_handler.py @@ -0,0 +1,273 @@ +from datetime import datetime +from typing import Dict, List, Optional, Tuple + +from mhq.exapi.jira import JiraApiService +from mhq.exapi.models.jira import JiraChangelogEntry, JiraIssue, JiraSprint +from mhq.service.project.sync.etl_provider_handler import ProjectProviderETLHandler +from mhq.store.models import UserIdentityProvider +from mhq.store.models.projects import OrgProject, Sprint, Ticket, TicketState +from mhq.store.repos.core import CoreRepoService +from mhq.store.repos.jira_connection import JiraConnectionRepoService +from mhq.store.repos.projects import ProjectRepoService +from mhq.utils.log import LOG +from mhq.utils.string import uuid4_str + + +class JiraETLHandler(ProjectProviderETLHandler): + def __init__( + self, + org_id: str, + jira_api_service: JiraApiService, + project_repo_service: ProjectRepoService, + connection_id: Optional[str] = None, + ): + self.org_id = org_id + self._api = jira_api_service + self._project_repo_service = project_repo_service + # CLUSTOX: None means "the legacy, single-account Integration row" -- + # see docs/JIRA_MULTI_ACCOUNT_PLAN.md. Both the project scope + # (get_org_projects_to_sync) and the ticket idempotency key branch on + # this; a JiraConnection-backed handler always passes its own id. + self.connection_id = connection_id + + def check_pat_validity(self) -> bool: + return self._api.check_pat() + + def get_org_projects_to_sync(self, org_id: str) -> List[OrgProject]: + if self.connection_id: + return self._project_repo_service.get_active_org_projects_for_connection( + self.connection_id + ) + return self._project_repo_service.get_active_org_projects_for_provider( + org_id, UserIdentityProvider.JIRA.value + ) + + def get_project_issues_data( + self, org_project: OrgProject, bookmark: datetime + ) -> Tuple[List[Ticket], List[TicketState]]: + # JQL's own "updated >=" filter does the incremental narrowing + # server-side (same idea as GitLab's updated_after param) rather + # than fetching everything and truncating client-side the way the + # GitHub PR sync has to. + jql = ( + f'project = "{org_project.key}" ' + f'AND updated >= "{bookmark.strftime("%Y-%m-%d %H:%M")}" ' + f"ORDER BY updated ASC" + ) + jira_issues = self._api.get_all_issues(jql) + if not jira_issues: + return [], [] + + tickets = self._to_tickets(org_project, jira_issues) + ticket_states = self._to_ticket_states(tickets, jira_issues) + return tickets, ticket_states + + def _to_tickets( + self, org_project: OrgProject, jira_issues: List[JiraIssue] + ) -> List[Ticket]: + idempotency_keys = [ + self._ticket_idempotency_key(issue) for issue in jira_issues + ] + # One batch lookup for every ticket in this page, not one query + # per ticket -- avoids the per-item DB round trip the existing + # GitHub PR sync pays (get_repo_pr_by_number, called once per PR). + existing_id_by_key = { + ticket.idempotency_key: ticket.id + for ticket in self._project_repo_service.get_tickets_by_idempotency_keys( + idempotency_keys + ) + } + + return [ + Ticket( + id=existing_id_by_key.get(idempotency_key, uuid4_str()), + org_project_id=org_project.id, + key=issue.key, + provider=UserIdentityProvider.JIRA.value, + status=issue.status, + status_category=issue.status_category, + idempotency_key=idempotency_key, + data=issue.data, + created_at=issue.created, + updated_at=issue.updated, + ) + for issue, idempotency_key in zip(jira_issues, idempotency_keys) + ] + + def _to_ticket_states( + self, tickets: List[Ticket], jira_issues: List[JiraIssue] + ) -> List[TicketState]: + entries_by_ticket: List[Tuple[Ticket, JiraChangelogEntry]] = [ + (ticket, entry) + for ticket, issue in zip(tickets, jira_issues) + for entry in issue.changelog + ] + if not entries_by_ticket: + return [] + + idempotency_keys = [ + self._ticket_state_idempotency_key(ticket, entry) + for ticket, entry in entries_by_ticket + ] + existing_id_by_key = { + state.idempotency_key: state.id + for state in self._project_repo_service.get_ticket_states_by_idempotency_keys( + idempotency_keys + ) + } + + return [ + TicketState( + id=existing_id_by_key.get(idempotency_key, uuid4_str()), + ticket_id=ticket.id, + from_status=entry.from_status, + to_status=entry.to_status, + changed_at=entry.changed_at, + idempotency_key=idempotency_key, + data=entry.data, + ) + for (ticket, entry), idempotency_key in zip( + entries_by_ticket, idempotency_keys + ) + ] + + def get_project_sprints_data(self, org_project: OrgProject) -> List[Sprint]: + boards = self._api.get_boards_for_project(org_project.key) + scrum_boards = [board for board in boards if board.board_type == "scrum"] + if not scrum_boards: + return [] + + jira_sprints = [ + sprint + for board in scrum_boards + for sprint in self._api.get_sprints_for_board(board.id) + ] + if not jira_sprints: + return [] + + return self._to_sprints(org_project, jira_sprints) + + def _to_sprints( + self, org_project: OrgProject, jira_sprints: List[JiraSprint] + ) -> List[Sprint]: + idempotency_keys = [self._sprint_idempotency_key(s) for s in jira_sprints] + existing_id_by_key = { + sprint.idempotency_key: sprint.id + for sprint in self._project_repo_service.get_sprints_by_idempotency_keys( + idempotency_keys + ) + } + + sprints = [] + for sprint, idempotency_key in zip(jira_sprints, idempotency_keys): + planned_count, completed_count = self._api.get_sprint_issue_counts( + sprint.id + ) + sprints.append( + Sprint( + id=existing_id_by_key.get(idempotency_key, uuid4_str()), + org_project_id=org_project.id, + provider=UserIdentityProvider.JIRA.value, + external_id=str(sprint.id), + name=sprint.name, + state=sprint.state, + start_date=sprint.start_date, + end_date=sprint.end_date, + planned_count=planned_count, + completed_count=completed_count, + idempotency_key=idempotency_key, + ) + ) + return sprints + + def _sprint_idempotency_key(self, sprint: JiraSprint) -> str: + # Scoped by org_id, same reasoning as _ticket_idempotency_key -- + # each org's Jira site is independent. + return f"jira:{self.org_id}:sprint:{sprint.id}" + + def _ticket_idempotency_key(self, issue: JiraIssue) -> str: + # Scoped by org_id, not the bare Jira issue id -- same reasoning as + # OrgProject's idempotency_key: each org's Jira site is + # independent, so two orgs' sites can land on the same id. + # + # CLUSTOX: also scoped by connection_id when this handler is + # JiraConnection-backed -- two connections in the same org are two + # independent Jira sites too, and a bare org_id scope would collide + # their small, site-local issue ids (see docs/JIRA_MULTI_ACCOUNT_PLAN.md, + # "Known risks" #1). The legacy (connection_id is None) branch keeps + # the exact pre-existing key format: changing it would orphan every + # ticket already synced through the single-account flow, which the + # plan's backward-compat requirement (Task 7) rules out. + if self.connection_id: + return f"jira:{self.org_id}:{self.connection_id}:{issue.id}" + return f"jira:{self.org_id}:{issue.id}" + + def _ticket_state_idempotency_key( + self, ticket: Ticket, entry: JiraChangelogEntry + ) -> str: + return f"{ticket.idempotency_key}:{entry.idempotency_key}" + + +def get_jira_etl_handler(org_id: str) -> JiraETLHandler: + """ + The legacy, single-account handler -- unchanged, so an org with zero + JiraConnection rows syncs exactly as it did before this feature existed + (docs/JIRA_MULTI_ACCOUNT_PLAN.md Task 7). connection_id is left at its + default (None): this handler's projects and idempotency keys use the + original, org-scoped-only shape. + """ + core_repo_service = CoreRepoService() + site_url, email = _get_jira_site_and_email(core_repo_service, org_id) + api_token = core_repo_service.get_access_token(org_id, UserIdentityProvider.JIRA) + + if not (site_url and email and api_token): + LOG.error(f"Jira credentials incomplete for org {org_id}") + + return JiraETLHandler( + org_id, + JiraApiService(email, api_token, site_url), + ProjectRepoService(), + ) + + +def get_jira_etl_handlers(org_id: str) -> List[JiraETLHandler]: + """ + docs/JIRA_MULTI_ACCOUNT_PLAN.md Task 4: an org with any JiraConnection + rows syncs *only* those connections, one handler each, every handler + scoped to its own connection's projects (get_org_projects_to_sync) and + its own ticket idempotency-key namespace. An org with none falls back to + the single legacy handler above -- this is a strict either/or, not + "legacy plus whatever connections exist": the two flows are + deliberately independent (see the plan's Global Constraints), so a + project the legacy flow already syncs is not re-synced here just + because the org later adds a JiraConnection: moving it over is Task 6's + project picker, not something sync does on an org's behalf. + """ + jira_connection_repo_service = JiraConnectionRepoService() + connections = jira_connection_repo_service.list_jira_connections(org_id) + if not connections: + return [get_jira_etl_handler(org_id)] + + project_repo_service = ProjectRepoService() + handlers = [] + for connection in connections: + access_token = jira_connection_repo_service.decrypt_access_token(connection) + handlers.append( + JiraETLHandler( + org_id, + JiraApiService(connection.email, access_token, connection.site_url), + project_repo_service, + connection_id=str(connection.id), + ) + ) + return handlers + + +def _get_jira_site_and_email( + core_repo_service: CoreRepoService, org_id: str +) -> Tuple[str, str]: + integrations = core_repo_service.get_org_integrations_for_names( + org_id, [UserIdentityProvider.JIRA.value] + ) + provider_meta: Dict = (integrations[0].provider_meta or {}) if integrations else {} + return provider_meta.get("site_url"), provider_meta.get("email") diff --git a/backend/analytics_server/mhq/service/project/sync/etl_project_factory.py b/backend/analytics_server/mhq/service/project/sync/etl_project_factory.py new file mode 100644 index 000000000..110c38a1a --- /dev/null +++ b/backend/analytics_server/mhq/service/project/sync/etl_project_factory.py @@ -0,0 +1,19 @@ +from typing import List + +from mhq.service.project.sync.etl_jira_handler import get_jira_etl_handlers +from mhq.service.project.sync.etl_provider_handler import ProjectProviderETLHandler +from mhq.store.models import UserIdentityProvider + + +class ProjectETLFactory: + def __init__(self, org_id: str): + self.org_id = org_id + + # CLUSTOX: returns a list, not a single handler -- Jira can now resolve + # to several (one per JiraConnection). See + # docs/JIRA_MULTI_ACCOUNT_PLAN.md Task 4. + def __call__(self, provider: str) -> List[ProjectProviderETLHandler]: + if provider == UserIdentityProvider.JIRA.value: + return get_jira_etl_handlers(self.org_id) + + raise NotImplementedError(f"Unknown provider - {provider}") diff --git a/backend/analytics_server/mhq/service/project/sync/etl_provider_handler.py b/backend/analytics_server/mhq/service/project/sync/etl_provider_handler.py new file mode 100644 index 000000000..64407a901 --- /dev/null +++ b/backend/analytics_server/mhq/service/project/sync/etl_provider_handler.py @@ -0,0 +1,54 @@ +from abc import ABC, abstractmethod +from datetime import datetime +from typing import List, Tuple + +from mhq.store.models.projects import OrgProject, Sprint, Ticket, TicketState + + +class ProjectProviderETLHandler(ABC): + """ + Mirrors CodeProviderETLHandler -- one implementation per + project-tracking tool (Jira, to start). See + docs/JIRA_INTEGRATION_PROPOSAL.md. + """ + + @abstractmethod + def check_pat_validity(self) -> bool: + """ + :return: whether the stored credentials are still valid. + """ + + @abstractmethod + def get_org_projects_to_sync(self, org_id: str) -> List[OrgProject]: + """ + Active projects this handler instance is responsible for. Not just + "every active project for the org+provider" any more -- a + connection-scoped handler (see docs/JIRA_MULTI_ACCOUNT_PLAN.md) + must only touch the projects synced from *its* connection, or two + connections' handlers would each re-sync the other's projects. + """ + + @abstractmethod + def get_project_issues_data( + self, org_project: OrgProject, bookmark: datetime + ) -> Tuple[List[Ticket], List[TicketState]]: + """ + Tickets updated after `bookmark`, and their status-transition + history, for the given project. + :param org_project: the OrgProject to sync issues for. + :param bookmark: only issues updated at or after this time. + :return: Tickets and TicketStates, ready to persist -- ids are + already reconciled against any existing rows (same idempotency_key + reuses the same id) by the implementation, not by the caller. + """ + + @abstractmethod + def get_project_sprints_data(self, org_project: OrgProject) -> List[Sprint]: + """ + Every sprint (any state -- future/active/closed) for the Scrum + board(s) backing this project, planned/completed issue counts + already computed -- ready to persist. Re-fetched and upserted in + full every cycle, no bookmark (see Sprint's own docstring for + why). A project with no Scrum board (Kanban-only) returns an + empty list, not an error. + """ diff --git a/backend/analytics_server/mhq/service/settings/benchmarks.py b/backend/analytics_server/mhq/service/settings/benchmarks.py new file mode 100644 index 000000000..117e1f6d3 --- /dev/null +++ b/backend/analytics_server/mhq/service/settings/benchmarks.py @@ -0,0 +1,134 @@ +# CLUSTOX: benchmark resolution and validation. +# +# Resolution lives in the backend rather than the browser for three reasons: +# every consumer would otherwise reimplement the fallback and eventually +# disagree; the response can report WHICH benchmark applied, so an admin who +# thinks they set a target can see that they did not; and a fifth metric later +# is one more key here rather than one more reimplementation. +from datetime import datetime +from typing import Dict, List, Optional + +from werkzeug.exceptions import BadRequest + +from mhq.service.settings.models import BenchmarkSetting, ConfigurationSettings +from mhq.store.models.settings.enums import EntityType +from mhq.utils.time import time_now + +# The global baseline belongs to no team and no workspace, but Settings. +# entity_id is NOT NULL. A fixed sentinel is the documented cost of reusing +# that table instead of adding one. +GLOBAL_BENCHMARK_ENTITY_ID = "00000000-0000-4000-8000-000000000001" + +BENCHMARK_METRICS: List[str] = [ + "lead_time", + "deployment_frequency", + "change_failure_rate", + "mean_time_to_recovery", + "lines_of_code", +] + +# Percent, so it has an upper bound the others do not. +_BOUNDED_METRICS = {"change_failure_rate": 100} + + +def resolve_benchmarks( + team_setting: Optional[BenchmarkSetting], + global_setting: Optional[BenchmarkSetting], +) -> Dict[str, Dict]: + resolved = {} + + for metric in BENCHMARK_METRICS: + team_value = getattr(team_setting, metric, None) if team_setting else None + global_value = getattr(global_setting, metric, None) if global_setting else None + + # `is not None` rather than truthiness: 0 is a deliberate target. + if team_value is not None: + resolved[metric] = {"target": team_value, "source": "team"} + elif global_value is not None: + resolved[metric] = {"target": global_value, "source": "global"} + else: + resolved[metric] = {"target": None, "source": None} + + return resolved + + +def validate_benchmark_payload(data: Dict) -> Dict: + unknown = set(data.keys()) - set(BENCHMARK_METRICS) + if unknown: + raise BadRequest(f"Unknown benchmark metrics: {', '.join(sorted(unknown))}") + + cleaned = {} + for metric, value in data.items(): + if value is None: + continue + + if not isinstance(value, (int, float)) or isinstance(value, bool): + raise BadRequest(f"{metric} must be a number") + + if value < 0: + raise BadRequest(f"{metric} must not be negative") + + upper = _BOUNDED_METRICS.get(metric) + if upper is not None and value > upper: + raise BadRequest(f"{metric} must be between 0 and {upper}") + + cleaned[metric] = value + + # CLUSTOX: always every key in BENCHMARK_METRICS, None for the ones the + # form cleared. + # The form omits empty fields, so clearing every field posts `{}` -- and + # `save_settings` treats a falsy setting_data as "no data supplied" and + # substitutes get_default_setting_data(). "Clear everything to go back to + # inheriting" therefore used to *write* whatever the defaults were. A + # dict of all-None values is truthy, round-trips through + # _adapt_benchmark_setting_from_json unchanged, and resolve_benchmarks' + # `is not None` check then falls back per metric, which is the point. + return {metric: cleaned.get(metric) for metric in BENCHMARK_METRICS} + + +def empty_benchmark_settings( + entity_type: EntityType, entity_id: str +) -> ConfigurationSettings: + """An all-`None` benchmark setting that is NOT persisted. + + CLUSTOX: a GET must never materialise a row. Reading the config form for + a team that has never set a benchmark used to write a TEAM-scoped row, + which permanently killed that team's per-metric inheritance -- and the + same on the global route meant one admin opening one form created the + installation-wide baseline. Returning this instead keeps "nothing + configured anywhere" reachable, which is the state the spec requires + every card to render unchanged in. + """ + now: datetime = time_now() + return ConfigurationSettings( + entity_id=entity_id, + entity_type=entity_type, + specific_settings=BenchmarkSetting(), + updated_by=None, + created_at=now, + updated_at=now, + ) + + +def get_resolved_benchmarks_for_team(team_id: str, settings_service=None) -> Dict: + """Resolve a team's benchmarks against the global baseline. + + CLUSTOX: `settings_service` is injectable so the resolution order can be + tested without a database. + """ + from mhq.service.settings.configuration_settings import get_settings_service + from mhq.store.models.settings import SettingType + + settings_service = settings_service or get_settings_service() + + team_setting = settings_service.get_settings( + SettingType.BENCHMARK_SETTING, EntityType.TEAM, team_id + ) + global_setting = settings_service.get_settings( + SettingType.BENCHMARK_SETTING, EntityType.GLOBAL, GLOBAL_BENCHMARK_ENTITY_ID + ) + + return resolve_benchmarks( + getattr(team_setting, "specific_settings", None), + getattr(global_setting, "specific_settings", None), + ) diff --git a/backend/analytics_server/mhq/service/settings/configuration_settings.py b/backend/analytics_server/mhq/service/settings/configuration_settings.py index 14d046619..8405c3378 100644 --- a/backend/analytics_server/mhq/service/settings/configuration_settings.py +++ b/backend/analytics_server/mhq/service/settings/configuration_settings.py @@ -3,6 +3,7 @@ from mhq.service.settings.default_settings_data import get_default_setting_data from mhq.service.settings.models import ( + BenchmarkSetting, ConfigurationSettings, DefaultSyncDaysSetting, ExcludedPRsSetting, @@ -10,6 +11,7 @@ IncidentSourcesSetting, IncidentTypesSetting, IncidentPRsSetting, + JiraIncidentIssueTypesSetting, ) from mhq.store.models.core.users import Users from mhq.store.models.incidents import IncidentSource, IncidentType @@ -73,6 +75,26 @@ def _adapt_incident_prs_setting_from_setting_data(self, data: Dict[str, any]): filters=data.get("filters", []), ) + def _adapt_jira_incident_issue_types_setting_from_setting_data( + self, data: Dict[str, any] + ) -> JiraIncidentIssueTypesSetting: + """ + Adapts the json data in Settings.data to JiraIncidentIssueTypesSetting + """ + return JiraIncidentIssueTypesSetting(issue_types=data.get("issue_types") or []) + + # CLUSTOX: `data.get(key)` rather than `data.get(key, 0)` -- absent means + # inherit, and 0 is a deliberate target. Collapsing them would make every + # unset metric read as a target of zero. + def _adapt_benchmark_setting_from_setting_data(self, data: Dict[str, any]): + return BenchmarkSetting( + lead_time=data.get("lead_time"), + deployment_frequency=data.get("deployment_frequency"), + change_failure_rate=data.get("change_failure_rate"), + mean_time_to_recovery=data.get("mean_time_to_recovery"), + lines_of_code=data.get("lines_of_code"), + ) + # ADD NEW DICT TO DATACLASS ADAPTERS HERE def _handle_config_setting_from_db_setting( @@ -98,6 +120,14 @@ def _handle_config_setting_from_db_setting( if setting_type == SettingType.INCIDENT_PRS_SETTING: return self._adapt_incident_prs_setting_from_setting_data(setting_data) + if setting_type == SettingType.JIRA_INCIDENT_ISSUE_TYPES_SETTING: + return self._adapt_jira_incident_issue_types_setting_from_setting_data( + setting_data + ) + + if setting_type == SettingType.BENCHMARK_SETTING: + return self._adapt_benchmark_setting_from_setting_data(setting_data) + # ADD NEW HANDLE FROM DB SETTINGS HERE raise Exception(f"Invalid Setting Type: {setting_type}") @@ -198,6 +228,26 @@ def _adapt_incident_prs_setting_from_json(self, data: Dict[str, any]): filters=data.get("filters", []), ) + def _adapt_jira_incident_issue_types_setting_from_json( + self, data: Dict[str, any] + ) -> JiraIncidentIssueTypesSetting: + """ + Adapts the json data from API to JiraIncidentIssueTypesSetting + """ + return JiraIncidentIssueTypesSetting(issue_types=data.get("issue_types") or []) + + # CLUSTOX: `data.get(key)` rather than `data.get(key, 0)` -- absent means + # inherit, and 0 is a deliberate target. Collapsing them would make every + # unset metric read as a target of zero. + def _adapt_benchmark_setting_from_json(self, data: Dict[str, any]): + return BenchmarkSetting( + lead_time=data.get("lead_time"), + deployment_frequency=data.get("deployment_frequency"), + change_failure_rate=data.get("change_failure_rate"), + mean_time_to_recovery=data.get("mean_time_to_recovery"), + lines_of_code=data.get("lines_of_code"), + ) + # ADD NEW DICT TO API ADAPTERS HERE def _handle_config_setting_from_json_data( @@ -223,6 +273,12 @@ def _handle_config_setting_from_json_data( if setting_type == SettingType.INCIDENT_PRS_SETTING: return self._adapt_incident_prs_setting_from_json(setting_data) + if setting_type == SettingType.JIRA_INCIDENT_ISSUE_TYPES_SETTING: + return self._adapt_jira_incident_issue_types_setting_from_json(setting_data) + + if setting_type == SettingType.BENCHMARK_SETTING: + return self._adapt_benchmark_setting_from_json(setting_data) + # ADD NEW HANDLE FROM JSON DATA HERE raise Exception(f"Invalid Setting Type: {setting_type}") @@ -269,6 +325,22 @@ def _adapt_incident_prs_setting_json_data( "filters": specific_setting.filters, } + def _adapt_jira_incident_issue_types_setting_json_data( + self, specific_setting: JiraIncidentIssueTypesSetting + ) -> Dict: + return {"issue_types": specific_setting.issue_types} + + def _adapt_benchmark_setting_json_data( + self, specific_setting: BenchmarkSetting + ) -> Dict: + return { + "lead_time": specific_setting.lead_time, + "deployment_frequency": specific_setting.deployment_frequency, + "change_failure_rate": specific_setting.change_failure_rate, + "mean_time_to_recovery": specific_setting.mean_time_to_recovery, + "lines_of_code": specific_setting.lines_of_code, + } + # ADD NEW DATACLASS TO JSON DATA ADAPTERS HERE def _handle_config_setting_to_db_setting( @@ -305,6 +377,19 @@ def _handle_config_setting_to_db_setting( ): return self._adapt_incident_prs_setting_json_data(specific_setting) + if ( + setting_type == SettingType.JIRA_INCIDENT_ISSUE_TYPES_SETTING + and isinstance(specific_setting, JiraIncidentIssueTypesSetting) + ): + return self._adapt_jira_incident_issue_types_setting_json_data( + specific_setting + ) + + if setting_type == SettingType.BENCHMARK_SETTING and isinstance( + specific_setting, BenchmarkSetting + ): + return self._adapt_benchmark_setting_json_data(specific_setting) + # ADD NEW HANDLE TO DB SETTINGS HERE raise Exception(f"Invalid Setting Type: {setting_type}") diff --git a/backend/analytics_server/mhq/service/settings/default_settings_data.py b/backend/analytics_server/mhq/service/settings/default_settings_data.py index 0e151a707..2b6e85c1e 100644 --- a/backend/analytics_server/mhq/service/settings/default_settings_data.py +++ b/backend/analytics_server/mhq/service/settings/default_settings_data.py @@ -1,7 +1,6 @@ from mhq.store.models.incidents import IncidentSource, IncidentType from mhq.store.models.settings import SettingType - MIN_CYCLE_TIME_THRESHOLD = 3600 @@ -13,7 +12,30 @@ def get_default_setting_data(setting_type: SettingType): return {"excluded_pr_ids": []} if setting_type == SettingType.INCIDENT_SOURCES_SETTING: - incident_sources = list(IncidentSource) + # CLUSTOX: deliberately NOT `list(IncidentSource)` -- that would + # silently opt every org with no saved setting into any new source + # value added to the enum sight-unseen. Every value below is + # explicitly chosen. + # + # JIRA_ISSUE used to be the one opt-in exception here (an admin had + # to explicitly turn it on via ConfigureJiraIncidentSourceModalBody). + # Default flipped to on, matching GIT_REPO's own default: a ticket + # reopened after being marked resolved is the same kind of "the fix + # didn't hold" signal a revert PR already is, so it shouldn't need + # an admin to first discover the switch exists. Unlike GIT_REPO + # (which has no toggle at all), the on/off switch for this one + # stays in the UI -- an org can still opt back out, this just + # changes what a *new* org (or one that hasn't touched this + # setting yet) starts with. See + # migrations/20260813120000_clustox_jira_incident_source_unconditional.sql + # for the counterpart that brings already-saved settings rows for + # existing orgs up to this same default. + incident_sources = [ + IncidentSource.INCIDENT_SERVICE, + IncidentSource.INCIDENT_TEAM, + IncidentSource.GIT_REPO, + IncidentSource.JIRA_ISSUE, + ] return { "incident_sources": [ incident_source.value for incident_source in incident_sources @@ -35,6 +57,30 @@ def get_default_setting_data(setting_type: SettingType): "filters": [], } + if setting_type == SettingType.JIRA_INCIDENT_ISSUE_TYPES_SETTING: + # CLUSTOX: which Jira issue types count as an incident when + # IncidentSource.JIRA_ISSUE is enabled -- see + # docs/JIRA_INTEGRATION_PROPOSAL.md. + return {"issue_types": ["Bug"]} + + # CLUSTOX: deliberately empty, not the DORA "High" band. The spec's + # zero-config guarantee -- "no benchmark at any level means no line, no + # caption, and the card renders exactly as it does today" -- is only + # reachable if nothing materialises a target nobody asked for. Shipping + # numbers here made every card in every workspace sprout a target line + # and a "the default benchmark" caption the first time any admin opened + # any settings form, and made a team that had never set a benchmark + # report source: "team" for every metric. A superadmin types the + # baseline in; the code does not guess it. + if setting_type == SettingType.BENCHMARK_SETTING: + return { + "lead_time": None, + "deployment_frequency": None, + "change_failure_rate": None, + "mean_time_to_recovery": None, + "lines_of_code": None, + } + # ADD NEW DEFAULT SETTING HERE raise Exception(f"Invalid Setting Type: {setting_type}") diff --git a/backend/analytics_server/mhq/service/settings/models.py b/backend/analytics_server/mhq/service/settings/models.py index d568cbea9..6349a2ac8 100644 --- a/backend/analytics_server/mhq/service/settings/models.py +++ b/backend/analytics_server/mhq/service/settings/models.py @@ -1,6 +1,6 @@ from dataclasses import dataclass from datetime import datetime -from typing import List, TypedDict, Literal +from typing import List, TypedDict, Literal, Optional from mhq.store.models import EntityType from mhq.store.models.incidents.enums import IncidentSource, IncidentType @@ -57,6 +57,26 @@ class IncidentPRsSetting(BaseSetting): filters: List[IncidentPRFilter] +@dataclass +class JiraIncidentIssueTypesSetting(BaseSetting): + issue_types: List[str] + + +# CLUSTOX: every field is optional so fallback is per metric, not +# all-or-nothing -- a team may set a lead-time target and inherit the rest. +@dataclass +class BenchmarkSetting(BaseSetting): + lead_time: Optional[float] = None + deployment_frequency: Optional[float] = None + change_failure_rate: Optional[float] = None + mean_time_to_recovery: Optional[float] = None + # CLUSTOX: average gross lines (additions + deletions) per merged PR -- + # NOT weekly volume. Its unit is lines, so unlike lead_time and + # mean_time_to_recovery it is stored exactly as the admin typed it, with + # no seconds/hours conversion at any boundary. + lines_of_code: Optional[float] = None + + # ADD NEW SETTING CLASS HERE # Sample Future Settings diff --git a/backend/analytics_server/mhq/service/settings/setting_type_validator.py b/backend/analytics_server/mhq/service/settings/setting_type_validator.py index 7d71ba7ee..405d2c318 100644 --- a/backend/analytics_server/mhq/service/settings/setting_type_validator.py +++ b/backend/analytics_server/mhq/service/settings/setting_type_validator.py @@ -22,6 +22,12 @@ def settings_type_validator(setting_type: str): if setting_type == SettingType.INCIDENT_PRS_SETTING.value: return SettingType.INCIDENT_PRS_SETTING + if setting_type == SettingType.JIRA_INCIDENT_ISSUE_TYPES_SETTING.value: + return SettingType.JIRA_INCIDENT_ISSUE_TYPES_SETTING + + if setting_type == SettingType.BENCHMARK_SETTING.value: + return SettingType.BENCHMARK_SETTING + # ADD NEW VALIDATOR HERE raise BadRequest(f"Invalid Setting Type: {setting_type}") diff --git a/backend/analytics_server/mhq/service/sprints/__init__.py b/backend/analytics_server/mhq/service/sprints/__init__.py new file mode 100644 index 000000000..24661f136 --- /dev/null +++ b/backend/analytics_server/mhq/service/sprints/__init__.py @@ -0,0 +1 @@ +from .service import get_sprint_service, SprintService diff --git a/backend/analytics_server/mhq/service/sprints/service.py b/backend/analytics_server/mhq/service/sprints/service.py new file mode 100644 index 000000000..f8aedf5a1 --- /dev/null +++ b/backend/analytics_server/mhq/service/sprints/service.py @@ -0,0 +1,45 @@ +from datetime import datetime, timezone +from typing import List + +from mhq.store.models.core import Team +from mhq.store.models.projects import Sprint +from mhq.store.repos.projects import ProjectRepoService + +# CLUSTOX: Jira integration -- the Sprint rollup chart (docs/ +# JIRA_INTEGRATION_PROPOSAL.md §6D). "Planned vs. shipped, per sprint" -- +# no date-range scoping like ticket_insights/lead_time have, since a +# sprint's own start/end dates are its natural window; the DORA Metrics +# page's selected period doesn't apply here. +DEFAULT_SPRINT_LIMIT = 6 + + +class SprintService: + def __init__(self, project_repo_service: ProjectRepoService): + self._project_repo_service = project_repo_service + + def get_team_sprints( + self, team: Team, limit: int = DEFAULT_SPRINT_LIMIT + ) -> List[Sprint]: + org_projects = self._project_repo_service.get_team_projects(str(team.id)) + project_ids = [str(project.id) for project in org_projects] + + # The repo query orders most-recent-first so a `limit` always + # keeps the N sprints that matter, not an arbitrary N out of + # however many exist -- re-sorted ascending here since a rollup + # chart reads left-to-right, oldest to newest. + sprints = self._project_repo_service.get_sprints_for_projects( + project_ids, limit + ) + # tz-aware fallback -- sprint.start_date, when present, is + # tz-aware (dt_from_iso_time_string always attaches an offset); + # a naive datetime.max here would raise on comparison against a + # real, tz-aware sprint date the moment both appear in the same + # sort. + undated_sorts_last = datetime.max.replace(tzinfo=timezone.utc) + return sorted( + sprints, key=lambda sprint: sprint.start_date or undated_sorts_last + ) + + +def get_sprint_service() -> SprintService: + return SprintService(ProjectRepoService()) diff --git a/backend/analytics_server/mhq/service/sync_data.py b/backend/analytics_server/mhq/service/sync_data.py index 35e9abdc4..933dd9928 100644 --- a/backend/analytics_server/mhq/service/sync_data.py +++ b/backend/analytics_server/mhq/service/sync_data.py @@ -1,6 +1,8 @@ from mhq.service.code import sync_code_repos from mhq.service.incidents import sync_org_incidents from mhq.service.merge_to_deploy_broker import process_merge_to_deploy_cache +from mhq.service.project import sync_project_issues +from mhq.service.ticket_matching import match_tickets_to_prs from mhq.service.workflows import sync_org_workflows from mhq.utils.log import LOG @@ -9,6 +11,13 @@ sync_org_workflows, process_merge_to_deploy_cache, sync_org_incidents, + # CLUSTOX: Jira integration, Phase 2/3 (issue sync, then ticket-PR + # matching) -- no-ops for an org with no project-tracking integration + # linked. match_tickets_to_prs must run after sync_code_repos AND + # sync_project_issues, since it only links rows both of those have + # already written this cycle. See docs/JIRA_INTEGRATION_PROPOSAL.md. + sync_project_issues, + match_tickets_to_prs, ] diff --git a/backend/analytics_server/mhq/service/ticket_insights/__init__.py b/backend/analytics_server/mhq/service/ticket_insights/__init__.py new file mode 100644 index 000000000..ca2872163 --- /dev/null +++ b/backend/analytics_server/mhq/service/ticket_insights/__init__.py @@ -0,0 +1,2 @@ +from .cycle_time import compute_cycle_time_by_project, ProjectCycleTime +from .service import get_ticket_insights_service, TicketInsightsService diff --git a/backend/analytics_server/mhq/service/ticket_insights/cycle_time.py b/backend/analytics_server/mhq/service/ticket_insights/cycle_time.py new file mode 100644 index 000000000..574079df3 --- /dev/null +++ b/backend/analytics_server/mhq/service/ticket_insights/cycle_time.py @@ -0,0 +1,169 @@ +from collections import defaultdict +from dataclasses import dataclass, field +from typing import Dict, List, Tuple + +from mhq.store.models.projects import OrgProject, Ticket, TicketState + +# Jira Cloud's 3 status categories are fixed platform-wide, not +# customizable per workflow -- unlike individual status names, which +# vary per team (see docs/JIRA_INTEGRATION_PROPOSAL.md's design +# reference: this groups by these 3 categories, one segmented bar per +# project, not a flat list of a team's own literal status names). +CATEGORIES = ("To Do", "In Progress", "Done") + + +@dataclass +class ProjectCycleTime: + project_key: str + project_name: str + ticket_count: int + avg_total_seconds: float + avg_seconds_by_category: Dict[str, float] = field(default_factory=dict) + + +def compute_cycle_time_by_project( + tickets: List[Ticket], + ticket_states: List[TicketState], + projects_by_id: Dict[str, OrgProject], +) -> List[ProjectCycleTime]: + """ + Average cycle time per project, each broken into the 3 status + categories -- docs/JIRA_INTEGRATION_PROPOSAL.md §6C. + + Callers must only pass tickets that have actually reached a + "Done"-category status (ProjectRepoService.get_tickets_with_states_ + for_projects already filters for this). A ticket still open has no + bounded end time -- an early version of this measured every ticket up + to "now", and a single item sitting untouched in a backlog for + months dominated the average with a duration that had nothing to do + with how long finished work actually took. Every segment's end + boundary here is a ticket's own last recorded activity, never live + time. + + Every category average is computed over the *same* ticket set (a + ticket that never had an "In Progress" segment counts as 0 seconds + there, not excluded), so the 3 category averages always sum to + avg_total_seconds -- needed for a stacked bar whose segment widths + are supposed to add up to the whole bar. + """ + states_by_ticket: Dict[str, List[TicketState]] = defaultdict(list) + for state in ticket_states: + states_by_ticket[str(state.ticket_id)].append(state) + + tickets_by_project: Dict[str, List[Ticket]] = defaultdict(list) + for ticket in tickets: + tickets_by_project[str(ticket.org_project_id)].append(ticket) + + results: List[ProjectCycleTime] = [] + for project_id, project_tickets in tickets_by_project.items(): + project = projects_by_id.get(project_id) + if not project: + continue + + per_ticket_totals: List[float] = [] + category_sums = {category: 0.0 for category in CATEGORIES} + + for ticket in project_tickets: + states = sorted( + states_by_ticket.get(str(ticket.id), []), key=lambda s: s.changed_at + ) + segments = [ + (status, seconds) + for status, seconds in _status_segments(ticket, states) + if seconds >= 0 + ] + if not segments: + continue + + for category, seconds in _bucket_by_category(segments).items(): + category_sums[category] += seconds + per_ticket_totals.append(sum(seconds for _, seconds in segments)) + + ticket_count = len(per_ticket_totals) + if not ticket_count: + continue + + results.append( + ProjectCycleTime( + project_key=project.key, + project_name=project.name, + ticket_count=ticket_count, + avg_total_seconds=sum(per_ticket_totals) / ticket_count, + avg_seconds_by_category={ + category: category_sums[category] / ticket_count + for category in CATEGORIES + }, + ) + ) + + return sorted(results, key=lambda p: p.project_key) + + +def _bucket_by_category(segments: List[Tuple[str, float]]) -> Dict[str, float]: + """ + First segment -> To Do (the status a ticket is created in), last + segment -> Done (guaranteed -- callers only pass completed tickets, + so a ticket's final recorded status is always Done-category), + everything in between -> In Progress. + + Positional, not a literal-status-name lookup: the changelog data this + integration syncs only has the literal from/to status strings, not + each one's category (that would need a separate call to Jira's + /rest/api/3/status to build a name -> category map, which nothing + else needs yet). Every real Jira workflow's first status is To Do + category and last is Done category by definition, so position is a + reliable enough proxy without that extra call. + + Known, accepted limitation: only the very first and very last + segments get special treatment. A ticket reopened *after* reaching + Done (Done -> In Progress -> Done again) has its middle Done period + counted as "In Progress", since this function has no way to + recognize a middle segment as Done on its own -- see + test_a_reopened_ticket_puts_its_middle_done_period_into_in_progress + for the documented, deliberate behavior this produces. + """ + if len(segments) == 1: + return {"Done": segments[0][1]} + + result: Dict[str, float] = defaultdict(float) + result["To Do"] += segments[0][1] + for _, seconds in segments[1:-1]: + result["In Progress"] += seconds + result["Done"] += segments[-1][1] + return dict(result) + + +def _status_segments(ticket: Ticket, states: List[TicketState]): + """ + (status, seconds-spent-in-it) for every segment of a completed + ticket's life, from creation to its own last recorded activity. + """ + if not states: + # Never transitioned since sync started tracking it -- it's spent + # its whole observed life (creation to last update) in this one + # status. + return [ + (ticket.status, (ticket.updated_at - ticket.created_at).total_seconds()) + ] + + segments = [] + # Jira's changelog is complete back to creation, so the first + # transition's from_status is the status the ticket was created in -- + # this segment is real, recoverable data, not a guess. + if states[0].from_status: + segments.append( + ( + states[0].from_status, + (states[0].changed_at - ticket.created_at).total_seconds(), + ) + ) + + for i, state in enumerate(states): + segment_end = ( + states[i + 1].changed_at if i + 1 < len(states) else ticket.updated_at + ) + segments.append( + (state.to_status, (segment_end - state.changed_at).total_seconds()) + ) + + return segments diff --git a/backend/analytics_server/mhq/service/ticket_insights/service.py b/backend/analytics_server/mhq/service/ticket_insights/service.py new file mode 100644 index 000000000..9e99f48fa --- /dev/null +++ b/backend/analytics_server/mhq/service/ticket_insights/service.py @@ -0,0 +1,85 @@ +from typing import Dict, List + +from mhq.service.ticket_insights.cycle_time import compute_cycle_time_by_project +from mhq.store.models.code import PullRequest +from mhq.store.models.core import Team +from mhq.store.repos.code import CodeRepoService +from mhq.store.repos.projects import ProjectRepoService +from mhq.store.repos.ticket_matching import TicketMatchingRepoService +from mhq.utils.time import Interval + + +class TicketInsightsService: + """ + Phase 4 (§6C/§6E) read-side aggregations for the DORA Metrics page's + Jira widget -- deliberately separate from the 4 existing DORA cards' + own services (lead time, deployment frequency, change failure rate, + MTTR), which this doesn't touch. Composes the code and project store + services rather than living inside either: a "how many merged PRs + have no ticket" count is inherently a join across both. + """ + + def __init__( + self, + project_repo_service: ProjectRepoService, + code_repo_service: CodeRepoService, + ticket_matching_repo_service: TicketMatchingRepoService, + ): + self._project_repo_service = project_repo_service + self._code_repo_service = code_repo_service + self._ticket_matching_repo_service = ticket_matching_repo_service + + def get_team_ticket_insights(self, team: Team, interval: Interval) -> Dict: + org_projects = self._project_repo_service.get_team_projects(str(team.id)) + project_ids = [str(project.id) for project in org_projects] + projects_by_id = {str(project.id): project for project in org_projects} + + tickets, ticket_states = ( + self._project_repo_service.get_tickets_with_states_for_projects( + project_ids, interval.from_time, interval.to_time + ) + ) + cycle_time_by_project = compute_cycle_time_by_project( + tickets, ticket_states, projects_by_id + ) + + repo_ids = self._team_repo_ids(team) + prs_without_ticket_count = ( + self._ticket_matching_repo_service.get_unlinked_merged_pr_count( + repo_ids, interval.from_time, interval.to_time + ) + ) + + return { + "cycle_time_by_project": cycle_time_by_project, + "prs_without_ticket_count": prs_without_ticket_count, + } + + def get_team_unlinked_prs( + self, team: Team, interval: Interval + ) -> List[PullRequest]: + """ + The actual merged PRs behind the Data Hygiene count above -- + title, branch, merge time -- so a real gap in the ticket-key + convention (a typo, a PR that references a ticket in its + description instead of its title/branch, or one that genuinely + never had a ticket) can be found and fixed by a person, not just + counted. Deliberately its own call, not folded into + get_team_ticket_insights: the count is cheap and used on every + DORA Metrics page load, this list is only fetched when someone + actually opens the drill-down. + """ + repo_ids = self._team_repo_ids(team) + return self._ticket_matching_repo_service.get_unlinked_merged_prs( + repo_ids, interval.from_time, interval.to_time + ) + + def _team_repo_ids(self, team: Team) -> List[str]: + org_repos = self._code_repo_service.get_team_repos(str(team.id)) + return [str(repo.id) for repo in org_repos] + + +def get_ticket_insights_service() -> TicketInsightsService: + return TicketInsightsService( + ProjectRepoService(), CodeRepoService(), TicketMatchingRepoService() + ) diff --git a/backend/analytics_server/mhq/service/ticket_matching/__init__.py b/backend/analytics_server/mhq/service/ticket_matching/__init__.py new file mode 100644 index 000000000..6c2bf780b --- /dev/null +++ b/backend/analytics_server/mhq/service/ticket_matching/__init__.py @@ -0,0 +1,6 @@ +from .matcher import extract_ticket_keys +from .service import ( + get_ticket_matching_service, + match_tickets_to_prs, + TicketMatchingService, +) diff --git a/backend/analytics_server/mhq/service/ticket_matching/matcher.py b/backend/analytics_server/mhq/service/ticket_matching/matcher.py new file mode 100644 index 000000000..3776d137f --- /dev/null +++ b/backend/analytics_server/mhq/service/ticket_matching/matcher.py @@ -0,0 +1,44 @@ +import re +from typing import List + +# Jira's own key format: 2-10 letters, a dash, a number (e.g. PZDA-543). +# Case-insensitive since branch names conventionally lowercase it +# (fix/pzda-543-...) while titles keep it uppercase. +# +# The trailing (?:[/,]\d+)* handles a real pattern found in this org's +# own PR history -- a title referencing "PZDA-544/546" to mean two +# tickets sharing one prefix, not one ticket numbered "544/546". +_TICKET_KEY_PATTERN = re.compile(r"\b([A-Za-z]{2,10})-(\d+(?:[/,]\d+)*)\b") + + +def extract_ticket_keys(*texts: str) -> List[str]: + """ + Every ticket-key-shaped token across the given strings, uppercased + and de-duplicated (order-preserving) -- a PR referencing the same + key in both its title and branch name should only match once, and + "PZDA-544/546" expands to ["PZDA-544", "PZDA-546"]. + + This is a heuristic, not a guarantee: something shaped like a ticket + key (e.g. "ISO-27001" in an unrelated branch name) would be extracted + too. The caller is expected to only keep matches against ticket keys + that actually exist in the org's synced tickets -- that check, not + this regex, is what keeps a coincidental key-shaped substring from + becoming a false match. + """ + seen = set() + keys: List[str] = [] + + for text in texts: + if not text: + continue + + for match in _TICKET_KEY_PATTERN.finditer(text): + prefix = match.group(1).upper() + for number in match.group(2).split("/"): + for sub_number in number.split(","): + key = f"{prefix}-{sub_number}" + if key not in seen: + seen.add(key) + keys.append(key) + + return keys diff --git a/backend/analytics_server/mhq/service/ticket_matching/service.py b/backend/analytics_server/mhq/service/ticket_matching/service.py new file mode 100644 index 000000000..b866da3a7 --- /dev/null +++ b/backend/analytics_server/mhq/service/ticket_matching/service.py @@ -0,0 +1,123 @@ +from typing import Dict, List, Optional, Set, Tuple + +from mhq.service.ticket_matching.matcher import extract_ticket_keys +from mhq.store.models.ticket_matching import PullRequestTicketMapping +from mhq.store.repos.ticket_matching import TicketMatchingRepoService +from mhq.utils.log import LOG + + +class TicketMatchingService: + """ + Step 4 of docs/JIRA_INTEGRATION_PROPOSAL.md -- links a PR to the + ticket(s) its title references. Provider-agnostic on purpose: this + only reads already-synced PullRequest/Ticket rows, so it doesn't need + an ETL-handler-per-provider the way the Jira/GitHub/GitLab syncs do -- + there's no external API variability to abstract over here, just + internal data already in our own DB. + + CLUSTOX: the PR title is the *only* authoritative source for this + association -- branch name, description/body, and commit messages + are deliberately never scanned, even though extract_ticket_keys + itself is happy to take more than one text and a real ticket key can + genuinely appear in any of them. A title-only PR ("Improve checkout + error handling") whose branch is "feature/PAY-123-..." or whose body + says "Closes PAY-123" must stay unlinked -- see the product + requirement this enforces (Jira multi-account support work, + 2026-09-10) for the full spec and its worked examples. An earlier + version of this method also scanned head_branch and description, + on the reasoning that real data showed PRs referencing a ticket only + in the body; that traded correctness (a title-shaped guarantee an + admin can see and trust) for recall, which the product spec now + explicitly rules out. + """ + + def __init__(self, repo_service: TicketMatchingRepoService): + self._repo = repo_service + + def match_org_prs_to_tickets(self, org_id: str) -> None: + # One batch lookup for every ticket in the org, one batch lookup + # for the (normally empty) set of colliding keys, and one batch + # fetch for every PR that doesn't have a mapping yet -- matching + # itself is then pure in-memory work (regex + dict lookups), not + # a query per PR. See docs/JIRA_MULTI_ACCOUNT_PLAN.md Task 5 for + # why colliding keys are looked up separately from the main map. + ticket_id_by_key = self._repo.get_org_tickets_key_map(org_id) + colliding_candidates = self._repo.get_colliding_ticket_key_candidates(org_id) + if not ticket_id_by_key and not colliding_candidates: + LOG.info(f"No tickets synced yet for org {org_id}, skipping PR matching") + return + + unmatched_prs = self._repo.get_unmatched_prs_for_org(org_id) + if not unmatched_prs: + return + + # Relevant-project-ids is itself a query (see + # get_relevant_org_project_ids_for_repo) -- cached per repo_id so + # N unmatched PRs from the same repo, each referencing a colliding + # key, cost one lookup, not N. + relevant_project_ids_by_repo: Dict[str, Set[str]] = {} + + # Title only -- see this class's own docstring for why branch name + # and description are deliberately never passed here. + # extract_ticket_keys' own filter-against-real-keys step (the `if + # key in ticket_id_by_key` check below) is what keeps something + # merely key-shaped in the title (e.g. "ISO-27001") from becoming + # a false match. + mappings = [] + for pr in unmatched_prs: + for key in extract_ticket_keys(pr.title): + ticket_id = ticket_id_by_key.get(key) + if ticket_id is None: + candidates = colliding_candidates.get(key) + if not candidates: + continue + ticket_id = self._resolve_colliding_key( + pr, candidates, relevant_project_ids_by_repo + ) + if ticket_id is None: + continue + mappings.append( + PullRequestTicketMapping(pr_id=pr.id, ticket_id=ticket_id) + ) + if not mappings: + return + + self._repo.save_mappings(mappings) + LOG.info(f"Matched {len(mappings)} PR-ticket link(s) for org {org_id}") + + def _resolve_colliding_key( + self, + pr, + candidates: List[Tuple[str, str]], + relevant_project_ids_by_repo: Dict[str, Set[str]], + ) -> Optional[str]: + """ + candidates is every (ticket_id, org_project_id) sharing this key + org-wide -- two or more, by construction (see + get_colliding_ticket_key_candidates). Disambiguates using the + project(s) actually relevant to the PR's own repo (its tracking + team's selected projects); if that still doesn't narrow it to + exactly one, fails closed -- no match, rather than a guess that + could silently attribute a PR to another team's ticket. + """ + repo_id = str(pr.repo_id) + if repo_id not in relevant_project_ids_by_repo: + relevant_project_ids_by_repo[repo_id] = ( + self._repo.get_relevant_org_project_ids_for_repo(repo_id) + ) + relevant_project_ids = relevant_project_ids_by_repo[repo_id] + + matches = [ + ticket_id + for ticket_id, org_project_id in candidates + if org_project_id in relevant_project_ids + ] + return matches[0] if len(matches) == 1 else None + + +def get_ticket_matching_service() -> TicketMatchingService: + return TicketMatchingService(TicketMatchingRepoService()) + + +def match_tickets_to_prs(org_id: str) -> None: + get_ticket_matching_service().match_org_prs_to_tickets(org_id) diff --git a/backend/analytics_server/mhq/service/workflows/integration.py b/backend/analytics_server/mhq/service/workflows/integration.py index 5a9aba7bb..1c2c4ec1c 100644 --- a/backend/analytics_server/mhq/service/workflows/integration.py +++ b/backend/analytics_server/mhq/service/workflows/integration.py @@ -4,8 +4,14 @@ from mhq.store.models.code import RepoWorkflowProviders from mhq.store.repos.core import CoreRepoService +# CLUSTOX: Jenkins is a deployment-detection provider alongside GitHub Actions. +# It has to be in this bucket or the workflow sync never sees it: a Jenkins-only +# workspace gets "No workflow integrations found" and returns early, and a mixed +# workspace has its Jenkins rows filtered out of +# get_active_repo_workflows_by_repo_ids_and_providers. WORKFLOW_INTEGRATION_BUCKET = [ RepoWorkflowProviders.GITHUB_ACTIONS.value, + RepoWorkflowProviders.JENKINS.value, ] diff --git a/backend/analytics_server/mhq/service/workflows/sync/etl_jenkins_handler.py b/backend/analytics_server/mhq/service/workflows/sync/etl_jenkins_handler.py new file mode 100644 index 000000000..8138aad7e --- /dev/null +++ b/backend/analytics_server/mhq/service/workflows/sync/etl_jenkins_handler.py @@ -0,0 +1,210 @@ +# CLUSTOX: Jenkins deployment detection. Implements the same two-method +# contract as GitHub Actions, so nothing downstream of the sync changes. +from datetime import datetime +from typing import Dict, List, Optional, Tuple +from uuid import uuid4 + +import pytz + +from mhq.service.workflows.sync.etl_provider_handler import WorkflowProviderETLHandler +from mhq.store.models.code import ( + OrgRepo, + RepoWorkflow, + RepoWorkflowProviders, + RepoWorkflowRuns, + RepoWorkflowRunsStatus, +) +from mhq.utils.log import LOG +from mhq.utils.time import time_now + +REFS_HEADS_PREFIX = "refs/heads/" +REFS_REMOTES_PREFIX = "refs/remotes/" + +# Remote names stripped from a bare "/" value. Deliberately a +# whitelist rather than "drop the first segment": branch names legitimately +# contain slashes, and release/2.0 or feature/origin-story must survive intact. +# Anything after an explicit refs/remotes/ prefix is a remote by definition and +# does not need this list. +KNOWN_REMOTE_NAMES = frozenset({"origin", "upstream"}) + + +class JenkinsETLHandler(WorkflowProviderETLHandler): + def __init__(self, org_id: str, jenkins_api_service, workflow_repo_service): + self.org_id = org_id + self._api = jenkins_api_service + self._workflow_repo_service = workflow_repo_service + self._provider = RepoWorkflowProviders.JENKINS.value + + def check_pat_validity(self) -> bool: + if not self._api.check_pat(): + raise Exception("Jenkins credentials are invalid or Jenkins is unreachable") + return True + + def get_workflow_runs( + self, + org_repo: OrgRepo, + repo_workflow: RepoWorkflow, + bookmark: datetime, + ) -> Tuple[List[RepoWorkflowRuns], datetime]: + try: + builds = self._api.get_builds(repo_workflow.provider_workflow_id, bookmark) + except Exception as e: + # Bookmark is returned unchanged by raising: the caller does not + # advance it, so this window is re-fetched next cycle. + raise Exception( + f"[Jenkins Sync Repo Workflow Worker] Error fetching job " + f"{repo_workflow.provider_workflow_id} for repo " + f"{str(org_repo.repo_id)}: {str(e)}" + ) + + if not builds: + LOG.info( + f"[Jenkins Sync Repo Workflow Worker] No builds found for job " + f"{repo_workflow.provider_workflow_id}. Org: {self.org_id}" + ) + return [], bookmark + + runs = [] + for build in builds: + try: + runs.append( + self._adapt_jenkins_build_to_workflow_run( + str(repo_workflow.id), build + ) + ) + except Exception as e: + # One malformed build must not lose the rest of the batch. + LOG.warn( + f"[Jenkins Sync Repo Workflow Worker] Skipping build " + f"{build.get('number')}: {str(e)}" + ) + + return runs, self._get_new_bookmark_time_stamp(builds) + + def _get_new_bookmark_time_stamp(self, builds: List[Dict]) -> datetime: + """ + Rewind to the oldest still-running build so it is re-fetched once it + finishes. Mirrors the GitHub Actions handler. + """ + pending = [ + self._get_datetime_from_epoch_ms(build["timestamp"]) + for build in builds + if build.get("building") or build.get("result") is None + ] + return min(pending) if pending else time_now() + + def _adapt_jenkins_build_to_workflow_run( + self, repo_workflow_id: str, build: Dict + ) -> RepoWorkflowRuns: + existing = self._workflow_repo_service.get_repo_workflow_run_by_provider_workflow_run_id( + repo_workflow_id, str(build["number"]) + ) + run_id = existing.id if existing else uuid4() + + duration = build.get("duration") + return RepoWorkflowRuns( + id=run_id, + repo_workflow_id=repo_workflow_id, + provider_workflow_run_id=str(build["number"]), + event_actor=self._get_actor(build), + head_branch=self._get_branch(build), + status=self._get_repo_workflow_status(build), + created_at=time_now(), + updated_at=time_now(), + conducted_at=self._get_datetime_from_epoch_ms(build["timestamp"]), + duration=int(duration / 1000) if duration else None, + meta=build, + html_url=build.get("url"), + ) + + @staticmethod + def _get_repo_workflow_status(build: Dict) -> RepoWorkflowRunsStatus: + result = build.get("result") + if build.get("building") or result is None: + return RepoWorkflowRunsStatus.PENDING + if result == "SUCCESS": + return RepoWorkflowRunsStatus.SUCCESS + if result == "ABORTED": + return RepoWorkflowRunsStatus.CANCELLED + # FAILURE and UNSTABLE. UNSTABLE means the build finished but something + # -- usually tests -- failed; counting it as a deployment would report a + # red build as a successful ship. + return RepoWorkflowRunsStatus.FAILURE + + @staticmethod + def _get_datetime_from_epoch_ms(ms: int) -> datetime: + return datetime.fromtimestamp(ms / 1000, tz=pytz.UTC) + + @staticmethod + def _get_branch(build: Dict) -> Optional[str]: + # Contributed by the git plugin. A freestyle job with no SCM has none. + for action in build.get("actions", []): + revision = (action or {}).get("lastBuiltRevision") + if revision and revision.get("branch"): + return JenkinsETLHandler._normalise_branch( + revision["branch"][0].get("name") + ) + return None + + @staticmethod + def _normalise_branch(branch: Optional[str]) -> Optional[str]: + """ + The git plugin reports 'origin/main', sometimes 'refs/remotes/origin/main'. + Every DORA query filters head_branch against prod_branches, which + defaults to the Postgres regex '^$' -- and + 'origin/main' ~ '^main$' is false, so an un-normalised value drops the + run out of Deployment Frequency, Lead Time, CFR and MTTR with no error. + """ + if not branch: + return None + branch = branch.strip() + if not branch: + return None + + if branch.startswith(REFS_HEADS_PREFIX): + return branch.split(REFS_HEADS_PREFIX, 1)[1] or None + + if branch.startswith(REFS_REMOTES_PREFIX): + # The segment straight after refs/remotes/ is a remote name by + # definition, whatever it is called. + rest = branch.split(REFS_REMOTES_PREFIX, 1)[1] + _, separator, after_remote = rest.partition("/") + return (after_remote if separator and after_remote else rest) or None + + remote, separator, rest = branch.partition("/") + if separator and rest and remote in KNOWN_REMOTE_NAMES: + return rest + return branch + + @staticmethod + def _get_actor(build: Dict) -> Optional[str]: + for action in build.get("actions", []): + causes = (action or {}).get("causes") + if causes: + return causes[0].get("userId") or causes[0].get("userName") + return None + + +def get_jenkins_etl_handler(org_id: str) -> JenkinsETLHandler: + from mhq.exapi.jenkins import JenkinsApiService + from mhq.store.models import UserIdentityProvider + from mhq.store.repos.core import CoreRepoService + from mhq.store.repos.workflows import WorkflowRepoService + from mhq.utils.jenkins import get_jenkins_config + + api_token = CoreRepoService().get_access_token(org_id, UserIdentityProvider.JENKINS) + base_url, username = get_jenkins_config(org_id) + + if not (api_token and base_url and username): + LOG.error( + f"Jenkins is not fully configured for org {org_id}: " + f"base_url={'set' if base_url else 'missing'}, " + f"username={'set' if username else 'missing'}, " + f"token={'set' if api_token else 'missing'}" + ) + + return JenkinsETLHandler( + org_id, + JenkinsApiService(base_url or "", username or "", api_token or ""), + WorkflowRepoService(), + ) diff --git a/backend/analytics_server/mhq/service/workflows/sync/etl_workflows_factory.py b/backend/analytics_server/mhq/service/workflows/sync/etl_workflows_factory.py index ea05cc02f..d207dc505 100644 --- a/backend/analytics_server/mhq/service/workflows/sync/etl_workflows_factory.py +++ b/backend/analytics_server/mhq/service/workflows/sync/etl_workflows_factory.py @@ -1,6 +1,9 @@ from mhq.service.workflows.sync.etl_github_actions_handler import ( get_github_actions_etl_handler, ) + +# CLUSTOX: Jenkins as a third deployment source. +from mhq.service.workflows.sync.etl_jenkins_handler import get_jenkins_etl_handler from mhq.service.workflows.sync.etl_provider_handler import WorkflowProviderETLHandler from mhq.store.models.code import RepoWorkflowProviders @@ -12,4 +15,6 @@ def __init__(self, org_id: str): def __call__(self, provider: str) -> WorkflowProviderETLHandler: if provider == RepoWorkflowProviders.GITHUB_ACTIONS.name: return get_github_actions_etl_handler(self.org_id) + if provider == RepoWorkflowProviders.JENKINS.name: + return get_jenkins_etl_handler(self.org_id) raise NotImplementedError(f"Unknown provider - {provider}") diff --git a/backend/analytics_server/mhq/service/workflows/workflow_filter.py b/backend/analytics_server/mhq/service/workflows/workflow_filter.py index b4466e82d..bf057580e 100644 --- a/backend/analytics_server/mhq/service/workflows/workflow_filter.py +++ b/backend/analytics_server/mhq/service/workflows/workflow_filter.py @@ -8,12 +8,27 @@ class ParseWorkflowFilterProcessor: def apply(self, workflow_filter: Dict = None) -> WorkflowFilter: + # CLUSTOX: contributor filter for deployment frequency. This guard is a + # behaviour change, not just an addition: upstream dereferenced the + # argument straight away, so `apply(None)` raised AttributeError even + # though the signature defaults to None. Making the documented default + # actually work is what lets the event_actors lookup below be + # unconditional. + workflow_filter = workflow_filter or {} + # END CLUSTOX head_branches: List[str] = self._parse_head_branches(workflow_filter) repo_filters: Dict[str, Dict] = self._parse_repo_filters(workflow_filter) + # CLUSTOX: contributor filter -- the actor who triggered the run. Read + # from inside the workflow_filter blob, the same level as head_branches. + event_actors: List[str] = workflow_filter.get("event_actors") + # END CLUSTOX return WorkflowFilter( head_branches=head_branches, repo_filters=repo_filters, + # CLUSTOX: contributor filter for deployment frequency. + event_actors=event_actors, + # END CLUSTOX ) def _parse_head_branches(self, workflow_filter: Dict) -> List[str]: diff --git a/backend/analytics_server/mhq/store/models/code/enums.py b/backend/analytics_server/mhq/store/models/code/enums.py index 032854c91..125b1780e 100644 --- a/backend/analytics_server/mhq/store/models/code/enums.py +++ b/backend/analytics_server/mhq/store/models/code/enums.py @@ -4,6 +4,7 @@ class CodeProvider(Enum): GITHUB = "github" GITLAB = "gitlab" + BITBUCKET = "bitbucket" class CodeBookmarkType(Enum): diff --git a/backend/analytics_server/mhq/store/models/code/filter.py b/backend/analytics_server/mhq/store/models/code/filter.py index c39357ec1..d10cea934 100644 --- a/backend/analytics_server/mhq/store/models/code/filter.py +++ b/backend/analytics_server/mhq/store/models/code/filter.py @@ -53,6 +53,15 @@ def _base_branch_query(): @property def filter_query(self) -> List: + # CLUSTOX: authors was declared on PRFilter and parsed from the request + # payload upstream, but never added to the conditions dict below -- so + # sending it raised KeyError rather than filtering. This applies it. + def _authors_query(): + if not self.authors: + return None + + return PullRequest.author.in_(self.authors) + def _base_branch_query(): if not self.base_branches: return None @@ -95,6 +104,7 @@ def _incident_pr_filters_query(): ) conditions = { + "authors": _authors_query(), "base_branches": _base_branch_query(), "repo_filters": _repo_filters_query(), "excluded_pr_ids": _excluded_pr_ids_query(), diff --git a/backend/analytics_server/mhq/store/models/code/pull_requests.py b/backend/analytics_server/mhq/store/models/code/pull_requests.py index f7f4f2fd6..5d656a653 100644 --- a/backend/analytics_server/mhq/store/models/code/pull_requests.py +++ b/backend/analytics_server/mhq/store/models/code/pull_requests.py @@ -75,6 +75,21 @@ def comments(self) -> int: def username(self) -> str: return self.meta.get("user_profile", {}).get("username", "") + @property + def description(self) -> str: + """ + The PR's own body text -- "body" on GitHub's payload, + "description" on GitLab's, provider-agnostically. CLUSTOX: Jira + integration -- ticket-PR matching (mhq/service/ticket_matching) + scans this in addition to title/branch, since real PRs + frequently reference their ticket(s) only here, under a + "Linked Issue(s)" section or similar -- see + docs/JIRA_INTEGRATION_PROPOSAL.md. + """ + if not self.data: + return "" + return self.data.get("body") or self.data.get("description") or "" + class PullRequestEvent(db.Model): __tablename__ = "PullRequestEvent" diff --git a/backend/analytics_server/mhq/store/models/code/workflows/enums.py b/backend/analytics_server/mhq/store/models/code/workflows/enums.py index cfcbd7584..82588e056 100644 --- a/backend/analytics_server/mhq/store/models/code/workflows/enums.py +++ b/backend/analytics_server/mhq/store/models/code/workflows/enums.py @@ -4,6 +4,9 @@ class RepoWorkflowProviders(Enum): GITHUB_ACTIONS = "github" CIRCLE_CI = "circle_ci" + # CLUSTOX: Jenkins as a deployment source. Persisted as varchar, so no + # migration is required despite the ENUM() wrapper on the column. + JENKINS = "jenkins" @classmethod def get_workflow_providers(cls): diff --git a/backend/analytics_server/mhq/store/models/code/workflows/filter.py b/backend/analytics_server/mhq/store/models/code/workflows/filter.py index baf220efa..eb3d27fa5 100644 --- a/backend/analytics_server/mhq/store/models/code/workflows/filter.py +++ b/backend/analytics_server/mhq/store/models/code/workflows/filter.py @@ -48,6 +48,10 @@ def _head_branches_query(): class WorkflowFilter: head_branches: List[str] = None repo_filters: Dict[str, Dict] = None + # CLUSTOX: who triggered the deployment. Lead Time filters on the pull + # request author; deployments have no author, so the nearest equivalent is + # the actor who ran the build. + event_actors: List[str] = None @property def filter_query(self) -> List: @@ -55,10 +59,17 @@ def _head_branches_query(): if not self.head_branches: return None - return or_( - RepoWorkflowRuns.head_branch.op("~")(term) - for term in self.head_branches - ) + # CLUSTOX: drop None entries, matching RepoWorkflowFilter above. + # A repo whose prod_branches is null arrives here as [None], and + # `head_branch ~ NULL` is NULL in Postgres -- never true -- so every + # deployment was filtered out and the dashboard read "No + # Deployments" with no error. Absent branch config must mean "do not + # filter", not "match nothing". + terms = [term for term in self.head_branches if term is not None] + if not terms: + return None + + return or_(RepoWorkflowRuns.head_branch.op("~")(term) for term in terms) def _repo_filters_query(): if not self.repo_filters: @@ -70,9 +81,20 @@ def _repo_filters_query(): if repo_filters ) + # CLUSTOX: runs with a null event_actor -- Jenkins builds without the + # git plugin, for instance -- are excluded by IN, so per-contributor + # deployment counts will not sum to the team total. That is intended: + # an unattributable deploy belongs to nobody. + def _event_actors_query(): + if not self.event_actors: + return None + + return RepoWorkflowRuns.event_actor.in_(self.event_actors) + conditions = { "head_branches": _head_branches_query(), "repo_filters": _repo_filters_query(), + "event_actors": _event_actors_query(), } return [ conditions[x] diff --git a/backend/analytics_server/mhq/store/models/incidents/enums.py b/backend/analytics_server/mhq/store/models/incidents/enums.py index 6b46cf80d..11b62e3d1 100644 --- a/backend/analytics_server/mhq/store/models/incidents/enums.py +++ b/backend/analytics_server/mhq/store/models/incidents/enums.py @@ -4,12 +4,23 @@ class IncidentProvider(Enum): GITHUB = "github" GITLAB = "gitlab" + # CLUSTOX: Jira issues (of a configurable issue type) as an incident + # source for Change Failure Rate / MTTR -- see + # docs/JIRA_INTEGRATION_PROPOSAL.md. + JIRA = "jira" class IncidentSource(Enum): INCIDENT_SERVICE = "INCIDENT_SERVICE" INCIDENT_TEAM = "INCIDENT_TEAM" GIT_REPO = "GIT_REPO" + # CLUSTOX: on by default, same as GIT_REPO -- see + # default_settings_data.py's INCIDENT_SOURCES_SETTING default. A ticket + # reopened after being marked resolved is the same kind of "the fix + # didn't hold" signal a revert PR is for GIT_REPO. Unlike GIT_REPO, + # still has an admin-facing on/off toggle + # (ConfigureJiraIncidentSourceModalBody) -- only the *default* changed. + JIRA_ISSUE = "JIRA_ISSUE" class ServiceStatus(Enum): @@ -30,6 +41,9 @@ class IncidentType(Enum): INCIDENT = "INCIDENT" REVERT_PR = "REVERT_PR" ALERT = "ALERT" + # CLUSTOX: a Ticket (of a configured Jira issue type) synced as an + # Incident -- see JiraIncidentsETLHandler. + JIRA_ISSUE = "JIRA_ISSUE" class IncidentBookmarkType(Enum): diff --git a/backend/analytics_server/mhq/store/models/integrations/__init__.py b/backend/analytics_server/mhq/store/models/integrations/__init__.py index 0192a8591..f335db872 100644 --- a/backend/analytics_server/mhq/store/models/integrations/__init__.py +++ b/backend/analytics_server/mhq/store/models/integrations/__init__.py @@ -1,2 +1,3 @@ from .enums import UserIdentityProvider from .integrations import Integration, UserIdentity +from .jira_connection import JiraConnection diff --git a/backend/analytics_server/mhq/store/models/integrations/enums.py b/backend/analytics_server/mhq/store/models/integrations/enums.py index 423bde139..2531d0c51 100644 --- a/backend/analytics_server/mhq/store/models/integrations/enums.py +++ b/backend/analytics_server/mhq/store/models/integrations/enums.py @@ -4,6 +4,16 @@ class UserIdentityProvider(Enum): GITHUB = "github" GITLAB = "gitlab" + BITBUCKET = "bitbucket" + # CLUSTOX: Jenkins credentials live in Integration like any other provider. + JENKINS = "jenkins" + # CLUSTOX: Jira is a project-tracker integration, not a code provider -- + # added here (Phase 1: link + store the token) so CoreRepoService's + # generic get_access_token(org_id, provider) can resolve a Jira token + # the same way it already does for GITHUB/GITLAB. Sync/fetch logic is a + # separate, later phase (see docs/JIRA_INTEGRATION_PROPOSAL.md) -- this + # value on its own does not add Jira to any code-sync path. + JIRA = "jira" @classmethod def get_enum(self, provider: str): diff --git a/backend/analytics_server/mhq/store/models/integrations/jira_connection.py b/backend/analytics_server/mhq/store/models/integrations/jira_connection.py new file mode 100644 index 000000000..690dc479b --- /dev/null +++ b/backend/analytics_server/mhq/store/models/integrations/jira_connection.py @@ -0,0 +1,36 @@ +import uuid + +from sqlalchemy import func +from sqlalchemy.dialects.postgresql import UUID, ARRAY, JSONB + +from mhq.store import db + + +class JiraConnection(db.Model): + """ + One connected Jira site+account for an org. Unlike Integration (one row + per provider per org, PK (org_id, name)), an org can have many of these -- + see docs/JIRA_MULTI_ACCOUNT_PLAN.md. The legacy Integration(name="jira") + row is untouched and unrelated to this table; there is deliberately no + sync between the two. + """ + + __tablename__ = "JiraConnection" + + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + org_id = db.Column(UUID(as_uuid=True), db.ForeignKey("Organization.id")) + site_url = db.Column(db.String) + email = db.Column(db.String) + access_token_enc_chunks = db.Column(ARRAY(db.String)) + provider_meta = db.Column(JSONB) + is_default = db.Column(db.Boolean, default=False) + generated_by = db.Column( + UUID(as_uuid=True), db.ForeignKey("Users.id"), nullable=True + ) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + def __hash__(self): + return hash(self.id) diff --git a/backend/analytics_server/mhq/store/models/projects/__init__.py b/backend/analytics_server/mhq/store/models/projects/__init__.py new file mode 100644 index 000000000..8ccb7179e --- /dev/null +++ b/backend/analytics_server/mhq/store/models/projects/__init__.py @@ -0,0 +1,11 @@ +from .repository import ( + OrgProject, + OrgProjectConnection, + ProjectIssuesBookmark, + TeamProjects, +) +from .sprints import Sprint +from .tickets import ( + Ticket, + TicketState, +) diff --git a/backend/analytics_server/mhq/store/models/projects/repository.py b/backend/analytics_server/mhq/store/models/projects/repository.py new file mode 100644 index 000000000..04c55e708 --- /dev/null +++ b/backend/analytics_server/mhq/store/models/projects/repository.py @@ -0,0 +1,94 @@ +import uuid + +from sqlalchemy import func +from sqlalchemy.dialects.postgresql import UUID + +from mhq.store import db + + +class OrgProject(db.Model): + """ + Org-level catalog of a project-tracking tool's projects (Jira, to start). + + Mirrors OrgRepo -- same "org-wide catalog, team join table" shape -- + minus the code-specific columns (default_branch, language, contributors) + that have no equivalent for a project-tracking tool. See + docs/JIRA_INTEGRATION_PROPOSAL.md for the phase this belongs to. + """ + + __tablename__ = "OrgProject" + + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + org_id = db.Column(UUID(as_uuid=True), db.ForeignKey("Organization.id")) + key = db.Column(db.String) + name = db.Column(db.String) + provider = db.Column(db.String) + idempotency_key = db.Column(db.String) + is_active = db.Column(db.Boolean, default=True) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + def __hash__(self): + return hash(self.id) + + +class TeamProjects(db.Model): + """Join table: which OrgProject(s) a team tracks. Mirrors TeamRepos.""" + + __tablename__ = "TeamProjects" + + team_id = db.Column(UUID(as_uuid=True), db.ForeignKey("Team.id"), primary_key=True) + org_project_id = db.Column( + UUID(as_uuid=True), db.ForeignKey("OrgProject.id"), primary_key=True + ) + is_active = db.Column(db.Boolean, default=True) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + +class OrgProjectConnection(db.Model): + """ + Which JiraConnection an OrgProject was synced from. A join table rather + than a column on OrgProject itself, so OrgProject (and everything that + already reads it -- TeamProjects, Ticket, ProjectIssuesBookmark) stays + unchanged. No row for a given org_project_id means it came from the + legacy single-Jira-account flow (Integration), not a JiraConnection -- + treat "no row" as "the org's legacy/default Jira", not as an error. See + docs/JIRA_MULTI_ACCOUNT_PLAN.md. + """ + + __tablename__ = "OrgProjectConnection" + + org_project_id = db.Column( + UUID(as_uuid=True), db.ForeignKey("OrgProject.id"), primary_key=True + ) + jira_connection_id = db.Column( + UUID(as_uuid=True), db.ForeignKey("JiraConnection.id") + ) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + + +class ProjectIssuesBookmark(db.Model): + """ + Incremental-sync watermark for a project's issue sync, one per + (project, provider) -- mirrors Bookmark (org repos), just scoped to + OrgProject instead of OrgRepo. Not reusing Bookmark itself: its + repo_id column has a real FK to OrgRepo at the DB level, so an + OrgProject id would be rejected by that constraint. + """ + + __tablename__ = "ProjectIssuesBookmark" + + org_project_id = db.Column( + UUID(as_uuid=True), db.ForeignKey("OrgProject.id"), primary_key=True + ) + provider = db.Column(db.String, primary_key=True) + bookmark = db.Column(db.String) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) diff --git a/backend/analytics_server/mhq/store/models/projects/sprints.py b/backend/analytics_server/mhq/store/models/projects/sprints.py new file mode 100644 index 000000000..be0d27598 --- /dev/null +++ b/backend/analytics_server/mhq/store/models/projects/sprints.py @@ -0,0 +1,42 @@ +from sqlalchemy import func +from sqlalchemy.dialects.postgresql import UUID + +from mhq.store import db + + +class Sprint(db.Model): + """ + A Jira (Scrum) board's sprint, with planned/completed issue counts + already computed -- the Sprint rollup chart's "planned vs. shipped, + per sprint" needs exactly these two numbers, never per-ticket + detail, so this deliberately doesn't have a per-ticket + sprint-membership join table (see the migration's own comment). + + Re-fetched and upserted in full every sync cycle, not + incrementally bookmarked: a project typically has a small, bounded + number of sprints, and a sprint's completed count can keep changing + after it closes (a straggler ticket resolved late) -- unlike a + ticket's own append-only status history, there's no safe "only + look at what changed since X" watermark here. + """ + + __tablename__ = "Sprint" + + id = db.Column(UUID(as_uuid=True), primary_key=True) + org_project_id = db.Column(UUID(as_uuid=True), db.ForeignKey("OrgProject.id")) + provider = db.Column(db.String) + external_id = db.Column(db.String) + name = db.Column(db.String) + state = db.Column(db.String) + start_date = db.Column(db.DateTime(timezone=True)) + end_date = db.Column(db.DateTime(timezone=True)) + planned_count = db.Column(db.Integer, default=0) + completed_count = db.Column(db.Integer, default=0) + idempotency_key = db.Column(db.String) + created_in_db_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_in_db_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + def __hash__(self): + return hash(self.id) diff --git a/backend/analytics_server/mhq/store/models/projects/tickets.py b/backend/analytics_server/mhq/store/models/projects/tickets.py new file mode 100644 index 000000000..b51bec67b --- /dev/null +++ b/backend/analytics_server/mhq/store/models/projects/tickets.py @@ -0,0 +1,85 @@ +from typing import Optional + +from sqlalchemy import func +from sqlalchemy.dialects.postgresql import UUID, JSONB + +from mhq.store import db + + +class Ticket(db.Model): + """ + Current-state row for a Jira (or, later, other project-tracking + tool's) issue. Mirrors PullRequest -- a current-state row, with + TicketState as its append-only status-transition history, the same + split PullRequest/PullRequestEvent already use. + + No rigid column per Jira field: summary/assignee/reporter/issue type + live in `data` and are exposed as properties, same choice + PullRequest made for its own provider payload (see `meta`). Jira + custom fields vary per instance, so a fixed column per field would + mean a migration every time a new one is wanted. + """ + + __tablename__ = "Ticket" + + id = db.Column(UUID(as_uuid=True), primary_key=True) + org_project_id = db.Column(UUID(as_uuid=True), db.ForeignKey("OrgProject.id")) + key = db.Column(db.String) + provider = db.Column(db.String) + status = db.Column(db.String) + status_category = db.Column(db.String) + idempotency_key = db.Column(db.String) + data = db.Column(JSONB) + # The provider's own created/updated timestamps -- not this row's own + # bookkeeping (created_in_db_at/updated_in_db_at below). updated_at is + # the incremental-sync bookmark cursor. + created_at = db.Column(db.DateTime(timezone=True)) + updated_at = db.Column(db.DateTime(timezone=True)) + created_in_db_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_in_db_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + def __hash__(self): + return hash(self.id) + + @property + def summary(self) -> Optional[str]: + return (self.data or {}).get("summary") + + @property + def issue_type(self) -> Optional[str]: + return (self.data or {}).get("issue_type") + + @property + def assignee(self) -> Optional[str]: + return (self.data or {}).get("assignee") + + @property + def reporter(self) -> Optional[str]: + return (self.data or {}).get("reporter") + + +class TicketState(db.Model): + """ + One row per status transition for a Ticket -- append-only, same + pattern as PullRequestEvent. `from_status` is null for a ticket's + very first recorded state. + """ + + __tablename__ = "TicketState" + + id = db.Column(UUID(as_uuid=True), primary_key=True) + ticket_id = db.Column(UUID(as_uuid=True), db.ForeignKey("Ticket.id")) + from_status = db.Column(db.String) + to_status = db.Column(db.String) + changed_at = db.Column(db.DateTime(timezone=True)) + idempotency_key = db.Column(db.String) + data = db.Column(JSONB) + created_in_db_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_in_db_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + def __hash__(self): + return hash(self.id) diff --git a/backend/analytics_server/mhq/store/models/settings/configuration_settings.py b/backend/analytics_server/mhq/store/models/settings/configuration_settings.py index e33c14634..b8203d957 100644 --- a/backend/analytics_server/mhq/store/models/settings/configuration_settings.py +++ b/backend/analytics_server/mhq/store/models/settings/configuration_settings.py @@ -18,6 +18,15 @@ class SettingType(Enum): EXCLUDED_PRS_SETTING = "EXCLUDED_PRS_SETTING" DEFAULT_SYNC_DAYS_SETTING = "DEFAULT_SYNC_DAYS_SETTING" INCIDENT_PRS_SETTING = "INCIDENT_PRS_SETTING" + # CLUSTOX: which Jira issue types count as an incident, gating + # IncidentSource.JIRA_ISSUE -- see docs/JIRA_INTEGRATION_PROPOSAL.md. + JIRA_INCIDENT_ISSUE_TYPES_SETTING = "JIRA_INCIDENT_ISSUE_TYPES_SETTING" + + # CLUSTOX: per-team DORA targets. Stored here rather than in a new table + # because Settings already has the entity scoping, service and API this + # needs, and its JSONB data column means adding a fifth metric later costs + # no migration. + BENCHMARK_SETTING = "BENCHMARK_SETTING" # ADD NEW SETTING TYPE ENUM HERE diff --git a/backend/analytics_server/mhq/store/models/settings/enums.py b/backend/analytics_server/mhq/store/models/settings/enums.py index caa351ebe..197f66e48 100644 --- a/backend/analytics_server/mhq/store/models/settings/enums.py +++ b/backend/analytics_server/mhq/store/models/settings/enums.py @@ -5,3 +5,7 @@ class EntityType(Enum): USER = "USER" TEAM = "TEAM" ORG = "ORG" + # CLUSTOX: the superadmin's baseline benchmark belongs to no team and no + # workspace. entity_type is character varying with no native enum, so this + # needs no migration. + GLOBAL = "GLOBAL" diff --git a/backend/analytics_server/mhq/store/models/team_repo_project_mapping/__init__.py b/backend/analytics_server/mhq/store/models/team_repo_project_mapping/__init__.py new file mode 100644 index 000000000..6ff1fb7bd --- /dev/null +++ b/backend/analytics_server/mhq/store/models/team_repo_project_mapping/__init__.py @@ -0,0 +1 @@ +from .mapping import TeamRepoProjectMapping diff --git a/backend/analytics_server/mhq/store/models/team_repo_project_mapping/mapping.py b/backend/analytics_server/mhq/store/models/team_repo_project_mapping/mapping.py new file mode 100644 index 000000000..1f32abd0a --- /dev/null +++ b/backend/analytics_server/mhq/store/models/team_repo_project_mapping/mapping.py @@ -0,0 +1,42 @@ +from sqlalchemy import func +from sqlalchemy.dialects.postgresql import UUID + +from mhq.store import db + + +class TeamRepoProjectMapping(db.Model): + """ + Explicit, informational pairing of one of a team's repos to one of its + Jira projects -- see docs/JIRA_MULTI_ACCOUNT_PLAN.md's follow-up on + Jira<->repo relationships. A dedicated package rather than living + under either `code` or `projects`, mirroring PullRequestTicketMapping: + it's a join across those two domains, and cramming it into either + one's package would make that package responsible for a model it + only half owns. + + Deliberately informational only: nothing in TicketMatchingService or + any sync handler reads this table. PR<->ticket association keeps + resolving purely from the Jira issue key found in the PR title. This + exists so an admin can record and see which repo goes with which + project once a team has more than one of each -- organization/ + reporting, not a matching constraint. + + Many-to-many, not a single project_id on OrgRepo (or vice versa) -- a + repo can reasonably pair with more than one project and a project + with more than one repo (see Test Scenario 1 in the product spec this + was requested against: one Jira project, four repos). + """ + + __tablename__ = "TeamRepoProjectMapping" + + team_id = db.Column(UUID(as_uuid=True), db.ForeignKey("Team.id"), primary_key=True) + org_repo_id = db.Column( + UUID(as_uuid=True), db.ForeignKey("OrgRepo.id"), primary_key=True + ) + org_project_id = db.Column( + UUID(as_uuid=True), db.ForeignKey("OrgProject.id"), primary_key=True + ) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) diff --git a/backend/analytics_server/mhq/store/models/ticket_matching/__init__.py b/backend/analytics_server/mhq/store/models/ticket_matching/__init__.py new file mode 100644 index 000000000..f12055fe5 --- /dev/null +++ b/backend/analytics_server/mhq/store/models/ticket_matching/__init__.py @@ -0,0 +1 @@ +from .mapping import PullRequestTicketMapping diff --git a/backend/analytics_server/mhq/store/models/ticket_matching/mapping.py b/backend/analytics_server/mhq/store/models/ticket_matching/mapping.py new file mode 100644 index 000000000..007b89d36 --- /dev/null +++ b/backend/analytics_server/mhq/store/models/ticket_matching/mapping.py @@ -0,0 +1,32 @@ +from sqlalchemy import func +from sqlalchemy.dialects.postgresql import UUID + +from mhq.store import db + + +class PullRequestTicketMapping(db.Model): + """ + Which Ticket(s) a PullRequest references, found by scanning its title + and branch name for a ticket key -- see + docs/JIRA_INTEGRATION_PROPOSAL.md step 4. A dedicated package rather + than living under either `code` or `projects`: it's a join across + those two domains, and cramming it into either one's package would + make that package responsible for a model it only half owns. + + Many-to-many, not a single ticket_id on PullRequest -- a PR can + reference more than one ticket (seen in this org's own PR history: + a title referencing "PZDA-544/546"). + """ + + __tablename__ = "PullRequestTicketMapping" + + pr_id = db.Column( + UUID(as_uuid=True), db.ForeignKey("PullRequest.id"), primary_key=True + ) + ticket_id = db.Column( + UUID(as_uuid=True), db.ForeignKey("Ticket.id"), primary_key=True + ) + created_at = db.Column(db.DateTime(timezone=True), server_default=func.now()) + updated_at = db.Column( + db.DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) diff --git a/backend/analytics_server/mhq/store/repos/code.py b/backend/analytics_server/mhq/store/repos/code.py index 310be35c3..ea436e938 100644 --- a/backend/analytics_server/mhq/store/repos/code.py +++ b/backend/analytics_server/mhq/store/repos/code.py @@ -1,9 +1,9 @@ from datetime import datetime from operator import and_ -from typing import Optional, List +from typing import Dict, Optional, List, Tuple from mhq.store.models.code.enums import CodeProvider -from sqlalchemy import or_ +from sqlalchemy import or_, func from sqlalchemy.orm import defer from mhq.store.models.core import Team @@ -21,6 +21,7 @@ BookmarkMergeToDeployBroker, CodeBookmarkType, ) +from mhq.utils.string import is_bot_author from mhq.utils.time import Interval @@ -275,6 +276,21 @@ def get_active_team_repos_by_team_ids(self, team_ids: List[str]) -> List[TeamRep .all() ) + # CLUSTOX: every team that tracks this repo, not just one. DeploymentsService + # splits a *team's* repos on TeamRepos.deployment_type, so a repo shared by + # three teams has three rows and a Jenkins mapping that switched only one of + # them would report deployments to one team and nothing to the other two. + @rollback_on_exc + def get_active_team_repos_by_repo_id(self, repo_id: str) -> List[TeamRepos]: + return ( + self._db.session.query(TeamRepos) + .filter( + TeamRepos.org_repo_id == repo_id, + TeamRepos.is_active.is_(True), + ) + .all() + ) + @rollback_on_exc def get_active_org_repos_by_ids(self, repo_ids: List[str]) -> List[OrgRepo]: return ( @@ -359,12 +375,19 @@ def get_repos_by_ids(self, ids: List[str]) -> List[OrgRepo]: @rollback_on_exc def get_repos_by_idempotency_keys( - self, idempotency_keys: List[str] + self, org_id: str, idempotency_keys: List[str] ) -> List[OrgRepo]: - + # CLUSTOX: scoped by org. A GitHub repo id is the same string in every + # workspace that links the repo, so an unscoped lookup returned another + # workspace's row -- and update_org_repos turned that into a permanent + # 500 on team save. Pairs with the migration replacing the global + # UNIQUE (idempotency_key) with UNIQUE (org_id, idempotency_key). return ( self._db.session.query(OrgRepo) - .filter(OrgRepo.idempotency_key.in_(idempotency_keys)) + .filter( + OrgRepo.org_id == org_id, + OrgRepo.idempotency_key.in_(idempotency_keys), + ) .all() ) @@ -490,6 +513,37 @@ def get_reverted_prs_by_merge_commit_hash( return query.all() + @rollback_on_exc + def get_first_commit_at_by_pr_ids(self, pr_ids: List[str]) -> Dict[str, datetime]: + """ + Each PR's own earliest recorded commit timestamp, keyed by pr_id. + + CLUSTOX: Jira integration -- the extended Lead Time breakdown's + "ticket created -> first commit" segment (docs/ + JIRA_INTEGRATION_PROPOSAL.md §6A) needs this real timestamp, not + PullRequest.first_commit_to_open (a duration measured to + ready-for-review time, which is *not* pr.created_at for a PR + that was opened as a draft -- see + CodeETLAnalyticsService.get_pr_performance). Deriving "first + commit at" as `pr.created_at - first_commit_to_open` would + silently be wrong for exactly those draft PRs, so this reads the + real PullRequestCommit rows instead. One batched query, not one + per PR. + """ + if not pr_ids: + return {} + + rows = ( + self._db.session.query( + PullRequestCommit.pull_request_id, + func.min(PullRequestCommit.created_at), + ) + .filter(PullRequestCommit.pull_request_id.in_(pr_ids)) + .group_by(PullRequestCommit.pull_request_id) + .all() + ) + return {str(pr_id): first_commit_at for pr_id, first_commit_at in rows} + def _filter_prs_by_repo_ids(self, query, repo_ids: List[str]): return query.filter(PullRequest.repo_id.in_(repo_ids)) @@ -511,3 +565,33 @@ def _filter_base_branch_on_regex(self, query, base_branches: List[str] = None): ] return query.filter(or_(*conditions)) return query + + # CLUSTOX: distinct authors for the contributor filter, scoped to the + # repos and window currently on screen so the dropdown can never offer + # someone with no data in view. + def get_contributors_for_repos( + self, repo_ids: List[str], from_time: datetime, to_time: datetime + ) -> List[Tuple[str, int]]: + if not repo_ids: + return [] + + rows = ( + self._db.session.query( + PullRequest.author, func.count(PullRequest.id).label("pr_count") + ) + .filter( + PullRequest.repo_id.in_(repo_ids), + PullRequest.author.isnot(None), + PullRequest.state_changed_at.between(from_time, to_time), + # Merged only, matching _filter_prs_merged_in_interval. Every + # metric this dropdown filters counts merged PRs, so counting + # anything else here inflates the numbers next to each name + # and can offer someone whose Lead Time card comes back empty. + PullRequest.state == PullRequestState.MERGED, + ) + .group_by(PullRequest.author) + .order_by(func.count(PullRequest.id).desc()) + .all() + ) + + return [(author, count) for author, count in rows if not is_bot_author(author)] diff --git a/backend/analytics_server/mhq/store/repos/core.py b/backend/analytics_server/mhq/store/repos/core.py index fe4d3e039..c19dc969c 100644 --- a/backend/analytics_server/mhq/store/repos/core.py +++ b/backend/analytics_server/mhq/store/repos/core.py @@ -21,6 +21,16 @@ def get_org(self, org_id): .one_or_none() ) + # CLUSTOX: every workspace. Upstream never needed this because only one + # Organization ever existed; multitenancy syncs each of them. + @rollback_on_exc + def get_all_orgs(self) -> List[Organization]: + return ( + self._db.session.query(Organization) + .order_by(Organization.created_at.asc()) + .all() + ) + @rollback_on_exc def get_org_by_name(self, org_name: str): return ( diff --git a/backend/analytics_server/mhq/store/repos/incidents.py b/backend/analytics_server/mhq/store/repos/incidents.py index 38b43edad..a878460b3 100644 --- a/backend/analytics_server/mhq/store/repos/incidents.py +++ b/backend/analytics_server/mhq/store/repos/incidents.py @@ -181,6 +181,52 @@ def get_incident_by_key_type_and_provider( .one_or_none() ) + @rollback_on_exc + def get_incident_by_key_type_provider_and_org( + self, + org_id: str, + key: str, + incident_type: IncidentType, + provider: IncidentProvider, + ) -> Incident: + """ + Same lookup as get_incident_by_key_type_and_provider, scoped to + one org via the incident's OrgIncidentService link. + + Incident itself carries no org_id column -- org scoping only + exists through IncidentOrgIncidentServiceMap -- and a Jira + ticket's key ("PROJ-123") is site-local, not globally unique the + way this codebase's other cross-org identifiers are (see + OrgProject/OrgRepo/Ticket's own org-scoped idempotency_key, e.g. + f"jira:{org_id}:{issue.id}"). Two different orgs' Jira sites can + land on the exact same key, and the unscoped lookup either lets + one org's sync silently adopt the other's incident row, or -- + once both rows exist -- makes .one_or_none() raise + MultipleResultsFound and kill the sync outright. Use this instead + of the unscoped method wherever the key's source (Jira, here) is + not already guaranteed globally unique. + """ + return ( + self._db.session.query(Incident) + .join( + IncidentOrgIncidentServiceMap, + IncidentOrgIncidentServiceMap.incident_id == Incident.id, + ) + .join( + OrgIncidentService, + OrgIncidentService.id == IncidentOrgIncidentServiceMap.service_id, + ) + .filter( + and_( + Incident.key == key, + Incident.incident_type == incident_type, + Incident.provider == provider.value, + OrgIncidentService.org_id == org_id, + ) + ) + .one_or_none() + ) + def _get_team_incidents_query( self, team_id: str, incident_filter: IncidentFilter = None ): diff --git a/backend/analytics_server/mhq/store/repos/jira_connection.py b/backend/analytics_server/mhq/store/repos/jira_connection.py new file mode 100644 index 000000000..24346a41b --- /dev/null +++ b/backend/analytics_server/mhq/store/repos/jira_connection.py @@ -0,0 +1,164 @@ +import re +from typing import List, Optional +from urllib.parse import urlparse + +from mhq.store import db, rollback_on_exc +from mhq.store.models.integrations import JiraConnection +from mhq.store.models.projects import OrgProjectConnection +from mhq.utils.cryptography import get_crypto_service + +# CLUSTOX: mirrors web-server's auth-supplementary.ts CHUNK_SIZE -- both sides +# encrypt against the same RSA keypair (mhq/config/config.ini), so the chunk +# size only needs to fit the key's OAEP message limit, not match byte-for-byte +# between languages. Kept equal anyway so a token's chunk count doesn't +# depend on which side happened to write it. +CHUNK_SIZE = 127 + + +def _normalize_site_url(site_url: str) -> str: + """ + A bare host, e.g. "acme.atlassian.net" -- no scheme, no trailing + slash, matching the migration's own column comment. Enforced here, + the one place every caller's create_jira_connection funnels through, + rather than trusted to each caller (the web-server's connection-add + form doesn't do this itself -- pasting a URL straight from the + browser's address bar, e.g. "https://acme.atlassian.net/", used to be + stored verbatim and then built into a doubly-prefixed, unreachable + URL at search/sync time: "https://https://acme.atlassian.net//..."). + Mirrors web-server's normalizeJiraSiteUrl (src/utils/auth.ts). + """ + trimmed = site_url.strip() + with_scheme = ( + trimmed if re.match(r"^https?://", trimmed, re.I) else (f"https://{trimmed}") + ) + host = urlparse(with_scheme).hostname + return host or trimmed + + +class JiraConnectionInUseError(Exception): + """ + Raised by delete_jira_connection when an OrgProjectConnection still + references the connection. The DB FK (NO ACTION, no cascade) would + raise its own IntegrityError anyway, but callers -- the Task 3 API + routes -- should never let a user hit a raw constraint violation, so + this is checked explicitly and raised first, with a message that + names the actual problem. + """ + + +class JiraConnectionNotFoundError(Exception): + """Raised when a connection_id doesn't resolve to a row for the given org_id.""" + + +class JiraConnectionRepoService: + """ + Store layer for JiraConnection -- one connected Jira site+account for an + org. See docs/JIRA_MULTI_ACCOUNT_PLAN.md. Deliberately independent of + IntegrationsRepoService/Integration: JiraConnection is a new, Jira-only, + surrogate-keyed table, not a replacement for the legacy singleton row. + """ + + def __init__(self): + self._db = db + self._crypto = get_crypto_service() + + @rollback_on_exc + def create_jira_connection( + self, + org_id: str, + site_url: str, + email: str, + access_token: str, + provider_meta: dict, + generated_by: Optional[str], + ) -> JiraConnection: + connection = JiraConnection( + org_id=org_id, + site_url=_normalize_site_url(site_url), + email=email, + access_token_enc_chunks=self._crypto.encrypt(access_token, CHUNK_SIZE), + provider_meta=provider_meta, + generated_by=generated_by, + ) + self._db.session.add(connection) + self._db.session.commit() + return self.get_jira_connection(org_id, str(connection.id)) + + def decrypt_access_token(self, connection: JiraConnection) -> str: + # Not @rollback_on_exc -- no DB access here, just the same decrypt + # call CoreRepoService.get_access_token makes for Integration. + return self._crypto.decrypt_chunks(connection.access_token_enc_chunks) + + @rollback_on_exc + def list_jira_connections(self, org_id: str) -> List[JiraConnection]: + return ( + self._db.session.query(JiraConnection) + .filter(JiraConnection.org_id == org_id) + .order_by(JiraConnection.created_at.asc()) + .all() + ) + + @rollback_on_exc + def get_jira_connection( + self, org_id: str, connection_id: str + ) -> Optional[JiraConnection]: + # Scoped by org_id, not just id -- a connection_id alone must never + # resolve across orgs, whether the caller is a delete/set-default + # check here or an API route trusting this as its authz boundary. + return ( + self._db.session.query(JiraConnection) + .filter( + JiraConnection.id == connection_id, + JiraConnection.org_id == org_id, + ) + .one_or_none() + ) + + @rollback_on_exc + def delete_jira_connection(self, org_id: str, connection_id: str) -> None: + connection = self.get_jira_connection(org_id, connection_id) + if not connection: + raise JiraConnectionNotFoundError( + f"No JiraConnection {connection_id} for org {org_id}" + ) + + still_referenced = ( + self._db.session.query(OrgProjectConnection) + .filter(OrgProjectConnection.jira_connection_id == connection_id) + .first() + is not None + ) + if still_referenced: + raise JiraConnectionInUseError( + f"JiraConnection {connection_id} still has projects synced " + "from it -- unlink those projects first" + ) + + self._db.session.delete(connection) + self._db.session.commit() + + @rollback_on_exc + def set_default_jira_connection(self, org_id: str, connection_id: str) -> None: + connection = self.get_jira_connection(org_id, connection_id) + if not connection: + raise JiraConnectionNotFoundError( + f"No JiraConnection {connection_id} for org {org_id}" + ) + + # CLUSTOX: unset every prior default *before* setting the new one, in + # that order, in one transaction. The partial unique index + # (jira_connection_one_default_per_org) allows zero default rows at + # any instant but never two -- so the intermediate state after the + # first statement (nobody default) is legal, while the reverse order + # (set new True first, while the old row is still True) would trip + # the index. Both statements commit together: no window exists where + # a reader could observe zero or two defaults. + self._db.session.query(JiraConnection).filter( + JiraConnection.org_id == org_id, + JiraConnection.id != connection_id, + JiraConnection.is_default.is_(True), + ).update({JiraConnection.is_default: False}, synchronize_session=False) + + connection.is_default = True + self._db.session.add(connection) + self._db.session.commit() diff --git a/backend/analytics_server/mhq/store/repos/projects.py b/backend/analytics_server/mhq/store/repos/projects.py new file mode 100644 index 000000000..138016b5a --- /dev/null +++ b/backend/analytics_server/mhq/store/repos/projects.py @@ -0,0 +1,433 @@ +from datetime import datetime +from typing import List, Optional, Tuple + +from sqlalchemy import and_ + +from mhq.store import db, rollback_on_exc +from mhq.store.models.core import Team +from mhq.store.models.projects import ( + OrgProject, + OrgProjectConnection, + ProjectIssuesBookmark, + Sprint, + TeamProjects, + Ticket, + TicketState, +) + + +class ProjectRepoService: + """ + Store layer for Jira (and, later, other project-tracking tools') org + project catalog + team selection. Mirrors CodeRepoService's repo/team + methods -- see docs/JIRA_INTEGRATION_PROPOSAL.md. + """ + + def __init__(self): + self._db = db + + @rollback_on_exc + def get_active_org_projects(self, org_id: str) -> List[OrgProject]: + return ( + self._db.session.query(OrgProject) + .filter(OrgProject.org_id == org_id, OrgProject.is_active.is_(True)) + .all() + ) + + @rollback_on_exc + def get_projects_by_idempotency_keys( + self, idempotency_keys: List[str] + ) -> List[OrgProject]: + if not idempotency_keys: + return [] + + return ( + self._db.session.query(OrgProject) + .filter(OrgProject.idempotency_key.in_(idempotency_keys)) + .all() + ) + + @rollback_on_exc + def get_projects_by_ids(self, ids: List[str]) -> List[OrgProject]: + if not ids: + return [] + + return self._db.session.query(OrgProject).filter(OrgProject.id.in_(ids)).all() + + @rollback_on_exc + def update_org_projects(self, org_projects: List[OrgProject]) -> List[OrgProject]: + [self._db.session.merge(org_project) for org_project in org_projects] + self._db.session.commit() + return self.get_projects_by_ids([str(project.id) for project in org_projects]) + + @rollback_on_exc + def save_org_project_connections( + self, connections: List[OrgProjectConnection] + ) -> None: + """ + Upserts which JiraConnection each given OrgProject came from. Called + only for projects picked under a connection (see + ProjectService._update_org_projects) -- a project saved without one + is left alone here, not de-associated, since the two flows + (legacy Integration vs JiraConnection) are deliberately independent. + Must run after the OrgProject rows it references are already + committed (update_org_projects), or the FK has nothing to point at. + """ + for connection in connections: + self._db.session.merge(connection) + self._db.session.commit() + + @rollback_on_exc + def get_existing_team_projects(self, team: Team) -> List[TeamProjects]: + return ( + self._db.session.query(TeamProjects) + .filter(TeamProjects.team_id == team.id) + .all() + ) + + @rollback_on_exc + def get_team_projects_by_project_id_for_team( + self, team_id: str, project_ids: List[str] + ) -> List[TeamProjects]: + if not project_ids: + return [] + + return ( + self._db.session.query(TeamProjects) + .filter( + TeamProjects.team_id == team_id, + TeamProjects.org_project_id.in_(project_ids), + ) + .all() + ) + + @rollback_on_exc + def update_team_projects(self, updated_team_projects: List[TeamProjects]): + for team_project in updated_team_projects: + self._db.session.merge(team_project) + + self._db.session.commit() + + @rollback_on_exc + def get_team_projects_by_team_id(self, team_id: str) -> List[TeamProjects]: + return ( + self._db.session.query(TeamProjects) + .filter( + and_( + TeamProjects.team_id == team_id, + TeamProjects.is_active == True, # noqa E712 + ) + ) + .all() + ) + + @rollback_on_exc + def get_team_projects(self, team_id: str) -> List[OrgProject]: + team_projects = self.get_team_projects_by_team_id(team_id) + if not team_projects: + return [] + + project_ids = [tp.org_project_id for tp in team_projects] + return self.get_projects_by_ids(project_ids) + + @rollback_on_exc + def get_org_projects_used_across_teams(self, org_id: str) -> List[OrgProject]: + """ + Active org projects that are also actively selected by at least one + non-deleted team -- mirrors + CodeRepoService.get_org_repos_used_across_teams, used to retire + catalog rows nothing points at anymore. + """ + return ( + self._db.session.query(OrgProject) + .join(TeamProjects, TeamProjects.org_project_id == OrgProject.id) + .join(Team, TeamProjects.team_id == Team.id) + .filter( + OrgProject.org_id == org_id, + OrgProject.is_active.is_(True), + TeamProjects.is_active.is_(True), + Team.is_deleted.is_(False), + ) + .all() + ) + + @rollback_on_exc + def get_team_ids_for_org_project(self, org_project_id: str) -> List[str]: + """ + Every non-deleted team that has this project actively selected -- + the reverse of get_org_projects_used_across_teams. Powers + JiraIncidentsETLHandler's team-linking: a team's existing MID-3 + Jira project selection is the one source of truth for "which + Jira project's issues feed this team's incidents," rather than a + second, separate config surface for the same choice. + """ + return [ + str(team_id) + for (team_id,) in self._db.session.query(TeamProjects.team_id) + .join(Team, TeamProjects.team_id == Team.id) + .filter( + TeamProjects.org_project_id == org_project_id, + TeamProjects.is_active.is_(True), + Team.is_deleted.is_(False), + ) + .all() + ] + + @rollback_on_exc + def get_active_org_projects_for_provider( + self, org_id: str, provider: str + ) -> List[OrgProject]: + return ( + self._db.session.query(OrgProject) + .filter( + OrgProject.org_id == org_id, + OrgProject.is_active.is_(True), + OrgProject.provider == provider, + ) + .all() + ) + + @rollback_on_exc + def get_active_org_projects_for_connection( + self, jira_connection_id: str + ) -> List[OrgProject]: + """ + Active projects synced from one JiraConnection -- the read side of + the OrgProjectConnection join. See + docs/JIRA_MULTI_ACCOUNT_PLAN.md Task 4; the write side (assigning a + project to a connection in the first place) is Task 6's project + picker, not sync -- sync only ever reads this join, never creates + the row. + """ + return ( + self._db.session.query(OrgProject) + .join( + OrgProjectConnection, + OrgProjectConnection.org_project_id == OrgProject.id, + ) + .filter( + OrgProjectConnection.jira_connection_id == jira_connection_id, + OrgProject.is_active.is_(True), + ) + .all() + ) + + # -- Tickets ------------------------------------------------------ + + @rollback_on_exc + def get_tickets_by_idempotency_keys( + self, idempotency_keys: List[str] + ) -> List[Ticket]: + if not idempotency_keys: + return [] + + return ( + self._db.session.query(Ticket) + .filter(Ticket.idempotency_key.in_(idempotency_keys)) + .all() + ) + + @rollback_on_exc + def get_ticket_states_by_idempotency_keys( + self, idempotency_keys: List[str] + ) -> List[TicketState]: + if not idempotency_keys: + return [] + + return ( + self._db.session.query(TicketState) + .filter(TicketState.idempotency_key.in_(idempotency_keys)) + .all() + ) + + @rollback_on_exc + def save_tickets_data( + self, tickets: List[Ticket], ticket_states: List[TicketState] + ): + # One merge loop per table, one commit -- mirrors + # CodeRepoService.save_pull_requests_data. Callers are responsible + # for id reconciliation (reusing an existing row's id when its + # idempotency_key already exists) before this is called -- see + # JiraETLHandler, which does that via a single batch lookup per + # table rather than a per-ticket query. + [self._db.session.merge(ticket) for ticket in tickets] + [self._db.session.merge(ticket_state) for ticket_state in ticket_states] + self._db.session.commit() + + # -- Issue-sync bookmark ------------------------------------------- + + @rollback_on_exc + def get_project_issues_bookmark( + self, org_project_id: str, provider: str + ) -> Optional[ProjectIssuesBookmark]: + return ( + self._db.session.query(ProjectIssuesBookmark) + .filter( + and_( + ProjectIssuesBookmark.org_project_id == org_project_id, + ProjectIssuesBookmark.provider == provider, + ) + ) + .one_or_none() + ) + + @rollback_on_exc + def update_project_issues_bookmark(self, bookmark: ProjectIssuesBookmark): + self._db.session.merge(bookmark) + self._db.session.commit() + + @rollback_on_exc + def get_all_org_project_issues_bookmarks( + self, org_id: str + ) -> List[ProjectIssuesBookmark]: + return ( + self._db.session.query(ProjectIssuesBookmark) + .join(OrgProject, OrgProject.id == ProjectIssuesBookmark.org_project_id) + .filter(OrgProject.org_id == org_id) + .all() + ) + + @rollback_on_exc + def update_project_issues_bookmarks(self, bookmarks: List[ProjectIssuesBookmark]): + for bookmark in bookmarks: + self._db.session.merge(bookmark) + self._db.session.commit() + + # -- Ticket cycle time (Phase 4, §6C) ------------------------------ + + @rollback_on_exc + def get_tickets_with_states_for_projects( + self, + org_project_ids: List[str], + from_time: datetime, + to_time: datetime, + ) -> Tuple[List[Ticket], List[TicketState]]: + """ + Completed tickets (status_category == "Done" -- Jira Cloud's 3 + category names are fixed, not customizable per workflow, unlike + individual status names) updated within [from_time, to_time] for + the given projects, plus every TicketState for those tickets (not + clipped to the window -- a ticket's full status history is needed + to compute its time-in-status correctly). Two queries total + regardless of how many tickets/states exist, not one per ticket. + + Only completed tickets: an open ticket has no bounded end time, + and including it would let a single item sitting untouched in a + backlog for months dominate a cycle-time average with a duration + that has nothing to do with how long finished work actually took + -- see compute_cycle_time_by_project's own docstring for the + concrete case that surfaced this. + """ + if not org_project_ids: + return [], [] + + tickets = ( + self._db.session.query(Ticket) + .filter( + Ticket.org_project_id.in_(org_project_ids), + Ticket.status_category == "Done", + Ticket.updated_at.between(from_time, to_time), + ) + .all() + ) + if not tickets: + return [], [] + + ticket_ids = [ticket.id for ticket in tickets] + ticket_states = ( + self._db.session.query(TicketState) + .filter(TicketState.ticket_id.in_(ticket_ids)) + .order_by(TicketState.changed_at.asc()) + .all() + ) + return tickets, ticket_states + + @rollback_on_exc + def get_tickets_by_issue_types_for_project( + self, + org_project_id: str, + issue_types: List[str], + from_time: datetime, + to_time: datetime, + ) -> Tuple[List[Ticket], List[TicketState]]: + """ + Tickets of the given issue type(s) (open or done -- unlike + get_tickets_with_states_for_projects above, this is for incident + tracking, where an as-yet-unresolved ticket is a real, ongoing + incident, not something to exclude) updated within + [from_time, to_time] for one project, plus every TicketState for + those tickets. Powers JiraIncidentsETLHandler -- see + docs/JIRA_INTEGRATION_PROPOSAL.md. + + issue_type lives in Ticket.data (a JSONB property, not its own + column -- same reasoning as Ticket.issue_type's docstring), so + this filters via the JSONB accessor directly rather than the + Python property. + """ + if not issue_types: + return [], [] + + tickets = ( + self._db.session.query(Ticket) + .filter( + Ticket.org_project_id == org_project_id, + Ticket.data["issue_type"].astext.in_(issue_types), + Ticket.updated_at.between(from_time, to_time), + ) + .all() + ) + if not tickets: + return [], [] + + ticket_ids = [ticket.id for ticket in tickets] + ticket_states = ( + self._db.session.query(TicketState) + .filter(TicketState.ticket_id.in_(ticket_ids)) + .order_by(TicketState.changed_at.asc()) + .all() + ) + return tickets, ticket_states + + # -- Sprints (§6D) -------------------------------------------------- + + @rollback_on_exc + def get_sprints_by_idempotency_keys( + self, idempotency_keys: List[str] + ) -> List[Sprint]: + if not idempotency_keys: + return [] + + return ( + self._db.session.query(Sprint) + .filter(Sprint.idempotency_key.in_(idempotency_keys)) + .all() + ) + + @rollback_on_exc + def save_sprints(self, sprints: List[Sprint]): + for sprint in sprints: + self._db.session.merge(sprint) + self._db.session.commit() + + @rollback_on_exc + def get_sprints_for_projects( + self, org_project_ids: List[str], limit: int + ) -> List[Sprint]: + """ + Most recent `limit` sprints (by start_date, nulls last -- a + sprint that somehow synced with no start_date shouldn't crowd + out real, dated ones) across the given projects, for the Sprint + rollup chart. Most-recent-first at the query level so a caller + capping to "last N" always gets the N that actually matter, not + an arbitrary N out of however many exist. + """ + if not org_project_ids: + return [] + + return ( + self._db.session.query(Sprint) + .filter(Sprint.org_project_id.in_(org_project_ids)) + .order_by(Sprint.start_date.desc().nullslast()) + .limit(limit) + .all() + ) diff --git a/backend/analytics_server/mhq/store/repos/team_repo_project_mapping.py b/backend/analytics_server/mhq/store/repos/team_repo_project_mapping.py new file mode 100644 index 000000000..bdecdef0e --- /dev/null +++ b/backend/analytics_server/mhq/store/repos/team_repo_project_mapping.py @@ -0,0 +1,66 @@ +from typing import List, Tuple +from uuid import UUID + +from mhq.store import db, rollback_on_exc +from mhq.store.models.team_repo_project_mapping import TeamRepoProjectMapping + + +class TeamRepoProjectMappingRepoService: + """ + Store layer for TeamRepoProjectMapping -- an explicit, informational + pairing of a team's repo to a team's Jira project. See + docs/JIRA_MULTI_ACCOUNT_PLAN.md's follow-up on Jira<->repo + relationships, and the model's own docstring for why this never + drives PR<->ticket matching. + """ + + def __init__(self): + self._db = db + + @rollback_on_exc + def get_mappings_for_team(self, team_id: str) -> List[TeamRepoProjectMapping]: + return ( + self._db.session.query(TeamRepoProjectMapping) + .filter(TeamRepoProjectMapping.team_id == team_id) + .all() + ) + + @rollback_on_exc + def set_mappings_for_team( + self, team_id: str, pairs: List[Tuple[str, str]] + ) -> List[TeamRepoProjectMapping]: + """ + Replaces the team's full set of repo<->project pairings with + exactly the given (org_repo_id, org_project_id) pairs. A plain + join table with no other columns worth preserving across a save + (unlike TeamRepos/TeamProjects, which carry their own is_active + history), so a full delete-then-insert is simpler than diffing + and just as correct -- there's nothing on an old row a caller + could have wanted kept. + """ + team_id = UUID(str(team_id)) + + self._db.session.query(TeamRepoProjectMapping).filter( + TeamRepoProjectMapping.team_id == team_id + ).delete(synchronize_session=False) + + # CLUSTOX: normalize every PK column to the same Python type + # (uuid.UUID) before the bulk insert below. org_repo_id/ + # org_project_id arrive as plain strings from the API layer + # (uuid_validator only validates the format, it doesn't convert -- + # see request_utils.py); team_id can be either, depending on the + # caller. Bulk-inserting a composite primary key with mixed str/ + # UUID values across rows breaks SQLAlchemy 2's insertmanyvalues + # sentinel matching ("Can't match sentinel values in result set to + # parameter sets") -- every column of the PK needs to be the same + # Python type in every row, not just internally consistent per row. + for repo_id, project_id in pairs: + self._db.session.add( + TeamRepoProjectMapping( + team_id=team_id, + org_repo_id=UUID(str(repo_id)), + org_project_id=UUID(str(project_id)), + ) + ) + self._db.session.commit() + return self.get_mappings_for_team(team_id) diff --git a/backend/analytics_server/mhq/store/repos/ticket_matching.py b/backend/analytics_server/mhq/store/repos/ticket_matching.py new file mode 100644 index 000000000..fa46a911e --- /dev/null +++ b/backend/analytics_server/mhq/store/repos/ticket_matching.py @@ -0,0 +1,218 @@ +from datetime import datetime +from typing import Dict, List, Set, Tuple + +from sqlalchemy import func +from sqlalchemy.orm import defer + +from mhq.store import db, rollback_on_exc +from mhq.store.models.code import OrgRepo, PullRequest, PullRequestState +from mhq.store.models.code.repository import TeamRepos +from mhq.store.models.projects import OrgProject, TeamProjects, Ticket +from mhq.store.models.ticket_matching import PullRequestTicketMapping + + +class TicketMatchingRepoService: + def __init__(self): + self._db = db + + def _ticket_candidates_by_key( + self, org_id: str + ) -> Dict[str, List[Tuple[str, str]]]: + """ + Every (ticket_id, org_project_id) synced for this org, grouped by + uppercased ticket key -- not scoped to a project's is_active (a + ticket that was already synced should stay matchable even if its + project is later deselected). One query; get_org_tickets_key_map + and get_colliding_ticket_key_candidates both group its result in + memory rather than each querying the tickets table themselves. + """ + rows = ( + self._db.session.query(Ticket.id, Ticket.key, Ticket.org_project_id) + .join(OrgProject, Ticket.org_project_id == OrgProject.id) + .filter(OrgProject.org_id == org_id) + .all() + ) + by_key: Dict[str, List[Tuple[str, str]]] = {} + for ticket_id, key, org_project_id in rows: + by_key.setdefault(key.upper(), []).append( + (str(ticket_id), str(org_project_id)) + ) + return by_key + + @rollback_on_exc + def get_org_tickets_key_map(self, org_id: str) -> Dict[str, str]: + """ + Uppercased ticket key -> ticket id, for every key that maps to + exactly one ticket org-wide -- the overwhelmingly common case, + and the only one multi-account Jira (docs/JIRA_MULTI_ACCOUNT_PLAN.md) + changes nothing about. A key two or more tickets share (two + connections' sites landing on the same project key -- "Known + risks" #2 in that plan) is deliberately excluded here rather than + resolved to "whichever the query happened to return last", which + is what silently mismatched a PR to the wrong connection's ticket + before this existed. get_colliding_ticket_key_candidates is what + TicketMatchingService uses to resolve those, per PR. + """ + by_key = self._ticket_candidates_by_key(org_id) + return { + key: candidates[0][0] + for key, candidates in by_key.items() + if len(candidates) == 1 + } + + @rollback_on_exc + def get_colliding_ticket_key_candidates( + self, org_id: str + ) -> Dict[str, List[Tuple[str, str]]]: + """ + Uppercased ticket key -> [(ticket_id, org_project_id), ...], for + every key shared by two or more tickets in this org. See + get_org_tickets_key_map's own docstring for why these are held + back from the main map. + """ + by_key = self._ticket_candidates_by_key(org_id) + return {key: c for key, c in by_key.items() if len(c) > 1} + + @rollback_on_exc + def get_relevant_org_project_ids_for_repo(self, repo_id: str) -> Set[str]: + """ + Project ids selected (TeamProjects) by any team that also tracks + this repo (TeamRepos) -- used only to disambiguate a ticket-key + collision across connections (get_colliding_ticket_key_candidates + above), never as the default matching scope. Matching stays + org-wide otherwise: scoping every PR to its repo's teams' projects + by default would break the existing, deliberate "stay matchable + even if the project was since deselected" guarantee for the + overwhelming majority of orgs that never have a colliding key at + all. See docs/JIRA_MULTI_ACCOUNT_PLAN.md Task 5. + """ + rows = ( + self._db.session.query(TeamProjects.org_project_id) + .join(TeamRepos, TeamRepos.team_id == TeamProjects.team_id) + .filter( + TeamRepos.org_repo_id == repo_id, + TeamRepos.is_active.is_(True), + TeamProjects.is_active.is_(True), + ) + .distinct() + .all() + ) + return {str(project_id) for (project_id,) in rows} + + @rollback_on_exc + def get_unmatched_prs_for_org(self, org_id: str) -> List[PullRequest]: + """ + PRs for this org with no PullRequestTicketMapping row yet. Not + "not yet closed" or "recently updated" -- a PR that genuinely + references no ticket gets re-scanned every cycle (cheap: no API + calls, just a regex against PullRequest.title), but a PR whose + match was already found never gets re-queried. + + data (which holds description/body) is deferred here, same as + get_unlinked_merged_prs below -- matching only ever reads + PullRequest.title, its own plain column (see + TicketMatchingService's own docstring for why description and + head_branch are deliberately never scanned), so there's no + reason to pull a potentially large JSONB blob off every unmatched + PR just to leave it unused. + """ + return ( + self._db.session.query(PullRequest) + .join(OrgRepo, PullRequest.repo_id == OrgRepo.id) + .outerjoin( + PullRequestTicketMapping, + PullRequestTicketMapping.pr_id == PullRequest.id, + ) + .filter( + OrgRepo.org_id == org_id, + PullRequestTicketMapping.pr_id.is_(None), + ) + .options(defer(PullRequest.data)) + .all() + ) + + @rollback_on_exc + def save_mappings(self, mappings: List[PullRequestTicketMapping]): + [self._db.session.merge(mapping) for mapping in mappings] + self._db.session.commit() + + @rollback_on_exc + def get_unlinked_merged_pr_count( + self, repo_ids: List[str], from_time: datetime, to_time: datetime + ) -> int: + """ + How many PRs merged in [from_time, to_time] across these repos + have no PullRequestTicketMapping row -- the data-hygiene callout + in docs/JIRA_INTEGRATION_PROPOSAL.md §6E. A single count query, + not "fetch all merged PRs and count client-side". + """ + if not repo_ids: + return 0 + + return self._unlinked_merged_prs_query(repo_ids, from_time, to_time).count() + + @rollback_on_exc + def get_unlinked_merged_prs( + self, repo_ids: List[str], from_time: datetime, to_time: datetime + ) -> List[PullRequest]: + """ + The actual rows behind get_unlinked_merged_pr_count -- the + drill-down a person needs to see *which* PRs fell through the + ticket-key regex (a typo, a non-standard branch name, or a PR + that genuinely never referenced a ticket) rather than just how + many. Same filter, same bounded window; only the projection and + `.all()` vs `.count()` differ, so the two can never disagree + about which PRs qualify. + """ + if not repo_ids: + return [] + + return ( + self._unlinked_merged_prs_query(repo_ids, from_time, to_time) + .options(defer(PullRequest.data)) + .order_by(PullRequest.state_changed_at.desc()) + .all() + ) + + @rollback_on_exc + def get_ticket_created_at_by_pr_ids(self, pr_ids: List[str]) -> Dict[str, datetime]: + """ + For PRs that already have a PullRequestTicketMapping row, the + earliest linked ticket's created_at, keyed by pr_id -- the + extended Lead Time breakdown's "idea was written up" anchor + (docs/JIRA_INTEGRATION_PROPOSAL.md §6A). A PR can reference more + than one ticket (real data: "PZDA-544/546" in one title) -- the + *earliest* ticket is the right anchor for an idea-to-production + span, not an arbitrary or last one. One batched query, not one + per PR. + """ + if not pr_ids: + return {} + + rows = ( + self._db.session.query( + PullRequestTicketMapping.pr_id, func.min(Ticket.created_at) + ) + .join(Ticket, Ticket.id == PullRequestTicketMapping.ticket_id) + .filter(PullRequestTicketMapping.pr_id.in_(pr_ids)) + .group_by(PullRequestTicketMapping.pr_id) + .all() + ) + return {str(pr_id): created_at for pr_id, created_at in rows} + + def _unlinked_merged_prs_query( + self, repo_ids: List[str], from_time: datetime, to_time: datetime + ): + return ( + self._db.session.query(PullRequest) + .outerjoin( + PullRequestTicketMapping, + PullRequestTicketMapping.pr_id == PullRequest.id, + ) + .filter( + PullRequest.repo_id.in_(repo_ids), + PullRequest.state == PullRequestState.MERGED, + PullRequest.state_changed_at.between(from_time, to_time), + PullRequestTicketMapping.pr_id.is_(None), + ) + ) diff --git a/backend/analytics_server/mhq/store/repos/workflows.py b/backend/analytics_server/mhq/store/repos/workflows.py index a950a032c..a248296ea 100644 --- a/backend/analytics_server/mhq/store/repos/workflows.py +++ b/backend/analytics_server/mhq/store/repos/workflows.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import List, Tuple +from typing import List, Optional, Tuple from sqlalchemy.orm import defer from sqlalchemy import and_ @@ -16,7 +16,10 @@ RepoWorkflowRuns, RepoWorkflowRunsBookmark, ) -from mhq.store.models.code.repository import OrgRepo + +# CLUSTOX: TeamRepos is written here so the deployment_type switch shares a +# commit with the Jenkins mapping -- see create_jenkins_repo_workflow. +from mhq.store.models.code.repository import OrgRepo, TeamRepos from mhq.utils.time import Interval @@ -118,6 +121,148 @@ def get_repo_workflow_by_repo_ids( .all() ) + # CLUSTOX: used by the Jenkins mapping routes to resolve a workflow by id. + # Deliberately not filtered by provider -- callers that are provider + # specific must check RepoWorkflow.provider themselves. meta is loaded + # rather than deferred: unmapping reads the displaced-workflow record out + # of it, and a deferred load would just cost a second round trip. + @rollback_on_exc + def get_repo_workflow_by_id(self, repo_workflow_id: str) -> Optional[RepoWorkflow]: + return ( + self._db.session.query(RepoWorkflow) + .filter(RepoWorkflow.id == repo_workflow_id) + .one_or_none() + ) + + # CLUSTOX: resolves the row that occupies the uniquely indexed + # (org_repo_id, provider_workflow_id) pair, active or not. Mapping a job + # that was previously mapped and then unmapped has to reuse this row; + # inserting a second one violates + # repoworkflow_orgrepoid_provider_workflow_id. meta is loaded rather than + # deferred: mapping rewrites the displaced-workflow record it holds. + # + # Not filtered by provider, because the index is not either. Filtering by + # provider made the lookup narrower than the constraint it exists to avoid + # tripping: a Jenkins job whose full name matched an existing non-Jenkins + # provider_workflow_id came back as "no row here", and the insert then hit + # the index and 500'd. Unlikely -- GitHub Actions ids are numeric -- but + # the caller can only handle a collision it is told about. Callers that + # care about the provider must check RepoWorkflow.provider themselves; + # returning a foreign row is the point. + @rollback_on_exc + def get_repo_workflow_by_repo_id_and_provider_workflow_id( + self, + repo_id: str, + provider_workflow_id: str, + ) -> Optional[RepoWorkflow]: + return ( + self._db.session.query(RepoWorkflow) + .filter( + RepoWorkflow.org_repo_id == repo_id, + RepoWorkflow.provider_workflow_id == provider_workflow_id, + ) + .one_or_none() + ) + + # CLUSTOX: unlike get_repo_workflow_by_repo_ids this returns inactive rows + # too, because the inactive ones are exactly what a Jenkins mapping turned + # off and an unmapping has to turn back on. + @rollback_on_exc + def get_repo_workflows_by_repo_id_and_provider( + self, + repo_id: str, + provider: RepoWorkflowProviders, + type: RepoWorkflowType, + ) -> List[RepoWorkflow]: + return ( + self._db.session.query(RepoWorkflow) + .options(defer(RepoWorkflow.meta)) + .filter( + RepoWorkflow.org_repo_id == repo_id, + RepoWorkflow.provider == provider, + RepoWorkflow.type == type, + ) + .all() + ) + + # CLUSTOX: the workspace's live mappings, for the route that lists them. + # Scoped by joining OrgRepo rather than by filtering ids the caller supplies: + # RepoWorkflow has no org column, so the join is the only thing standing + # between an admin and another workspace's mappings, and doing it in SQL + # means there is no path where the filter is forgotten. Returns OrgRepo + # alongside because the mapping table shows repo names, and fetching them + # one by one afterwards is a query per row. + @rollback_on_exc + def get_active_repo_workflows_by_org_id_and_provider( + self, + org_id: str, + provider: RepoWorkflowProviders, + type: RepoWorkflowType, + ) -> List[Tuple[RepoWorkflow, OrgRepo]]: + return ( + self._db.session.query(RepoWorkflow, OrgRepo) + .options(defer(RepoWorkflow.meta)) + .join(OrgRepo, RepoWorkflow.org_repo_id == OrgRepo.id) + .filter( + OrgRepo.org_id == org_id, + RepoWorkflow.provider == provider, + RepoWorkflow.type == type, + RepoWorkflow.is_active.is_(True), + ) + .all() + ) + + # CLUSTOX: writes the Jenkins mapping, the deactivation of the repo's other + # deployment workflows and the TeamRepos.deployment_type switch in a single + # commit. If any part applied on its own, a repo could be left with two + # active deployment sources -- silently double-counting deployments, see + # deactivate_deployment_workflows_for_repo in mhq/api/integrations.py -- or + # switched to WORKFLOW with no workflow behind it, which reads as zero + # deployments. + # + # TeamRepos is written from here rather than through CodeRepoService because + # both services share one db.session and only one of them can own the + # commit; splitting the write across two commits is the partial-failure this + # method exists to prevent. + @rollback_on_exc + def create_jenkins_repo_workflow( + self, + jenkins_workflow: RepoWorkflow, + workflows_to_deactivate: List[RepoWorkflow], + team_repos_to_update: List[TeamRepos] = None, + ) -> RepoWorkflow: + # add() on an instance already loaded in this session is a no-op, so + # this covers both a brand new mapping and the reactivation of a row + # that a previous unmapping left inactive. + self._db.session.add(jenkins_workflow) + for workflow in workflows_to_deactivate: + self._db.session.merge(workflow) + for team_repo in team_repos_to_update or []: + self._db.session.merge(team_repo) + self._db.session.commit() + return jenkins_workflow + + # CLUSTOX: removing a Jenkins mapping, restoring the GitHub Actions rows it + # displaced and restoring the TeamRepos.deployment_type values it overwrote + # happen in one commit. Half of this applied on its own would leave the repo + # with no active deployment source at all, and its Deployment Frequency at + # zero. + @rollback_on_exc + def deactivate_repo_workflow( + self, + repo_workflow: RepoWorkflow, + workflows_to_reactivate: List[RepoWorkflow] = None, + team_repos_to_update: List[TeamRepos] = None, + ) -> RepoWorkflow: + repo_workflow.is_active = False + self._db.session.merge(repo_workflow) + for workflow in workflows_to_reactivate or []: + self._db.session.merge(workflow) + for team_repo in team_repos_to_update or []: + self._db.session.merge(team_repo) + self._db.session.commit() + return repo_workflow + @rollback_on_exc def get_repo_workflows_by_repo_id(self, repo_id: str) -> List[RepoWorkflow]: return ( diff --git a/backend/analytics_server/mhq/utils/jenkins.py b/backend/analytics_server/mhq/utils/jenkins.py new file mode 100644 index 000000000..300fff698 --- /dev/null +++ b/backend/analytics_server/mhq/utils/jenkins.py @@ -0,0 +1,20 @@ +# CLUSTOX: Jenkins needs a base URL and username alongside its API token. +# The token is a secret and lives encrypted in access_token_enc_chunks; these +# two are not, and follow the existing provider_meta precedent used for +# GitHub's custom domain. +from typing import List, Optional, Tuple + +from mhq.store.models import Integration, UserIdentityProvider +from mhq.store.repos.core import CoreRepoService + + +def get_jenkins_config(org_id: str) -> Tuple[Optional[str], Optional[str]]: + core_repo_service = CoreRepoService() + integrations: List[Integration] = core_repo_service.get_org_integrations_for_names( + org_id, [UserIdentityProvider.JENKINS.value] + ) + if not integrations or not integrations[0].provider_meta: + return None, None + + meta = integrations[0].provider_meta + return meta.get("base_url"), meta.get("username") diff --git a/backend/analytics_server/mhq/utils/string.py b/backend/analytics_server/mhq/utils/string.py index 64041ef9e..ca741b6d6 100644 --- a/backend/analytics_server/mhq/utils/string.py +++ b/backend/analytics_server/mhq/utils/string.py @@ -1,5 +1,6 @@ from uuid import uuid4 import re +from typing import Optional def uuid4_str(): @@ -11,3 +12,28 @@ def is_bot_name(name: str) -> bool: r"(?i)(\b[\w@-]*[-_\[\]@ ]+bot[-_\d\[\]]*\b|\[bot\]|_bot_|_bot$|^bot_)" ) return bool(pattern.search(name)) + + +# CLUSTOX: bot accounts author real pull requests, so they appear in the +# contributor list unless excluded. Matching is on the exact "[bot]" suffix +# plus an explicit list -- a substring match on "bot" would exclude real +# people called robotnik or abbott. +KNOWN_BOT_AUTHORS = frozenset( + { + "dependabot", + "renovate", + "github-actions", + "snyk-bot", + "greenkeeper", + "codecov", + "imgbot", + } +) + + +def is_bot_author(username: Optional[str]) -> bool: + if not username: + return False + + lowered = username.lower() + return lowered.endswith("[bot]") or lowered in KNOWN_BOT_AUTHORS diff --git a/backend/analytics_server/sync_app.py b/backend/analytics_server/sync_app.py index c67117577..5007c10a3 100644 --- a/backend/analytics_server/sync_app.py +++ b/backend/analytics_server/sync_app.py @@ -10,6 +10,10 @@ from mhq.api.hello import app as core_api from mhq.api.sync import app as sync_api +# CLUSTOX: restrict this API to the BFF and cron. +# See mhq/clustox_auth/internal_token.py +from mhq.clustox_auth.internal_token import register_internal_token_guard + SYNC_SERVER_PORT = getenv("SYNC_SERVER_PORT") app = Flask(__name__) @@ -17,6 +21,9 @@ app.register_blueprint(core_api) app.register_blueprint(sync_api) +# CLUSTOX: reject any request that did not come from the BFF or cron. +register_internal_token_guard(app) + configure_db_with_app(app) if __name__ == "__main__": diff --git a/backend/analytics_server/tests/api/__init__.py b/backend/analytics_server/tests/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/api/resources/__init__.py b/backend/analytics_server/tests/api/resources/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/api/resources/test_sprint_resources.py b/backend/analytics_server/tests/api/resources/test_sprint_resources.py new file mode 100644 index 000000000..07de9ead1 --- /dev/null +++ b/backend/analytics_server/tests/api/resources/test_sprint_resources.py @@ -0,0 +1,43 @@ +from unittest.mock import MagicMock + +from mhq.api.resources.sprint_resources import adapt_sprints +from mhq.utils.time import time_now + +# CLUSTOX: Jira integration -- the Sprint rollup chart. See +# docs/JIRA_INTEGRATION_PROPOSAL.md §6D. + + +def _sprint(**overrides): + sprint = MagicMock() + sprint.name = "PZDA Sprint 1" + sprint.state = "closed" + sprint.start_date = time_now() + sprint.end_date = time_now() + sprint.planned_count = 355 + sprint.completed_count = 272 + for key, value in overrides.items(): + setattr(sprint, key, value) + return sprint + + +class TestAdaptSprints: + def test_maps_every_field(self): + sprint = _sprint() + + [adapted] = adapt_sprints([sprint]) + + assert adapted == { + "name": "PZDA Sprint 1", + "state": "closed", + "start_date": sprint.start_date.isoformat(), + "end_date": sprint.end_date.isoformat(), + "planned_count": 355, + "completed_count": 272, + } + + def test_a_missing_end_date_adapts_to_none_not_an_error(self): + [adapted] = adapt_sprints([_sprint(end_date=None)]) + assert adapted["end_date"] is None + + def test_returns_an_empty_list_for_no_sprints(self): + assert adapt_sprints([]) == [] diff --git a/backend/analytics_server/tests/api/resources/test_ticket_insights_resources.py b/backend/analytics_server/tests/api/resources/test_ticket_insights_resources.py new file mode 100644 index 000000000..a412724f9 --- /dev/null +++ b/backend/analytics_server/tests/api/resources/test_ticket_insights_resources.py @@ -0,0 +1,47 @@ +from unittest.mock import MagicMock + +from mhq.api.resources.ticket_insights_resources import adapt_unlinked_prs +from mhq.utils.time import time_now + +# CLUSTOX: Jira integration, Phase 4 (§6E) -- the Data Hygiene +# drill-down. See docs/JIRA_INTEGRATION_PROPOSAL.md. + + +def _pr(**overrides): + pr = MagicMock() + pr.id = "pr-1" + pr.title = "feat(payments): add refund flow" + pr.url = "https://github.com/org/repo/pull/42" + pr.head_branch = "feat/refund-flow" + pr.author = "jordan" + pr.state_changed_at = time_now() + for key, value in overrides.items(): + setattr(pr, key, value) + return pr + + +class TestAdaptUnlinkedPrs: + def test_maps_every_field_a_person_needs_to_recognize_the_pr(self): + pr = _pr() + + [adapted] = adapt_unlinked_prs([pr]) + + assert adapted == { + "id": "pr-1", + "title": pr.title, + "url": pr.url, + "head_branch": pr.head_branch, + "author": pr.author, + "merged_at": pr.state_changed_at.isoformat(), + } + + def test_returns_an_empty_list_for_no_prs(self): + assert adapt_unlinked_prs([]) == [] + + def test_preserves_order(self): + newer = _pr(id="pr-new") + older = _pr(id="pr-old") + + adapted = adapt_unlinked_prs([newer, older]) + + assert [pr["id"] for pr in adapted] == ["pr-new", "pr-old"] diff --git a/backend/analytics_server/tests/api/resources/test_ticket_lead_time_resources.py b/backend/analytics_server/tests/api/resources/test_ticket_lead_time_resources.py new file mode 100644 index 000000000..173880c0a --- /dev/null +++ b/backend/analytics_server/tests/api/resources/test_ticket_lead_time_resources.py @@ -0,0 +1,33 @@ +from mhq.api.resources.ticket_lead_time_resources import ( + adapt_ticket_lead_time_metrics, +) +from mhq.service.code.models.ticket_lead_time import TicketLeadTimeMetrics + +# CLUSTOX: Jira integration -- the extended Lead Time breakdown. See +# docs/JIRA_INTEGRATION_PROPOSAL.md §6A. + + +class TestAdaptTicketLeadTimeMetrics: + def test_maps_every_field_and_derives_the_extended_total(self): + metrics = TicketLeadTimeMetrics( + matched_pr_count=5, + avg_ticket_to_first_commit_seconds=86400.4, + avg_commit_only_lead_time_seconds=3600.6, + ) + + adapted = adapt_ticket_lead_time_metrics(metrics) + + assert adapted == { + "matched_pr_count": 5, + "avg_ticket_to_first_commit_seconds": 86400, + "avg_commit_only_lead_time_seconds": 3601, + "avg_extended_lead_time_seconds": 90001, + } + + def test_zeroed_metrics_adapt_to_all_zeros(self): + assert adapt_ticket_lead_time_metrics(TicketLeadTimeMetrics()) == { + "matched_pr_count": 0, + "avg_ticket_to_first_commit_seconds": 0, + "avg_commit_only_lead_time_seconds": 0, + "avg_extended_lead_time_seconds": 0, + } diff --git a/backend/analytics_server/tests/api/test_benchmark_routes.py b/backend/analytics_server/tests/api/test_benchmark_routes.py new file mode 100644 index 000000000..86c4db8a5 --- /dev/null +++ b/backend/analytics_server/tests/api/test_benchmark_routes.py @@ -0,0 +1,285 @@ +# CLUSTOX: route-level tests for the benchmark settings surface. +# +# These deliberately go through Flask's test client rather than calling the +# service functions, because every benchmark defect this branch shipped lived +# in the seam between a correct function and its caller: resolved targets +# attached to a response no card read, a GET that wrote a row, a cleared form +# that saved defaults. Unit tests on resolve_benchmarks and +# validate_benchmark_payload cannot see any of that -- they were passing the +# whole time. +import pytest +from flask import Flask + +import mhq.api.settings as settings_module +from mhq.service.settings.benchmarks import ( + BENCHMARK_METRICS, + GLOBAL_BENCHMARK_ENTITY_ID, +) +from mhq.service.settings.models import BenchmarkSetting, ConfigurationSettings +from mhq.store.models.settings import SettingType +from mhq.store.models.settings.enums import EntityType +from mhq.utils.time import time_now + +TEAM_ID = "b8b069ff-048d-4635-b859-9a6609e34cf4" +ORG_ID = "e2a1d0ea-9f0e-4d3e-9b23-4a7c1e5d0f11" + + +class FakeTeam: + id = TEAM_ID + org_id = ORG_ID + + +class FakeQueryValidator: + def team_validator(self, team_id): + return FakeTeam() + + def user_validator(self, user_id): + raise AssertionError("no setter in these tests") + + +class FakeSettingsService: + """Records writes so a test can assert a GET did not perform one.""" + + def __init__(self, rows=None): + self.rows = rows or {} + self.saved = [] + + def get_settings(self, setting_type, entity_type, entity_id): + return self.rows.get((setting_type, entity_type, str(entity_id))) + + def save_settings( + self, setting_type, entity_type, entity_id, setter=None, setting_data=None + ): + self.saved.append((setting_type, entity_type, str(entity_id), setting_data)) + saved = _settings_row(entity_type, entity_id, setting_data or {}) + self.rows[(setting_type, entity_type, str(entity_id))] = saved + return saved + + +def _settings_row(entity_type, entity_id, data): + now = time_now() + return ConfigurationSettings( + entity_id=entity_id, + entity_type=entity_type, + specific_settings=BenchmarkSetting( + lead_time=data.get("lead_time"), + deployment_frequency=data.get("deployment_frequency"), + change_failure_rate=data.get("change_failure_rate"), + mean_time_to_recovery=data.get("mean_time_to_recovery"), + lines_of_code=data.get("lines_of_code"), + ), + updated_by=None, + created_at=now, + updated_at=now, + ) + + +@pytest.fixture +def routes(monkeypatch): + service = FakeSettingsService() + + monkeypatch.setattr( + settings_module, "get_query_validator", lambda: FakeQueryValidator() + ) + monkeypatch.setattr(settings_module, "get_settings_service", lambda: service) + # get_resolved_benchmarks_for_team resolves its service lazily, from the + # module it lives next to. + monkeypatch.setattr( + "mhq.service.settings.configuration_settings.get_settings_service", + lambda: service, + ) + + app = Flask(__name__) + app.register_blueprint(settings_module.app) + return app.test_client(), service + + +def _team_row(service, **values): + service.rows[(SettingType.BENCHMARK_SETTING, EntityType.TEAM, TEAM_ID)] = ( + _settings_row(EntityType.TEAM, TEAM_ID, values) + ) + + +def _global_row(service, **values): + service.rows[ + (SettingType.BENCHMARK_SETTING, EntityType.GLOBAL, GLOBAL_BENCHMARK_ENTITY_ID) + ] = _settings_row(EntityType.GLOBAL, GLOBAL_BENCHMARK_ENTITY_ID, values) + + +def test_the_benchmarks_route_serves_resolved_targets_over_http(routes): + # The shape the four DORA cards read out of metrics_summary.benchmarks: + # `.target` and `.source`. If this body changes, every + # card silently renders nothing. + client, service = routes + _team_row(service, lead_time=3600) + _global_row(service, lead_time=86400, deployment_frequency=5) + + response = client.get(f"/teams/{TEAM_ID}/benchmarks") + + assert response.status_code == 200 + assert response.json == { + "lead_time": {"target": 3600, "source": "team"}, + "deployment_frequency": {"target": 5, "source": "global"}, + "change_failure_rate": {"target": None, "source": None}, + "mean_time_to_recovery": {"target": None, "source": None}, + "lines_of_code": {"target": None, "source": None}, + } + + +def test_a_team_put_of_only_lines_of_code_round_trips_as_a_team_target(routes): + # CLUSTOX: the fifth metric has to survive the same PUT -> store -> GET + # path as the original four. Every previous regression here was an + # adapter that silently dropped the new key, which no unit test on + # resolve_benchmarks could see: the value went in, came back absent, and + # the metric quietly reverted to the global baseline. + client, service = routes + _global_row(service, lead_time=86400) + + put = client.put( + f"/teams/{TEAM_ID}/settings", + json={ + "setting_type": "BENCHMARK_SETTING", + "setting_data": {"lines_of_code": 250}, + }, + ) + + assert put.status_code == 200 + assert put.json["setting"]["lines_of_code"] == 250 + + resolved = client.get(f"/teams/{TEAM_ID}/benchmarks") + + assert resolved.status_code == 200 + assert resolved.json["lines_of_code"] == {"target": 250, "source": "team"} + assert resolved.json["lead_time"] == {"target": 86400, "source": "global"} + assert resolved.json["change_failure_rate"] == {"target": None, "source": None} + + +def test_the_benchmarks_route_is_all_null_when_nothing_is_configured(routes): + # The zero-config state, over the wire. Every card must be able to see + # "no target" and render exactly as it did before this feature. + client, service = routes + + response = client.get(f"/teams/{TEAM_ID}/benchmarks") + + assert response.status_code == 200 + assert response.json == { + metric: {"target": None, "source": None} for metric in BENCHMARK_METRICS + } + assert service.saved == [], "resolving benchmarks must not write anything" + + +def test_reading_a_team_benchmark_does_not_create_a_team_row(routes): + client, service = routes + + response = client.get( + f"/teams/{TEAM_ID}/settings", query_string={"setting_type": "BENCHMARK_SETTING"} + ) + + assert response.status_code == 200 + assert response.json["setting"] == {metric: None for metric in BENCHMARK_METRICS} + # The bug: an auto-created row of defaults ended that team's per-metric + # inheritance forever, and made the card claim "your team's benchmark". + assert service.saved == [] + + +def test_reading_the_global_baseline_does_not_create_the_global_row(routes): + client, service = routes + + response = client.get( + "/settings/global", query_string={"setting_type": "BENCHMARK_SETTING"} + ) + + assert response.status_code == 200 + assert response.json["setting"] == {metric: None for metric in BENCHMARK_METRICS} + assert response.json["scope"] == "global" + assert service.saved == [] + + +def test_a_non_benchmark_read_still_auto_creates_its_row(routes): + # The auto-create is pre-existing behaviour other setting types rely on; + # only benchmarks opt out of it. + client, service = routes + + client.get( + f"/teams/{TEAM_ID}/settings", query_string={"setting_type": "INCIDENT_SETTING"} + ) + + assert service.saved == [ + (SettingType.INCIDENT_SETTING, EntityType.TEAM, TEAM_ID, None) + ] + + +def test_clearing_every_field_on_a_team_writes_nulls_not_defaults(routes): + client, service = routes + _team_row(service, lead_time=3600, change_failure_rate=10) + + response = client.put( + f"/teams/{TEAM_ID}/settings", + json={"setting_type": "BENCHMARK_SETTING", "setting_data": {}}, + ) + + assert response.status_code == 200 + assert response.json["setting"] == {metric: None for metric in BENCHMARK_METRICS} + _, _, _, written = service.saved[-1] + assert written == {metric: None for metric in BENCHMARK_METRICS} + + +def test_a_team_put_rejects_an_out_of_range_value(routes): + client, service = routes + + response = client.put( + f"/teams/{TEAM_ID}/settings", + json={ + "setting_type": "BENCHMARK_SETTING", + "setting_data": {"change_failure_rate": 5000}, + }, + ) + + assert response.status_code == 400 + assert service.saved == [] + + +def test_a_team_put_rejects_a_typoed_metric_key(routes): + # Silently dropping this key stored nothing and reported success. + client, service = routes + + response = client.put( + f"/teams/{TEAM_ID}/settings", + json={"setting_type": "BENCHMARK_SETTING", "setting_data": {"leadtime": 3600}}, + ) + + assert response.status_code == 400 + assert service.saved == [] + + +def test_a_team_put_of_a_non_benchmark_setting_skips_benchmark_validation(routes): + client, service = routes + + response = client.put( + f"/teams/{TEAM_ID}/settings", + json={ + "setting_type": "INCIDENT_SETTING", + "setting_data": {"title_includes": ["outage"]}, + }, + ) + + assert response.status_code == 200 + assert service.saved[-1][0] == SettingType.INCIDENT_SETTING + + +@pytest.mark.parametrize("method", ["get", "put"]) +def test_the_global_scope_refuses_non_benchmark_setting_types(routes, method): + client, service = routes + + if method == "get": + response = client.get( + "/settings/global", query_string={"setting_type": "INCIDENT_SETTING"} + ) + else: + response = client.put( + "/settings/global", + json={"setting_type": "INCIDENT_SETTING", "setting_data": {}}, + ) + + assert response.status_code == 400 + assert service.saved == [] diff --git a/backend/analytics_server/tests/api/test_jenkins_routes.py b/backend/analytics_server/tests/api/test_jenkins_routes.py new file mode 100644 index 000000000..3424af013 --- /dev/null +++ b/backend/analytics_server/tests/api/test_jenkins_routes.py @@ -0,0 +1,866 @@ +from typing import List, Optional +from unittest.mock import MagicMock, patch + +import pytest +import requests +from flask import Flask +from sqlalchemy.exc import IntegrityError + +from mhq.api import integrations as integrations_module +from mhq.api.integrations import ( + deactivate_deployment_workflows_for_repo, + reactivate_github_actions_workflows_for_repo, + restore_team_repo_deployment_types, + switch_team_repos_to_workflow_deployments, +) +from mhq.exapi.jenkins import JenkinsAddressNotAllowed +from mhq.service.deployments.deployment_service import DeploymentsService +from mhq.store.models.code import RepoWorkflowProviders, RepoWorkflowType +from mhq.store.models.code.enums import TeamReposDeploymentType +from mhq.utils.string import uuid4_str + +ORG_ID = "11111111-1111-4111-8111-111111111111" +OTHER_ORG_ID = "22222222-2222-4222-8222-222222222222" +REPO_ID = "33333333-3333-4333-8333-333333333333" +TEAM_ID = "44444444-4444-4444-8444-444444444444" +OTHER_TEAM_ID = "55555555-5555-4555-8555-555555555555" + + +class FakeWorkflow: + def __init__( + self, + provider, + is_active=True, + org_repo_id=REPO_ID, + provider_workflow_id="deploy-api", + type=RepoWorkflowType.DEPLOYMENT, + id=None, + ): + self.id = id or uuid4_str() + self.provider = provider + self.is_active = is_active + self.org_repo_id = org_repo_id + self.provider_workflow_id = provider_workflow_id + self.type = type + self.name = provider_workflow_id + # Matches the column default on RepoWorkflow: a row written before the + # displaced-workflow record existed carries no dict at all. + self.meta = None + + +def test_deactivates_every_active_deployment_workflow_whatever_the_provider(): + workflows = [ + FakeWorkflow(RepoWorkflowProviders.GITHUB_ACTIONS), + FakeWorkflow(RepoWorkflowProviders.GITHUB_ACTIONS), + FakeWorkflow(RepoWorkflowProviders.JENKINS), + ] + + deactivated = deactivate_deployment_workflows_for_repo(workflows) + + # One active deployment source per repo is the invariant. Leaving the + # Jenkins row active -- as the GitHub-only version did -- lets a repo end + # up mapped to two Jenkins jobs and count every deploy twice. + assert deactivated == workflows + assert [w.is_active for w in workflows] == [False, False, False] + + +def test_deactivating_is_idempotent(): + workflows = [FakeWorkflow(RepoWorkflowProviders.GITHUB_ACTIONS, is_active=False)] + + deactivated = deactivate_deployment_workflows_for_repo(workflows) + + # Already inactive, so nothing changed and nothing is reported -- and, + # crucially, nothing is recorded as displaced, because it was not. + assert deactivated == [] + + +def test_reactivates_only_the_inactive_workflows_the_mapping_displaced(): + displaced = FakeWorkflow(RepoWorkflowProviders.GITHUB_ACTIONS, is_active=False) + deselected = FakeWorkflow(RepoWorkflowProviders.GITHUB_ACTIONS, is_active=False) + already_on = FakeWorkflow(RepoWorkflowProviders.GITHUB_ACTIONS, is_active=True) + jenkins = FakeWorkflow(RepoWorkflowProviders.JENKINS, is_active=False) + + reactivated = reactivate_github_actions_workflows_for_repo( + [displaced, deselected, already_on, jenkins], [displaced.id] + ) + + assert reactivated == [displaced] + assert displaced.is_active is True + # Inactive because the admin deselected it in the team config, not because + # the Jenkins mapping touched it. Turning it on is a deployment the repo + # never had. + assert deselected.is_active is False + assert already_on.is_active is True + assert jenkins.is_active is False + + +def test_reactivates_nothing_when_the_mapping_recorded_no_displacement(): + # A mapping created before the displaced-workflow record existed. Restoring + # nothing is the safe reading: too few active sources under-reports + # visibly, too many over-reports silently. + github = FakeWorkflow(RepoWorkflowProviders.GITHUB_ACTIONS, is_active=False) + + assert reactivate_github_actions_workflows_for_repo([github], None) == [] + assert github.is_active is False + + +class FakeTeamRepo: + """ + A row of TeamRepos. deployment_type is the column DeploymentsService splits + on: repos on PR_MERGE never have their RepoWorkflowRuns looked at, whatever + has been ingested for them. + """ + + def __init__( + self, + team_id=TEAM_ID, + org_repo_id=REPO_ID, + deployment_type=TeamReposDeploymentType.PR_MERGE, + is_active=True, + ): + self.team_id = team_id + self.org_repo_id = org_repo_id + self.deployment_type = deployment_type + self.is_active = is_active + + +def test_switching_records_only_the_rows_it_changed(): + on_pr_merge = FakeTeamRepo() + already_workflow = FakeTeamRepo( + team_id=OTHER_TEAM_ID, deployment_type=TeamReposDeploymentType.WORKFLOW + ) + + switched, previous = switch_team_repos_to_workflow_deployments( + [on_pr_merge, already_workflow] + ) + + assert switched == [on_pr_merge] + assert on_pr_merge.deployment_type == TeamReposDeploymentType.WORKFLOW + # Recording the already-WORKFLOW row would make unmapping demote a repo the + # admin configured that way himself. + assert previous == {TEAM_ID: "PR_MERGE"} + + +def test_restoring_leaves_a_row_the_admin_has_since_changed_alone(): + changed_by_hand = FakeTeamRepo(deployment_type=TeamReposDeploymentType.PR_MERGE) + untouched = FakeTeamRepo( + team_id=OTHER_TEAM_ID, deployment_type=TeamReposDeploymentType.WORKFLOW + ) + + restored = restore_team_repo_deployment_types( + [changed_by_hand, untouched], + {TEAM_ID: "PR_MERGE", OTHER_TEAM_ID: "PR_MERGE"}, + ) + + # The first row no longer holds the value the mapping wrote, so the mapping + # is not what is on it any more and there is nothing here to undo. + assert restored == [untouched] + assert untouched.deployment_type == TeamReposDeploymentType.PR_MERGE + + +class FakeOrgRepo: + def __init__(self, repo_id=REPO_ID, org_id=ORG_ID, name="middleware"): + self.id = repo_id + self.org_id = org_id + self.name = name + + +def org_repo_for(repo_id) -> FakeOrgRepo: + """ + Stands in for the OrgRepo join. Only REPO_ID belongs to ORG_ID; every other + repo belongs to another workspace, which is what the scoping tests turn on. + """ + if str(repo_id) == REPO_ID: + return FakeOrgRepo() + return FakeOrgRepo(repo_id=repo_id, org_id=OTHER_ORG_ID, name="someone-elses-repo") + + +class FakeCodeRepoService: + """ + Resolves REPO_ID into ORG_ID and reads the repo's TeamRepos rows out of the + same in-memory store the workflow service writes to -- in production both + services share one db.session, and the mapping routes depend on that. + """ + + def __init__(self, store): + self._store = store + + def get_repo_by_id(self, repo_id) -> Optional[FakeOrgRepo]: + return FakeOrgRepo(repo_id=repo_id) if str(repo_id) == REPO_ID else None + + def get_active_team_repos_by_repo_id(self, repo_id) -> List[FakeTeamRepo]: + return [ + team_repo + for team_repo in self._store.team_repos + if str(team_repo.org_repo_id) == str(repo_id) and team_repo.is_active + ] + + +class FakeWorkflowRepoService: + """ + In-memory stand-in that reproduces the two database behaviours these routes + depend on: get_repo_workflow_by_repo_ids returns only active rows, and the + (org_repo_id, provider_workflow_id) pair is uniquely indexed + (repoworkflow_orgrepoid_provider_workflow_id), so a second insert for a + pair that already has a row raises IntegrityError. + """ + + def __init__( + self, + workflows: Optional[List[FakeWorkflow]] = None, + team_repos: Optional[List[FakeTeamRepo]] = None, + ): + self.workflows = list(workflows or []) + # Defaults to a single team on PR_MERGE: what repository_service.py + # gives a newly added repo, and the state the live workspace was in + # while its 14 ingested Jenkins builds counted for nothing. + self.team_repos = list( + team_repos if team_repos is not None else [FakeTeamRepo()] + ) + + def get_active_repo_workflows_by_org_id_and_provider(self, org_id, provider, type): + # Reproduces the OrgRepo join: a workflow belongs to the workspace its + # repo belongs to, and no query the route makes can widen that. + return [ + (w, org_repo_for(w.org_repo_id)) + for w in self.workflows + if w.provider == provider + and w.type == type + and w.is_active + and str(org_repo_for(w.org_repo_id).org_id) == str(org_id) + ] + + def get_repo_workflow_by_repo_ids(self, repo_ids, type) -> List[FakeWorkflow]: + return [ + w + for w in self.workflows + if str(w.org_repo_id) in [str(r) for r in repo_ids] + and w.type == type + and w.is_active + ] + + def get_repo_workflow_by_repo_id_and_provider_workflow_id( + self, repo_id, provider_workflow_id + ) -> Optional[FakeWorkflow]: + # Matches repoworkflow_orgrepoid_provider_workflow_id, which is not + # scoped by provider -- so neither is this. + for w in self.workflows: + if ( + str(w.org_repo_id) == str(repo_id) + and w.provider_workflow_id == provider_workflow_id + ): + return w + return None + + def get_repo_workflows_by_repo_id_and_provider( + self, repo_id, provider, type + ) -> List[FakeWorkflow]: + return [ + w + for w in self.workflows + if str(w.org_repo_id) == str(repo_id) + and w.provider == provider + and w.type == type + ] + + def get_repo_workflow_by_id(self, repo_workflow_id) -> Optional[FakeWorkflow]: + for w in self.workflows: + if str(w.id) == str(repo_workflow_id): + return w + return None + + def create_jenkins_repo_workflow( + self, jenkins_workflow, workflows_to_deactivate, team_repos_to_update=None + ): + if jenkins_workflow not in self.workflows: + for w in self.workflows: + if ( + str(w.org_repo_id) == str(jenkins_workflow.org_repo_id) + and w.provider_workflow_id == jenkins_workflow.provider_workflow_id + ): + raise IntegrityError( + 'duplicate key value violates unique constraint "' + 'repoworkflow_orgrepoid_provider_workflow_id"', + None, + Exception(), + ) + # The database assigns the primary key on insert. + if getattr(jenkins_workflow, "id", None) is None: + jenkins_workflow.id = uuid4_str() + self.workflows.append(jenkins_workflow) + return jenkins_workflow + + def deactivate_repo_workflow( + self, repo_workflow, workflows_to_reactivate=None, team_repos_to_update=None + ): + repo_workflow.is_active = False + return repo_workflow + + @property + def deployment_types(self) -> List[TeamReposDeploymentType]: + return [team_repo.deployment_type for team_repo in self.team_repos] + + @property + def active_deployment_workflows(self) -> List[FakeWorkflow]: + return [w for w in self.workflows if w.is_active] + + +def _build_client(): + app = Flask(__name__) + app.register_blueprint(integrations_module.app) + return app.test_client() + + +@pytest.fixture +def routes(request): + """ + Wires the Jenkins mapping routes to in-memory services. The repo lookup + resolves REPO_ID into ORG_ID unless the test overrides it. + + The parameter is either a list of workflows or a dict of + {"workflows": [...], "team_repos": [...]}. + """ + param = getattr(request, "param", None) or {} + if isinstance(param, list): + param = {"workflows": param} + workflow_repo_service = FakeWorkflowRepoService( + param.get("workflows"), param.get("team_repos") + ) + code_repo_service = FakeCodeRepoService(workflow_repo_service) + + with patch.object( + integrations_module, "get_query_validator", return_value=MagicMock() + ), patch.object( + integrations_module, "CodeRepoService", return_value=code_repo_service + ), patch.object( + integrations_module, "WorkflowRepoService", return_value=workflow_repo_service + ): + yield _build_client(), workflow_repo_service + + +def _map(client, job_full_name, org_id=ORG_ID, repo_id=REPO_ID): + return client.post( + f"/orgs/{org_id}/integrations/jenkins/mappings", + json={"org_repo_id": repo_id, "job_full_name": job_full_name}, + ) + + +def _unmap(client, repo_workflow_id, org_id=ORG_ID): + return client.delete( + f"/orgs/{org_id}/integrations/jenkins/mappings", + json={"repo_workflow_id": str(repo_workflow_id)}, + ) + + +def _list(client, org_id=ORG_ID): + return client.get(f"/orgs/{org_id}/integrations/jenkins/mappings") + + +def test_listing_returns_the_workspaces_live_mappings(routes): + # The mapping table reloaded showing "Select a Jenkins job" for a repo that + # was mapped and ingesting deployments, and nothing could supply the + # repo_workflow_id the DELETE route needs -- so a mapping could not be + # undone from the UI at all. + client, service = routes + _map(client, "platform/deploy-api") + jenkins_workflow = service.get_repo_workflow_by_repo_id_and_provider_workflow_id( + REPO_ID, "platform/deploy-api" + ) + + response = _list(client) + + assert response.status_code == 200 + assert response.json == [ + { + "repo_workflow_id": str(jenkins_workflow.id), + "org_repo_id": REPO_ID, + "job_full_name": "platform/deploy-api", + "repo_name": "middleware", + } + ] + # The id it hands back is the one the DELETE route accepts. + assert _unmap(client, response.json[0]["repo_workflow_id"]).status_code == 200 + + +def test_listing_is_empty_for_a_workspace_with_no_mappings(routes): + client, _ = routes + + response = _list(client) + + assert response.status_code == 200 + assert response.json == [] + + +@pytest.mark.parametrize( + "routes", + [ + [ + FakeWorkflow( + RepoWorkflowProviders.JENKINS, + org_repo_id="66666666-6666-4666-8666-666666666666", + provider_workflow_id="their-deploy-job", + ) + ] + ], + indirect=True, +) +def test_listing_does_not_leak_another_workspaces_mappings(routes): + client, _ = routes + _map(client, "our-deploy-job") + + ours = _list(client) + theirs = _list(client, org_id=OTHER_ORG_ID) + + assert [m["job_full_name"] for m in ours.json] == ["our-deploy-job"] + # Same authenticated admin, another workspace's org id in the path. The + # scoping is the OrgRepo join, not anything the caller passes. + assert [m["job_full_name"] for m in theirs.json] == ["their-deploy-job"] + + +@pytest.mark.parametrize( + "routes", + [[FakeWorkflow(RepoWorkflowProviders.GITHUB_ACTIONS, provider_workflow_id="9931")]], + indirect=True, +) +def test_listing_covers_neither_other_providers_nor_removed_mappings(routes): + client, service = routes + _map(client, "deploy-api") + jenkins_workflow = service.get_repo_workflow_by_repo_id_and_provider_workflow_id( + REPO_ID, "deploy-api" + ) + + assert len(_list(client).json) == 1 + + _unmap(client, jenkins_workflow.id) + + # The GitHub Actions workflow is active again, but it is not a Jenkins + # mapping and the removed Jenkins row is not one any more. + assert _list(client).json == [] + + +def test_mapping_a_second_job_leaves_exactly_one_active_deployment_source(routes): + client, service = routes + + assert _map(client, "deploy-api").status_code == 200 + # The wrong job first, then the right one -- two clicks, no warning in the + # UI the second time round. + assert _map(client, "deploy-api-v2").status_code == 200 + + active = service.active_deployment_workflows + assert [w.provider_workflow_id for w in active] == ["deploy-api-v2"] + + +@pytest.mark.parametrize( + "routes", + [[FakeWorkflow(RepoWorkflowProviders.GITHUB_ACTIONS, provider_workflow_id="9931")]], + indirect=True, +) +def test_mapping_deactivates_the_repos_github_actions_workflow(routes): + client, service = routes + + response = _map(client, "deploy-api") + + assert response.status_code == 200 + assert response.json["deactivated_workflows"] == 1 + active = service.active_deployment_workflows + assert [w.provider for w in active] == [RepoWorkflowProviders.JENKINS] + + +@pytest.mark.parametrize( + "routes", + [[FakeWorkflow(RepoWorkflowProviders.GITHUB_ACTIONS, provider_workflow_id="9931")]], + indirect=True, +) +def test_unmapping_restores_the_github_actions_workflow(routes): + client, service = routes + _map(client, "deploy-api") + jenkins_workflow = service.get_repo_workflow_by_repo_id_and_provider_workflow_id( + REPO_ID, "deploy-api" + ) + + response = _unmap(client, jenkins_workflow.id) + + assert response.status_code == 200 + assert response.json["reactivated_github_workflows"] == 1 + # The repo is back to exactly one active deployment source: the GitHub + # Actions workflow the mapping displaced. Without this the repo has none, + # and its Deployment Frequency stays at zero. + active = service.active_deployment_workflows + assert [w.provider for w in active] == [RepoWorkflowProviders.GITHUB_ACTIONS] + + +def test_mapping_switches_the_repo_to_workflow_deployments(routes): + # The gap that made the whole integration invisible: 14 builds ingested, + # zero deployments counted, because DeploymentsService only consults + # RepoWorkflowRuns for team repos on WORKFLOW and a mapped repo sat on + # PR_MERGE. + client, service = routes + assert service.deployment_types == [TeamReposDeploymentType.PR_MERGE] + + response = _map(client, "deploy-api") + + assert response.status_code == 200 + assert response.json["switched_team_repos"] == 1 + assert service.deployment_types == [TeamReposDeploymentType.WORKFLOW] + + +def test_mapping_moves_the_repo_into_the_workflow_side_of_the_real_splitter(routes): + """ + The assertion above is only meaningful because of this one. DeploymentsService + splits a team's repos on deployment_type and asks the workflow-based service + for nothing but the WORKFLOW side, so a mapped repo on the PR_MERGE side + counts none of its RepoWorkflowRuns however many were ingested. Driven + through the real splitter rather than a restatement of it. + """ + client, service = routes + deployments_service = DeploymentsService(None, None, None, None) + + before, _ = deployments_service.get_filtered_team_repos_by_deployment_config( + service.team_repos + ) + assert before == [] + + _map(client, "deploy-api") + + after, on_pr_merge = ( + deployments_service.get_filtered_team_repos_by_deployment_config( + service.team_repos + ) + ) + assert after == service.team_repos + assert on_pr_merge == [] + + +@pytest.mark.parametrize( + "routes", + [ + { + "team_repos": [ + FakeTeamRepo(team_id=TEAM_ID), + FakeTeamRepo(team_id=OTHER_TEAM_ID), + # Another repo's row, and one this repo no longer belongs to. + FakeTeamRepo(team_id=TEAM_ID, org_repo_id=uuid4_str()), + FakeTeamRepo(team_id=OTHER_TEAM_ID, is_active=False), + ] + } + ], + indirect=True, +) +def test_mapping_switches_every_team_that_tracks_the_repo(routes): + client, service = routes + + assert _map(client, "deploy-api").json["switched_team_repos"] == 2 + + assert service.deployment_types == [ + TeamReposDeploymentType.WORKFLOW, + TeamReposDeploymentType.WORKFLOW, + # A different repo, and an inactive membership: neither is this + # mapping's business. + TeamReposDeploymentType.PR_MERGE, + TeamReposDeploymentType.PR_MERGE, + ] + + +def test_unmapping_restores_the_deployment_type_the_mapping_overwrote(routes): + client, service = routes + _map(client, "deploy-api") + jenkins_workflow = service.get_repo_workflow_by_repo_id_and_provider_workflow_id( + REPO_ID, "deploy-api" + ) + assert jenkins_workflow.meta["jenkins_previous_deployment_types"] == { + TEAM_ID: "PR_MERGE" + } + + response = _unmap(client, jenkins_workflow.id) + + assert response.status_code == 200 + assert response.json["restored_team_repos"] == 1 + assert service.deployment_types == [TeamReposDeploymentType.PR_MERGE] + + +@pytest.mark.parametrize( + "routes", + [{"team_repos": [FakeTeamRepo(deployment_type=TeamReposDeploymentType.WORKFLOW)]}], + indirect=True, +) +def test_unmapping_leaves_a_repo_that_was_already_workflow_on_workflow(routes): + # The repo was on WORKFLOW before Jenkins ever touched it -- an admin ran it + # off GitHub Actions. Mapping changed nothing there, so unmapping must not + # either: demoting it to PR_MERGE would silently rewrite a setting the + # mapping never owned. + client, service = routes + + assert _map(client, "deploy-api").json["switched_team_repos"] == 0 + jenkins_workflow = service.get_repo_workflow_by_repo_id_and_provider_workflow_id( + REPO_ID, "deploy-api" + ) + assert jenkins_workflow.meta["jenkins_previous_deployment_types"] == {} + + response = _unmap(client, jenkins_workflow.id) + + assert response.status_code == 200 + assert response.json["restored_team_repos"] == 0 + assert service.deployment_types == [TeamReposDeploymentType.WORKFLOW] + + +def test_remapping_carries_the_original_deployment_types_to_the_new_row(routes): + # Map the wrong job, then the right one: the second mapping finds the repo + # already on WORKFLOW and records nothing of its own. Without carrying the + # first mapping's record forward, unmapping strands the repo on WORKFLOW + # with no active workflow behind it -- zero deployments, no error. + client, service = routes + _map(client, "deploy-api") + _map(client, "deploy-api-v2") + + second = service.get_repo_workflow_by_repo_id_and_provider_workflow_id( + REPO_ID, "deploy-api-v2" + ) + assert second.meta["jenkins_previous_deployment_types"] == {TEAM_ID: "PR_MERGE"} + + assert _unmap(client, second.id).json["restored_team_repos"] == 1 + assert service.deployment_types == [TeamReposDeploymentType.PR_MERGE] + + +def test_unmapping_a_mapping_with_no_deployment_type_record_changes_nothing(routes): + # A mapping written before the record existed. Guessing PR_MERGE would + # demote repos the record cannot speak for. + client, service = routes + _map(client, "deploy-api") + jenkins_workflow = service.get_repo_workflow_by_repo_id_and_provider_workflow_id( + REPO_ID, "deploy-api" + ) + jenkins_workflow.meta = {} + + response = _unmap(client, jenkins_workflow.id) + + assert response.status_code == 200 + assert response.json["restored_team_repos"] == 0 + assert service.deployment_types == [TeamReposDeploymentType.WORKFLOW] + + +@pytest.mark.parametrize( + "routes", + [ + [ + FakeWorkflow( + RepoWorkflowProviders.GITHUB_ACTIONS, provider_workflow_id="9931" + ), + ] + + [ + FakeWorkflow( + RepoWorkflowProviders.GITHUB_ACTIONS, + provider_workflow_id=str(9932 + offset), + is_active=False, + ) + for offset in range(4) + ] + ], + indirect=True, +) +def test_unmapping_leaves_deselected_github_actions_workflows_inactive(routes): + """ + One selected GitHub Actions deployment workflow and four the admin + deselected. teams/v2.ts deactivates all of a repo's deployment workflows + and re-enables only the selected ones, so those four sit inactive for a + reason that has nothing to do with Jenkins. Mapping switches off exactly + one row, so unmapping must switch on exactly one row -- restoring all five + gives the repo five active deployment sources and inflates Deployment + Frequency fivefold, which is the double-counting the invariant exists to + prevent, reached through the GitHub path instead of the Jenkins one. + """ + client, service = routes + selected = service.workflows[0] + deselected = service.workflows[1:] + + assert _map(client, "deploy-api").json["deactivated_workflows"] == 1 + jenkins_workflow = service.get_repo_workflow_by_repo_id_and_provider_workflow_id( + REPO_ID, "deploy-api" + ) + # Only the row the mapping actually switched off is recorded as displaced. + assert jenkins_workflow.meta["jenkins_displaced_workflow_ids"] == [str(selected.id)] + + response = _unmap(client, jenkins_workflow.id) + + assert response.status_code == 200 + assert response.json["reactivated_github_workflows"] == 1 + assert selected.is_active is True + assert [w.is_active for w in deselected] == [False, False, False, False] + active = service.active_deployment_workflows + assert [w.provider_workflow_id for w in active] == ["9931"] + + +@pytest.mark.parametrize( + "routes", + [ + [ + FakeWorkflow( + RepoWorkflowProviders.JENKINS, provider_workflow_id="deploy-api" + ), + FakeWorkflow( + RepoWorkflowProviders.GITHUB_ACTIONS, + provider_workflow_id="9931", + is_active=False, + ), + ] + ], + indirect=True, +) +def test_unmapping_a_mapping_with_no_displacement_record_restores_nothing(routes): + # A mapping written before the record existed. There is no way to tell + # which rows it displaced, and guessing "all the inactive ones" is how the + # over-restore happened in the first place. + client, service = routes + jenkins_workflow, github_workflow = service.workflows + assert jenkins_workflow.meta is None + + response = _unmap(client, jenkins_workflow.id) + + assert response.status_code == 200 + assert response.json["reactivated_github_workflows"] == 0 + assert github_workflow.is_active is False + + +def test_remapping_the_same_job_after_unmapping_does_not_hit_the_unique_index(routes): + client, service = routes + assert _map(client, "deploy-api").status_code == 200 + jenkins_workflow = service.get_repo_workflow_by_repo_id_and_provider_workflow_id( + REPO_ID, "deploy-api" + ) + assert _unmap(client, jenkins_workflow.id).status_code == 200 + + # This used to insert a second row for the same + # (org_repo_id, provider_workflow_id) pair, raising IntegrityError -> 500. + response = _map(client, "deploy-api") + + assert response.status_code == 200 + assert len(service.workflows) == 1 + assert service.workflows[0].is_active is True + + +@pytest.mark.parametrize( + "routes", + [ + [ + FakeWorkflow( + RepoWorkflowProviders.GITHUB_ACTIONS, provider_workflow_id="deploy-api" + ) + ] + ], + indirect=True, +) +def test_mapping_a_job_whose_name_collides_with_another_provider_is_409(routes): + # repoworkflow_orgrepoid_provider_workflow_id is on + # (org_repo_id, provider_workflow_id), with no provider in it. A lookup + # narrower than the index reported "no row here" and the insert then hit + # the constraint -> IntegrityError -> 500. Reusing the row instead would be + # worse: a GitHub Actions workflow, its runs and its history silently + # become a Jenkins mapping. + client, service = routes + github_workflow = service.workflows[0] + + response = _map(client, "deploy-api") + + assert response.status_code == 409 + assert "deploy-api" in response.json["error"] + assert len(service.workflows) == 1 + assert github_workflow.provider == RepoWorkflowProviders.GITHUB_ACTIONS + assert github_workflow.is_active is True + + +@pytest.mark.parametrize( + "routes", + [[FakeWorkflow(RepoWorkflowProviders.GITHUB_ACTIONS, provider_workflow_id="9931")]], + indirect=True, +) +def test_deleting_a_github_actions_workflow_through_the_jenkins_route_is_404(routes): + client, service = routes + github_workflow = service.workflows[0] + + response = _unmap(client, github_workflow.id) + + assert response.status_code == 404 + # Untouched: the Jenkins route must not be a way to disable a GitHub + # Actions deployment workflow. + assert github_workflow.is_active is True + + +@pytest.mark.parametrize( + "routes", + [[FakeWorkflow(RepoWorkflowProviders.JENKINS, org_repo_id="nope")]], + indirect=True, +) +def test_deleting_a_workflow_from_another_workspace_is_404(routes): + client, service = routes + foreign_workflow = service.workflows[0] + + response = _unmap(client, foreign_workflow.id, org_id=OTHER_ORG_ID) + + assert response.status_code == 404 + assert foreign_workflow.is_active is True + + +def test_deleting_an_unknown_workflow_is_404(routes): + client, _ = routes + + assert _unmap(client, uuid4_str()).status_code == 404 + + +def test_mapping_a_repo_from_another_workspace_is_404(routes): + client, service = routes + + response = _map(client, "deploy-api", repo_id=uuid4_str()) + + assert response.status_code == 404 + assert service.workflows == [] + + +def _jobs_route(get_jobs_result): + """Drives GET .../jenkins/jobs against a fully configured workspace.""" + client = _build_client() + api_service = MagicMock() + if isinstance(get_jobs_result, Exception): + api_service.get_jobs.side_effect = get_jobs_result + else: + api_service.get_jobs.return_value = get_jobs_result + + with patch.object( + integrations_module, "get_query_validator", return_value=MagicMock() + ), patch("mhq.store.repos.core.CoreRepoService") as core_repo_service_cls, patch( + "mhq.utils.jenkins.get_jenkins_config", + return_value=("https://jenkins.example.com", "user"), + ), patch( + "mhq.exapi.jenkins.JenkinsApiService", return_value=api_service + ): + core_repo_service_cls.return_value.get_access_token.return_value = "token" + return client.get(f"/orgs/{ORG_ID}/integrations/jenkins/jobs") + + +def test_an_unreachable_jenkins_is_a_502_not_a_500(): + response = _jobs_route(requests.ConnectionError("connection refused")) + + # A 500 reached the setup form as "check your credentials", which sends the + # admin looking in entirely the wrong place. + assert response.status_code == 502 + assert "Could not reach Jenkins" in response.json["error"] + + +def test_a_refused_address_is_told_apart_from_an_unreachable_jenkins(): + response = _jobs_route( + JenkinsAddressNotAllowed( + "Jenkins host metadata resolves to 169.254.169.254, which is a " + "private, loopback or link-local address." + ) + ) + + # A 502 "Could not reach Jenkins" would send the admin to look at a + # firewall for a URL this server is refusing on purpose. + assert response.status_code == 400 + assert "169.254.169.254" in response.json["error"] + assert "Could not reach Jenkins" not in response.json["error"] + + +def test_a_reachable_jenkins_returns_its_jobs(): + response = _jobs_route([{"name": "deploy-api", "full_name": "deploy-api"}]) + + assert response.status_code == 200 + assert response.json[0]["full_name"] == "deploy-api" diff --git a/backend/analytics_server/tests/api/test_jira_connections_routes.py b/backend/analytics_server/tests/api/test_jira_connections_routes.py new file mode 100644 index 000000000..c45ca270c --- /dev/null +++ b/backend/analytics_server/tests/api/test_jira_connections_routes.py @@ -0,0 +1,262 @@ +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch +from uuid import uuid4 + +import pytest +from flask import Flask +from sqlalchemy.exc import IntegrityError + +from mhq.api import integrations as integrations_module +from mhq.store.repos.jira_connection import ( + JiraConnectionInUseError, + JiraConnectionNotFoundError, +) + +ORG_ID = "11111111-1111-4111-8111-111111111111" +OTHER_ORG_ID = "22222222-2222-4222-8222-222222222222" + + +class FakeJiraConnection: + def __init__(self, org_id, site_url, email, provider_meta=None, generated_by=None): + self.id = str(uuid4()) + self.org_id = org_id + self.site_url = site_url + self.email = email + # Never read by the routes -- present only to prove the serializer + # (_serialize_jira_connection) never touches it. + self.access_token_enc_chunks = ["enc-chunk-1", "enc-chunk-2"] + self.provider_meta = provider_meta or {} + self.is_default = False + self.generated_by = generated_by + self.created_at = datetime(2026, 1, 1, tzinfo=timezone.utc) + + +class FakeJiraConnectionRepoService: + """ + In-memory stand-in reproducing the three invariants the routes depend on + from the real JiraConnectionRepoService: (org_id, site_url, email) + uniqueness on create, delete blocked while an OrgProjectConnection + references the row, and default-switch atomicity. + """ + + def __init__(self, connections=None, referenced_ids=None): + self.connections = list(connections or []) + self.referenced_ids = set(referenced_ids or []) + + def list_jira_connections(self, org_id): + return [c for c in self.connections if str(c.org_id) == str(org_id)] + + def get_jira_connection(self, org_id, connection_id): + for c in self.connections: + if str(c.id) == str(connection_id) and str(c.org_id) == str(org_id): + return c + return None + + def create_jira_connection( + self, org_id, site_url, email, access_token, provider_meta, generated_by + ): + if any( + str(c.org_id) == str(org_id) and c.site_url == site_url and c.email == email + for c in self.connections + ): + raise IntegrityError( + "duplicate key value violates unique constraint " + '"jira_connection_unique_account"', + None, + Exception(), + ) + connection = FakeJiraConnection( + org_id, site_url, email, provider_meta, generated_by + ) + self.connections.append(connection) + return connection + + def delete_jira_connection(self, org_id, connection_id): + connection = self.get_jira_connection(org_id, connection_id) + if not connection: + raise JiraConnectionNotFoundError( + f"No JiraConnection {connection_id} for org {org_id}" + ) + if connection.id in self.referenced_ids: + raise JiraConnectionInUseError( + f"JiraConnection {connection_id} still has projects synced from it" + ) + self.connections.remove(connection) + + def set_default_jira_connection(self, org_id, connection_id): + connection = self.get_jira_connection(org_id, connection_id) + if not connection: + raise JiraConnectionNotFoundError( + f"No JiraConnection {connection_id} for org {org_id}" + ) + for c in self.connections: + if str(c.org_id) == str(org_id) and c.id != connection.id: + c.is_default = False + connection.is_default = True + + +def _build_client(): + app = Flask(__name__) + app.register_blueprint(integrations_module.app) + return app.test_client() + + +@pytest.fixture +def routes(request): + """ + Wires the jira-connections routes to an in-memory service. The parameter + is a dict of {"connections": [...], "referenced_ids": [...]}. + """ + param = getattr(request, "param", None) or {} + service = FakeJiraConnectionRepoService( + param.get("connections"), param.get("referenced_ids") + ) + + with patch.object( + integrations_module, "get_query_validator", return_value=MagicMock() + ), patch.object( + integrations_module, "JiraConnectionRepoService", return_value=service + ): + yield _build_client(), service + + +def _list(client, org_id=ORG_ID): + return client.get(f"/orgs/{org_id}/integrations/jira-connections") + + +def _create(client, site_url="acme.atlassian.net", email="a@acme.com", org_id=ORG_ID): + return client.post( + f"/orgs/{org_id}/integrations/jira-connections", + json={"site_url": site_url, "email": email, "access_token": "raw-token"}, + ) + + +def _delete(client, connection_id, org_id=ORG_ID): + return client.delete( + f"/orgs/{org_id}/integrations/jira-connections/{connection_id}" + ) + + +def _set_default(client, connection_id, org_id=ORG_ID): + return client.patch(f"/orgs/{org_id}/integrations/jira-connections/{connection_id}") + + +def test_listing_is_empty_for_a_workspace_with_no_connections(routes): + client, _ = routes + + response = _list(client) + + assert response.status_code == 200 + assert response.json == [] + + +def test_creating_returns_201_with_the_serialized_row_and_no_token(routes): + client, service = routes + + response = _create(client) + + assert response.status_code == 201 + body = response.json + assert body["site_url"] == "acme.atlassian.net" + assert body["email"] == "a@acme.com" + assert body["is_default"] is False + # The raw token was passed through to the service (which is what would + # encrypt it); nothing in the response carries any form of it. + assert "access_token" not in body + assert "access_token_enc_chunks" not in body + assert service.connections[0].id == body["id"] + + +def test_creating_a_duplicate_account_is_409(routes): + client, service = routes + _create(client) + + response = _create(client) + + assert response.status_code == 409 + assert len(service.connections) == 1 + + +def test_listing_scopes_to_the_given_org(routes): + client, _ = routes + _create(client, site_url="ours.atlassian.net") + + ours = _list(client) + theirs = _list(client, org_id=OTHER_ORG_ID) + + assert [c["site_url"] for c in ours.json] == ["ours.atlassian.net"] + assert theirs.json == [] + + +@pytest.mark.parametrize( + "routes", + [{"connections": [FakeJiraConnection(ORG_ID, "acme.atlassian.net", "a@acme.com")]}], + indirect=True, +) +def test_deleting_an_unreferenced_connection_removes_it(routes): + client, service = routes + connection = service.connections[0] + + response = _delete(client, connection.id) + + assert response.status_code == 200 + assert service.connections == [] + + +@pytest.mark.parametrize( + "routes", + [{"connections": [FakeJiraConnection(ORG_ID, "acme.atlassian.net", "a@acme.com")]}], + indirect=True, +) +def test_deleting_a_referenced_connection_is_409_and_leaves_it_in_place(routes): + client, service = routes + connection = service.connections[0] + service.referenced_ids = {connection.id} + + response = _delete(client, connection.id) + + assert response.status_code == 409 + assert service.connections == [connection] + + +def test_deleting_an_unknown_connection_is_404(routes): + client, _ = routes + + assert _delete(client, str(uuid4())).status_code == 404 + + +@pytest.mark.parametrize( + "routes", + [{"connections": [FakeJiraConnection(ORG_ID, "acme.atlassian.net", "a@acme.com")]}], + indirect=True, +) +def test_deleting_from_another_workspace_is_404(routes): + client, service = routes + connection = service.connections[0] + + response = _delete(client, connection.id, org_id=OTHER_ORG_ID) + + assert response.status_code == 404 + assert service.connections == [connection] + + +def test_setting_default_on_an_unknown_connection_is_404(routes): + client, _ = routes + + assert _set_default(client, str(uuid4())).status_code == 404 + + +def test_setting_default_swaps_atomically(routes): + client, service = routes + _create(client, site_url="first.atlassian.net") + _create(client, site_url="second.atlassian.net") + first, second = service.connections + service.set_default_jira_connection(ORG_ID, first.id) + assert first.is_default is True + + response = _set_default(client, second.id) + + assert response.status_code == 200 + # Never both, never neither -- exactly the second one now. + assert first.is_default is False + assert second.is_default is True diff --git a/backend/analytics_server/tests/clustox_auth/__init__.py b/backend/analytics_server/tests/clustox_auth/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/clustox_auth/test_internal_token.py b/backend/analytics_server/tests/clustox_auth/test_internal_token.py new file mode 100644 index 000000000..f9a9493f5 --- /dev/null +++ b/backend/analytics_server/tests/clustox_auth/test_internal_token.py @@ -0,0 +1,59 @@ +import os + +from flask import Flask + +from mhq.clustox_auth.internal_token import register_internal_token_guard + + +def build_app(token): + if token is None: + os.environ.pop("INTERNAL_API_TOKEN", None) + else: + os.environ["INTERNAL_API_TOKEN"] = token + + app = Flask(__name__) + register_internal_token_guard(app) + + @app.route("/protected") + def protected(): + return {"ok": True} + + @app.route("/") + def health(): + return "hello" + + return app + + +def test_request_with_valid_token_is_allowed(): + app = build_app("secret-token") + client = app.test_client() + res = client.get("/protected", headers={"X-Internal-Token": "secret-token"}) + assert res.status_code == 200 + + +def test_request_without_token_is_rejected(): + app = build_app("secret-token") + res = app.test_client().get("/protected") + assert res.status_code == 403 + + +def test_request_with_wrong_token_is_rejected(): + app = build_app("secret-token") + client = app.test_client() + res = client.get("/protected", headers={"X-Internal-Token": "wrong"}) + assert res.status_code == 403 + + +def test_unset_env_fails_closed(): + """An unconfigured token must reject everything, never allow everything.""" + app = build_app(None) + client = app.test_client() + res = client.get("/protected", headers={"X-Internal-Token": "anything"}) + assert res.status_code == 403 + + +def test_health_endpoint_is_exempt(): + app = build_app("secret-token") + res = app.test_client().get("/") + assert res.status_code == 200 diff --git a/backend/analytics_server/tests/clustox_auth/test_sync_run.py b/backend/analytics_server/tests/clustox_auth/test_sync_run.py new file mode 100644 index 000000000..59e59dbda --- /dev/null +++ b/backend/analytics_server/tests/clustox_auth/test_sync_run.py @@ -0,0 +1,156 @@ +""" +CLUSTOX: workspace isolation during sync. + +The behaviour that matters is that one workspace failing does not prevent the +others from syncing, and that the failure is recorded rather than swallowed. +Upstream returned 200 whatever happened inside. +""" + +from unittest.mock import MagicMock, patch + +import pytest + + +class FakeRun: + def __init__(self, org_id): + self.org_id = org_id + self.status = "RUNNING" + self.detail = None + self.started_at = None + self.finished_at = None + + +class FakeSyncRunService: + """Records outcomes in memory so assertions do not need a database.""" + + def __init__(self): + self.runs = {} + + def start(self, org_id): + run = FakeRun(org_id) + self.runs[org_id] = run + return run + + def finish(self, run, status, detail=None): + run.status = status + run.detail = detail + + def latest_for_org(self, org_id): + return self.runs.get(org_id) + + +class FakeOrg: + def __init__(self, org_id, name): + self.id = org_id + self.name = name + + +class FakeLock: + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + +class FakeLockService: + def acquire_lock(self, _key): + return FakeLock() + + +@pytest.fixture +def sync_module(): + from mhq.api import sync as sync_module + + return sync_module + + +def build_app(sync_module): + from flask import Flask + + app = Flask(__name__) + app.register_blueprint(sync_module.app) + return app + + +def test_one_failing_workspace_does_not_stop_the_others(sync_module): + orgs = [FakeOrg("org-a", "A"), FakeOrg("org-b", "B"), FakeOrg("org-c", "C")] + fake_service = FakeSyncRunService() + + def trigger(org_id): + if org_id == "org-b": + raise RuntimeError("provider token rejected") + + with patch.object(sync_module, "CoreRepoService") as repo_cls, patch.object( + sync_module, "get_sync_run_service", return_value=fake_service + ), patch.object( + sync_module, "get_redis_lock_service", return_value=FakeLockService() + ), patch.object( + sync_module, "trigger_data_sync", side_effect=trigger + ) as trigger_mock: + repo_cls.return_value = MagicMock(get_all_orgs=lambda: orgs) + + client = build_app(sync_module).test_client() + res = client.post("/sync") + + assert res.status_code == 200 + # every workspace was attempted, including the one after the failure + assert trigger_mock.call_count == 3 + + assert fake_service.runs["org-a"].status == "SUCCESS" + assert fake_service.runs["org-b"].status == "FAILED" + assert fake_service.runs["org-c"].status == "SUCCESS" + + +def test_failure_detail_is_recorded_not_swallowed(sync_module): + orgs = [FakeOrg("org-a", "A")] + fake_service = FakeSyncRunService() + + with patch.object(sync_module, "CoreRepoService") as repo_cls, patch.object( + sync_module, "get_sync_run_service", return_value=fake_service + ), patch.object( + sync_module, "get_redis_lock_service", return_value=FakeLockService() + ), patch.object( + sync_module, + "trigger_data_sync", + side_effect=RuntimeError("provider token rejected"), + ): + repo_cls.return_value = MagicMock(get_all_orgs=lambda: orgs) + + client = build_app(sync_module).test_client() + res = client.post("/sync") + + body = res.get_json() + assert "0/1" in body["message"] + assert fake_service.runs["org-a"].status == "FAILED" + assert "provider token rejected" in fake_service.runs["org-a"].detail + + +def test_reports_how_many_workspaces_succeeded(sync_module): + orgs = [FakeOrg("org-a", "A"), FakeOrg("org-b", "B")] + fake_service = FakeSyncRunService() + + with patch.object(sync_module, "CoreRepoService") as repo_cls, patch.object( + sync_module, "get_sync_run_service", return_value=fake_service + ), patch.object( + sync_module, "get_redis_lock_service", return_value=FakeLockService() + ), patch.object( + sync_module, "trigger_data_sync" + ): + repo_cls.return_value = MagicMock(get_all_orgs=lambda: orgs) + + client = build_app(sync_module).test_client() + res = client.post("/sync") + + assert "2/2" in res.get_json()["message"] + + +def test_no_workspaces_is_not_an_error(sync_module): + with patch.object(sync_module, "CoreRepoService") as repo_cls: + repo_cls.return_value = MagicMock(get_all_orgs=lambda: []) + + client = build_app(sync_module).test_client() + res = client.post("/sync") + + assert res.status_code == 200 + assert res.get_json()["results"] == [] diff --git a/backend/analytics_server/tests/exapi/test_bitbucket_api.py b/backend/analytics_server/tests/exapi/test_bitbucket_api.py new file mode 100644 index 000000000..c8a259aca --- /dev/null +++ b/backend/analytics_server/tests/exapi/test_bitbucket_api.py @@ -0,0 +1,217 @@ +from datetime import datetime, timezone +from unittest.mock import MagicMock + +import pytest + +from mhq.exapi.bitbucket import BitbucketApiService, BitbucketRateLimitExceeded +from mhq.exapi.models.bitbucket import BitbucketPR, BitbucketRepo + +# CLUSTOX: fixtures are shaped exactly like Bitbucket Cloud's documented v2 +# payloads, not hand-simplified. A test asserting against a simplified shape +# would pass while the adapter misread the real API -- the encode-the-broken- +# shape-back-to-itself failure this project has shipped twice. + +BB_PR = { + "id": 42, + "title": "feat: add rate limiter", + "state": "MERGED", + "author": { + "uuid": "{a1b2c3d4-0000-4000-8000-000000000001}", + "nickname": "hamadr", + }, + "source": {"branch": {"name": "feat/rate-limiter"}}, + "destination": {"branch": {"name": "main"}}, + "created_on": "2026-08-20T10:00:00+00:00", + "updated_on": "2026-08-21T15:30:00+00:00", + "merge_commit": {"hash": "abc123def456"}, + "participants": [ + { + "role": "REVIEWER", + "approved": True, + "user": { + "uuid": "{b2c3d4e5-0000-4000-8000-000000000002}", + "nickname": "muzz", + }, + "participated_on": "2026-08-21T12:00:00+00:00", + } + ], + "links": {"html": {"href": "https://bitbucket.org/ws/repo/pull-requests/42"}}, +} + +BB_REPO = { + "uuid": "{c3d4e5f6-0000-4000-8000-000000000003}", + "name": "middleware", + "slug": "middleware", + "full_name": "clustox/middleware", + "description": "DORA metrics", + "mainbranch": {"name": "main"}, + "workspace": {"slug": "clustox"}, + "links": {"html": {"href": "https://bitbucket.org/clustox/middleware"}}, +} + + +def _response(json_body, status=200, headers=None): + response = MagicMock() + response.status_code = status + response.json.return_value = json_body + response.headers = headers or {} + return response + + +def _service_with_pages(pages): + service = BitbucketApiService("hamad@clustox.com", "token-123") + service._session = MagicMock() + service._session.get.side_effect = [_response(page) for page in pages] + return service + + +def test_pagination_follows_next_links(): + page_one = { + "values": [dict(BB_PR, id=1)], + "next": "https://api.bitbucket.org/2.0/repositories/ws/repo/pullrequests?page=2", + } + page_two = {"values": [dict(BB_PR, id=2)]} + service = _service_with_pages([page_one, page_two]) + + prs = service.get_repo_pull_requests( + "ws", "repo", datetime(2026, 1, 1, tzinfo=timezone.utc) + ) + + assert [pr["id"] for pr in prs] == [1, 2] + assert service._session.get.call_count == 2 + # The second call follows the `next` URL verbatim. + second_url = service._session.get.call_args_list[1].args[0] + assert second_url == page_one["next"] + + +def test_pr_listing_requests_all_states(): + # CLUSTOX: without an explicit state list Bitbucket returns ONLY open PRs. + # Every merged PR would silently vanish from lead time -- data fetched + # "successfully" and wrong, with no error anywhere to notice. + service = _service_with_pages([{"values": []}]) + + service.get_repo_pull_requests( + "ws", "repo", datetime(2026, 1, 1, tzinfo=timezone.utc) + ) + + params = service._session.get.call_args_list[0].kwargs["params"] + assert "MERGED" in str(params.get("state")) + assert "updated_on" in str(params.get("q")) + + +def test_429_raises_rate_limit_with_retry_after(): + service = BitbucketApiService("hamad@clustox.com", "token-123") + service._session = MagicMock() + service._session.get.return_value = _response( + {"error": {"message": "Rate limit exceeded"}}, + status=429, + headers={"Retry-After": "1800"}, + ) + + with pytest.raises(BitbucketRateLimitExceeded) as exc: + service.get_workspaces() + + assert exc.value.retry_after_seconds == 1800 + + +def test_auth_is_basic_session_auth(): + service = BitbucketApiService("hamad@clustox.com", "token-123") + + assert service._session.auth == ("hamad@clustox.com", "token-123") + + # And the token never leaks into a URL. + service._session = MagicMock() + service._session.auth = ("hamad@clustox.com", "token-123") + service._session.get.return_value = _response({"values": []}) + service.get_workspace_repos("clustox") + for call in service._session.get.call_args_list: + assert "token-123" not in call.args[0] + + +def test_repo_model_adapts_the_v2_shape(): + repo = BitbucketRepo(BB_REPO) + + assert repo.name == "middleware" + assert repo.slug == "middleware" + assert repo.org_name == "clustox" + assert repo.default_branch == "main" + assert repo.idempotency_key == "{c3d4e5f6-0000-4000-8000-000000000003}" + assert repo.web_url == "https://bitbucket.org/clustox/middleware" + + +def test_pr_model_adapts_the_v2_shape(): + pr = BitbucketPR(BB_PR) + + assert pr.number == 42 + assert pr.state == "MERGED" + assert pr.author_uuid == "{a1b2c3d4-0000-4000-8000-000000000001}" + assert pr.author_nickname == "hamadr" + assert pr.head_branch == "feat/rate-limiter" + assert pr.base_branch == "main" + assert pr.merge_commit_sha == "abc123def456" + assert pr.url == "https://bitbucket.org/ws/repo/pull-requests/42" + assert pr.updated_on == datetime(2026, 8, 21, 15, 30, tzinfo=timezone.utc) + + +def test_pr_model_tolerates_null_merge_commit(): + # CLUSTOX: `merge_commit` is null on every unmerged PR -- the common case, + # not an edge case. + open_pr = dict(BB_PR, state="OPEN", merge_commit=None) + + pr = BitbucketPR(open_pr) + + assert pr.state == "OPEN" + assert pr.merge_commit_sha is None + + +def test_pr_commits_paginate_like_everything_else(): + # CLUSTOX: commits feed first_commit_to_open in lead time -- a handler + # that skips them would degrade one lead-time stage to zero silently. + page = { + "values": [ + { + "hash": "abc123", + "message": "feat: thing", + "date": "2026-08-20T09:00:00+00:00", + "author": {"raw": "Hamad "}, + "links": { + "html": {"href": "https://bitbucket.org/ws/repo/commits/abc123"} + }, + } + ] + } + service = _service_with_pages([page]) + + commits = service.get_pr_commits("ws", "repo", 42) + + assert commits[0]["hash"] == "abc123" + + +def test_workspaces_use_the_permissions_endpoint_not_the_dead_one(): + # CLUSTOX: /2.0/workspaces returns 410 "CHANGE-2770 - Functionality has + # been deprecated" -- discovered live, after every fixture test passed + # against it. This pins the replacement URL so a refactor cannot drift + # back to the dead endpoint. + service = _service_with_pages( + [{"values": [{"workspace": {"slug": "clustox"}}, {"not_a_workspace": 1}]}] + ) + + workspaces = service.get_workspaces() + + url = service._session.get.call_args_list[0].args[0] + assert url.endswith("/2.0/user/permissions/workspaces") + assert workspaces == [{"slug": "clustox"}] + + +def test_check_pat_raises_rate_limit_instead_of_reading_as_invalid(): + # CLUSTOX: False here becomes "token is invalid, revoked or expired" at + # sync start -- and the post-429 resume window is exactly when that check + # runs next. A rate limit must stay a rate limit. + service = BitbucketApiService("hamad@clustox.com", "token-123") + service._session = MagicMock() + service._session.get.return_value = _response( + {}, status=429, headers={"Retry-After": "600"} + ) + + with pytest.raises(BitbucketRateLimitExceeded): + service.check_pat() diff --git a/backend/analytics_server/tests/exapi/test_jenkins.py b/backend/analytics_server/tests/exapi/test_jenkins.py new file mode 100644 index 000000000..f1cfdd48a --- /dev/null +++ b/backend/analytics_server/tests/exapi/test_jenkins.py @@ -0,0 +1,423 @@ +import gzip +import json +import socket +import threading +from datetime import datetime +from time import monotonic, sleep + +import pytest +import pytz +import requests + +from unittest.mock import patch + +from mhq.exapi.jenkins import ( + JOB_TREE, + JenkinsAddressNotAllowed, + JenkinsApiService, + assert_url_is_allowed, + job_path, +) +from tests.factories.models.exapi.jenkins import get_jenkins_nested_jobs_dict + + +def test_job_path_encodes_a_top_level_job(): + assert job_path("deploy-api") == "job/deploy-api" + + +def test_job_path_encodes_a_folder_job(): + # Jenkins addresses nested jobs by repeating /job/ per segment. + assert job_path("platform/deploy-api") == "job/platform/job/deploy-api" + + +def test_get_builds_filters_out_builds_at_or_before_the_bookmark(): + class FakeResponse: + status_code = 200 + + @staticmethod + def json(): + return { + "builds": [ + {"number": 3, "timestamp": 3000, "result": "SUCCESS"}, + {"number": 2, "timestamp": 2000, "result": "SUCCESS"}, + {"number": 1, "timestamp": 1000, "result": "SUCCESS"}, + ] + } + + @staticmethod + def raise_for_status(): + return None + + service = JenkinsApiService("https://jenkins.example.com", "user", "token") + service._get = lambda path: FakeResponse() + + bookmark = datetime.fromtimestamp(2, tz=pytz.UTC) + builds = service.get_builds("deploy-api", bookmark) + + assert [b["number"] for b in builds] == [3] + + +class FakeJsonResponse: + status_code = 200 + + def __init__(self, payload): + self._payload = payload + + def json(self): + return self._payload + + @staticmethod + def raise_for_status(): + return None + + +def _service_returning(payload): + service = JenkinsApiService("https://jenkins.example.com", "user", "token") + service.requested_paths = [] + + def _get(path): + service.requested_paths.append(path) + return FakeJsonResponse(payload) + + service._get = _get + return service + + +def test_the_job_tree_recurses_far_enough_for_folder_layouts(): + # A flat "jobs[...]" tree stops at the top level, so on a folder-organised + # Jenkins the picker offers folders instead of jobs. + assert JOB_TREE.count("jobs[") == 3 + + +def test_get_jobs_returns_nested_jobs_and_not_their_containers(): + service = _service_returning(get_jenkins_nested_jobs_dict()) + + jobs = service.get_jobs() + + assert [job["full_name"] for job in jobs] == [ + "deploy-legacy", + "platform/deploy-api", + "platform/web/main", + "platform/tooling/lint", + ] + # Mapping a folder or a multibranch project produces a URL with no "builds" + # key, which the handler reports as "No builds found" -- zero deployments + # forever, and no error anywhere. + assert not {"platform", "platform/web", "platform/tooling"} & { + job["full_name"] for job in jobs + } + assert jobs[2]["name"] == "main" + assert jobs[1]["url"].endswith("/job/platform/job/deploy-api/") + + +def test_get_jobs_survives_an_instance_with_no_jobs(): + assert _service_returning({}).get_jobs() == [] + + +# CLUSTOX: the ceiling tests below drive a real socket, not a fake response. +# The fake they replace yielded one byte per iter_content() iteration, which +# requests never does: urllib3 blocks until it has the whole requested chunk, +# so the deadline between chunks was not consulted until 64 KiB had arrived. +# The fake passed against an implementation that could not bound a real +# connection -- confidence with nothing behind it. +class LocalJenkins: + """ + An HTTP server on 127.0.0.1 that declares a Content-Length and then hands + the body over in pieces, optionally slowly. Content-Length (not chunked + transfer encoding) is the case that matters: chunked responses happen to + yield per HTTP chunk, so they hide the bug. + """ + + def __init__( + self, + body: bytes, + chunk_size: int = None, + interval: float = 0.0, + status_line: bytes = b"HTTP/1.1 200 OK", + headers=(), + ): + self._body = body + self._chunk_size = chunk_size or max(len(body), 1) + self._interval = interval + self._status_line = status_line + self._headers = list(headers) + self._stop = threading.Event() + self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._socket.bind(("127.0.0.1", 0)) + self._socket.listen(8) + self._socket.settimeout(0.25) + self._thread = threading.Thread(target=self._serve, daemon=True) + + @property + def url(self) -> str: + return "http://127.0.0.1:{}".format(self._socket.getsockname()[1]) + + def __enter__(self): + self._thread.start() + return self + + def __exit__(self, *exc_info): + self._stop.set() + self._thread.join(timeout=5) + self._socket.close() + return False + + def _serve(self): + while not self._stop.is_set(): + try: + connection, _ = self._socket.accept() + except socket.timeout: + continue + except OSError: + return + with connection: + try: + self._respond(connection) + except OSError: + # The client hit its ceiling and hung up mid-dribble, which + # is exactly what these tests are checking for. + pass + + def _respond(self, connection): + connection.settimeout(5) + request = b"" + while b"\r\n\r\n" not in request: + received = connection.recv(4096) + if not received: + return + request += received + + head = [ + self._status_line, + b"Content-Length: %d" % len(self._body), + b"Connection: close", + ] + head.extend(self._headers) + connection.sendall(b"\r\n".join(head) + b"\r\n\r\n") + + for start in range(0, len(self._body), self._chunk_size): + if self._stop.is_set(): + return + end = start + self._chunk_size + connection.sendall(self._body[start:end]) + if self._interval: + sleep(self._interval) + + +def _service(url, max_seconds=5): + return JenkinsApiService( + url, "user", "token", timeout=(2, 2), max_seconds=max_seconds + ) + + +# CLUSTOX: LocalJenkins listens on 127.0.0.1, which assert_url_is_allowed exists +# to refuse. These tests are about reading a real socket under a deadline, not +# about which addresses are allowed -- that has its own tests below -- so they +# stand the guard down rather than working around it with a fake response, which +# is what previously hid a bug in this exact code path. +@pytest.fixture +def loopback_allowed(): + with patch("mhq.exapi.jenkins.assert_url_is_allowed"): + yield + + +def test_a_dribbling_jenkins_is_cut_off_at_the_ceiling(loopback_allowed): + # 4000 bytes handed over 50 at a time every 50ms: four seconds to complete, + # with every gap comfortably inside the read timeout, so nothing but the + # total ceiling can stop it. The whole body is under one 64 KiB chunk, so + # an implementation that checks its deadline between iter_content() chunks + # blocks for the full four seconds before looking at the clock even once. + with LocalJenkins(b"x" * 4000, chunk_size=50, interval=0.05) as jenkins: + service = _service(jenkins.url, max_seconds=1) + + started = monotonic() + with pytest.raises(requests.exceptions.Timeout) as raised: + service.check_pat() + elapsed = monotonic() - started + + assert "ceiling" in str(raised.value) + # Generous, and still nowhere near the 4s the body needs to finish. The + # sequential sync loop is the thing being protected: every workspace behind + # this one waits for it. + assert elapsed < 2.5, f"ceiling did not bound the read: took {elapsed:.1f}s" + + +def test_a_prompt_jenkins_is_read_in_full(loopback_allowed): + with LocalJenkins(b'{"jobs": []}') as jenkins: + assert _service(jenkins.url).get_jobs() == [] + + +def test_a_body_larger_than_one_read_is_reassembled(loopback_allowed): + # read1() returns whatever is available rather than a full chunk, so the + # loop has to keep going until EOF instead of stopping at the first short + # read. + payload = { + "jobs": [ + { + "_class": "hudson.model.FreeStyleProject", + "name": f"deploy-{index}", + "fullName": f"deploy-{index}", + "url": f"http://jenkins/job/deploy-{index}/", + } + for index in range(2000) + ] + } + body = json.dumps(payload).encode() + assert len(body) > 64 * 1024 + + with LocalJenkins(body, chunk_size=1024) as jenkins: + jobs = _service(jenkins.url).get_jobs() + + assert len(jobs) == 2000 + assert jobs[-1]["full_name"] == "deploy-1999" + + +def test_a_gzipped_response_is_decoded(loopback_allowed): + # requests advertises gzip on every request. Reading response.raw without + # decode_content hands json.loads a gzip stream. + body = gzip.compress(json.dumps(get_jenkins_nested_jobs_dict()).encode()) + + with LocalJenkins(body, headers=[b"Content-Encoding: gzip"]) as jenkins: + jobs = _service(jenkins.url).get_jobs() + + assert [job["full_name"] for job in jobs] == [ + "deploy-legacy", + "platform/deploy-api", + "platform/web/main", + "platform/tooling/lint", + ] + + +def test_an_http_error_is_raised_rather_than_parsed(loopback_allowed): + with LocalJenkins(b"forbidden", status_line=b"HTTP/1.1 403 Forbidden") as jenkins: + service = _service(jenkins.url) + + assert service.check_pat() is False + with pytest.raises(requests.HTTPError): + service.get_jobs() + + +# CLUSTOX: base_url is admin-supplied and the server fetches it. ClustoxJenkinsSetup.tsx +# checks it in the browser, which is no check at all: an authenticated admin +# could point a workspace at http://169.254.169.254/latest and read the cloud +# instance credentials back out of the jobs endpoint. None of these tests +# resolve a name for real -- the resolver is patched, so they say what the guard +# does rather than what this machine's DNS happens to answer. +def _resolving_to(*addresses): + """Patches the resolver the guard uses, in the form getaddrinfo returns.""" + return patch( + "mhq.exapi.jenkins.socket.getaddrinfo", + return_value=[ + (socket.AF_INET, socket.SOCK_STREAM, 6, "", (address, 8080)) + for address in addresses + ], + ) + + +@pytest.mark.parametrize( + "url, address", + [ + ("http://127.0.0.1:8080", "127.0.0.1"), + ("http://10.0.0.5", "10.0.0.5"), + ("https://192.168.1.1", "192.168.1.1"), + # The cloud metadata endpoint: the reason this guard exists. + ("http://169.254.169.254/latest", "169.254.169.254"), + ("http://localhost:8080", "127.0.0.1"), + ("http://jenkins.internal", "172.16.4.9"), + ("http://[::1]", "::1"), + # An IPv4 private address wearing an IPv6 hat. + ("http://jenkins.internal", "::ffff:10.0.0.5"), + ], +) +def test_an_internal_address_is_refused(url, address): + with _resolving_to(address): + with pytest.raises(JenkinsAddressNotAllowed) as raised: + assert_url_is_allowed(url) + + assert address in str(raised.value) + + +def test_a_host_resolving_to_both_a_public_and_a_private_address_is_refused(): + # Rejecting on the first address only lets a hostname whose A records are + # public-then-private through, and which one is connected to is not ours to + # predict. + with _resolving_to("69.30.247.141", "10.0.0.5"): + with pytest.raises(JenkinsAddressNotAllowed): + assert_url_is_allowed("https://jenkins.example.com") + + +def test_the_real_jenkins_this_was_built_against_is_still_allowed(): + # https://jenkins-gpu.theclustox.com resolves to a public address. A guard + # that blocks the one Jenkins in use is not a fix. + with _resolving_to("69.30.247.141"): + assert_url_is_allowed("https://jenkins-gpu.theclustox.com/api/json") + + +@pytest.mark.parametrize( + "url", + [ + "file:///etc/passwd", + "gopher://jenkins.example.com", + "ftp://jenkins.example.com", + "jenkins.example.com", + "https://", + ], +) +def test_only_http_urls_with_a_host_are_accepted(url): + with pytest.raises(JenkinsAddressNotAllowed): + assert_url_is_allowed(url) + + +def test_a_name_that_does_not_resolve_reads_as_unreachable_not_as_refused(): + # Two different problems for the admin: an address we will not fetch is one + # he retypes, a name that does not resolve is one he checks DNS for. + with patch( + "mhq.exapi.jenkins.socket.getaddrinfo", side_effect=socket.gaierror("no such") + ): + with pytest.raises(requests.ConnectionError): + assert_url_is_allowed("https://jenkins.example.com") + + +def test_nothing_is_requested_when_the_address_is_refused(): + # The check is worth nothing after the fact: a request to 169.254.169.254 + # has leaked whatever it was going to leak by the time a response arrives. + with _resolving_to("169.254.169.254"), patch( + "mhq.exapi.jenkins.requests.get" + ) as requests_get: + with pytest.raises(JenkinsAddressNotAllowed): + _service("http://169.254.169.254").get_jobs() + + requests_get.assert_not_called() + + +def test_redirects_are_not_followed(): + # A public host answering with a redirect to an internal one walks straight + # past an address check performed on the URL we asked for. + with _resolving_to("69.30.247.141"), patch( + "mhq.exapi.jenkins.requests.get" + ) as requests_get: + requests_get.return_value.status_code = 302 + requests_get.return_value.headers = {"Location": "http://169.254.169.254/"} + requests_get.return_value.raw.read1.return_value = b"" + + with pytest.raises(requests.HTTPError) as raised: + _service("https://jenkins.example.com").get_jobs() + + assert requests_get.call_args.kwargs["allow_redirects"] is False + # Falling through as a 200-with-no-body would have parsed as {} and reported + # a redirecting Jenkins as an empty one. + assert "169.254.169.254" in str(raised.value) + + +def test_tls_verification_is_not_negotiable(): + with _resolving_to("69.30.247.141"), patch( + "mhq.exapi.jenkins.requests.get" + ) as requests_get: + requests_get.return_value.status_code = 200 + requests_get.return_value.headers = {} + requests_get.return_value.raw.read1.return_value = b"" + + _service("https://jenkins.example.com").check_pat() + + assert requests_get.call_args.kwargs["verify"] is True diff --git a/backend/analytics_server/tests/exapi/test_jira.py b/backend/analytics_server/tests/exapi/test_jira.py new file mode 100644 index 000000000..831185609 --- /dev/null +++ b/backend/analytics_server/tests/exapi/test_jira.py @@ -0,0 +1,289 @@ +from unittest.mock import MagicMock, patch + +import pytest + +from mhq.exapi.jira import ( + JiraApiError, + JiraApiService, + JiraRateLimitExceeded, +) + +# CLUSTOX: Jira integration, Phase 2 (issue sync). See +# docs/JIRA_INTEGRATION_PROPOSAL.md. + + +def _service() -> JiraApiService: + return JiraApiService( + email="jordan@mycompany.com", + api_token="tok", + site_url="mycompany.atlassian.net", + ) + + +def _response(status_code=200, json_data=None, text="", headers=None): + response = MagicMock() + response.status_code = status_code + response.json.return_value = json_data or {} + response.text = text + response.headers = headers or {} + return response + + +def _issue(issue_id="1", key="PAY-1", histories=None): + return { + "id": issue_id, + "key": key, + "fields": { + "summary": "Fix refund rounding", + "status": { + "name": "In Progress", + "statusCategory": {"name": "In Progress"}, + }, + "issuetype": {"name": "Bug"}, + "created": "2024-01-01T10:00:00.000+0000", + "updated": "2024-01-02T10:00:00.000+0000", + }, + "changelog": {"histories": histories or []}, + } + + +class TestCheckPat: + @patch("mhq.exapi.jira.requests.get") + def test_returns_true_on_200(self, mock_get): + mock_get.return_value = _response(200) + assert _service().check_pat() is True + + @patch("mhq.exapi.jira.requests.get") + def test_returns_false_on_anything_else(self, mock_get): + mock_get.return_value = _response(401) + assert _service().check_pat() is False + + +class TestSearchIssues: + @patch("mhq.exapi.jira.requests.post") + def test_posts_jql_and_expand_changelog_to_the_new_search_endpoint(self, mock_post): + mock_post.return_value = _response(200, {"issues": []}) + + _service().search_issues("project = PAY") + + url, kwargs = mock_post.call_args + assert url[0] == "https://mycompany.atlassian.net/rest/api/3/search/jql" + assert kwargs["json"]["jql"] == "project = PAY" + assert kwargs["json"]["expand"] == "changelog" + assert "nextPageToken" not in kwargs["json"] + + @patch("mhq.exapi.jira.requests.post") + def test_includes_next_page_token_when_given(self, mock_post): + mock_post.return_value = _response(200, {"issues": []}) + + _service().search_issues("project = PAY", next_page_token="abc") + + _, kwargs = mock_post.call_args + assert kwargs["json"]["nextPageToken"] == "abc" + + @patch("mhq.exapi.jira.requests.post") + def test_raises_rate_limit_error_on_429_with_retry_after(self, mock_post): + mock_post.return_value = _response(429, headers={"Retry-After": "30"}) + + with pytest.raises(JiraRateLimitExceeded) as exc_info: + _service().search_issues("project = PAY") + assert exc_info.value.retry_after == "30" + + @pytest.mark.parametrize("status", [401, 403]) + @patch("mhq.exapi.jira.requests.post") + def test_raises_api_error_on_auth_failure(self, mock_post, status): + mock_post.return_value = _response(status) + + with pytest.raises(JiraApiError) as exc_info: + _service().search_issues("project = PAY") + assert exc_info.value.status_code == status + + @patch("mhq.exapi.jira.requests.post") + def test_raises_api_error_on_other_failures(self, mock_post): + mock_post.return_value = _response(500, text="boom") + + with pytest.raises(JiraApiError): + _service().search_issues("project = PAY") + + +class TestGetAllIssues: + @patch("mhq.exapi.jira.requests.post") + def test_stops_when_there_is_no_next_page_token(self, mock_post): + mock_post.return_value = _response(200, {"issues": [_issue("1"), _issue("2")]}) + + issues = _service().get_all_issues("project = PAY") + + assert [i.id for i in issues] == ["1", "2"] + mock_post.assert_called_once() + + @patch("mhq.exapi.jira.requests.post") + def test_pages_until_the_token_runs_out(self, mock_post): + mock_post.side_effect = [ + _response(200, {"issues": [_issue("1")], "nextPageToken": "page2"}), + _response(200, {"issues": [_issue("2")]}), + ] + + issues = _service().get_all_issues("project = PAY") + + assert [i.id for i in issues] == ["1", "2"] + assert mock_post.call_count == 2 + + @patch("mhq.exapi.jira.requests.post") + def test_stops_on_an_empty_page_even_if_a_token_is_somehow_present(self, mock_post): + # Defensive: an empty `issues` page should halt pagination + # regardless of what nextPageToken says, so a misbehaving/unclear + # response can't spin this into an infinite loop. + mock_post.return_value = _response( + 200, {"issues": [], "nextPageToken": "page2"} + ) + + issues = _service().get_all_issues("project = PAY") + + assert issues == [] + mock_post.assert_called_once() + + @patch("mhq.exapi.jira.requests.post") + def test_parses_changelog_status_transitions(self, mock_post): + histories = [ + { + "id": "100", + "created": "2024-01-01T12:00:00.000+0000", + "items": [ + { + "field": "status", + "fromString": "To Do", + "toString": "In Progress", + } + ], + }, + # Not a status change -- should be dropped, not turned into a + # transition with a null to_status. + { + "id": "101", + "created": "2024-01-01T13:00:00.000+0000", + "items": [{"field": "assignee", "toString": "Jordan"}], + }, + ] + mock_post.return_value = _response( + 200, {"issues": [_issue("1", histories=histories)]} + ) + + issues = _service().get_all_issues("project = PAY") + + assert len(issues[0].changelog) == 1 + transition = issues[0].changelog[0] + assert transition.from_status == "To Do" + assert transition.to_status == "In Progress" + assert transition.idempotency_key == "100" + + +# CLUSTOX: Jira integration -- the Sprint rollup chart. See +# docs/JIRA_INTEGRATION_PROPOSAL.md §6D. +class TestGetBoardsForProject: + @patch("mhq.exapi.jira.requests.get") + def test_maps_every_board_field(self, mock_get): + mock_get.return_value = _response( + 200, + { + "isLast": True, + "values": [{"id": 1076, "name": "PZDA board", "type": "scrum"}], + }, + ) + + [board] = _service().get_boards_for_project("PZDA") + + assert board.id == 1076 + assert board.name == "PZDA board" + assert board.board_type == "scrum" + + @patch("mhq.exapi.jira.requests.get") + def test_paginates_via_start_at_until_is_last(self, mock_get): + mock_get.side_effect = [ + _response(200, {"isLast": False, "values": [{"id": 1, "type": "scrum"}]}), + _response(200, {"isLast": True, "values": [{"id": 2, "type": "scrum"}]}), + ] + + boards = _service().get_boards_for_project("PZDA") + + assert [b.id for b in boards] == [1, 2] + assert mock_get.call_args_list[1].kwargs["params"]["startAt"] == 1 + + @patch("mhq.exapi.jira.requests.get") + def test_returns_empty_list_when_the_project_has_no_boards(self, mock_get): + mock_get.return_value = _response(200, {"isLast": True, "values": []}) + assert _service().get_boards_for_project("PZDA") == [] + + +class TestGetSprintsForBoard: + @patch("mhq.exapi.jira.requests.get") + def test_maps_every_sprint_field_including_dates(self, mock_get): + mock_get.return_value = _response( + 200, + { + "isLast": True, + "values": [ + { + "id": 299, + "name": "PZDA Sprint 1", + "state": "closed", + "startDate": "2026-07-20T07:55:00.130Z", + "endDate": "2026-08-03T05:00:00.000Z", + } + ], + }, + ) + + [sprint] = _service().get_sprints_for_board(1076) + + assert sprint.id == 299 + assert sprint.name == "PZDA Sprint 1" + assert sprint.state == "closed" + assert sprint.start_date is not None + assert sprint.end_date is not None + + @patch("mhq.exapi.jira.requests.get") + def test_a_sprint_with_no_end_date_yet_does_not_error(self, mock_get): + # A currently-active sprint's endDate can be absent/still open. + mock_get.return_value = _response( + 200, + { + "isLast": True, + "values": [ + { + "id": 332, + "name": "Active sprint", + "state": "active", + "startDate": "2026-08-04T08:19:44.893Z", + } + ], + }, + ) + + [sprint] = _service().get_sprints_for_board(1076) + + assert sprint.end_date is None + + +class TestGetSprintIssueCounts: + @patch("mhq.exapi.jira.requests.get") + def test_reads_total_from_each_response_without_fetching_issue_bodies( + self, mock_get + ): + mock_get.side_effect = [ + _response(200, {"total": 355}), + _response(200, {"total": 272}), + ] + + planned, completed = _service().get_sprint_issue_counts(299) + + assert planned == 355 + assert completed == 272 + # maxResults=0 on both calls -- issue bodies are never requested. + for call in mock_get.call_args_list: + assert call.kwargs["params"]["maxResults"] == 0 + # The second call is the one scoped to done issues. + assert ( + mock_get.call_args_list[1].kwargs["params"]["jql"] + == "statusCategory = Done" + ) + assert "jql" not in mock_get.call_args_list[0].kwargs["params"] diff --git a/backend/analytics_server/tests/factories/models/exapi/jenkins.py b/backend/analytics_server/tests/factories/models/exapi/jenkins.py new file mode 100644 index 000000000..8e730b08b --- /dev/null +++ b/backend/analytics_server/tests/factories/models/exapi/jenkins.py @@ -0,0 +1,92 @@ +# CLUSTOX: recorded from a real Jenkins pipeline build. Shape matters more than +# values -- these encode the assumptions the handler makes about the API. +from typing import Dict + + +def get_jenkins_build_dict( + number: int = 42, + result: str = "SUCCESS", + timestamp: int = 1754827200000, # 2025-08-10T12:00:00Z + duration: int = 125000, + url: str = "https://jenkins.example.com/job/deploy-api/42/", + building: bool = False, + user_id: str = "hamad", + branch_name: str = "origin/main", + sha: str = "a1b2c3d4e5f6", +) -> Dict: + return { + "number": number, + "result": result, + "timestamp": timestamp, + "duration": duration, + "url": url, + "building": building, + "actions": [ + {"causes": [{"userId": user_id, "shortDescription": "Started by user"}]}, + {"lastBuiltRevision": {"SHA1": sha, "branch": [{"name": branch_name}]}}, + ], + } + + +def get_jenkins_build_dict_without_git_plugin(**kwargs) -> Dict: + """A freestyle job with no SCM: no branch, no revision, no user cause.""" + build = get_jenkins_build_dict(**kwargs) + build["actions"] = [{}] + return build + + +BASE_URL = "https://jenkins.example.com" + +FOLDER_CLASS = "com.cloudbees.hudson.plugins.folder.Folder" +MULTIBRANCH_CLASS = ( + "org.jenkinsci.plugins.workflow.multibranch.WorkflowMultiBranchProject" +) +PIPELINE_CLASS = "org.jenkinsci.plugins.workflow.job.WorkflowJob" +FREESTYLE_CLASS = "hudson.model.FreeStyleProject" + + +def _node(job_class: str, full_name: str, jobs=None) -> Dict: + node = { + "_class": job_class, + "name": full_name.split("/")[-1], + "fullName": full_name, + "url": f"{BASE_URL}/{'/'.join('job/' + s for s in full_name.split('/'))}/", + } + if jobs is not None: + node["jobs"] = jobs + return node + + +def get_jenkins_nested_jobs_dict() -> Dict: + """ + A folder-organised Jenkins, the layout the flat tree could not see past. + Note "platform/tooling/nested": a folder sitting at the deepest level the + tree asks for, so it comes back with no "jobs" key at all and only its + _class marks it as a container. + """ + return { + "jobs": [ + _node(FREESTYLE_CLASS, "deploy-legacy"), + _node( + FOLDER_CLASS, + "platform", + jobs=[ + _node(PIPELINE_CLASS, "platform/deploy-api"), + _node( + MULTIBRANCH_CLASS, + "platform/web", + jobs=[_node(PIPELINE_CLASS, "platform/web/main")], + ), + _node( + FOLDER_CLASS, + "platform/tooling", + jobs=[ + _node(FOLDER_CLASS, "platform/tooling/nested"), + _node(PIPELINE_CLASS, "platform/tooling/lint"), + ], + ), + ], + ), + _node(FOLDER_CLASS, "empty", jobs=[]), + ] + } diff --git a/backend/analytics_server/tests/service/Incidents/sync/test_etl_jira_incidents_handler.py b/backend/analytics_server/tests/service/Incidents/sync/test_etl_jira_incidents_handler.py new file mode 100644 index 000000000..991d976f1 --- /dev/null +++ b/backend/analytics_server/tests/service/Incidents/sync/test_etl_jira_incidents_handler.py @@ -0,0 +1,403 @@ +from datetime import datetime, timezone + +from mhq.service.incidents.sync.etl_jira_incidents_handler import ( + JiraIncidentsETLHandler, +) +from mhq.service.settings.models import JiraIncidentIssueTypesSetting +from mhq.store.models.incidents import ( + IncidentProvider, + IncidentSource, + IncidentStatus, + IncidentType, + TeamIncidentService, +) +from mhq.store.models.projects import OrgProject, Ticket, TicketState +from mhq.utils.string import uuid4_str +from tests.factories.models.incidents import get_org_incident_service +from tests.utilities import compare_objects_as_dicts + +ORG_ID = uuid4_str() +PROJECT_ID = uuid4_str() +SITE_URL = "https://clustox.atlassian.net" + + +def _org_project(project_id=PROJECT_ID, org_id=ORG_ID, name="Middleware"): + return OrgProject(id=project_id, org_id=org_id, name=name, provider="jira") + + +def _ticket( + ticket_id="t1", + key="MID-1", + project_id=PROJECT_ID, + status="Done", + status_category="Done", + issue_type="Bug", + summary="Something broke", + assignee=None, + created_at=None, + updated_at=None, +): + return Ticket( + id=ticket_id, + key=key, + org_project_id=project_id, + status=status, + status_category=status_category, + data={"issue_type": issue_type, "summary": summary, "assignee": assignee}, + created_at=created_at or datetime(2026, 1, 1, tzinfo=timezone.utc), + updated_at=updated_at or datetime(2026, 1, 10, tzinfo=timezone.utc), + ) + + +def _state(ticket_id, to_status, changed_at): + return TicketState(ticket_id=ticket_id, to_status=to_status, changed_at=changed_at) + + +class FakeProjectRepoService: + def __init__( + self, + active_projects=None, + tickets_and_states=(None, None), + team_ids_by_project=None, + ): + self._active_projects = active_projects or [] + self._tickets, self._states = tickets_and_states + self._team_ids_by_project = team_ids_by_project or {} + self.fetched_with = None + + def get_active_org_projects_for_provider(self, org_id, provider): + return self._active_projects + + def get_tickets_by_issue_types_for_project( + self, org_project_id, issue_types, from_time, to_time + ): + self.fetched_with = (org_project_id, issue_types, from_time, to_time) + if self._tickets is None: + return [], [] + return self._tickets, self._states + + def get_team_ids_for_org_project(self, org_project_id): + return self._team_ids_by_project.get(org_project_id, []) + + +class FakeIncidentsRepoService: + def __init__(self, existing_incident=None, existing_team_links=None): + self._existing_incident = existing_incident + self._existing_team_links = existing_team_links or [] + self.added_team_links = [] + self.looked_up_with = None + + def get_incident_by_key_type_provider_and_org(self, *args, **kwargs): + self.looked_up_with = args + return self._existing_incident + + def get_team_incident_services(self, team): + return [ + link for link in self._existing_team_links if str(link.team_id) == team.id + ] + + def add_team_incident_services(self, services): + self.added_team_links.extend(services) + + +class FakeSettingsService: + def __init__(self, issue_types=("Bug",)): + self._issue_types = list(issue_types) + + def get_or_set_default_settings(self, **kwargs): + class _Wrapper: + specific_settings = JiraIncidentIssueTypesSetting( + issue_types=self._issue_types + ) + + return _Wrapper() + + +def _handler( + project_repo_service=None, + incidents_repo_service=None, + settings_service=None, +): + return JiraIncidentsETLHandler( + ORG_ID, + None, + project_repo_service or FakeProjectRepoService(), + incidents_repo_service or FakeIncidentsRepoService(), + settings_service or FakeSettingsService(), + SITE_URL, + ) + + +class TestGetUpdatedIncidentServices: + def test_creates_one_service_per_active_jira_project(self): + project = _org_project() + handler = _handler( + project_repo_service=FakeProjectRepoService(active_projects=[project]) + ) + + services = handler.get_updated_incident_services([]) + + assert len(services) == 1 + service = services[0] + assert service.key == str(PROJECT_ID) + assert service.org_id == ORG_ID + assert service.provider == IncidentProvider.JIRA.value + assert service.source_type == IncidentSource.JIRA_ISSUE + assert service.name == "Middleware" + + def test_reuses_existing_service_id_for_the_same_project(self): + project = _org_project() + existing = get_org_incident_service( + service_id="existing-id", key=str(PROJECT_ID) + ) + existing.source_type = IncidentSource.JIRA_ISSUE + handler = _handler( + project_repo_service=FakeProjectRepoService(active_projects=[project]) + ) + + services = handler.get_updated_incident_services([existing]) + + assert services[0].id == "existing-id" + + def test_ignores_non_jira_incident_services(self): + project = _org_project() + github_service = get_org_incident_service(service_id="gh-1", key="some-repo") + handler = _handler( + project_repo_service=FakeProjectRepoService(active_projects=[project]) + ) + + services = handler.get_updated_incident_services([github_service]) + + # A new id was minted -- the github service was never considered a match. + assert services[0].id != "gh-1" + + +class TestToIncident: + def test_open_ticket_becomes_a_triggered_unresolved_incident(self): + ticket = _ticket(status="In Progress", status_category="In Progress") + handler = _handler() + + incident = handler._to_incident(ticket, states=[]) + + assert incident.status == IncidentStatus.TRIGGERED.value + assert incident.resolved_date is None + assert incident.creation_date == ticket.created_at + assert incident.key == "MID-1" + assert incident.incident_type == IncidentType.JIRA_ISSUE + assert incident.provider == IncidentProvider.JIRA.value + assert incident.url == f"{SITE_URL}/browse/MID-1" + + def test_done_ticket_with_states_resolves_at_its_last_transition(self): + ticket = _ticket(status="Done", status_category="Done") + states = [ + _state(ticket.id, "In Progress", datetime(2026, 1, 2, tzinfo=timezone.utc)), + _state(ticket.id, "Done", datetime(2026, 1, 5, tzinfo=timezone.utc)), + ] + handler = _handler() + + incident = handler._to_incident(ticket, states) + + assert incident.status == IncidentStatus.RESOLVED.value + assert incident.resolved_date == datetime(2026, 1, 5, tzinfo=timezone.utc) + + def test_done_ticket_with_no_states_falls_back_to_updated_at(self): + ticket = _ticket(status="Done", status_category="Done") + handler = _handler() + + incident = handler._to_incident(ticket, states=[]) + + assert incident.resolved_date == ticket.updated_at + + def test_looks_up_the_existing_incident_scoped_to_this_org(self): + # Cross-tenant collision regression: ticket.key ("MID-1") is + # site-local, not globally unique -- two orgs' Jira sites can + # land on the same key, so the lookup must be scoped by org_id, + # not just (key, incident_type, provider). + ticket = _ticket(key="MID-1") + incidents_repo_service = FakeIncidentsRepoService() + handler = _handler(incidents_repo_service=incidents_repo_service) + + handler._to_incident(ticket, states=[]) + + assert incidents_repo_service.looked_up_with == ( + ORG_ID, + "MID-1", + IncidentType.JIRA_ISSUE, + IncidentProvider.JIRA, + ) + + def test_upsert_reuses_existing_incident_id_and_created_at(self): + existing = get_org_incident_service(service_id=uuid4_str()) # unrelated fixture + from tests.factories.models import get_incident + + existing_incident = get_incident( + id="existing-incident-id", + created_at=datetime(2020, 1, 1, tzinfo=timezone.utc), + ) + ticket = _ticket() + handler = _handler( + incidents_repo_service=FakeIncidentsRepoService( + existing_incident=existing_incident + ) + ) + + incident = handler._to_incident(ticket, states=[]) + + assert incident.id == "existing-incident-id" + assert incident.created_at == datetime(2020, 1, 1, tzinfo=timezone.utc) + + def test_assignee_is_carried_through_when_present(self): + ticket = _ticket(assignee="jdoe") + handler = _handler() + + incident = handler._to_incident(ticket, states=[]) + + assert incident.assigned_to == "jdoe" + assert incident.assignees == ["jdoe"] + + def test_no_site_url_means_no_ticket_url(self): + ticket = _ticket() + handler = JiraIncidentsETLHandler( + ORG_ID, + None, + FakeProjectRepoService(), + FakeIncidentsRepoService(), + FakeSettingsService(), + None, + ) + + incident = handler._to_incident(ticket, states=[]) + + assert incident.url is None + + +class TestProcessServiceIncidents: + def test_no_configured_issue_types_short_circuits_without_querying(self): + project_repo = FakeProjectRepoService(tickets_and_states=([], [])) + handler = _handler( + project_repo_service=project_repo, + settings_service=FakeSettingsService(issue_types=[]), + ) + service = get_org_incident_service(service_id=uuid4_str(), key=str(PROJECT_ID)) + bookmark = datetime(2026, 1, 1, tzinfo=timezone.utc) + + incidents, maps, new_bookmark = handler.process_service_incidents( + service, bookmark + ) + + assert incidents == [] + assert maps == [] + assert new_bookmark == bookmark + assert project_repo.fetched_with is None + + def test_no_qualifying_tickets_returns_empty_and_keeps_bookmark(self): + project_repo = FakeProjectRepoService(tickets_and_states=([], [])) + handler = _handler(project_repo_service=project_repo) + service = get_org_incident_service(service_id=uuid4_str(), key=str(PROJECT_ID)) + bookmark = datetime(2026, 1, 1, tzinfo=timezone.utc) + + incidents, maps, new_bookmark = handler.process_service_incidents( + service, bookmark + ) + + assert incidents == [] + assert maps == [] + assert new_bookmark == bookmark + + def test_qualifying_tickets_become_incidents_and_advance_the_bookmark(self): + ticket = _ticket( + ticket_id="t1", + key="MID-1", + updated_at=datetime(2026, 2, 1, tzinfo=timezone.utc), + ) + project_repo = FakeProjectRepoService(tickets_and_states=([ticket], [])) + handler = _handler(project_repo_service=project_repo) + service = get_org_incident_service(service_id=uuid4_str(), key=str(PROJECT_ID)) + bookmark = datetime(2026, 1, 1, tzinfo=timezone.utc) + + incidents, maps, new_bookmark = handler.process_service_incidents( + service, bookmark + ) + + assert len(incidents) == 1 + assert incidents[0].key == "MID-1" + assert len(maps) == 1 + assert maps[0].incident_id == incidents[0].id + assert maps[0].service_id == service.id + assert new_bookmark == datetime(2026, 2, 1, tzinfo=timezone.utc) + + def test_passes_the_configured_issue_types_to_the_ticket_query(self): + project_repo = FakeProjectRepoService(tickets_and_states=([], [])) + handler = _handler( + project_repo_service=project_repo, + settings_service=FakeSettingsService(issue_types=["Bug", "Incident"]), + ) + service = get_org_incident_service(service_id=uuid4_str(), key=str(PROJECT_ID)) + + handler.process_service_incidents( + service, datetime(2026, 1, 1, tzinfo=timezone.utc) + ) + + fetched_project_id, fetched_issue_types, _, _ = project_repo.fetched_with + assert fetched_project_id == str(PROJECT_ID) + assert fetched_issue_types == ["Bug", "Incident"] + + +class TestEnsureTeamLinks: + def test_links_teams_with_the_project_actively_selected(self): + team_id = uuid4_str() + project_repo = FakeProjectRepoService( + tickets_and_states=([], []), + team_ids_by_project={str(PROJECT_ID): [team_id]}, + ) + incidents_repo = FakeIncidentsRepoService() + handler = _handler( + project_repo_service=project_repo, incidents_repo_service=incidents_repo + ) + service = get_org_incident_service(service_id=uuid4_str(), key=str(PROJECT_ID)) + + handler.process_service_incidents( + service, datetime(2026, 1, 1, tzinfo=timezone.utc) + ) + + assert len(incidents_repo.added_team_links) == 1 + link = incidents_repo.added_team_links[0] + assert link.team_id == team_id + assert link.service_id == service.id + + def test_does_not_duplicate_an_existing_link(self): + team_id = uuid4_str() + service_id = uuid4_str() + project_repo = FakeProjectRepoService( + tickets_and_states=([], []), + team_ids_by_project={str(PROJECT_ID): [team_id]}, + ) + incidents_repo = FakeIncidentsRepoService( + existing_team_links=[ + TeamIncidentService(team_id=team_id, service_id=service_id) + ] + ) + handler = _handler( + project_repo_service=project_repo, incidents_repo_service=incidents_repo + ) + service = get_org_incident_service(service_id=service_id, key=str(PROJECT_ID)) + + handler.process_service_incidents( + service, datetime(2026, 1, 1, tzinfo=timezone.utc) + ) + + assert incidents_repo.added_team_links == [] + + def test_no_teams_selected_the_project_links_nothing(self): + project_repo = FakeProjectRepoService(tickets_and_states=([], [])) + incidents_repo = FakeIncidentsRepoService() + handler = _handler( + project_repo_service=project_repo, incidents_repo_service=incidents_repo + ) + service = get_org_incident_service(service_id=uuid4_str(), key=str(PROJECT_ID)) + + handler.process_service_incidents( + service, datetime(2026, 1, 1, tzinfo=timezone.utc) + ) + + assert incidents_repo.added_team_links == [] diff --git a/backend/analytics_server/tests/service/Incidents/test_integration.py b/backend/analytics_server/tests/service/Incidents/test_integration.py new file mode 100644 index 000000000..b97b1bb29 --- /dev/null +++ b/backend/analytics_server/tests/service/Incidents/test_integration.py @@ -0,0 +1,94 @@ +from mhq.service.incidents.integration import IncidentsIntegrationService +from mhq.service.settings.models import IncidentSourcesSetting +from mhq.store.models.incidents import IncidentProvider, IncidentSource +from mhq.utils.string import uuid4_str + +ORG_ID = uuid4_str() + + +class FakeSettingsService: + def __init__(self, incident_sources): + self._incident_sources = incident_sources + self.saved = False + + def get_settings(self, **kwargs): + class _Wrapper: + specific_settings = IncidentSourcesSetting( + incident_sources=self._incident_sources + ) + + return _Wrapper() if self._incident_sources is not None else None + + def save_settings(self, **kwargs): + self.saved = True + return self.get_settings() + + +class FakeCoreRepoService: + def __init__(self, integrations_by_name): + self._integrations_by_name = integrations_by_name + + def get_org_integrations_for_names(self, org_id, names): + return [ + self._integrations_by_name[name] + for name in names + if name in self._integrations_by_name + ] + + +class FakeIntegration: + def __init__(self, name): + self.name = name + + +def _service(incident_sources, linked_providers): + settings_service = FakeSettingsService(incident_sources) + core_repo_service = FakeCoreRepoService( + {name: FakeIntegration(name) for name in linked_providers} + ) + return IncidentsIntegrationService(core_repo_service, settings_service) + + +class TestGetOrgProviders: + def test_jira_issue_source_enabled_and_linked_returns_jira(self): + service = _service( + incident_sources=[IncidentSource.JIRA_ISSUE], + linked_providers=[IncidentProvider.JIRA.value], + ) + + assert service.get_org_providers(ORG_ID) == [IncidentProvider.JIRA.value] + + def test_jira_issue_source_disabled_excludes_jira_even_if_linked(self): + service = _service( + incident_sources=[IncidentSource.GIT_REPO], + linked_providers=[ + IncidentProvider.JIRA.value, + IncidentProvider.GITHUB.value, + ], + ) + + # Jira is linked, but the org hasn't opted the source in -- + # get_org_providers must not surface it. + providers = service.get_org_providers(ORG_ID) + assert IncidentProvider.JIRA.value not in providers + + def test_jira_issue_source_enabled_but_not_linked_returns_nothing_for_jira(self): + service = _service( + incident_sources=[IncidentSource.JIRA_ISSUE], + linked_providers=[], + ) + + assert service.get_org_providers(ORG_ID) == [] + + def test_both_git_repo_and_jira_issue_enabled_returns_both(self): + service = _service( + incident_sources=[IncidentSource.GIT_REPO, IncidentSource.JIRA_ISSUE], + linked_providers=[ + IncidentProvider.JIRA.value, + IncidentProvider.GITHUB.value, + ], + ) + + providers = service.get_org_providers(ORG_ID) + assert IncidentProvider.JIRA.value in providers + assert IncidentProvider.GITHUB.value in providers diff --git a/backend/analytics_server/tests/service/Incidents/test_mean_time_to_recovery.py b/backend/analytics_server/tests/service/Incidents/test_mean_time_to_recovery.py index 22dc0671b..e8856223e 100644 --- a/backend/analytics_server/tests/service/Incidents/test_mean_time_to_recovery.py +++ b/backend/analytics_server/tests/service/Incidents/test_mean_time_to_recovery.py @@ -7,7 +7,6 @@ from tests.factories.models import get_incident - first_week_2024 = datetime(2024, 1, 1, 0, 0, 0, tzinfo=pytz.UTC) second_week_2024 = datetime(2024, 1, 8, 0, 0, 0, tzinfo=pytz.UTC) third_week_2024 = datetime(2024, 1, 15, 0, 0, 0, tzinfo=pytz.UTC) diff --git a/backend/analytics_server/tests/service/bookmark/__init__.py b/backend/analytics_server/tests/service/bookmark/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/service/bookmark/test_bookmark.py b/backend/analytics_server/tests/service/bookmark/test_bookmark.py new file mode 100644 index 000000000..94b7c3640 --- /dev/null +++ b/backend/analytics_server/tests/service/bookmark/test_bookmark.py @@ -0,0 +1,103 @@ +from datetime import datetime, timedelta, timezone +from unittest.mock import MagicMock + +from mhq.service.bookmark.bookmark import BookmarkService +from mhq.service.bookmark.bookmark_types import BookmarkType +from mhq.store.models.projects import ProjectIssuesBookmark + +# CLUSTOX: Jira integration, Phase 2 (issue sync). This repo has no +# existing test coverage for BookmarkService at all -- scoped to just the +# PROJECT_ISSUES_BOOKMARK path added for Jira issue sync, not a full +# backfill of the pre-existing (also untested) bookmark types. See +# docs/JIRA_INTEGRATION_PROPOSAL.md. + +ORG_PROJECT_ID = "proj-1" +PROVIDER = "jira" + + +def _service(project_repo_service=None) -> BookmarkService: + return BookmarkService( + code_repo_service=MagicMock(), + workflow_repo_service=MagicMock(), + incident_repo_service=MagicMock(), + project_repo_service=project_repo_service or MagicMock(), + ) + + +class TestGetBookmark: + def test_defaults_to_default_sync_days_ago_when_no_bookmark_row_exists(self): + repo = MagicMock() + repo.get_project_issues_bookmark.return_value = None + + bookmark = _service(repo).get_bookmark( + ORG_PROJECT_ID, BookmarkType.PROJECT_ISSUES_BOOKMARK, PROVIDER + ) + + expected = datetime.now(timezone.utc) - timedelta( + days=BookmarkService.DEFAULT_SYNC_DAYS + ) + assert abs((expected - bookmark).total_seconds()) < 5 + + def test_uses_a_custom_default_sync_days_when_given(self): + repo = MagicMock() + repo.get_project_issues_bookmark.return_value = None + + bookmark = _service(repo).get_bookmark( + ORG_PROJECT_ID, + BookmarkType.PROJECT_ISSUES_BOOKMARK, + PROVIDER, + default_sync_days=90, + ) + + expected = datetime.now(timezone.utc) - timedelta(days=90) + assert abs((expected - bookmark).total_seconds()) < 5 + + def test_returns_the_stored_bookmark_when_one_exists(self): + stored = datetime(2024, 6, 1, tzinfo=timezone.utc) + repo = MagicMock() + repo.get_project_issues_bookmark.return_value = ProjectIssuesBookmark( + org_project_id=ORG_PROJECT_ID, + provider=PROVIDER, + bookmark=stored.isoformat(), + ) + + bookmark = _service(repo).get_bookmark( + ORG_PROJECT_ID, BookmarkType.PROJECT_ISSUES_BOOKMARK, PROVIDER + ) + + assert bookmark == stored + + +class TestUpdateBookmark: + def test_persists_the_new_timestamp_for_the_right_project_and_provider(self): + repo = MagicMock() + repo.get_project_issues_bookmark.return_value = None + new_bookmark = datetime(2024, 7, 1, tzinfo=timezone.utc) + + _service(repo).update_bookmark( + ORG_PROJECT_ID, + BookmarkType.PROJECT_ISSUES_BOOKMARK, + PROVIDER, + new_bookmark, + ) + + repo.update_project_issues_bookmark.assert_called_once() + saved = repo.update_project_issues_bookmark.call_args[0][0] + assert str(saved.org_project_id) == ORG_PROJECT_ID + assert saved.provider == PROVIDER + assert saved.bookmark == new_bookmark.isoformat() + + +class TestResetOrgBookmarks: + def test_resets_every_project_issues_bookmark_for_the_org(self): + existing = ProjectIssuesBookmark( + org_project_id=ORG_PROJECT_ID, provider=PROVIDER, bookmark="old" + ) + repo = MagicMock() + repo.get_all_org_project_issues_bookmarks.return_value = [existing] + reset_to = datetime(2024, 1, 1, tzinfo=timezone.utc) + + _service(repo).reset_org_bookmarks("org-1", reset_to) + + assert existing.bookmark == reset_to.isoformat() + repo.update_project_issues_bookmarks.assert_called_once_with([existing]) diff --git a/backend/analytics_server/tests/service/code/sync/test_etl_bitbucket_handler.py b/backend/analytics_server/tests/service/code/sync/test_etl_bitbucket_handler.py new file mode 100644 index 000000000..36ed75e4b --- /dev/null +++ b/backend/analytics_server/tests/service/code/sync/test_etl_bitbucket_handler.py @@ -0,0 +1,455 @@ +from datetime import datetime, timezone +from uuid import uuid4 + +from mhq.service.code.sync.etl_bitbucket_handler import BitbucketETLHandler +from mhq.store.models.code import ( + CodeProvider, + PullRequestEventState, + PullRequestState, +) +from tests.exapi.test_bitbucket_api import BB_PR + + +# CLUSTOX: a fake, not a mock -- it narrows and returns exactly what the real +# client's contract promises, so these tests exercise the handler's adaptation +# rather than restating its internals. +class FakeBitbucketApiService: + def __init__(self, prs=None, activity=None, diffstat=None, commits=None): + self._prs = prs or [] + self._activity = activity if activity is not None else [] + self._diffstat = diffstat if diffstat is not None else [] + self._commits = commits if commits is not None else [] + + def get_repo_pull_requests(self, workspace, repo_slug, updated_since): + return self._prs + + def get_pr_activity(self, workspace, repo_slug, pr_id): + return self._activity + + def get_pr_diffstat(self, workspace, repo_slug, pr_id): + return self._diffstat + + def get_pr_commits(self, workspace, repo_slug, pr_id): + return self._commits + + +class FakeCodeRepoService: + def get_repo_pr_by_number(self, repo_id, number): + return None + + def get_pr_events(self, pr_model): + return [] + + +class _Repo: + def __init__(self): + self.id = uuid4() + self.org_name = "clustox" + self.slug = "middleware" + self.idempotency_key = "{c3d4e5f6-0000-4000-8000-000000000003}" + + +def _handler(api): + from mhq.service.code.sync.etl_code_analytics import CodeETLAnalyticsService + + return BitbucketETLHandler( + org_id="org-1", + bitbucket_api_service=api, + code_repo_service=FakeCodeRepoService(), + code_etl_analytics_service=CodeETLAnalyticsService(), + bitbucket_revert_pr_sync_handler=None, + ) + + +def _sync(prs, **api_kwargs): + handler = _handler(FakeBitbucketApiService(prs=prs, **api_kwargs)) + return handler.get_repo_pull_requests_data( + _Repo(), datetime(2026, 1, 1, tzinfo=timezone.utc) + ) + + +def test_state_mapping_covers_all_four_bitbucket_states(): + prs = [ + dict(BB_PR, id=1, state="MERGED"), + dict(BB_PR, id=2, state="OPEN", merge_commit=None), + dict(BB_PR, id=3, state="DECLINED", merge_commit=None), + dict(BB_PR, id=4, state="SUPERSEDED", merge_commit=None), + ] + + pull_requests, _, _ = _sync(prs) + + states = {pr.number: pr.state for pr in pull_requests} + assert states[1] == PullRequestState.MERGED + assert states[2] == PullRequestState.OPEN + # CLUSTOX: DECLINED and SUPERSEDED both fold into CLOSED -- our model has + # three states, Bitbucket four. + assert states[3] == PullRequestState.CLOSED + assert states[4] == PullRequestState.CLOSED + + +def test_unknown_state_skips_that_pr_not_the_batch(): + prs = [ + dict(BB_PR, id=1, state="MERGED"), + dict(BB_PR, id=2, state="SOMETHING_NEW", merge_commit=None), + dict(BB_PR, id=3, state="OPEN", merge_commit=None), + ] + + pull_requests, _, _ = _sync(prs) + + assert [pr.number for pr in pull_requests] == [1, 3] + + +def test_merged_pr_state_changed_at_is_updated_on(): + # CLUSTOX: Bitbucket has no merged_at field anywhere in the PR object. + # updated_on at the moment of merge is the closest truth, and it is what + # lead time keys on -- this pin is the whole reason the test exists. + pull_requests, _, _ = _sync([BB_PR]) + + assert pull_requests[0].state_changed_at == datetime( + 2026, 8, 21, 15, 30, tzinfo=timezone.utc + ) + + +def test_open_pr_has_no_merge_commit_sha(): + pull_requests, _, _ = _sync([dict(BB_PR, state="OPEN", merge_commit=None)]) + + assert pull_requests[0].merge_commit_sha is None + + +def test_author_is_the_nickname_with_uuid_fallback(): + # CLUSTOX: nickname, not uuid. The contributor dropdown lists `author` + # strings verbatim -- storing the uuid would put "{a1b2c3d4-...}" in the + # UI. The uuid stays available in `data` for a future identity layer. + pull_requests, _, _ = _sync([BB_PR]) + assert pull_requests[0].author == "hamadr" + + no_nickname = dict(BB_PR, author={"uuid": "{u-1}"}) + pull_requests, _, _ = _sync([no_nickname]) + assert pull_requests[0].author == "{u-1}" + + +def test_provider_and_url_come_through(): + pull_requests, _, _ = _sync([BB_PR]) + + assert pull_requests[0].provider == CodeProvider.BITBUCKET.value + assert pull_requests[0].url == "https://bitbucket.org/ws/repo/pull-requests/42" + + +def test_reviewers_on_a_merged_pr_come_from_review_events(): + # CLUSTOX: create_pr_metrics -- shared with GitHub and GitLab -- REBUILDS + # `reviewers` from the review events' actors for every non-open PR. The + # participants list only survives on OPEN PRs. Asserting participants on a + # merged PR would pin a contract the shared analytics layer deliberately + # overrides. + activity = [ + { + "approval": { + "date": "2026-08-21T12:00:00+00:00", + "user": {"uuid": "{u-2}", "nickname": "muzz"}, + } + } + ] + + pull_requests, _, _ = _sync([BB_PR], activity=activity) + + assert pull_requests[0].reviewers == ["muzz"] + + +def test_reviewers_on_an_open_pr_are_the_reviewer_role_participants(): + pr = dict( + BB_PR, + state="OPEN", + merge_commit=None, + participants=[ + { + "role": "REVIEWER", + "approved": False, + "user": {"uuid": "{u-2}", "nickname": "muzz"}, + "participated_on": "2026-08-21T12:00:00+00:00", + }, + { + "role": "PARTICIPANT", + "approved": False, + "user": {"uuid": "{u-3}", "nickname": "drive-by"}, + "participated_on": "2026-08-21T13:00:00+00:00", + }, + ], + ) + + pull_requests, _, _ = _sync([pr]) + + assert pull_requests[0].reviewers == ["muzz"] + + +class RateLimitingApiService(FakeBitbucketApiService): + """Raises 429 when asked for the activity of `explode_at_pr`.""" + + def __init__(self, prs, explode_at_pr): + super().__init__(prs=prs) + self._explode_at_pr = explode_at_pr + + def get_pr_activity(self, workspace, repo_slug, pr_id): + if pr_id == self._explode_at_pr: + from mhq.exapi.bitbucket import BitbucketRateLimitExceeded + + raise BitbucketRateLimitExceeded("limited", retry_after_seconds=1800) + return [] + + +class BrokenDiffstatApiService(FakeBitbucketApiService): + def get_pr_diffstat(self, workspace, repo_slug, pr_id): + raise RuntimeError("diffstat exploded") + + +def test_activity_becomes_review_events_with_states_and_actors(): + activity = [ + { + "comment": { + "id": 9001, + "created_on": "2026-08-20T11:00:00+00:00", + "user": {"nickname": "muzz"}, + "content": {"raw": "nit: rename this"}, + } + }, + { + "approval": { + "date": "2026-08-21T12:00:00+00:00", + "user": {"nickname": "muzz"}, + } + }, + # An update entry -- present in every real feed, not a review. + {"update": {"state": "OPEN", "date": "2026-08-20T10:30:00+00:00"}}, + ] + + pull_requests, _, events = _sync([BB_PR], activity=activity) + + assert len(events) == 2 + states = sorted(e.data["state"] for e in events) + # CLUSTOX: asserted via the enum, because the analytics layer compares + # data["state"] == PullRequestEventState.X.value -- the literal casing is + # the contract, and uppercase here would silently break first-response + # and rework detection downstream. + assert states == sorted( + [ + PullRequestEventState.APPROVED.value, + PullRequestEventState.COMMENTED.value, + ] + ) + assert all(e.actor_username == "muzz" for e in events) + # The earliest review drives first_response_time; it must be non-None + # once a review event exists on a merged PR. + assert pull_requests[0].first_response_time is not None + + +def test_malformed_activity_entry_skips_that_entry_not_the_batch(): + activity = [ + {"approval": {"date": None, "user": None}}, + {"comment": "not-even-a-dict"}, + { + "approval": { + "date": "2026-08-21T12:00:00+00:00", + "user": {"nickname": "muzz"}, + } + }, + ] + + _, _, events = _sync([BB_PR], activity=activity) + + assert len(events) == 1 + assert events[0].data["state"] == PullRequestEventState.APPROVED.value + + +def test_diffstat_sums_land_in_code_stats(): + diffstat = [ + {"lines_added": 100, "lines_removed": 20, "status": "modified"}, + {"lines_added": 30, "lines_removed": 5, "status": "added"}, + ] + + pull_requests, _, _ = _sync([BB_PR], diffstat=diffstat) + + stats = pull_requests[0].meta["code_stats"] + assert stats["additions"] == 130 + assert stats["deletions"] == 25 + assert stats["changed_files"] == 2 + + +def test_diffstat_failure_keeps_the_pr_and_omits_code_stats(): + # CLUSTOX: the diffstat is a separate request and can fail independently. + # The PR must stay (lead time, contributor filter) with code_stats absent + # -- LOC undercounts honestly. Dropping the PR would corrupt four metrics + # to protect one; writing zeros would claim an empty diff that never was. + handler = _handler(BrokenDiffstatApiService(prs=[BB_PR])) + + pull_requests, _, _ = handler.get_repo_pull_requests_data( + _Repo(), datetime(2026, 1, 1, tzinfo=timezone.utc) + ) + + assert len(pull_requests) == 1 + assert "code_stats" not in pull_requests[0].meta + # The model property reads 0 through its own default -- measured-nothing, + # not a crash. + assert pull_requests[0].additions == 0 + + +def test_429_keeps_already_processed_prs_and_stops(): + prs = [ + dict(BB_PR, id=1), + dict(BB_PR, id=2), + dict(BB_PR, id=3), + dict(BB_PR, id=4), + ] + handler = _handler(RateLimitingApiService(prs, explode_at_pr=3)) + + pull_requests, _, _ = handler.get_repo_pull_requests_data( + _Repo(), datetime(2026, 1, 1, tzinfo=timezone.utc) + ) + + assert [pr.number for pr in pull_requests] == [1, 2] + + +def test_429_does_not_advance_past_unfetched_prs(): + # CLUSTOX: the caller advances the bookmark to the newest returned PR's + # updated_on. PRs 3 and 4 were not returned, so the resume point must not + # cover them -- covering them would lose their data forever, the worse of + # the two bookmark failure directions. + prs = [ + dict(BB_PR, id=1, updated_on="2026-08-21T10:00:00+00:00"), + dict(BB_PR, id=2, updated_on="2026-08-21T11:00:00+00:00"), + dict(BB_PR, id=3, updated_on="2026-08-21T12:00:00+00:00"), + dict(BB_PR, id=4, updated_on="2026-08-21T13:00:00+00:00"), + ] + handler = _handler(RateLimitingApiService(prs, explode_at_pr=3)) + + pull_requests, _, _ = handler.get_repo_pull_requests_data( + _Repo(), datetime(2026, 1, 1, tzinfo=timezone.utc) + ) + + newest_returned = max(pr.updated_at for pr in pull_requests) + assert newest_returned == datetime(2026, 8, 21, 11, 0, tzinfo=timezone.utc) + + +def test_merged_pr_commits_are_adapted(): + commits = [ + { + "hash": "abc123", + "message": "feat: thing", + "date": "2026-08-20T09:00:00+00:00", + "author": {"raw": "Hamad "}, + "links": {"html": {"href": "https://bitbucket.org/ws/repo/commits/abc123"}}, + }, + {"message": "corrupt commit with no hash"}, + ] + + _, pr_commits, _ = _sync([BB_PR], commits=commits) + + assert len(pr_commits) == 1 + assert pr_commits[0].hash == "abc123" + assert pr_commits[0].author == "Hamad " + + +def test_every_provider_reaches_the_factory(): + # CLUSTOX: a provider enum value with no factory branch fails at sync + # time with NotImplementedError -- for the org that linked it, silently, + # in a scheduler log. This pins enum and factory together. + import inspect + + from mhq.service.code.sync.etl_code_factory import CodeETLFactory + + source = inspect.getsource(CodeETLFactory) + for provider in CodeProvider: + assert ( + f"CodeProvider.{provider.name}.value" in source + ), f"CodeProvider.{provider.name} has no CodeETLFactory branch" + + +def test_every_code_provider_is_in_the_sync_bucket(): + # CLUSTOX: CODE_INTEGRATION_BUCKET decides which orgs the scheduler syncs. + # A provider in the factory but not the bucket links fine in the UI and + # then never syncs -- no error, no log, just a dashboard that stays empty. + from mhq.service.code.integration import CODE_INTEGRATION_BUCKET + + for provider in CodeProvider: + assert ( + provider.value in CODE_INTEGRATION_BUCKET + ), f"CodeProvider.{provider.name} missing from CODE_INTEGRATION_BUCKET" + + +def test_a_bitbucket_capable_build_changes_nothing_for_other_orgs(): + # CLUSTOX: provider isolation. The bucket gained a third entry, and this + # pins what that must NOT mean: an org with only GitHub linked still gets + # exactly ['github'] from the scheduler's provider lookup -- the sync + # plan for every existing org is byte-identical to before this feature. + from mhq.service.code.integration import CodeIntegrationService + + class _Integration: + def __init__(self, name): + self.name = name + + class GithubOnlyCoreRepoService: + def get_org_integrations_for_names(self, org_id, names): + # The service asks for every bucket member; only what is actually + # linked comes back. + assert "bitbucket" in names + return [_Integration("github")] + + providers = CodeIntegrationService(GithubOnlyCoreRepoService()).get_org_providers( + "org-1" + ) + + assert providers == ["github"] + + +def test_open_pr_has_no_state_changed_at(): + # CLUSTOX: GitHub sets None for OPEN; a timestamp here would be a silent + # cross-provider inconsistency in a load-bearing column. + pull_requests, _, _ = _sync([dict(BB_PR, state="OPEN", merge_commit=None)]) + + assert pull_requests[0].state_changed_at is None + + +def test_approval_event_key_is_uuid_stable_across_nickname_renames(): + def event_key(nickname): + activity = [ + { + "approval": { + "date": "2026-08-21T12:00:00+00:00", + "user": {"uuid": "{u-stable}", "nickname": nickname}, + } + } + ] + _, _, events = _sync([BB_PR], activity=activity) + return events[0].idempotency_key + + # A rename must not change the key -- a changed key duplicates the event + # on the next re-sync of the PR. + assert event_key("muzz") == event_key("muzz-renamed") + + +def test_resync_diffstat_failure_keeps_previously_stored_code_stats(): + kept = {"commits": 1, "additions": 10, "deletions": 2, "changed_files": 1} + + class ExistingPRRepoService(FakeCodeRepoService): + def get_repo_pr_by_number(self, repo_id, number): + class _Existing: + id = uuid4() + meta = {"code_stats": kept} + + return _Existing() + + handler = BitbucketETLHandler( + org_id="org-1", + bitbucket_api_service=BrokenDiffstatApiService(prs=[BB_PR]), + code_repo_service=ExistingPRRepoService(), + code_etl_analytics_service=__import__( + "mhq.service.code.sync.etl_code_analytics", + fromlist=["CodeETLAnalyticsService"], + ).CodeETLAnalyticsService(), + bitbucket_revert_pr_sync_handler=None, + ) + + pull_requests, _, _ = handler.get_repo_pull_requests_data( + _Repo(), datetime(2026, 1, 1, tzinfo=timezone.utc) + ) + + # The re-sync must not erase what an earlier successful sync stored. + assert pull_requests[0].meta["code_stats"] == kept diff --git a/backend/analytics_server/tests/service/code/sync/test_revert_pr_bitbucket_sync.py b/backend/analytics_server/tests/service/code/sync/test_revert_pr_bitbucket_sync.py new file mode 100644 index 000000000..2c0842e61 --- /dev/null +++ b/backend/analytics_server/tests/service/code/sync/test_revert_pr_bitbucket_sync.py @@ -0,0 +1,86 @@ +from uuid import uuid4 + +from mhq.service.code.sync.revert_prs_bitbucket_sync import ( + RevertPRsBitbucketSyncHandler, +) +from mhq.store.models.code import PullRequestState + + +class _PR: + def __init__( + self, number, head_branch, state=PullRequestState.MERGED, repo_id=None + ): + self.id = uuid4() + self.number = number + self.head_branch = head_branch + self.state = state + self.repo_id = repo_id or uuid4() + + +class FakeCodeRepoService: + def __init__(self, prs_by_number=None, prs_by_branch=None): + self._by_number = prs_by_number or {} + self._by_branch = prs_by_branch or [] + + def get_repo_pr_by_number(self, repo_id, number): + return self._by_number.get(int(number)) + + def get_prs_by_head_branch_match_strings(self, repo_ids, match_strings): + return [pr for pr in self._by_branch if pr.head_branch in match_strings] + + +def test_a_revert_branch_maps_to_the_original_pr(): + repo_id = uuid4() + original = _PR(42, "feat/rate-limiter", repo_id=repo_id) + revert = _PR(57, "revert-pr-42", repo_id=repo_id) + handler = RevertPRsBitbucketSyncHandler( + FakeCodeRepoService(prs_by_number={42: original}) + ) + + mappings = handler.process_revert_prs([revert]) + + assert len(mappings) == 1 + assert mappings[0].pr_id == revert.id + assert mappings[0].reverted_pr == original.id + + +def test_an_original_pr_finds_its_earlier_synced_revert(): + repo_id = uuid4() + original = _PR(42, "feat/rate-limiter", repo_id=repo_id) + stored_revert = _PR(57, "revert-pr-42", repo_id=repo_id) + handler = RevertPRsBitbucketSyncHandler( + FakeCodeRepoService(prs_by_branch=[stored_revert]) + ) + + mappings = handler.process_revert_prs([original]) + + assert len(mappings) == 1 + assert mappings[0].pr_id == stored_revert.id + assert mappings[0].reverted_pr == original.id + + +def test_a_pr_merely_about_reverting_maps_to_nothing(): + # CLUSTOX: the negative case a branch/title heuristic must carry. A PR + # discussing reverts, or a hand-named branch, is not a revert marker -- + # a false mapping here becomes a false CFR incident. + prose = _PR(60, "chore/revert-the-tabs-decision") + titled = _PR(61, "revert-pr-abc") + handler = RevertPRsBitbucketSyncHandler(FakeCodeRepoService()) + + assert handler.process_revert_prs([prose, titled]) == [] + + +def test_a_revert_of_an_unsynced_original_maps_to_nothing(): + revert = _PR(57, "revert-pr-42") + handler = RevertPRsBitbucketSyncHandler(FakeCodeRepoService()) + + assert handler.process_revert_prs([revert]) == [] + + +def test_unmerged_originals_are_not_searched_for_reverts(): + open_pr = _PR(42, "feat/x", state=PullRequestState.OPEN) + handler = RevertPRsBitbucketSyncHandler( + FakeCodeRepoService(prs_by_branch=[_PR(57, "revert-pr-42")]) + ) + + assert handler.process_revert_prs([open_pr]) == [] diff --git a/backend/analytics_server/tests/service/code/test_contributors.py b/backend/analytics_server/tests/service/code/test_contributors.py new file mode 100644 index 000000000..d35bbd993 --- /dev/null +++ b/backend/analytics_server/tests/service/code/test_contributors.py @@ -0,0 +1,27 @@ +from mhq.utils.string import is_bot_author + + +def test_bracket_bot_suffix_is_a_bot(): + assert is_bot_author("dependabot[bot]") is True + assert is_bot_author("renovate[bot]") is True + + +def test_known_bot_names_are_bots(): + assert is_bot_author("dependabot") is True + assert is_bot_author("github-actions") is True + + +def test_a_human_whose_name_contains_bot_is_not_a_bot(): + # Substring matching on "bot" would wrongly exclude these people. + assert is_bot_author("robotnik") is False + assert is_bot_author("botond") is False + assert is_bot_author("abbott") is False + + +def test_none_and_empty_are_not_bots(): + assert is_bot_author(None) is False + assert is_bot_author("") is False + + +def test_matching_is_case_insensitive(): + assert is_bot_author("Dependabot[Bot]") is True diff --git a/backend/analytics_server/tests/service/code/test_lead_time_service.py b/backend/analytics_server/tests/service/code/test_lead_time_service.py index de2114dde..b27e19fc5 100644 --- a/backend/analytics_server/tests/service/code/test_lead_time_service.py +++ b/backend/analytics_server/tests/service/code/test_lead_time_service.py @@ -1,7 +1,11 @@ +from unittest.mock import MagicMock, patch + from mhq.utils.string import uuid4_str from tests.utilities import compare_objects_as_dicts from mhq.service.code.lead_time import LeadTimeService from mhq.service.code.models.lead_time import LeadTimeMetrics +from mhq.store.models.core import Team +from mhq.utils.time import Interval, time_now class FakeCodeRepoService: @@ -49,6 +53,33 @@ def test_get_avg_time_for_no_lead_time_metrics_returns_zero(): assert result == 0 +# CLUSTOX: Jira integration -- the extended Lead Time breakdown needs +# per-PR granularity (not the org-wide average) to scope itself to only +# ticket-matched PRs. See docs/JIRA_INTEGRATION_PROPOSAL.md §6A and +# mhq/service/code/ticket_lead_time.py. +def test_get_team_pr_lead_time_metrics_returns_the_per_pr_list_unaveraged(): + code_repo_service = MagicMock() + code_repo_service.get_active_team_repos_by_team_id.return_value = ["repo-1"] + lead_time_service = LeadTimeService(code_repo_service, FakeDeploymentsService) + + per_pr_metrics = [ + LeadTimeMetrics(pr_id="pr-1", pr_count=1), + LeadTimeMetrics(pr_id="pr-2", pr_count=1), + ] + team = Team(id=uuid4_str()) + interval = Interval(time_now(), time_now()) + + with patch.object( + lead_time_service, + "_get_team_repos_lead_time_metrics", + return_value=per_pr_metrics, + ) as mocked: + result = lead_time_service.get_team_pr_lead_time_metrics(team, interval) + + assert result == per_pr_metrics + mocked.assert_called_once_with(["repo-1"], interval, None) + + def test_get_avg_time_for_empty_lead_time_metrics_returns_zero(): lead_time_metrics = [LeadTimeMetrics(), LeadTimeMetrics()] field = "first_response_time" diff --git a/backend/analytics_server/tests/service/code/test_loc.py b/backend/analytics_server/tests/service/code/test_loc.py new file mode 100644 index 000000000..85ddff654 --- /dev/null +++ b/backend/analytics_server/tests/service/code/test_loc.py @@ -0,0 +1,402 @@ +import re +from datetime import datetime, timezone +from uuid import uuid4 + +from mhq.service.code.loc import LOCMetrics, LOCService, aggregate_loc +from mhq.store.models.code import PRFilter, PullRequestState +from mhq.utils.time import Interval +from tests.factories.models.code import get_pull_request + + +def _pr(additions, deletions, state="MERGED", state_changed_at=None): + class FakePR: + pass + + pr = FakePR() + pr.additions = additions + pr.deletions = deletions + pr.state = state + pr.state_changed_at = state_changed_at + return pr + + +def test_aggregate_sums_additions_and_deletions(): + result = aggregate_loc([_pr(10, 5), _pr(20, 1)]) + assert result.additions == 30 + assert result.deletions == 6 + + +def test_avg_pr_size_is_gross_lines_over_pr_count(): + # (10+5 + 20+1) / 2 == 18 + result = aggregate_loc([_pr(10, 5), _pr(20, 1)]) + assert result.avg_pr_size == 18 + + +def test_avg_pr_size_is_zero_not_a_crash_when_no_prs(): + # A team with no merged PRs in range must not divide by zero, and must not + # report None -- the card would then need a third empty state. + result = aggregate_loc([]) + assert result == LOCMetrics(additions=0, deletions=0, avg_pr_size=0) + + +class FakeTeamRepo: + def __init__(self, org_repo_id): + self.org_repo_id = org_repo_id + + +class FakeCodeRepoService: + def __init__(self, prs, team_repos=None): + self._prs = prs + self._team_repos = ( + team_repos if team_repos is not None else [FakeTeamRepo("repo-1")] + ) + self.merged_in_interval_calls = [] + + def get_active_team_repos_by_team_id(self, team_id): + return self._team_repos + + def get_prs_merged_in_interval(self, repo_ids, interval, pr_filter=None): + self.merged_in_interval_calls.append((repo_ids, interval, pr_filter)) + return self._prs + + +def test_team_loc_metrics_are_fetched_through_the_shared_pr_filter(): + # The contributor filter, branch mode and the excluded-PRs setting all live + # on PRFilter -- a hand-rolled query would silently drop all three. + prs = [_pr(10, 5), _pr(20, 1)] + code_repo_service = FakeCodeRepoService(prs) + interval = Interval(datetime(2024, 1, 1), datetime(2024, 1, 15)) + pr_filter = object() + + result = LOCService(code_repo_service).get_team_loc_metrics( + "team-1", interval, pr_filter + ) + + assert result == LOCMetrics(additions=30, deletions=6, avg_pr_size=18, avg_daily=3) + assert code_repo_service.merged_in_interval_calls == [ + (["repo-1"], interval, pr_filter) + ] + + +def test_team_loc_trends_bucket_prs_into_the_week_they_merged(): + # 2024-01-01 and 2024-01-08 are Mondays. + prs = [ + _pr(10, 5, state_changed_at=datetime(2024, 1, 3)), + _pr(20, 1, state_changed_at=datetime(2024, 1, 9)), + ] + interval = Interval(datetime(2024, 1, 1), datetime(2024, 1, 14)) + + trends = LOCService(FakeCodeRepoService(prs)).get_team_loc_trends( + "team-1", interval, None + ) + + assert trends[datetime(2024, 1, 1, tzinfo=timezone.utc)] == LOCMetrics( + additions=10, deletions=5, avg_pr_size=15, avg_daily=2 + ) + assert trends[datetime(2024, 1, 8, tzinfo=timezone.utc)] == LOCMetrics( + additions=20, deletions=1, avg_pr_size=21, avg_daily=3 + ) + + +def test_team_loc_trends_report_zero_for_a_week_with_no_merges(): + # Every week in the window must be present, so the chart has no gaps. + prs = [_pr(10, 5, state_changed_at=datetime(2024, 1, 3))] + interval = Interval(datetime(2024, 1, 1), datetime(2024, 1, 14)) + + trends = LOCService(FakeCodeRepoService(prs)).get_team_loc_trends( + "team-1", interval, None + ) + + assert trends[datetime(2024, 1, 8, tzinfo=timezone.utc)] == LOCMetrics() + + +def test_a_team_with_no_active_repos_reports_zero_rather_than_querying(): + code_repo_service = FakeCodeRepoService([], team_repos=[]) + interval = Interval(datetime(2024, 1, 1), datetime(2024, 1, 15)) + + result = LOCService(code_repo_service).get_team_loc_metrics( + "team-1", interval, None + ) + + assert result == LOCMetrics() + assert code_repo_service.merged_in_interval_calls == [] + + +# --------------------------------------------------------------------------- +# Filter inheritance. +# +# CLUSTOX: the tests above assert the PRFilter is handed over unmodified. That +# is necessary but not sufficient: it passes just as happily against a +# LOCService that forwards the object and then ignores what comes back, or that +# swaps in an unfiltered fetch. The only thing that catches those is asserting +# the numbers MOVE when a filter is applied, so every case below pins the +# unfiltered total as well as the narrowed one. +# +# The suite has no database, so the real WHERE clause cannot run. The fake +# below mirrors it in Python -- merged-only, inside the interval, plus each +# PRFilter condition -- and narrows using nothing but the filter it is handed. +# Drop the filter anywhere between get_team_loc_metrics and the fetch and the +# narrowed assertions collapse back onto the unfiltered ones. +# --------------------------------------------------------------------------- + +REPO_ID = "11111111-1111-4111-8111-111111111111" +OTHER_REPO_ID = "22222222-2222-4222-8222-222222222222" +INTERVAL = Interval(datetime(2024, 1, 1), datetime(2024, 1, 15)) + + +def _pull_request( + additions, + deletions, + author="alice", + base_branch="main", + state=PullRequestState.MERGED, + repo_id=REPO_ID, + pr_id=None, + state_changed_at=None, +): + # CLUSTOX: a real PullRequest, not the FakePR above, because `additions` + # and `deletions` are properties reading `meta["code_stats"]` -- a fake with + # plain attributes would not notice the aggregator reading the wrong key. + return get_pull_request( + id=pr_id or uuid4(), + repo_id=repo_id, + author=author, + base_branch=base_branch, + state=state, + state_changed_at=state_changed_at or datetime(2024, 1, 3), + meta={"code_stats": {"additions": additions, "deletions": deletions}}, + ) + + +def _matches_any(value, patterns): + # Postgres `~` is an unanchored regex match, which is `re.search`. + return any(re.search(p, value or "") for p in patterns or [] if p is not None) + + +class FilteringCodeRepoService: + """Stands in for CodeRepoService, narrowing the way its SQL does.""" + + def __init__(self, prs, team_repos=None): + self._prs = prs + self._team_repos = ( + team_repos if team_repos is not None else [FakeTeamRepo(REPO_ID)] + ) + + def get_active_team_repos_by_team_id(self, team_id): + return self._team_repos + + def get_prs_merged_in_interval(self, repo_ids, interval, pr_filter=None): + prs = [ + pr + for pr in self._prs + if str(pr.repo_id) in repo_ids + # Mirrors _filter_prs_merged_in_interval: merged, and merged in + # window. An abandoned 5,000-line PR is not delivered work. + and pr.state == PullRequestState.MERGED + and interval.from_time <= pr.state_changed_at <= interval.to_time + ] + return [pr for pr in prs if self._passes(pr, pr_filter)] + + @staticmethod + def _passes(pr, pr_filter: PRFilter): + if pr_filter is None: + return True + + if pr_filter.authors and pr.author not in pr_filter.authors: + return False + + if pr_filter.base_branches and not _matches_any( + pr.base_branch, pr_filter.base_branches + ): + return False + + if pr_filter.repo_filters and not any( + str(pr.repo_id) == repo_id + and _matches_any(pr.base_branch, (config or {}).get("base_branches")) + for repo_id, config in pr_filter.repo_filters.items() + ): + return False + + excluded = [str(pr_id) for pr_id in pr_filter.excluded_pr_ids or []] + if str(pr.id) in excluded: + return False + + return True + + +def _loc(prs, pr_filter=None): + return LOCService(FilteringCodeRepoService(prs)).get_team_loc_metrics( + "team-1", INTERVAL, pr_filter + ) + + +def test_the_contributor_filter_narrows_the_loc_totals(): + prs = [ + _pull_request(100, 10, author="alice"), + _pull_request(20, 2, author="bob"), + ] + + # (100+10 + 20+2) / 2 == 66 + assert _loc(prs) == LOCMetrics( + additions=120, deletions=12, avg_pr_size=66, avg_daily=9 + ) + # Only alice's PR survives, and the average is hers alone -- not the + # team-wide 66 that a card ignoring the contributor filter would show. + assert _loc(prs, PRFilter(authors=["alice"])) == LOCMetrics( + additions=100, deletions=10, avg_pr_size=110, avg_daily=8 + ) + + +def test_a_base_branches_filter_narrows_the_loc_totals(): + prs = [ + _pull_request(100, 10, base_branch="release/1.0"), + _pull_request(20, 2, base_branch="main"), + ] + + assert _loc(prs) == LOCMetrics( + additions=120, deletions=12, avg_pr_size=66, avg_daily=9 + ) + assert _loc(prs, PRFilter(base_branches=["^release/"])) == LOCMetrics( + additions=100, deletions=10, avg_pr_size=110, avg_daily=8 + ) + + +def test_prod_branch_mode_narrows_the_loc_totals(): + # CLUSTOX: PROD branch mode arrives as per-repo `repo_filters`, not as the + # flat `base_branches` above -- the dashboard builds it from each repo's + # configured prod branches. Two different shapes of the same setting, and + # only the flat one is exercised elsewhere. + prs = [ + _pull_request(100, 10, base_branch="main"), + _pull_request(20, 2, base_branch="feature/x"), + ] + prod_mode = PRFilter(repo_filters={REPO_ID: {"base_branches": ["^main$"]}}) + + assert _loc(prs) == LOCMetrics( + additions=120, deletions=12, avg_pr_size=66, avg_daily=9 + ) + assert _loc(prs, prod_mode) == LOCMetrics( + additions=100, deletions=10, avg_pr_size=110, avg_daily=8 + ) + + +def test_the_excluded_prs_setting_narrows_the_loc_totals(): + excluded_id = "33333333-3333-4333-8333-333333333333" + prs = [ + _pull_request(100, 10, pr_id=excluded_id), + _pull_request(20, 2), + ] + + assert _loc(prs) == LOCMetrics( + additions=120, deletions=12, avg_pr_size=66, avg_daily=9 + ) + # A generated-code PR an admin excluded must not drag the average up. + assert _loc(prs, PRFilter(excluded_pr_ids=[excluded_id])) == LOCMetrics( + additions=20, deletions=2, avg_pr_size=22, avg_daily=2 + ) + + +def test_an_unmerged_pull_request_is_never_counted(): + # CLUSTOX: no filter at all here. Merged-only is not something the caller + # opts into -- it comes from LOC choosing the merged-in-interval fetch, so + # a future rewrite reaching for an unfiltered "all PRs" query would show a + # 5,000-line abandoned PR as delivered work. + prs = [ + _pull_request(10, 5), + _pull_request(5000, 4000, state=PullRequestState.OPEN), + ] + + assert _loc(prs) == LOCMetrics( + additions=10, deletions=5, avg_pr_size=15, avg_daily=1 + ) + + +def test_a_pr_merged_outside_the_window_is_never_counted(): + prs = [ + _pull_request(10, 5), + _pull_request(5000, 4000, state_changed_at=datetime(2023, 12, 20)), + ] + + assert _loc(prs) == LOCMetrics( + additions=10, deletions=5, avg_pr_size=15, avg_daily=1 + ) + + +def test_a_repo_outside_the_team_is_never_counted(): + prs = [ + _pull_request(10, 5), + _pull_request(5000, 4000, repo_id=OTHER_REPO_ID), + ] + + assert _loc(prs) == LOCMetrics( + additions=10, deletions=5, avg_pr_size=15, avg_daily=1 + ) + + +def test_the_same_filter_narrows_the_loc_trend_buckets(): + # CLUSTOX: trends are a second public entry point, and the card draws the + # headline and the trend line together. If only one of them honoured the + # contributor filter the two would disagree on screen, which reads as a + # data bug rather than a filtering one. 2024-01-01 is a Monday. + prs = [ + _pull_request(100, 10, author="alice", state_changed_at=datetime(2024, 1, 3)), + _pull_request(20, 2, author="bob", state_changed_at=datetime(2024, 1, 4)), + ] + week = datetime(2024, 1, 1, tzinfo=timezone.utc) + + def trends(pr_filter): + return LOCService(FilteringCodeRepoService(prs)).get_team_loc_trends( + "team-1", INTERVAL, pr_filter + ) + + assert trends(None)[week] == LOCMetrics( + additions=120, deletions=12, avg_pr_size=66, avg_daily=19 + ) + assert trends(PRFilter(authors=["alice"]))[week] == LOCMetrics( + additions=100, deletions=10, avg_pr_size=110, avg_daily=16 + ) + + +def test_a_filter_that_matches_nothing_reports_zero_rather_than_the_team_total(): + # CLUSTOX: the empty result has to come back as a measured zero, not fall + # back to unfiltered numbers -- picking a contributor who merged nothing + # this period must empty the card, not quietly show everyone's totals. + prs = [_pull_request(100, 10, author="alice")] + + assert _loc(prs, PRFilter(authors=["nobody"])) == LOCMetrics() + + +def test_avg_daily_is_the_rate_over_the_selected_range(): + # CLUSTOX: 700 gross lines over 7 days is 100/day. The headline this + # replaces was a period total, which cannot be compared between a week and + # a month -- the month always looks four times larger for the same pace. + result = aggregate_loc([_pr(600, 100)], days=7) + assert result.avg_daily == 100 + + +def test_avg_daily_is_zero_without_a_day_count_rather_than_a_guess(): + # A caller with no interval has no rate. Defaulting to 1 would silently + # report the period total as if it were a daily figure. + assert aggregate_loc([_pr(600, 100)]).avg_daily == 0 + + +def test_avg_daily_never_divides_by_zero_on_a_same_day_range(): + from datetime import datetime, timezone + + from mhq.service.code.loc import interval_days + + same_day = Interval( + datetime(2026, 8, 12, tzinfo=timezone.utc), + datetime(2026, 8, 12, tzinfo=timezone.utc), + ) + assert interval_days(same_day) == 1 + assert aggregate_loc([_pr(60, 40)], days=interval_days(same_day)).avg_daily == 100 + + +def test_avg_daily_and_avg_pr_size_are_independent(): + # CLUSTOX: two PRs of 350 lines each over 7 days -- 350 lines/PR but + # 100 lines/day. They are different quantities and a card showing one + # while labelling it the other would be plausible and wrong. + result = aggregate_loc([_pr(300, 50), _pr(300, 50)], days=7) + assert result.avg_pr_size == 350 + assert result.avg_daily == 100 diff --git a/backend/analytics_server/tests/service/code/test_repository_service.py b/backend/analytics_server/tests/service/code/test_repository_service.py new file mode 100644 index 000000000..97cf31230 --- /dev/null +++ b/backend/analytics_server/tests/service/code/test_repository_service.py @@ -0,0 +1,100 @@ +from uuid import uuid4 + +from mhq.service.code.models.org_repo import RawTeamOrgRepo +from mhq.service.code.repository_service import RepositoryService +from mhq.store.models.code import CodeProvider, OrgRepo +from mhq.store.models.code.enums import TeamReposDeploymentType + + +class FakeCodeRepoService: + def __init__(self, existing_repos=None): + self._existing = existing_repos or [] + self.lookup_calls = [] + self.saved = None + + def get_repos_by_idempotency_keys(self, org_id, idempotency_keys): + self.lookup_calls.append((str(org_id), list(idempotency_keys))) + return [r for r in self._existing if str(r.org_id) == str(org_id)] + + def update_org_repos(self, org_repos): + self.saved = org_repos + return org_repos + + +def _raw_repo(idempotency_key="1030593440", name="urban-assembly"): + return RawTeamOrgRepo( + team_id=str(uuid4()), + provider=CodeProvider.GITHUB, + name=name, + org_name="Clustox", + slug=name, + idempotency_key=idempotency_key, + default_branch="main", + deployment_type=TeamReposDeploymentType.PR_MERGE, + ) + + +def _service(code_repo_service): + return RepositoryService(code_repo_service, incident_repo_service=None) + + +def test_the_lookup_is_scoped_to_the_org(): + # CLUSTOX: the lookup used to be global -- upstream's own ToDo. A GitHub + # repo id is the same string no matter which workspace links the repo, so + # a global lookup found the OTHER workspace's row and update_org_repos + # raised "Data integrity error, matching idempotency key across orgs": + # a permanent 500 on team save for the second workspace, specific to + # exactly the repos another workspace already tracks. + org_id = str(uuid4()) + fake = FakeCodeRepoService() + + _service(fake).update_org_repos(org_id, [_raw_repo()]) + + assert fake.lookup_calls == [(org_id, ["1030593440"])] + + +def test_a_repo_tracked_by_another_workspace_gets_its_own_row(): + my_org = str(uuid4()) + other_org = str(uuid4()) + other_orgs_row = OrgRepo( + id=str(uuid4()), + org_id=other_org, + name="urban-assembly", + provider=CodeProvider.GITHUB.value, + org_name="Clustox", + idempotency_key="1030593440", + slug="urban-assembly", + ) + fake = FakeCodeRepoService(existing_repos=[other_orgs_row]) + + _service(fake).update_org_repos(my_org, [_raw_repo()]) + + # No exception, and a NEW row for this org -- never the other org's row. + assert len(fake.saved) == 1 + saved = fake.saved[0] + assert str(saved.org_id) == my_org + assert saved.idempotency_key == "1030593440" + assert str(saved.id) != str(other_orgs_row.id) + + +def test_an_existing_row_in_the_same_org_is_updated_in_place(): + org_id = str(uuid4()) + existing = OrgRepo( + id=str(uuid4()), + org_id=org_id, + name="old-name", + provider=CodeProvider.GITHUB.value, + org_name="Clustox", + idempotency_key="1030593440", + slug="old-name", + is_active=False, + ) + fake = FakeCodeRepoService(existing_repos=[existing]) + + _service(fake).update_org_repos(org_id, [_raw_repo()]) + + assert len(fake.saved) == 1 + saved = fake.saved[0] + assert str(saved.id) == str(existing.id) + assert saved.is_active is True + assert saved.name == "urban-assembly" diff --git a/backend/analytics_server/tests/service/code/test_ticket_lead_time.py b/backend/analytics_server/tests/service/code/test_ticket_lead_time.py new file mode 100644 index 000000000..8a11ef93b --- /dev/null +++ b/backend/analytics_server/tests/service/code/test_ticket_lead_time.py @@ -0,0 +1,163 @@ +from datetime import timedelta +from unittest.mock import MagicMock + +from mhq.service.code.ticket_lead_time import TicketLeadTimeService +from mhq.store.models.core import Team +from mhq.utils.time import Interval, time_now + +# CLUSTOX: Jira integration -- the extended Lead Time breakdown's +# "ticket created -> first commit" leading phase (docs/ +# JIRA_INTEGRATION_PROPOSAL.md §6A). Deliberately additive alongside +# LeadTimeService -- every test here mocks that service rather than +# exercising its real deployment-config branching, which is already +# covered by test_lead_time_service.py. + + +def _metric(pr_id, lead_time_seconds): + metric = MagicMock() + metric.pr_id = pr_id + metric.lead_time = lead_time_seconds + return metric + + +def _service(lead_time_service=None, code_repo=None, ticket_matching_repo=None): + return TicketLeadTimeService( + lead_time_service or MagicMock(), + code_repo or MagicMock(), + ticket_matching_repo or MagicMock(), + ) + + +def _interval(): + now = time_now() + return Interval(now, now) + + +class TestGetTeamTicketLeadTimeMetrics: + def test_returns_zeroed_metrics_when_the_team_has_no_merged_prs(self): + lead_time_service = MagicMock() + lead_time_service.get_team_pr_lead_time_metrics.return_value = [] + + result = _service(lead_time_service).get_team_ticket_lead_time_metrics( + Team(id="team-1"), _interval() + ) + + assert result.matched_pr_count == 0 + assert result.avg_extended_lead_time_seconds == 0 + + def test_returns_zeroed_metrics_when_no_pr_has_a_matched_ticket(self): + lead_time_service = MagicMock() + lead_time_service.get_team_pr_lead_time_metrics.return_value = [ + _metric("pr-1", 1000) + ] + ticket_matching_repo = MagicMock() + ticket_matching_repo.get_ticket_created_at_by_pr_ids.return_value = {} + + result = _service( + lead_time_service, ticket_matching_repo=ticket_matching_repo + ).get_team_ticket_lead_time_metrics(Team(id="team-1"), _interval()) + + assert result.matched_pr_count == 0 + + def test_excludes_a_matched_pr_with_no_recorded_first_commit(self): + lead_time_service = MagicMock() + lead_time_service.get_team_pr_lead_time_metrics.return_value = [ + _metric("pr-1", 1000) + ] + ticket_matching_repo = MagicMock() + ticket_matching_repo.get_ticket_created_at_by_pr_ids.return_value = { + "pr-1": time_now() + } + code_repo = MagicMock() + code_repo.get_first_commit_at_by_pr_ids.return_value = ( + {} + ) # no commit row synced + + result = _service( + lead_time_service, code_repo, ticket_matching_repo + ).get_team_ticket_lead_time_metrics(Team(id="team-1"), _interval()) + + assert result.matched_pr_count == 0 + + def test_excludes_a_ticket_created_after_its_first_commit(self): + # A ticket filed retroactively (or a false-positive key match) -- + # not a real "idea to commit" span, so this must not surface as + # a negative duration. + first_commit_at = time_now() + ticket_created_at = first_commit_at + timedelta(days=1) + + lead_time_service = MagicMock() + lead_time_service.get_team_pr_lead_time_metrics.return_value = [ + _metric("pr-1", 1000) + ] + ticket_matching_repo = MagicMock() + ticket_matching_repo.get_ticket_created_at_by_pr_ids.return_value = { + "pr-1": ticket_created_at + } + code_repo = MagicMock() + code_repo.get_first_commit_at_by_pr_ids.return_value = {"pr-1": first_commit_at} + + result = _service( + lead_time_service, code_repo, ticket_matching_repo + ).get_team_ticket_lead_time_metrics(Team(id="team-1"), _interval()) + + assert result.matched_pr_count == 0 + + def test_averages_only_over_the_matched_and_resolvable_subset(self): + now = time_now() + + lead_time_service = MagicMock() + lead_time_service.get_team_pr_lead_time_metrics.return_value = [ + _metric("pr-matched-1", lead_time_seconds=1000), + _metric("pr-matched-2", lead_time_seconds=2000), + _metric("pr-unmatched", lead_time_seconds=999999), + ] + ticket_matching_repo = MagicMock() + ticket_matching_repo.get_ticket_created_at_by_pr_ids.return_value = { + "pr-matched-1": now - timedelta(days=2), + "pr-matched-2": now - timedelta(days=4), + # "pr-unmatched" deliberately absent -- no ticket match. + } + code_repo = MagicMock() + code_repo.get_first_commit_at_by_pr_ids.return_value = { + "pr-matched-1": now - timedelta(days=1), # 1 day ticket->commit + "pr-matched-2": now - timedelta(days=1), # 3 days ticket->commit + } + + result = _service( + lead_time_service, code_repo, ticket_matching_repo + ).get_team_ticket_lead_time_metrics(Team(id="team-1"), _interval()) + + assert result.matched_pr_count == 2 + # (1 day + 3 days) / 2 = 2 days, in seconds. + assert ( + result.avg_ticket_to_first_commit_seconds + == timedelta(days=2).total_seconds() + ) + # (1000 + 2000) / 2 -- the unmatched PR's 999999s must not leak in. + assert result.avg_commit_only_lead_time_seconds == 1500 + assert result.avg_extended_lead_time_seconds == ( + timedelta(days=2).total_seconds() + 1500 + ) + + def test_looks_up_first_commit_only_for_the_matched_subset_not_every_pr(self): + # A batch lookup scoped to matched PRs only -- not every merged + # PR in the window -- keeps this from doing unnecessary work for + # the (often larger) unmatched population. + lead_time_service = MagicMock() + lead_time_service.get_team_pr_lead_time_metrics.return_value = [ + _metric("pr-matched", 1000), + _metric("pr-unmatched", 1000), + ] + ticket_matching_repo = MagicMock() + ticket_matching_repo.get_ticket_created_at_by_pr_ids.return_value = { + "pr-matched": time_now() + } + code_repo = MagicMock() + code_repo.get_first_commit_at_by_pr_ids.return_value = {} + + _service( + lead_time_service, code_repo, ticket_matching_repo + ).get_team_ticket_lead_time_metrics(Team(id="team-1"), _interval()) + + code_repo.get_first_commit_at_by_pr_ids.assert_called_once_with(["pr-matched"]) diff --git a/backend/analytics_server/tests/service/project/__init__.py b/backend/analytics_server/tests/service/project/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/service/project/sync/__init__.py b/backend/analytics_server/tests/service/project/sync/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/service/project/sync/test_etl_jira_handler.py b/backend/analytics_server/tests/service/project/sync/test_etl_jira_handler.py new file mode 100644 index 000000000..14f39545c --- /dev/null +++ b/backend/analytics_server/tests/service/project/sync/test_etl_jira_handler.py @@ -0,0 +1,456 @@ +from datetime import datetime +from unittest.mock import MagicMock, patch + +from mhq.exapi.models.jira import JiraIssue +from mhq.service.project.sync.etl_jira_handler import ( + JiraETLHandler, + get_jira_etl_handlers, +) +from mhq.store.models.projects import OrgProject, Ticket, TicketState + +# CLUSTOX: Jira integration, Phase 2 (issue sync). See +# docs/JIRA_INTEGRATION_PROPOSAL.md. Uses the real JiraIssue/ +# JiraChangelogEntry parser (already covered by tests/exapi/test_jira.py) +# so this file stays focused on JiraETLHandler's own job: building the +# JQL, and reconciling ids against existing rows without a per-item query. + +ORG_ID = "org-1" + + +def _org_project(project_id="proj-1", key="PAY") -> OrgProject: + return OrgProject(id=project_id, org_id=ORG_ID, key=key, name="Payments") + + +def _issue(issue_id="1", key="PAY-1", status_history=None) -> JiraIssue: + return JiraIssue( + { + "id": issue_id, + "key": key, + "fields": { + "summary": "Fix refund rounding", + "status": { + "name": "In Progress", + "statusCategory": {"name": "In Progress"}, + }, + "issuetype": {"name": "Bug"}, + "created": "2024-01-01T10:00:00.000+0000", + "updated": "2024-01-02T10:00:00.000+0000", + }, + "changelog": {"histories": status_history or []}, + } + ) + + +def _status_change(history_id="100", from_status="To Do", to_status="In Progress"): + return { + "id": history_id, + "created": "2024-01-01T11:00:00.000+0000", + "items": [ + { + "field": "status", + "fromString": from_status, + "toString": to_status, + } + ], + } + + +def _handler(api=None, project_repo_service=None, connection_id=None) -> JiraETLHandler: + return JiraETLHandler( + ORG_ID, + api or MagicMock(), + project_repo_service or MagicMock(), + connection_id=connection_id, + ) + + +class TestCheckPatValidity: + def test_delegates_to_the_api_client(self): + api = MagicMock() + api.check_pat.return_value = True + + assert _handler(api).check_pat_validity() is True + api.check_pat.assert_called_once() + + +class TestGetProjectIssuesData: + def test_builds_jql_scoped_to_the_project_key_and_bookmark(self): + api = MagicMock() + api.get_all_issues.return_value = [] + org_project = _org_project(key="PAY") + bookmark = datetime(2024, 1, 1, 9, 30) + + _handler(api).get_project_issues_data(org_project, bookmark) + + jql = api.get_all_issues.call_args[0][0] + assert 'project = "PAY"' in jql + assert '"2024-01-01 09:30"' in jql + + def test_returns_empty_lists_without_any_batch_lookup_when_nothing_changed(self): + api = MagicMock() + api.get_all_issues.return_value = [] + repo = MagicMock() + + tickets, states = _handler(api, repo).get_project_issues_data( + _org_project(), datetime(2024, 1, 1) + ) + + assert tickets == [] + assert states == [] + repo.get_tickets_by_idempotency_keys.assert_not_called() + + def test_reuses_the_existing_tickets_id_when_the_idempotency_key_already_exists( + self, + ): + api = MagicMock() + api.get_all_issues.return_value = [_issue("1")] + existing = Ticket(id="existing-ticket-id", idempotency_key=f"jira:{ORG_ID}:1") + repo = MagicMock() + repo.get_tickets_by_idempotency_keys.return_value = [existing] + repo.get_ticket_states_by_idempotency_keys.return_value = [] + + tickets, _ = _handler(api, repo).get_project_issues_data( + _org_project(), datetime(2024, 1, 1) + ) + + assert len(tickets) == 1 + assert str(tickets[0].id) == "existing-ticket-id" + assert tickets[0].idempotency_key == f"jira:{ORG_ID}:1" + + def test_mints_a_new_id_for_a_ticket_never_seen_before(self): + api = MagicMock() + api.get_all_issues.return_value = [_issue("1")] + repo = MagicMock() + repo.get_tickets_by_idempotency_keys.return_value = [] + repo.get_ticket_states_by_idempotency_keys.return_value = [] + + tickets, _ = _handler(api, repo).get_project_issues_data( + _org_project(), datetime(2024, 1, 1) + ) + + assert len(tickets) == 1 + assert tickets[0].id is not None + + def test_does_one_batch_lookup_for_all_tickets_in_the_page_not_one_per_ticket( + self, + ): + api = MagicMock() + api.get_all_issues.return_value = [_issue("1"), _issue("2"), _issue("3")] + repo = MagicMock() + repo.get_tickets_by_idempotency_keys.return_value = [] + repo.get_ticket_states_by_idempotency_keys.return_value = [] + + _handler(api, repo).get_project_issues_data( + _org_project(), datetime(2024, 1, 1) + ) + + repo.get_tickets_by_idempotency_keys.assert_called_once() + looked_up_keys = repo.get_tickets_by_idempotency_keys.call_args[0][0] + assert set(looked_up_keys) == { + f"jira:{ORG_ID}:1", + f"jira:{ORG_ID}:2", + f"jira:{ORG_ID}:3", + } + + def test_builds_a_ticket_state_per_status_transition_linked_to_its_ticket(self): + api = MagicMock() + api.get_all_issues.return_value = [ + _issue("1", status_history=[_status_change()]) + ] + repo = MagicMock() + repo.get_tickets_by_idempotency_keys.return_value = [] + repo.get_ticket_states_by_idempotency_keys.return_value = [] + + tickets, states = _handler(api, repo).get_project_issues_data( + _org_project(), datetime(2024, 1, 1) + ) + + assert len(states) == 1 + assert states[0].ticket_id == tickets[0].id + assert states[0].from_status == "To Do" + assert states[0].to_status == "In Progress" + assert states[0].idempotency_key == f"jira:{ORG_ID}:1:100" + + def test_reuses_the_existing_ticket_states_id_when_its_idempotency_key_already_exists( + self, + ): + api = MagicMock() + api.get_all_issues.return_value = [ + _issue("1", status_history=[_status_change(history_id="100")]) + ] + existing_state = TicketState( + id="existing-state-id", idempotency_key=f"jira:{ORG_ID}:1:100" + ) + repo = MagicMock() + repo.get_tickets_by_idempotency_keys.return_value = [] + repo.get_ticket_states_by_idempotency_keys.return_value = [existing_state] + + _, states = _handler(api, repo).get_project_issues_data( + _org_project(), datetime(2024, 1, 1) + ) + + assert len(states) == 1 + assert str(states[0].id) == "existing-state-id" + + def test_does_one_batch_lookup_for_all_ticket_states_regardless_of_issue_count( + self, + ): + api = MagicMock() + api.get_all_issues.return_value = [ + _issue("1", status_history=[_status_change("100"), _status_change("101")]), + _issue("2", status_history=[_status_change("200")]), + ] + repo = MagicMock() + repo.get_tickets_by_idempotency_keys.return_value = [] + repo.get_ticket_states_by_idempotency_keys.return_value = [] + + _handler(api, repo).get_project_issues_data( + _org_project(), datetime(2024, 1, 1) + ) + + repo.get_ticket_states_by_idempotency_keys.assert_called_once() + looked_up_keys = repo.get_ticket_states_by_idempotency_keys.call_args[0][0] + assert set(looked_up_keys) == { + f"jira:{ORG_ID}:1:100", + f"jira:{ORG_ID}:1:101", + f"jira:{ORG_ID}:2:200", + } + + +def _board(board_id=1076, board_type="scrum"): + board = MagicMock() + board.id = board_id + board.board_type = board_type + return board + + +def _sprint(sprint_id=299, name="Sprint 1", state="closed"): + sprint = MagicMock() + sprint.id = sprint_id + sprint.name = name + sprint.state = state + sprint.start_date = datetime(2024, 1, 1) + sprint.end_date = datetime(2024, 1, 15) + return sprint + + +# CLUSTOX: Jira integration -- the Sprint rollup chart. See +# docs/JIRA_INTEGRATION_PROPOSAL.md §6D. +class TestGetProjectSprintsData: + def test_returns_empty_list_when_the_project_has_no_scrum_board(self): + api = MagicMock() + api.get_boards_for_project.return_value = [_board(board_type="kanban")] + + sprints = _handler(api).get_project_sprints_data(_org_project()) + + assert sprints == [] + api.get_sprints_for_board.assert_not_called() + + def test_returns_empty_list_when_the_project_has_no_boards_at_all(self): + api = MagicMock() + api.get_boards_for_project.return_value = [] + + sprints = _handler(api).get_project_sprints_data(_org_project()) + + assert sprints == [] + + def test_fetches_counts_and_builds_sprints_for_a_scrum_board(self): + api = MagicMock() + api.get_boards_for_project.return_value = [_board()] + api.get_sprints_for_board.return_value = [_sprint()] + api.get_sprint_issue_counts.return_value = (355, 272) + repo = MagicMock() + repo.get_sprints_by_idempotency_keys.return_value = [] + + [sprint] = _handler(api, repo).get_project_sprints_data( + _org_project("proj-1", "PZDA") + ) + + assert sprint.name == "Sprint 1" + assert sprint.state == "closed" + assert sprint.planned_count == 355 + assert sprint.completed_count == 272 + assert sprint.org_project_id == "proj-1" + assert sprint.idempotency_key == f"jira:{ORG_ID}:sprint:299" + api.get_sprint_issue_counts.assert_called_once_with(299) + + def test_reuses_the_existing_row_id_for_an_already_synced_sprint(self): + api = MagicMock() + api.get_boards_for_project.return_value = [_board()] + api.get_sprints_for_board.return_value = [_sprint()] + api.get_sprint_issue_counts.return_value = (10, 5) + existing = MagicMock() + existing.idempotency_key = f"jira:{ORG_ID}:sprint:299" + existing.id = "existing-row-id" + repo = MagicMock() + repo.get_sprints_by_idempotency_keys.return_value = [existing] + + [sprint] = _handler(api, repo).get_project_sprints_data(_org_project()) + + assert sprint.id == "existing-row-id" + + def test_pulls_sprints_from_every_scrum_board_when_a_project_has_several(self): + api = MagicMock() + api.get_boards_for_project.return_value = [ + _board(board_id=1), + _board(board_id=2), + ] + api.get_sprints_for_board.side_effect = [ + [_sprint(sprint_id=101)], + [_sprint(sprint_id=201)], + ] + api.get_sprint_issue_counts.return_value = (1, 1) + repo = MagicMock() + repo.get_sprints_by_idempotency_keys.return_value = [] + + sprints = _handler(api, repo).get_project_sprints_data(_org_project()) + + assert {s.external_id for s in sprints} == {"101", "201"} + + +# CLUSTOX: Jira multi-account support. See docs/JIRA_MULTI_ACCOUNT_PLAN.md +# Task 4. +class TestConnectionScopedIdempotencyKey: + def test_legacy_handler_keeps_the_original_key_shape(self): + api = MagicMock() + api.get_all_issues.return_value = [_issue("1")] + repo = MagicMock() + repo.get_tickets_by_idempotency_keys.return_value = [] + repo.get_ticket_states_by_idempotency_keys.return_value = [] + + tickets, _ = _handler(api, repo).get_project_issues_data( + _org_project(), datetime(2024, 1, 1) + ) + + # Not just "some format" -- byte-identical to what a pre-existing + # ticket for this org already has as its idempotency_key, or this + # sync would stop reusing the row and start duplicating it. + assert tickets[0].idempotency_key == f"jira:{ORG_ID}:1" + + def test_connection_scoped_handler_includes_the_connection_id(self): + api = MagicMock() + api.get_all_issues.return_value = [_issue("1")] + repo = MagicMock() + repo.get_tickets_by_idempotency_keys.return_value = [] + repo.get_ticket_states_by_idempotency_keys.return_value = [] + + tickets, _ = _handler( + api, repo, connection_id="conn-1" + ).get_project_issues_data(_org_project(), datetime(2024, 1, 1)) + + assert tickets[0].idempotency_key == f"jira:{ORG_ID}:conn-1:1" + + def test_two_connections_do_not_collide_on_the_same_bare_issue_id(self): + # The regression Task 4 exists to close: two connections' sites can + # each have their own small, site-local issue id "1". Without + # connection scoping both would idempotency-key to "jira:org-1:1" + # and the second connection's ticket would silently overwrite the + # first's. + api = MagicMock() + api.get_all_issues.return_value = [_issue("1")] + repo = MagicMock() + repo.get_tickets_by_idempotency_keys.return_value = [] + repo.get_ticket_states_by_idempotency_keys.return_value = [] + + tickets_a, _ = _handler( + api, repo, connection_id="conn-a" + ).get_project_issues_data(_org_project(), datetime(2024, 1, 1)) + tickets_b, _ = _handler( + api, repo, connection_id="conn-b" + ).get_project_issues_data(_org_project(), datetime(2024, 1, 1)) + + assert tickets_a[0].idempotency_key != tickets_b[0].idempotency_key + + +class TestGetOrgProjectsToSync: + def test_legacy_handler_reads_every_active_project_for_the_org_and_provider(self): + repo = MagicMock() + repo.get_active_org_projects_for_provider.return_value = ["proj-a", "proj-b"] + + projects = _handler(project_repo_service=repo).get_org_projects_to_sync(ORG_ID) + + repo.get_active_org_projects_for_provider.assert_called_once_with( + ORG_ID, "jira" + ) + repo.get_active_org_projects_for_connection.assert_not_called() + assert projects == ["proj-a", "proj-b"] + + def test_connection_scoped_handler_reads_only_its_own_connections_projects(self): + repo = MagicMock() + repo.get_active_org_projects_for_connection.return_value = ["proj-a"] + + projects = _handler( + project_repo_service=repo, connection_id="conn-1" + ).get_org_projects_to_sync(ORG_ID) + + repo.get_active_org_projects_for_connection.assert_called_once_with("conn-1") + repo.get_active_org_projects_for_provider.assert_not_called() + assert projects == ["proj-a"] + + +class TestGetJiraEtlHandlers: + def test_falls_back_to_the_single_legacy_handler_when_the_org_has_no_connections( + self, + ): + with patch( + "mhq.service.project.sync.etl_jira_handler.JiraConnectionRepoService" + ) as connection_repo_cls, patch( + "mhq.service.project.sync.etl_jira_handler.get_jira_etl_handler" + ) as get_legacy_handler: + connection_repo_cls.return_value.list_jira_connections.return_value = [] + legacy_handler = MagicMock() + get_legacy_handler.return_value = legacy_handler + + handlers = get_jira_etl_handlers(ORG_ID) + + assert handlers == [legacy_handler] + + def test_builds_one_handler_per_connection_with_its_own_credentials(self): + connection_a = MagicMock( + id="conn-a", email="a@acme.com", site_url="acme.atlassian.net" + ) + connection_b = MagicMock( + id="conn-b", email="b@other.com", site_url="other.atlassian.net" + ) + + with patch( + "mhq.service.project.sync.etl_jira_handler.JiraConnectionRepoService" + ) as connection_repo_cls, patch( + "mhq.service.project.sync.etl_jira_handler.JiraApiService" + ) as api_service_cls: + connection_repo_service = connection_repo_cls.return_value + connection_repo_service.list_jira_connections.return_value = [ + connection_a, + connection_b, + ] + connection_repo_service.decrypt_access_token.side_effect = [ + "token-a", + "token-b", + ] + + handlers = get_jira_etl_handlers(ORG_ID) + + assert [h.connection_id for h in handlers] == ["conn-a", "conn-b"] + assert api_service_cls.call_args_list == [ + (("a@acme.com", "token-a", "acme.atlassian.net"),), + (("b@other.com", "token-b", "other.atlassian.net"),), + ] + + def test_never_calls_the_legacy_handler_when_any_connection_exists(self): + # The plan's strict either/or: a JiraConnection-backed org does not + # also sync through the legacy Integration row, even if that row + # still exists (see get_jira_etl_handlers's own docstring). + with patch( + "mhq.service.project.sync.etl_jira_handler.JiraConnectionRepoService" + ) as connection_repo_cls, patch( + "mhq.service.project.sync.etl_jira_handler.JiraApiService" + ), patch( + "mhq.service.project.sync.etl_jira_handler.get_jira_etl_handler" + ) as get_legacy_handler: + connection_repo_cls.return_value.list_jira_connections.return_value = [ + MagicMock(id="conn-a") + ] + + get_jira_etl_handlers(ORG_ID) + + get_legacy_handler.assert_not_called() diff --git a/backend/analytics_server/tests/service/project/test_repository_service.py b/backend/analytics_server/tests/service/project/test_repository_service.py new file mode 100644 index 000000000..75d920c3f --- /dev/null +++ b/backend/analytics_server/tests/service/project/test_repository_service.py @@ -0,0 +1,303 @@ +from unittest.mock import MagicMock + +import pytest + +from mhq.service.project.models.org_project import RawTeamOrgProject +from mhq.service.project.repository_service import ProjectService +from mhq.store.models.projects import OrgProject, TeamProjects + +# CLUSTOX: Jira integration, Phase 2 (project selection). Mirrors this +# repo's existing service-layer test style (fake/mock collaborators, no +# real DB -- see tests/service/code/test_lead_time_service.py) since this +# codebase has no DB-backed test fixture to hook a store-layer integration +# test into. See docs/JIRA_INTEGRATION_PROPOSAL.md. + + +class FakeTeam: + def __init__(self, id="team-1", org_id="org-1"): + self.id = id + self.org_id = org_id + + +def _raw( + team_id="team-1", + key="PAY", + name="Payments", + idempotency_key="jira:org-1:10001", + provider="jira", + connection_id=None, +) -> RawTeamOrgProject: + return RawTeamOrgProject( + team_id=team_id, + provider=provider, + key=key, + name=name, + idempotency_key=idempotency_key, + connection_id=connection_id, + ) + + +def _service(project_repo_service=None) -> ProjectService: + return ProjectService(project_repo_service or MagicMock()) + + +def test_creates_a_new_org_project_for_an_unseen_idempotency_key(): + repo = MagicMock() + repo.get_projects_by_idempotency_keys.return_value = [] + repo.update_org_projects.side_effect = lambda projects: projects + repo.get_existing_team_projects.return_value = [] + repo.get_active_org_projects.return_value = [] + repo.get_org_projects_used_across_teams.return_value = [] + + team = FakeTeam() + raw = _raw() + + result = _service(repo).update_team_projects(team, [raw]) + + assert len(result) == 1 + created = result[0] + assert str(created.org_id) == team.org_id + assert created.key == raw.key + assert created.name == raw.name + assert created.idempotency_key == raw.idempotency_key + assert created.is_active is True + + +def test_reuses_and_reactivates_an_existing_org_project_matched_by_idempotency_key(): + # Previously unselected by every team (is_active=False) and, per the + # incoming payload, renamed/rekeyed in Jira since we last saw it. + existing = OrgProject( + id="proj-1", + org_id="org-1", + key="OLD-KEY", + name="Old Name", + provider="jira", + idempotency_key="jira:org-1:10001", + is_active=False, + ) + repo = MagicMock() + repo.get_projects_by_idempotency_keys.return_value = [existing] + repo.update_org_projects.side_effect = lambda projects: projects + repo.get_existing_team_projects.return_value = [] + repo.get_active_org_projects.return_value = [existing] + repo.get_org_projects_used_across_teams.return_value = [existing] + + team = FakeTeam() + raw = _raw(key="PAY", name="Payments") + + result = _service(repo).update_team_projects(team, [raw]) + + # Reused, not duplicated into a second OrgProject row. + assert result == [existing] + assert existing.key == "PAY" + assert existing.name == "Payments" + assert existing.is_active is True + + +def test_raises_if_the_same_idempotency_key_already_belongs_to_a_different_org(): + # Same construction as OrgRepo's cross-org idempotency-key guard -- + # this should never happen in practice (idempotency_key is scoped by + # org_id, see the migration comment), but a match across orgs means + # something is badly wrong and must fail loudly, not silently adopt + # another org's project row. + existing = OrgProject( + id="proj-1", + org_id="some-other-org", + key="PAY", + name="Payments", + provider="jira", + idempotency_key="jira:org-1:10001", + is_active=True, + ) + repo = MagicMock() + repo.get_projects_by_idempotency_keys.return_value = [existing] + + team = FakeTeam(org_id="org-1") + + with pytest.raises(Exception): + _service(repo).update_team_projects(team, [_raw()]) + + +def test_deselecting_every_project_deactivates_the_teams_link_and_the_org_catalog_row(): + org_project = OrgProject( + id="proj-1", + org_id="org-1", + key="PAY", + name="Payments", + provider="jira", + idempotency_key="jira:org-1:10001", + is_active=True, + ) + existing_team_project = TeamProjects( + team_id="team-1", org_project_id="proj-1", is_active=True + ) + repo = MagicMock() + repo.get_projects_by_idempotency_keys.return_value = [] + repo.update_org_projects.side_effect = lambda projects: projects + repo.get_existing_team_projects.return_value = [existing_team_project] + repo.get_active_org_projects.return_value = [org_project] + # Not used by any team anymore -- this was the only one. + repo.get_org_projects_used_across_teams.return_value = [] + + team = FakeTeam() + + result = _service(repo).update_team_projects(team, []) + + assert result == [] + # The deactivation itself is what matters -- *not* which exact list + # object gets passed to update_team_projects. existing_team_project + # came back from get_existing_team_projects already attached to the + # store's session (same as CodeRepoService.get_existing_team_repos), + # so mutating it here is enough for the next real commit to persist + # it regardless of whether it's also in this particular call's list -- + # mirrors RepositoryService._update_team_repos exactly. + assert existing_team_project.is_active is False + repo.update_team_projects.assert_called_once() + # Nothing points at it anymore, so the org-level catalog row itself is + # retired too, not just this team's link to it -- mirrors + # CodeRepoService.set_unused_repos_as_inactive. + assert org_project.is_active is False + + +def test_a_second_team_selecting_an_already_active_project_does_not_touch_the_first_teams_link(): + org_project = OrgProject( + id="proj-1", + org_id="org-1", + key="PAY", + name="Payments", + provider="jira", + idempotency_key="jira:org-1:10001", + is_active=True, + ) + repo = MagicMock() + repo.get_projects_by_idempotency_keys.return_value = [org_project] + repo.update_org_projects.side_effect = lambda projects: projects + # This team (team-2) has no existing rows of its own yet. + repo.get_existing_team_projects.return_value = [] + repo.get_active_org_projects.return_value = [org_project] + repo.get_org_projects_used_across_teams.return_value = [org_project] + + team_two = FakeTeam(id="team-2", org_id="org-1") + + _service(repo).update_team_projects(team_two, [_raw()]) + + created_team_projects = repo.update_team_projects.call_args[0][0] + assert len(created_team_projects) == 1 + assert str(created_team_projects[0].team_id) == "team-2" + assert str(created_team_projects[0].org_project_id) == "proj-1" + + +# CLUSTOX: Jira multi-account support -- see +# docs/JIRA_MULTI_ACCOUNT_PLAN.md Task 6 part 2. The write side of the gap +# flagged in Task 4: sync only ever reads the OrgProjectConnection join, +# this is where a project actually gets assigned to a connection. +class TestOrgProjectConnectionWrites: + def test_writes_a_connection_row_for_a_newly_created_project(self): + repo = MagicMock() + repo.get_projects_by_idempotency_keys.return_value = [] + repo.update_org_projects.side_effect = lambda projects: projects + repo.get_existing_team_projects.return_value = [] + repo.get_active_org_projects.return_value = [] + repo.get_org_projects_used_across_teams.return_value = [] + + team = FakeTeam() + raw = _raw(connection_id="conn-1") + + result = _service(repo).update_team_projects(team, [raw]) + + repo.save_org_project_connections.assert_called_once() + [saved] = repo.save_org_project_connections.call_args[0][0] + assert str(saved.org_project_id) == str(result[0].id) + assert saved.jira_connection_id == "conn-1" + + def test_writes_a_connection_row_for_an_existing_project_too(self): + existing = OrgProject( + id="proj-1", + org_id="org-1", + key="OLD-KEY", + name="Old Name", + provider="jira", + idempotency_key="jira:org-1:10001", + is_active=False, + ) + repo = MagicMock() + repo.get_projects_by_idempotency_keys.return_value = [existing] + repo.update_org_projects.side_effect = lambda projects: projects + repo.get_existing_team_projects.return_value = [] + repo.get_active_org_projects.return_value = [existing] + repo.get_org_projects_used_across_teams.return_value = [] + + team = FakeTeam() + raw = _raw(connection_id="conn-1") + + _service(repo).update_team_projects(team, [raw]) + + [saved] = repo.save_org_project_connections.call_args[0][0] + assert str(saved.org_project_id) == "proj-1" + + def test_does_not_write_anything_when_no_project_has_a_connection_id(self): + # The legacy single-account flow, and non-Jira providers, must be + # completely unaffected -- not even an empty-list call. + repo = MagicMock() + repo.get_projects_by_idempotency_keys.return_value = [] + repo.update_org_projects.side_effect = lambda projects: projects + repo.get_existing_team_projects.return_value = [] + repo.get_active_org_projects.return_value = [] + repo.get_org_projects_used_across_teams.return_value = [] + + team = FakeTeam() + + _service(repo).update_team_projects(team, [_raw()]) + + repo.save_org_project_connections.assert_not_called() + + def test_connection_rows_are_only_written_for_projects_that_have_one(self): + # Two projects in one save: one picked under a connection, one + # through the legacy flow. Only the first gets a connection row. + repo = MagicMock() + repo.get_projects_by_idempotency_keys.return_value = [] + repo.update_org_projects.side_effect = lambda projects: projects + repo.get_existing_team_projects.return_value = [] + repo.get_active_org_projects.return_value = [] + repo.get_org_projects_used_across_teams.return_value = [] + + team = FakeTeam() + with_connection = _raw(idempotency_key="jira:org-1:1", connection_id="conn-1") + without_connection = _raw(idempotency_key="jira:org-1:2", connection_id=None) + + result = _service(repo).update_team_projects( + team, [with_connection, without_connection] + ) + + [saved] = repo.save_org_project_connections.call_args[0][0] + connected_project = next( + p for p in result if p.idempotency_key == "jira:org-1:1" + ) + assert str(saved.org_project_id) == str(connected_project.id) + + def test_saves_org_projects_before_writing_connection_rows(self): + # OrgProjectConnection.org_project_id is a real FK -- writing it + # before the OrgProject row it points to is committed would fail + # in a real database, even though this mocked repo can't catch + # that itself. + repo = MagicMock() + repo.get_projects_by_idempotency_keys.return_value = [] + repo.get_existing_team_projects.return_value = [] + repo.get_active_org_projects.return_value = [] + repo.get_org_projects_used_across_teams.return_value = [] + calls = [] + repo.update_org_projects.side_effect = lambda projects: ( + calls.append("update_org_projects"), + projects, + )[1] + repo.save_org_project_connections.side_effect = lambda _: calls.append( + "save_org_project_connections" + ) + + team = FakeTeam() + _service(repo).update_team_projects(team, [_raw(connection_id="conn-1")]) + + # update_team_projects makes a second, unrelated update_org_projects + # call afterwards (_set_unused_projects_as_inactive) -- only the + # relative order of the first two matters here. + assert calls[:2] == ["update_org_projects", "save_org_project_connections"] diff --git a/backend/analytics_server/tests/service/settings/__init__.py b/backend/analytics_server/tests/service/settings/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/service/settings/test_benchmark_setting_type.py b/backend/analytics_server/tests/service/settings/test_benchmark_setting_type.py new file mode 100644 index 000000000..ef2bc2449 --- /dev/null +++ b/backend/analytics_server/tests/service/settings/test_benchmark_setting_type.py @@ -0,0 +1,41 @@ +from mhq.service.settings.default_settings_data import get_default_setting_data +from mhq.service.settings.models import BenchmarkSetting +from mhq.service.settings.setting_type_validator import settings_type_validator +from mhq.store.models.settings import SettingType +from mhq.store.models.settings.enums import EntityType + + +def test_benchmark_setting_type_exists(): + assert SettingType.BENCHMARK_SETTING.value == "BENCHMARK_SETTING" + + +def test_global_entity_type_exists(): + # The superadmin baseline belongs to no team and no workspace. + assert EntityType.GLOBAL.value == "GLOBAL" + + +def test_validator_accepts_the_new_type(): + assert settings_type_validator("BENCHMARK_SETTING") == SettingType.BENCHMARK_SETTING + + +def test_default_baseline_has_every_benchmark_metric(): + data = get_default_setting_data(SettingType.BENCHMARK_SETTING) + + assert set(data.keys()) == { + "lead_time", + "deployment_frequency", + "change_failure_rate", + "mean_time_to_recovery", + "lines_of_code", + } + + +def test_benchmark_setting_fields_default_to_none(): + # Every field is optional: a team may set one target and inherit the rest. + setting = BenchmarkSetting() + + assert setting.lead_time is None + assert setting.deployment_frequency is None + assert setting.change_failure_rate is None + assert setting.mean_time_to_recovery is None + assert setting.lines_of_code is None diff --git a/backend/analytics_server/tests/service/settings/test_benchmarks.py b/backend/analytics_server/tests/service/settings/test_benchmarks.py new file mode 100644 index 000000000..f9b8afee1 --- /dev/null +++ b/backend/analytics_server/tests/service/settings/test_benchmarks.py @@ -0,0 +1,288 @@ +from datetime import datetime + +import pytest +from werkzeug.exceptions import BadRequest + +from mhq.api.resources.settings_resource import adapt_configuration_settings_response +from mhq.service.settings.benchmarks import ( + BENCHMARK_METRICS, + resolve_benchmarks, + validate_benchmark_payload, +) +from mhq.service.settings.models import BenchmarkSetting, ConfigurationSettings +from mhq.store.models.settings.enums import EntityType + + +def test_team_value_wins_and_reports_its_source(): + team = BenchmarkSetting(lead_time=3600) + glob = BenchmarkSetting(lead_time=86400) + + resolved = resolve_benchmarks(team, glob) + + assert resolved["lead_time"] == {"target": 3600, "source": "team"} + + +def test_absent_team_value_falls_back_to_global(): + team = BenchmarkSetting(lead_time=3600) + glob = BenchmarkSetting(lead_time=86400, deployment_frequency=5) + + resolved = resolve_benchmarks(team, glob) + + assert resolved["deployment_frequency"] == {"target": 5, "source": "global"} + + +def test_neither_set_yields_no_target(): + resolved = resolve_benchmarks(BenchmarkSetting(), BenchmarkSetting()) + + assert resolved["change_failure_rate"] == {"target": None, "source": None} + + +def test_metrics_mix_sources_within_one_response(): + # The case that makes per-metric fallback worth having. + team = BenchmarkSetting(lead_time=3600) + glob = BenchmarkSetting(lead_time=86400, deployment_frequency=5) + + resolved = resolve_benchmarks(team, glob) + + assert resolved["lead_time"]["source"] == "team" + assert resolved["deployment_frequency"]["source"] == "global" + assert resolved["mean_time_to_recovery"]["source"] is None + + +def test_zero_is_a_real_target_not_an_absent_one(): + # A team targeting zero failures must not silently inherit 15%. + team = BenchmarkSetting(change_failure_rate=0) + glob = BenchmarkSetting(change_failure_rate=15) + + resolved = resolve_benchmarks(team, glob) + + assert resolved["change_failure_rate"] == {"target": 0, "source": "team"} + + +def test_missing_settings_are_treated_as_empty(): + resolved = resolve_benchmarks(None, None) + + assert set(resolved.keys()) == set(BENCHMARK_METRICS) + assert all(v["target"] is None for v in resolved.values()) + + +def test_every_metric_is_registered_at_every_extension_point(): + # CLUSTOX: adding a metric means editing a dataclass, a list, three + # adapters, the API resource and the shipped default -- and missing one + # of them fails somewhere unrelated, or not at all. Pinning them to each + # other means the next metric fails HERE, by name, instead of silently + # vanishing from a response. + import dataclasses + + from mhq.service.settings.configuration_settings import SettingsService + from mhq.service.settings.default_settings_data import get_default_setting_data + from mhq.store.models.settings import SettingType + + metrics = set(BENCHMARK_METRICS) + + assert {f.name for f in dataclasses.fields(BenchmarkSetting)} == metrics + assert set(get_default_setting_data(SettingType.BENCHMARK_SETTING)) == metrics + + populated = BenchmarkSetting(**{metric: 1 for metric in metrics}) + + # Every adapter must carry every metric, in both directions. + to_json = SettingsService._adapt_benchmark_setting_json_data(None, populated) + assert to_json == {metric: 1 for metric in metrics} + + from_db = SettingsService._adapt_benchmark_setting_from_setting_data(None, to_json) + from_api = SettingsService._adapt_benchmark_setting_from_json(None, to_json) + assert from_db == populated + assert from_api == populated + + api_response = adapt_configuration_settings_response( + ConfigurationSettings( + entity_id="team-1", + entity_type=EntityType.TEAM, + specific_settings=populated, + updated_by=None, + created_at=datetime(2026, 1, 1), + updated_at=datetime(2026, 1, 1), + ) + ) + assert set(api_response["setting"]) == metrics + + +def test_lines_of_code_resolves_like_every_other_metric(): + resolved = resolve_benchmarks( + BenchmarkSetting(lines_of_code=200), + BenchmarkSetting(lines_of_code=400, lead_time=86400), + ) + + assert resolved["lines_of_code"] == {"target": 200, "source": "team"} + assert resolved["lead_time"] == {"target": 86400, "source": "global"} + + +def test_zero_lines_of_code_is_a_real_target(): + resolved = resolve_benchmarks(BenchmarkSetting(lines_of_code=0), None) + + assert resolved["lines_of_code"] == {"target": 0, "source": "team"} + + +def test_lines_of_code_has_no_upper_bound(): + # Percent is the only metric with a ceiling. A 5,000-line average PR is + # awful, not invalid -- rejecting it would stop an admin recording where + # the team actually is before setting a target to climb down from. + cleaned = validate_benchmark_payload({"lines_of_code": 5000}) + + assert cleaned["lines_of_code"] == 5000 + + +def test_validation_rejects_negatives(): + with pytest.raises(BadRequest): + validate_benchmark_payload({"lead_time": -1}) + + +def test_validation_rejects_change_failure_rate_above_100(): + with pytest.raises(BadRequest): + validate_benchmark_payload({"change_failure_rate": 101}) + + +def test_validation_rejects_unknown_keys(): + # A typo would otherwise store silently and inherit forever. + with pytest.raises(BadRequest): + validate_benchmark_payload({"leadtime": 3600}) + + +def test_validation_rejects_booleans(): + # `False is not None`, so a boolean surviving validation would reach + # resolve_benchmarks and be emitted as {"target": false} -- a target the + # cards would then try to draw a line at. + with pytest.raises(BadRequest): + validate_benchmark_payload({"deployment_frequency": True}) + + with pytest.raises(BadRequest): + validate_benchmark_payload({"deployment_frequency": False}) + + +def test_validation_keeps_zero_and_nulls_the_rest(): + cleaned = validate_benchmark_payload({"change_failure_rate": 0}) + + # 0 survives as a real target; the untouched metrics come back as + # explicit Nones rather than absent keys. + assert cleaned == { + "lead_time": None, + "deployment_frequency": None, + "change_failure_rate": 0, + "mean_time_to_recovery": None, + "lines_of_code": None, + } + + +def test_validation_accepts_a_partial_payload(): + cleaned = validate_benchmark_payload({"lead_time": 3600}) + + assert cleaned["lead_time"] == 3600 + assert set(cleaned.keys()) == set(BENCHMARK_METRICS) + + +def test_clearing_every_field_is_a_truthy_all_none_payload(): + # The form omits empty fields, so clearing all four posts `{}`. + # save_settings substitutes get_default_setting_data() for a *falsy* + # setting_data, so an empty dict here would write defaults instead of + # clearing -- "clear everything to go back to inheriting" has to survive + # this function as something truthy that means "no target anywhere". + cleaned = validate_benchmark_payload({}) + + assert cleaned == {metric: None for metric in BENCHMARK_METRICS} + assert cleaned, "an all-None payload must stay truthy for save_settings" + + +def test_the_shipped_default_sets_no_target_at_any_scope(): + # Zero-config: nothing may invent a benchmark. A default with numbers in + # it gave every card a target line and a "the default benchmark" caption + # the first time any settings form was opened. + from mhq.service.settings.default_settings_data import get_default_setting_data + from mhq.store.models.settings import SettingType + + default = get_default_setting_data(SettingType.BENCHMARK_SETTING) + + assert default == {metric: None for metric in BENCHMARK_METRICS} + + +def test_an_unsaved_benchmark_setting_is_all_none_and_not_persisted(): + from mhq.service.settings.benchmarks import empty_benchmark_settings + from mhq.store.models.settings.enums import EntityType + + settings = empty_benchmark_settings(EntityType.TEAM, "team-1") + + response = adapt_configuration_settings_response(settings) + + assert response["setting"] == {metric: None for metric in BENCHMARK_METRICS} + assert response["team_id"] == "team-1" + + +def test_resolution_asks_for_the_team_row_and_the_global_row(): + from mhq.service.settings.benchmarks import GLOBAL_BENCHMARK_ENTITY_ID + from mhq.store.models.settings import SettingType + from mhq.store.models.settings.enums import EntityType + + asked = [] + + class FakeSettingsService: + def get_settings(self, setting_type, entity_type, entity_id): + asked.append((setting_type, entity_type, entity_id)) + return None + + from mhq.service.settings.benchmarks import get_resolved_benchmarks_for_team + + get_resolved_benchmarks_for_team("team-1", settings_service=FakeSettingsService()) + + assert ( + SettingType.BENCHMARK_SETTING, + EntityType.TEAM, + "team-1", + ) in asked + assert ( + SettingType.BENCHMARK_SETTING, + EntityType.GLOBAL, + GLOBAL_BENCHMARK_ENTITY_ID, + ) in asked + + +def test_benchmark_setting_response_always_has_every_metric_key(): + # Set, zero, and unset must all round-trip distinctly: a dropped or + # zero-coerced key would make the config form unable to tell "inherit" + # apart from "target of 0". + config_settings = ConfigurationSettings( + entity_id="team-1", + entity_type=EntityType.TEAM, + specific_settings=BenchmarkSetting(lead_time=3600, change_failure_rate=0), + updated_by=None, + created_at=datetime(2026, 1, 1), + updated_at=datetime(2026, 1, 1), + ) + + response = adapt_configuration_settings_response(config_settings) + + assert response["setting"] == { + "lead_time": 3600, + "deployment_frequency": None, + "change_failure_rate": 0, + "mean_time_to_recovery": None, + "lines_of_code": None, + } + + +def test_global_benchmark_row_reports_scope_not_the_sentinel_id(): + from mhq.service.settings.benchmarks import GLOBAL_BENCHMARK_ENTITY_ID + + config_settings = ConfigurationSettings( + entity_id=GLOBAL_BENCHMARK_ENTITY_ID, + entity_type=EntityType.GLOBAL, + specific_settings=BenchmarkSetting(lead_time=3600), + updated_by=None, + created_at=datetime(2026, 1, 1), + updated_at=datetime(2026, 1, 1), + ) + + response = adapt_configuration_settings_response(config_settings) + + assert response["scope"] == "global" + assert "team_id" not in response + assert "org_id" not in response + assert "user_id" not in response diff --git a/backend/analytics_server/tests/service/settings/test_configuration_settings.py b/backend/analytics_server/tests/service/settings/test_configuration_settings.py new file mode 100644 index 000000000..167f35dac --- /dev/null +++ b/backend/analytics_server/tests/service/settings/test_configuration_settings.py @@ -0,0 +1,67 @@ +from mhq.service.settings.configuration_settings import SettingsService +from mhq.service.settings.models import JiraIncidentIssueTypesSetting +from mhq.store.models.settings import SettingType + + +def _service() -> SettingsService: + # The adapter methods under test are pure -- they never touch + # self._settings_repo, so a real repo isn't needed here. + return SettingsService(None) + + +class TestJiraIncidentIssueTypesSettingAdapters: + def test_from_json_data(self): + service = _service() + + setting = service._adapt_jira_incident_issue_types_setting_from_json( + {"issue_types": ["Bug", "Incident"]} + ) + + assert setting == JiraIncidentIssueTypesSetting(issue_types=["Bug", "Incident"]) + + def test_from_json_data_missing_key_defaults_to_empty_list(self): + service = _service() + + setting = service._adapt_jira_incident_issue_types_setting_from_json({}) + + assert setting.issue_types == [] + + def test_from_db_setting_data(self): + service = _service() + + setting = service._adapt_jira_incident_issue_types_setting_from_setting_data( + {"issue_types": ["Bug"]} + ) + + assert setting == JiraIncidentIssueTypesSetting(issue_types=["Bug"]) + + def test_to_db_json_data(self): + service = _service() + + data = service._adapt_jira_incident_issue_types_setting_json_data( + JiraIncidentIssueTypesSetting(issue_types=["Bug", "Task"]) + ) + + assert data == {"issue_types": ["Bug", "Task"]} + + def test_full_round_trip_from_api_json_to_db_json(self): + service = _service() + + db_json = service._adapt_specific_setting_data_from_json( + SettingType.JIRA_INCIDENT_ISSUE_TYPES_SETTING, {"issue_types": ["Bug"]} + ) + + assert db_json == {"issue_types": ["Bug"]} + + def test_dispatches_via_the_generic_setting_type_handlers(self): + service = _service() + + from_db = service._handle_config_setting_from_db_setting( + SettingType.JIRA_INCIDENT_ISSUE_TYPES_SETTING, {"issue_types": ["Bug"]} + ) + assert isinstance(from_db, JiraIncidentIssueTypesSetting) + + to_db = service._handle_config_setting_to_db_setting( + SettingType.JIRA_INCIDENT_ISSUE_TYPES_SETTING, from_db + ) + assert to_db == {"issue_types": ["Bug"]} diff --git a/backend/analytics_server/tests/service/settings/test_default_settings_data.py b/backend/analytics_server/tests/service/settings/test_default_settings_data.py new file mode 100644 index 000000000..be4a316c0 --- /dev/null +++ b/backend/analytics_server/tests/service/settings/test_default_settings_data.py @@ -0,0 +1,36 @@ +from mhq.service.settings.default_settings_data import get_default_setting_data +from mhq.store.models.incidents import IncidentSource +from mhq.store.models.settings import SettingType + + +def test_incident_sources_default_is_explicit_not_computed_live(): + # Regression test: this default used to be `list(IncidentSource)`, + # computed live -- adding a new value to the enum would silently opt + # every org with no saved setting into it, sight-unseen. It's an + # explicit, pinned list instead, so a new source added to the enum + # in the future has to be a deliberate addition here too. + data = get_default_setting_data(SettingType.INCIDENT_SOURCES_SETTING) + + assert set(data["incident_sources"]) == { + IncidentSource.INCIDENT_SERVICE.value, + IncidentSource.INCIDENT_TEAM.value, + IncidentSource.GIT_REPO.value, + IncidentSource.JIRA_ISSUE.value, + } + + +def test_incident_sources_default_includes_jira_issue_unconditionally(): + # JIRA_ISSUE used to be the one opt-in exception in this list -- now on + # by default, same as GIT_REPO, since a reopened ticket is the same + # kind of "the fix didn't hold" signal a revert PR already is. The + # admin-facing toggle to turn it back off still exists + # (ConfigureJiraIncidentSourceModalBody); only this default changed. + data = get_default_setting_data(SettingType.INCIDENT_SOURCES_SETTING) + + assert IncidentSource.JIRA_ISSUE.value in data["incident_sources"] + + +def test_jira_incident_issue_types_default_is_bug_only(): + data = get_default_setting_data(SettingType.JIRA_INCIDENT_ISSUE_TYPES_SETTING) + + assert data == {"issue_types": ["Bug"]} diff --git a/backend/analytics_server/tests/service/sprints/__init__.py b/backend/analytics_server/tests/service/sprints/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/service/sprints/test_service.py b/backend/analytics_server/tests/service/sprints/test_service.py new file mode 100644 index 000000000..e3d1e9802 --- /dev/null +++ b/backend/analytics_server/tests/service/sprints/test_service.py @@ -0,0 +1,76 @@ +from datetime import datetime, timezone +from unittest.mock import MagicMock + +from mhq.service.sprints.service import SprintService +from mhq.store.models.core import Team + +# tz-aware, matching real Jira dates (dt_from_iso_time_string always +# attaches an offset) -- a naive datetime here wouldn't have caught the +# real "datetime.max has no tzinfo" bug this file's tests found. +_UTC = timezone.utc + +# CLUSTOX: Jira integration -- the Sprint rollup chart. See +# docs/JIRA_INTEGRATION_PROPOSAL.md §6D. + + +def _sprint(name, start_date): + sprint = MagicMock() + sprint.name = name + sprint.start_date = start_date + return sprint + + +def _service(repo=None) -> SprintService: + return SprintService(repo or MagicMock()) + + +class TestGetTeamSprints: + def test_scopes_the_lookup_to_the_teams_tracked_projects(self): + repo = MagicMock() + repo.get_team_projects.return_value = [ + MagicMock(id="proj-1"), + MagicMock(id="proj-2"), + ] + repo.get_sprints_for_projects.return_value = [] + + _service(repo).get_team_sprints(Team(id="team-1")) + + looked_up_project_ids = repo.get_sprints_for_projects.call_args[0][0] + assert set(looked_up_project_ids) == {"proj-1", "proj-2"} + + def test_passes_the_limit_through_to_the_repo_query(self): + repo = MagicMock() + repo.get_team_projects.return_value = [] + repo.get_sprints_for_projects.return_value = [] + + _service(repo).get_team_sprints(Team(id="team-1"), limit=3) + + assert repo.get_sprints_for_projects.call_args[0][1] == 3 + + def test_returns_sprints_oldest_first_for_a_left_to_right_chart(self): + repo = MagicMock() + repo.get_team_projects.return_value = [] + repo.get_sprints_for_projects.return_value = [ + _sprint("Sprint 3", datetime(2024, 3, 1, tzinfo=_UTC)), + _sprint("Sprint 1", datetime(2024, 1, 1, tzinfo=_UTC)), + _sprint("Sprint 2", datetime(2024, 2, 1, tzinfo=_UTC)), + ] + + sprints = _service(repo).get_team_sprints(Team(id="team-1")) + + assert [s.name for s in sprints] == ["Sprint 1", "Sprint 2", "Sprint 3"] + + def test_a_sprint_with_no_start_date_sorts_last_rather_than_erroring(self): + # Real sprint dates are tz-aware -- a naive fallback for the + # undated sprint would raise here, not just sort wrong, the + # moment it's compared against a real one. + repo = MagicMock() + repo.get_team_projects.return_value = [] + repo.get_sprints_for_projects.return_value = [ + _sprint("Dated", datetime(2024, 1, 1, tzinfo=_UTC)), + _sprint("Undated", None), + ] + + sprints = _service(repo).get_team_sprints(Team(id="team-1")) + + assert [s.name for s in sprints] == ["Dated", "Undated"] diff --git a/backend/analytics_server/tests/service/ticket_insights/__init__.py b/backend/analytics_server/tests/service/ticket_insights/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/service/ticket_insights/test_cycle_time.py b/backend/analytics_server/tests/service/ticket_insights/test_cycle_time.py new file mode 100644 index 000000000..c05d9391e --- /dev/null +++ b/backend/analytics_server/tests/service/ticket_insights/test_cycle_time.py @@ -0,0 +1,270 @@ +from datetime import datetime, timedelta, timezone + +from mhq.service.ticket_insights.cycle_time import compute_cycle_time_by_project +from mhq.store.models.projects import OrgProject, Ticket, TicketState + +# CLUSTOX: Jira integration, Phase 4 (§6C). See +# docs/JIRA_INTEGRATION_PROPOSAL.md. +# +# Callers (ProjectRepoService.get_tickets_with_states_for_projects) only +# ever pass completed (status_category == "Done") tickets in here -- an +# open ticket has no bounded end time, and an earlier version of this +# module measured every ticket up to "now", letting a single item +# sitting untouched in a backlog for months dominate the average. Every +# ticket below therefore has both created_at and updated_at set, the way +# a real completed ticket would. + +PROJECT_ID = "proj-1" + + +def _project(project_id=PROJECT_ID, key="PZDA", name="Project Zero Deposit Africa"): + return OrgProject(id=project_id, key=key, name=name) + + +def _ticket( + ticket_id="t1", + project_id=PROJECT_ID, + status="Done", + created_at=None, + updated_at=None, +): + return Ticket( + id=ticket_id, + org_project_id=project_id, + status=status, + created_at=created_at or datetime(2026, 1, 1, tzinfo=timezone.utc), + updated_at=updated_at or datetime(2026, 1, 10, tzinfo=timezone.utc), + ) + + +def _state(ticket_id, from_status, to_status, changed_at): + return TicketState( + ticket_id=ticket_id, + from_status=from_status, + to_status=to_status, + changed_at=changed_at, + ) + + +def _projects_by_id(*projects): + return {str(p.id): p for p in projects} + + +class TestComputeCycleTimeByProject: + def test_a_ticket_with_no_transitions_counts_entirely_as_done(self): + ticket = _ticket( + created_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + updated_at=datetime(2026, 1, 10, tzinfo=timezone.utc), + ) + project = _project() + + [result] = compute_cycle_time_by_project([ticket], [], _projects_by_id(project)) + + assert ( + result.avg_seconds_by_category["Done"] == timedelta(days=9).total_seconds() + ) + assert result.avg_seconds_by_category["To Do"] == 0 + assert result.avg_seconds_by_category["In Progress"] == 0 + + def test_the_three_categories_always_sum_to_the_total(self): + created_at = datetime(2026, 1, 1, tzinfo=timezone.utc) + updated_at = datetime(2026, 1, 20, tzinfo=timezone.utc) + ticket = _ticket(created_at=created_at, updated_at=updated_at) + states = [ + _state( + "t1", "To Do", "In Progress", datetime(2026, 1, 3, tzinfo=timezone.utc) + ), + _state( + "t1", "In Progress", "QA", datetime(2026, 1, 8, tzinfo=timezone.utc) + ), + _state("t1", "QA", "Done", datetime(2026, 1, 15, tzinfo=timezone.utc)), + ] + project = _project() + + [result] = compute_cycle_time_by_project( + [ticket], states, _projects_by_id(project) + ) + + assert sum(result.avg_seconds_by_category.values()) == result.avg_total_seconds + assert result.avg_total_seconds == (updated_at - created_at).total_seconds() + + def test_the_first_segment_is_to_do_and_the_last_is_done_regardless_of_literal_name( + self, + ): + # First status and last status use this project's own literal + # names ("Backlog", "Shipped") -- the category bucketing is + # positional, not a name lookup, so it must still work. + created_at = datetime(2026, 1, 1, tzinfo=timezone.utc) + updated_at = datetime(2026, 1, 6, tzinfo=timezone.utc) + ticket = _ticket(status="Shipped", created_at=created_at, updated_at=updated_at) + state = _state( + "t1", "Backlog", "Shipped", datetime(2026, 1, 4, tzinfo=timezone.utc) + ) + project = _project() + + [result] = compute_cycle_time_by_project( + [ticket], [state], _projects_by_id(project) + ) + + assert ( + result.avg_seconds_by_category["To Do"] == timedelta(days=3).total_seconds() + ) + assert ( + result.avg_seconds_by_category["Done"] == timedelta(days=2).total_seconds() + ) + assert result.avg_seconds_by_category["In Progress"] == 0 + + def test_every_middle_segment_becomes_in_progress_regardless_of_how_many_there_are( + self, + ): + created_at = datetime(2026, 1, 1, tzinfo=timezone.utc) + updated_at = datetime(2026, 1, 20, tzinfo=timezone.utc) + ticket = _ticket(created_at=created_at, updated_at=updated_at) + states = [ + _state( + "t1", "To Do", "In Progress", datetime(2026, 1, 2, tzinfo=timezone.utc) + ), + _state( + "t1", "In Progress", "QA", datetime(2026, 1, 5, tzinfo=timezone.utc) + ), + _state("t1", "QA", "In Testing", datetime(2026, 1, 9, tzinfo=timezone.utc)), + _state( + "t1", + "In Testing", + "In_Review", + datetime(2026, 1, 12, tzinfo=timezone.utc), + ), + _state( + "t1", "In_Review", "Done", datetime(2026, 1, 18, tzinfo=timezone.utc) + ), + ] + project = _project() + + [result] = compute_cycle_time_by_project( + [ticket], states, _projects_by_id(project) + ) + + # In Progress + QA + In Testing + In_Review segments, combined. + expected_in_progress = ( + timedelta(days=3) + + timedelta(days=4) + + timedelta(days=3) + + timedelta(days=6) + ).total_seconds() + assert result.avg_seconds_by_category["In Progress"] == expected_in_progress + + def test_groups_tickets_into_separate_rows_per_project(self): + project_a = _project("proj-a", key="PAY", name="Payments") + project_b = _project("proj-b", key="ZDA", name="Zero Deposit Africa") + ticket_a = _ticket("t1", project_id="proj-a") + ticket_b = _ticket("t2", project_id="proj-b") + + results = compute_cycle_time_by_project( + [ticket_a, ticket_b], [], _projects_by_id(project_a, project_b) + ) + + assert {r.project_key for r in results} == {"PAY", "ZDA"} + + def test_sorts_projects_by_key_for_a_stable_order(self): + project_z = _project("proj-z", key="ZDA") + project_a = _project("proj-a", key="ABC") + results = compute_cycle_time_by_project( + [_ticket("t1", project_id="proj-z"), _ticket("t2", project_id="proj-a")], + [], + _projects_by_id(project_z, project_a), + ) + + assert [r.project_key for r in results] == ["ABC", "ZDA"] + + def test_skips_a_ticket_whose_project_is_not_in_the_lookup(self): + # Defensive: shouldn't happen in practice (the store query already + # scopes tickets to the team's tracked projects), but must not + # crash if it did. + ticket = _ticket(project_id="unknown-project") + + results = compute_cycle_time_by_project([ticket], [], {}) + + assert results == [] + + def test_averages_multiple_tickets_in_the_same_project(self): + project = _project() + tickets = [ + _ticket( + "t1", + created_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + updated_at=datetime(2026, 1, 5, tzinfo=timezone.utc), + ), + _ticket( + "t2", + created_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + updated_at=datetime(2026, 1, 3, tzinfo=timezone.utc), + ), + ] + + [result] = compute_cycle_time_by_project(tickets, [], _projects_by_id(project)) + + assert result.ticket_count == 2 + assert result.avg_total_seconds == timedelta(days=3).total_seconds() + + def test_discards_a_negative_duration_instead_of_corrupting_the_average(self): + ticket = _ticket( + created_at=datetime(2026, 1, 5, tzinfo=timezone.utc), + updated_at=datetime(2026, 1, 10, tzinfo=timezone.utc), + ) + bad_state = _state( + "t1", "To Do", "In Progress", datetime(2026, 1, 1, tzinfo=timezone.utc) + ) + project = _project() + + [result] = compute_cycle_time_by_project( + [ticket], [bad_state], _projects_by_id(project) + ) + + # The bad (negative) "To Do" segment is dropped entirely, leaving + # only the valid "In Progress" -> now segment for this ticket. + assert result.avg_seconds_by_category["To Do"] == 0 + + def test_a_reopened_ticket_puts_its_middle_done_period_into_in_progress(self): + # Honest, documented limitation of the positional heuristic (see + # _bucket_by_category's docstring): only the very first and very + # last segments get special treatment, so a segment that's + # literally "Done" but sits in the *middle* of the sequence (the + # ticket was reopened afterward) still lands in "In Progress" -- + # this function has no name->category map to recognize it as + # Done on its own. Documenting the actual behavior here rather + # than asserting something this heuristic doesn't actually do. + created_at = datetime(2026, 1, 1, tzinfo=timezone.utc) + updated_at = datetime(2026, 1, 20, tzinfo=timezone.utc) + ticket = _ticket(created_at=created_at, updated_at=updated_at) + states = [ + _state( + "t1", "To Do", "In Progress", datetime(2026, 1, 2, tzinfo=timezone.utc) + ), + _state( + "t1", "In Progress", "Done", datetime(2026, 1, 5, tzinfo=timezone.utc) + ), + _state( + "t1", "Done", "In Progress", datetime(2026, 1, 10, tzinfo=timezone.utc) + ), + _state( + "t1", "In Progress", "Done", datetime(2026, 1, 15, tzinfo=timezone.utc) + ), + ] + project = _project() + + [result] = compute_cycle_time_by_project( + [ticket], states, _projects_by_id(project) + ) + + assert result.ticket_count == 1 + # In Progress (Jan2->Jan5, 3d) + the middle Done period, reopened + # afterward (Jan5->Jan10, 5d) + the reopened In Progress visit + # (Jan10->Jan15, 5d) = 13 days, all counted as "In Progress". + # Only the final Done segment (Jan15->Jan20, 5d) is excluded. + assert ( + result.avg_seconds_by_category["In Progress"] + == timedelta(days=13).total_seconds() + ) + assert ( + result.avg_seconds_by_category["Done"] == timedelta(days=5).total_seconds() + ) diff --git a/backend/analytics_server/tests/service/ticket_insights/test_service.py b/backend/analytics_server/tests/service/ticket_insights/test_service.py new file mode 100644 index 000000000..74a708470 --- /dev/null +++ b/backend/analytics_server/tests/service/ticket_insights/test_service.py @@ -0,0 +1,129 @@ +from datetime import timedelta +from unittest.mock import MagicMock + +from mhq.service.ticket_insights.service import TicketInsightsService +from mhq.store.models.core import Team +from mhq.utils.time import Interval, time_now + +# CLUSTOX: Jira integration, Phase 4 (§6C/§6E). See +# docs/JIRA_INTEGRATION_PROPOSAL.md. + + +def _service(project_repo=None, code_repo=None, ticket_matching_repo=None): + return TicketInsightsService( + project_repo or MagicMock(), + code_repo or MagicMock(), + ticket_matching_repo or MagicMock(), + ) + + +def _interval(): + now = time_now() + return Interval(now, now) + + +class TestGetTeamTicketInsights: + def test_scopes_the_ticket_lookup_to_the_teams_tracked_projects(self): + project_repo = MagicMock() + project_repo.get_team_projects.return_value = [ + MagicMock(id="proj-1"), + MagicMock(id="proj-2"), + ] + project_repo.get_tickets_with_states_for_projects.return_value = ([], []) + code_repo = MagicMock() + code_repo.get_team_repos.return_value = [] + ticket_matching_repo = MagicMock() + ticket_matching_repo.get_unlinked_merged_pr_count.return_value = 0 + + team = Team(id="team-1") + _service( + project_repo, code_repo, ticket_matching_repo + ).get_team_ticket_insights(team, _interval()) + + looked_up_project_ids = ( + project_repo.get_tickets_with_states_for_projects.call_args[0][0] + ) + assert set(looked_up_project_ids) == {"proj-1", "proj-2"} + + def test_scopes_the_unlinked_pr_count_to_the_teams_tracked_repos(self): + project_repo = MagicMock() + project_repo.get_team_projects.return_value = [] + project_repo.get_tickets_with_states_for_projects.return_value = ([], []) + code_repo = MagicMock() + code_repo.get_team_repos.return_value = [MagicMock(id="repo-1")] + ticket_matching_repo = MagicMock() + ticket_matching_repo.get_unlinked_merged_pr_count.return_value = 3 + + team = Team(id="team-1") + result = _service( + project_repo, code_repo, ticket_matching_repo + ).get_team_ticket_insights(team, _interval()) + + looked_up_repo_ids = ( + ticket_matching_repo.get_unlinked_merged_pr_count.call_args[0][0] + ) + assert looked_up_repo_ids == ["repo-1"] + assert result["prs_without_ticket_count"] == 3 + + def test_returns_cycle_time_grouped_by_project(self): + project = MagicMock(id="proj-1", key="PZDA", name="Project Zero Deposit Africa") + project_repo = MagicMock() + project_repo.get_team_projects.return_value = [project] + ticket = MagicMock(id="t1", status="Done", org_project_id="proj-1") + ticket.updated_at = time_now() + ticket.created_at = ticket.updated_at - timedelta(days=3) + project_repo.get_tickets_with_states_for_projects.return_value = ( + [ticket], + [], + ) + code_repo = MagicMock() + code_repo.get_team_repos.return_value = [] + ticket_matching_repo = MagicMock() + ticket_matching_repo.get_unlinked_merged_pr_count.return_value = 0 + + team = Team(id="team-1") + result = _service( + project_repo, code_repo, ticket_matching_repo + ).get_team_ticket_insights(team, _interval()) + + [project_result] = result["cycle_time_by_project"] + assert project_result.project_key == "PZDA" + assert project_result.ticket_count == 1 + assert project_result.avg_total_seconds == timedelta(days=3).total_seconds() + + +# CLUSTOX: Jira integration, Phase 4 (§6E) -- the Data Hygiene +# drill-down. See docs/JIRA_INTEGRATION_PROPOSAL.md. +class TestGetTeamUnlinkedPrs: + def test_scopes_the_lookup_to_the_teams_tracked_repos(self): + code_repo = MagicMock() + code_repo.get_team_repos.return_value = [ + MagicMock(id="repo-1"), + MagicMock(id="repo-2"), + ] + ticket_matching_repo = MagicMock() + ticket_matching_repo.get_unlinked_merged_prs.return_value = [] + + team = Team(id="team-1") + _service( + code_repo=code_repo, ticket_matching_repo=ticket_matching_repo + ).get_team_unlinked_prs(team, _interval()) + + looked_up_repo_ids = ticket_matching_repo.get_unlinked_merged_prs.call_args[0][ + 0 + ] + assert set(looked_up_repo_ids) == {"repo-1", "repo-2"} + + def test_returns_exactly_what_the_repo_service_returns(self): + code_repo = MagicMock() + code_repo.get_team_repos.return_value = [] + prs = [MagicMock(), MagicMock()] + ticket_matching_repo = MagicMock() + ticket_matching_repo.get_unlinked_merged_prs.return_value = prs + + team = Team(id="team-1") + result = _service( + code_repo=code_repo, ticket_matching_repo=ticket_matching_repo + ).get_team_unlinked_prs(team, _interval()) + + assert result == prs diff --git a/backend/analytics_server/tests/service/ticket_matching/__init__.py b/backend/analytics_server/tests/service/ticket_matching/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/service/ticket_matching/test_matcher.py b/backend/analytics_server/tests/service/ticket_matching/test_matcher.py new file mode 100644 index 000000000..c28ea743b --- /dev/null +++ b/backend/analytics_server/tests/service/ticket_matching/test_matcher.py @@ -0,0 +1,76 @@ +from mhq.service.ticket_matching.matcher import extract_ticket_keys + +# CLUSTOX: Jira integration, Phase 3 (ticket-PR matching). Every example +# here is either a real PR title/branch found in this org's own history +# (see docs/JIRA_INTEGRATION_PROPOSAL.md's status notes) or a real false- +# positive-shaped string chosen to prove the regex alone isn't the +# safety net (the caller cross-checking against real ticket keys is). + + +def test_finds_a_ticket_key_in_a_conventional_commit_style_title(): + keys = extract_ticket_keys("fix(PZDA-543): record the accepted policy version", "") + assert keys == ["PZDA-543"] + + +def test_finds_a_lowercased_ticket_key_in_a_branch_name(): + keys = extract_ticket_keys("", "fix/pzda-543-consent-policy-version") + assert keys == ["PZDA-543"] + + +def test_dedupes_the_same_key_appearing_twice_in_one_string(): + # "PAY-123 Fix PAY-123 payment validation" -- one mention, not two. + keys = extract_ticket_keys("PAY-123 Fix PAY-123 payment validation") + assert keys == ["PAY-123"] + + +def test_dedupes_the_same_key_appearing_in_both_title_and_branch(): + keys = extract_ticket_keys( + "fix(PZDA-543): record the accepted policy version", + "fix/pzda-543-consent-policy-version", + ) + assert keys == ["PZDA-543"] + + +def test_expands_a_slash_separated_multi_ticket_reference(): + # Real example: "feat(PZDA-544/546): measure the profile-completion + # reminder interval in working days" + keys = extract_ticket_keys("feat(PZDA-544/546): measure the reminder interval", "") + assert keys == ["PZDA-544", "PZDA-546"] + + +def test_expands_a_comma_separated_multi_ticket_reference(): + keys = extract_ticket_keys("fix(PZDA-544,546): two tickets, one fix", "") + assert keys == ["PZDA-544", "PZDA-546"] + + +def test_finds_a_trailing_parenthetical_reference(): + keys = extract_ticket_keys( + "fix(guarantees): TopUpCalculator must round half-up (PZDA-877)", "" + ) + assert keys == ["PZDA-877"] + + +def test_returns_nothing_for_a_pr_with_no_ticket_reference(): + keys = extract_ticket_keys( + "ci(frontend): fail the ESLint gate only on findings a change introduced", + "fix/eslint-gate-new-findings-only", + ) + assert keys == [] + + +def test_ignores_a_bare_number_with_no_letter_prefix(): + keys = extract_ticket_keys("Bump version to 2024", "") + assert keys == [] + + +def test_extracts_a_key_shaped_substring_that_is_not_actually_a_ticket(): + # The regex alone can't know "ISO-27001" isn't a Jira key -- that's + # deliberate (see the module docstring). The caller is responsible + # for discarding anything that isn't a real, currently-synced ticket + # key before treating it as a match. + keys = extract_ticket_keys("chore: note the ISO-27001 audit date", "") + assert keys == ["ISO-27001"] + + +def test_handles_none_and_empty_strings_without_raising(): + assert extract_ticket_keys(None, "", None) == [] diff --git a/backend/analytics_server/tests/service/ticket_matching/test_service.py b/backend/analytics_server/tests/service/ticket_matching/test_service.py new file mode 100644 index 000000000..2bb5c78ff --- /dev/null +++ b/backend/analytics_server/tests/service/ticket_matching/test_service.py @@ -0,0 +1,367 @@ +from unittest.mock import MagicMock + +from mhq.service.ticket_matching.service import TicketMatchingService +from mhq.store.models.code import PullRequest + +# CLUSTOX: Jira integration, Phase 3 (ticket-PR matching). See +# docs/JIRA_INTEGRATION_PROPOSAL.md. The collision-resolution tests further +# down are Phase 4 (docs/JIRA_MULTI_ACCOUNT_PLAN.md Task 5). + +ORG_ID = "org-1" + + +def _pr(pr_id="pr-1", title="", head_branch="", body=None, repo_id="repo-1"): + return PullRequest( + id=pr_id, + title=title, + head_branch=head_branch, + data={"body": body} if body else None, + repo_id=repo_id, + ) + + +def _repo(**overrides) -> MagicMock: + # CLUSTOX: get_colliding_ticket_key_candidates is called on every + # match_org_prs_to_tickets run, alongside get_org_tickets_key_map -- + # defaulted to "no collisions" here so the tests below that only care + # about the ordinary, non-colliding path don't each have to stub it. + repo = MagicMock() + repo.get_colliding_ticket_key_candidates.return_value = {} + for attr, value in overrides.items(): + setattr(repo, attr, value) + return repo + + +def _service(repo=None) -> TicketMatchingService: + return TicketMatchingService(repo if repo is not None else _repo()) + + +class TestMatchOrgPrsToTickets: + def test_skips_fetching_prs_entirely_when_no_tickets_are_synced_yet(self): + repo = _repo() + repo.get_org_tickets_key_map.return_value = {} + + _service(repo).match_org_prs_to_tickets(ORG_ID) + + repo.get_unmatched_prs_for_org.assert_not_called() + repo.save_mappings.assert_not_called() + + def test_does_nothing_when_there_are_no_unmatched_prs(self): + repo = _repo() + repo.get_org_tickets_key_map.return_value = {"PZDA-543": "ticket-1"} + repo.get_unmatched_prs_for_org.return_value = [] + + _service(repo).match_org_prs_to_tickets(ORG_ID) + + repo.save_mappings.assert_not_called() + + def test_maps_a_pr_to_the_ticket_its_title_references(self): + repo = _repo() + repo.get_org_tickets_key_map.return_value = {"PZDA-543": "ticket-1"} + repo.get_unmatched_prs_for_org.return_value = [ + _pr("pr-1", title="fix(PZDA-543): consent policy version") + ] + + _service(repo).match_org_prs_to_tickets(ORG_ID) + + mappings = repo.save_mappings.call_args[0][0] + assert len(mappings) == 1 + assert str(mappings[0].pr_id) == "pr-1" + assert mappings[0].ticket_id == "ticket-1" + + def test_maps_a_pr_to_every_ticket_a_multi_ticket_title_references(self): + repo = _repo() + repo.get_org_tickets_key_map.return_value = { + "PZDA-544": "ticket-544", + "PZDA-546": "ticket-546", + } + repo.get_unmatched_prs_for_org.return_value = [ + _pr("pr-1", title="feat(PZDA-544/546): reminder interval") + ] + + _service(repo).match_org_prs_to_tickets(ORG_ID) + + mappings = repo.save_mappings.call_args[0][0] + assert {m.ticket_id for m in mappings} == {"ticket-544", "ticket-546"} + + def test_does_not_match_a_key_shaped_string_that_is_not_a_real_ticket(self): + # The regex would extract "ISO-27001" as a candidate; since it's + # not in the org's real ticket key map, it must not become a + # mapping row. + repo = _repo() + repo.get_org_tickets_key_map.return_value = {"PZDA-543": "ticket-1"} + repo.get_unmatched_prs_for_org.return_value = [ + _pr("pr-1", title="chore: note the ISO-27001 audit date") + ] + + _service(repo).match_org_prs_to_tickets(ORG_ID) + + repo.save_mappings.assert_not_called() + + # CLUSTOX: the PR title is the *only* authoritative source for a Jira + # association -- product spec, 2026-09-10 (see TicketMatchingService's + # own docstring). These replace an earlier version of this suite that + # asserted the opposite: a body/branch reference used to be enough on + # its own, on the reasoning that real data showed roughly half of + # this org's "unmatched" PRs referenced a ticket only in the body. + # That traded a title-shaped guarantee an admin can see and trust for + # recall, which the product spec now explicitly rules out -- these + # scenario numbers below refer to that spec's own numbered examples. + + def test_ignores_a_ticket_key_present_only_in_the_branch_name(self): + # Scenario 6: branch "feature/PAY-123-payment-validation" must + # not be used as a fallback when the title has no key at all. + repo = _repo() + repo.get_org_tickets_key_map.return_value = {"PAY-123": "ticket-123"} + repo.get_unmatched_prs_for_org.return_value = [ + _pr( + "pr-1", + title="Improve checkout error handling", + head_branch="feature/PAY-123-payment-validation", + ) + ] + + _service(repo).match_org_prs_to_tickets(ORG_ID) + + repo.save_mappings.assert_not_called() + + def test_ignores_a_ticket_key_present_only_in_the_description(self): + # Scenario 8: a body saying "This fixes PAY-123" must not be used + # as a fallback when the title has no key at all. + repo = _repo() + repo.get_org_tickets_key_map.return_value = {"PAY-123": "ticket-123"} + repo.get_unmatched_prs_for_org.return_value = [ + _pr( + "pr-1", + title="Improve checkout error handling", + body="This fixes PAY-123.", + ) + ] + + _service(repo).match_org_prs_to_tickets(ORG_ID) + + repo.save_mappings.assert_not_called() + + def test_ignores_branch_and_description_together_when_the_title_has_no_key(self): + # Scenario 10: the combined case -- a key-free title with a real + # key sitting in both the branch and the description at once must + # still produce no association. + repo = _repo() + repo.get_org_tickets_key_map.return_value = {"PAY-123": "ticket-123"} + repo.get_unmatched_prs_for_org.return_value = [ + _pr( + "pr-1", + title="Improve checkout error handling", + head_branch="feature/checkout-errors", + body="This fixes PAY-123.", + ) + ] + + _service(repo).match_org_prs_to_tickets(ORG_ID) + + repo.save_mappings.assert_not_called() + + def test_matches_only_the_titles_key_when_branch_and_description_reference_a_different_real_ticket( + self, + ): + # Scenario 14 (adapted -- this schema has no commit-message field + # at all for a PR, so there is no code path that could scan one; + # description is the closest real field carrying the same risk, + # and the same guarantee must hold for it): the title's key wins, + # full stop, even when a *different, real, currently-synced* + # ticket key appears elsewhere on the same PR. + repo = _repo() + repo.get_org_tickets_key_map.return_value = { + "PAY-123": "ticket-123", + "PAY-456": "ticket-456", + } + repo.get_unmatched_prs_for_org.return_value = [ + _pr( + "pr-1", + title="PAY-123 Fix payment validation", + head_branch="feature/pay-456-refactor", + body="Related to PAY-456 refactor work.", + ) + ] + + _service(repo).match_org_prs_to_tickets(ORG_ID) + + mappings = repo.save_mappings.call_args[0][0] + assert [m.ticket_id for m in mappings] == ["ticket-123"] + + def test_deduplicates_the_same_key_repeated_within_one_title(self): + # Scenario 13: "PAY-123 Fix PAY-123 payment validation" must + # produce exactly one mapping, not two. + repo = _repo() + repo.get_org_tickets_key_map.return_value = {"PAY-123": "ticket-123"} + repo.get_unmatched_prs_for_org.return_value = [ + _pr("pr-1", title="PAY-123 Fix PAY-123 payment validation") + ] + + _service(repo).match_org_prs_to_tickets(ORG_ID) + + mappings = repo.save_mappings.call_args[0][0] + assert len(mappings) == 1 + assert mappings[0].ticket_id == "ticket-123" + + def test_a_later_pr_can_match_a_ticket_an_earlier_pr_already_matched(self): + # Scenario 12: a Jira issue can have more than one PR against it + # over time (e.g. reopened and fixed again) -- matching must not + # assume or enforce "one ticket, one PR". get_unmatched_prs_for_org + # already only returns PRs with no mapping row yet, so this is + # really a guarantee that matching doesn't skip a PR just because + # its ticket already has a mapping from a different PR. + repo = _repo() + repo.get_org_tickets_key_map.return_value = {"PAY-123": "ticket-123"} + repo.get_unmatched_prs_for_org.return_value = [ + _pr("pr-100", title="PAY-123 Initial payment fix"), + _pr("pr-120", title="PAY-123 Fix payment regression"), + ] + + _service(repo).match_org_prs_to_tickets(ORG_ID) + + mappings = repo.save_mappings.call_args[0][0] + assert {str(m.pr_id) for m in mappings} == {"pr-100", "pr-120"} + assert {m.ticket_id for m in mappings} == {"ticket-123"} + + def test_skips_saving_entirely_when_no_pr_matches_anything(self): + repo = _repo() + repo.get_org_tickets_key_map.return_value = {"PZDA-543": "ticket-1"} + repo.get_unmatched_prs_for_org.return_value = [ + _pr("pr-1", title="chore: bump dependencies") + ] + + _service(repo).match_org_prs_to_tickets(ORG_ID) + + repo.save_mappings.assert_not_called() + + def test_does_one_batch_lookup_and_one_batch_save_regardless_of_pr_count(self): + repo = _repo() + repo.get_org_tickets_key_map.return_value = {"PZDA-543": "ticket-1"} + repo.get_unmatched_prs_for_org.return_value = [ + _pr(f"pr-{i}", title="fix(PZDA-543): x") for i in range(50) + ] + + _service(repo).match_org_prs_to_tickets(ORG_ID) + + repo.get_org_tickets_key_map.assert_called_once() + repo.get_unmatched_prs_for_org.assert_called_once() + repo.save_mappings.assert_called_once() + assert len(repo.save_mappings.call_args[0][0]) == 50 + + +# CLUSTOX: Jira multi-account support. See docs/JIRA_MULTI_ACCOUNT_PLAN.md +# Task 5 -- two connections' sites landing on the same project key ("PROJ") +# means get_org_tickets_key_map alone can no longer resolve every key to one +# ticket; these are the collision path. +class TestCollidingTicketKeys: + def test_resolves_to_the_ticket_from_the_prs_own_relevant_connection(self): + # The plan's own regression case: two connections both have a + # project called "PROJ". The PR's repo is tracked by a team that + # selected connection A's "PROJ", not connection B's -- so + # "PROJ-123" must resolve to connection A's ticket specifically. + repo = _repo() + repo.get_org_tickets_key_map.return_value = {} + repo.get_colliding_ticket_key_candidates.return_value = { + "PROJ-123": [("ticket-a", "project-a"), ("ticket-b", "project-b")] + } + repo.get_relevant_org_project_ids_for_repo.return_value = {"project-a"} + repo.get_unmatched_prs_for_org.return_value = [ + _pr("pr-1", title="fix(PROJ-123): refund rounding", repo_id="repo-1") + ] + + _service(repo).match_org_prs_to_tickets(ORG_ID) + + mappings = repo.save_mappings.call_args[0][0] + assert len(mappings) == 1 + assert mappings[0].ticket_id == "ticket-a" + repo.get_relevant_org_project_ids_for_repo.assert_called_once_with("repo-1") + + def test_skips_the_match_when_no_candidate_project_is_relevant_to_the_repo(self): + # Fails closed: neither candidate's project is one this repo's + # teams actually use, so there is no principled way to pick -- + # a guess here could attribute the PR to a different team's + # ticket entirely. + repo = _repo() + repo.get_org_tickets_key_map.return_value = {} + repo.get_colliding_ticket_key_candidates.return_value = { + "PROJ-123": [("ticket-a", "project-a"), ("ticket-b", "project-b")] + } + repo.get_relevant_org_project_ids_for_repo.return_value = {"project-c"} + repo.get_unmatched_prs_for_org.return_value = [ + _pr("pr-1", title="fix(PROJ-123): refund rounding") + ] + + _service(repo).match_org_prs_to_tickets(ORG_ID) + + repo.save_mappings.assert_not_called() + + def test_skips_the_match_when_more_than_one_candidate_project_is_relevant(self): + # Still ambiguous even after scoping -- e.g. one team tracks this + # repo and has selected projects from both connections. No + # single correct answer exists, so this stays unmatched rather + # than picking one arbitrarily. + repo = _repo() + repo.get_org_tickets_key_map.return_value = {} + repo.get_colliding_ticket_key_candidates.return_value = { + "PROJ-123": [("ticket-a", "project-a"), ("ticket-b", "project-b")] + } + repo.get_relevant_org_project_ids_for_repo.return_value = { + "project-a", + "project-b", + } + repo.get_unmatched_prs_for_org.return_value = [ + _pr("pr-1", title="fix(PROJ-123): refund rounding") + ] + + _service(repo).match_org_prs_to_tickets(ORG_ID) + + repo.save_mappings.assert_not_called() + + def test_a_key_with_no_collision_never_triggers_the_relevant_project_lookup(self): + # The overwhelmingly common case -- no collision at all -- must + # stay exactly as cheap as it always was: no extra query. + repo = _repo() + repo.get_org_tickets_key_map.return_value = {"PZDA-543": "ticket-1"} + repo.get_unmatched_prs_for_org.return_value = [ + _pr("pr-1", title="fix(PZDA-543): x") + ] + + _service(repo).match_org_prs_to_tickets(ORG_ID) + + repo.get_relevant_org_project_ids_for_repo.assert_not_called() + + def test_caches_the_relevant_project_lookup_per_repo_not_per_pr(self): + repo = _repo() + repo.get_org_tickets_key_map.return_value = {} + repo.get_colliding_ticket_key_candidates.return_value = { + "PROJ-1": [("ticket-a", "project-a"), ("ticket-b", "project-b")], + "PROJ-2": [("ticket-c", "project-a"), ("ticket-d", "project-b")], + } + repo.get_relevant_org_project_ids_for_repo.return_value = {"project-a"} + repo.get_unmatched_prs_for_org.return_value = [ + _pr("pr-1", title="fix(PROJ-1): x", repo_id="repo-1"), + _pr("pr-2", title="fix(PROJ-2): y", repo_id="repo-1"), + ] + + _service(repo).match_org_prs_to_tickets(ORG_ID) + + repo.get_relevant_org_project_ids_for_repo.assert_called_once_with("repo-1") + mappings = repo.save_mappings.call_args[0][0] + assert {m.ticket_id for m in mappings} == {"ticket-a", "ticket-c"} + + def test_a_non_colliding_and_colliding_key_on_the_same_pr_both_resolve(self): + repo = _repo() + repo.get_org_tickets_key_map.return_value = {"PZDA-1": "ticket-plain"} + repo.get_colliding_ticket_key_candidates.return_value = { + "PROJ-1": [("ticket-a", "project-a"), ("ticket-b", "project-b")] + } + repo.get_relevant_org_project_ids_for_repo.return_value = {"project-a"} + repo.get_unmatched_prs_for_org.return_value = [ + _pr("pr-1", title="fix(PZDA-1): also touches PROJ-1") + ] + + _service(repo).match_org_prs_to_tickets(ORG_ID) + + mappings = repo.save_mappings.call_args[0][0] + assert {m.ticket_id for m in mappings} == {"ticket-plain", "ticket-a"} diff --git a/backend/analytics_server/tests/service/workflows/sync/test_etl_handler.py b/backend/analytics_server/tests/service/workflows/sync/test_etl_handler.py new file mode 100644 index 000000000..58647949c --- /dev/null +++ b/backend/analytics_server/tests/service/workflows/sync/test_etl_handler.py @@ -0,0 +1,182 @@ +# CLUSTOX: the Jenkins rows have to survive the provider filter in +# _get_active_repo_workflows, otherwise the whole integration is inert -- the +# mapping is stored, the picker works, and no build is ever synced. +from typing import List + +from mhq.service.workflows import integration as integration_module +from mhq.service.workflows.integration import WorkflowsIntegrationsService +from mhq.service.workflows.sync import etl_handler as etl_handler_module +from mhq.service.workflows.sync.etl_handler import WorkflowETLHandler +from mhq.store.models.code import ( + OrgRepo, + RepoWorkflow, + RepoWorkflowProviders, + RepoWorkflowType, +) +from mhq.utils.string import uuid4_str + + +class FakeIntegration: + def __init__(self, name: str): + self.name = name + + +class FakeCoreRepoService: + """ + Applies the same name filter the real query does, so a provider missing + from WORKFLOW_INTEGRATION_BUCKET is invisible to the service. + """ + + def __init__(self, linked_names: List[str]): + self._linked_names = linked_names + + def get_org_integrations_for_names(self, org_id: str, provider_names: List[str]): + return [FakeIntegration(n) for n in self._linked_names if n in provider_names] + + +class FakeCodeIntegrationService: + @staticmethod + def get_org_providers(org_id: str) -> List[str]: + return ["github"] + + +class FakeCodeRepoService: + def __init__(self, org_repos: List[OrgRepo]): + self._org_repos = org_repos + + def get_active_org_repos(self, org_id: str) -> List[OrgRepo]: + return self._org_repos + + +class FakeWorkflowRepoService: + """ + Applies the same provider filter as + get_active_repo_workflows_by_repo_ids_and_providers, and records the + provider list it was handed so the test can assert on it directly. + """ + + def __init__(self, repo_workflows: List[RepoWorkflow]): + self._repo_workflows = repo_workflows + self.providers_queried: List[RepoWorkflowProviders] = [] + + def get_active_repo_workflows_by_repo_ids_and_providers( + self, repo_ids: List[str], providers: List[RepoWorkflowProviders] + ) -> List[RepoWorkflow]: + self.providers_queried = providers + return [ + workflow + for workflow in self._repo_workflows + if str(workflow.org_repo_id) in repo_ids + and workflow.provider in providers + and workflow.is_active + ] + + +def _org_repo(org_id: str) -> OrgRepo: + return OrgRepo(id=uuid4_str(), org_id=org_id, name="api", provider="github") + + +def _repo_workflow(org_repo: OrgRepo, provider: RepoWorkflowProviders) -> RepoWorkflow: + return RepoWorkflow( + id=uuid4_str(), + org_repo_id=str(org_repo.id), + type=RepoWorkflowType.DEPLOYMENT, + provider=provider, + provider_workflow_id="deploy-api", + is_active=True, + name="deploy-api", + ) + + +def _build_handler( + monkeypatch, + linked_integration_names: List[str], + org_repos: List[OrgRepo], + repo_workflows: List[RepoWorkflow], +): + workflow_repo_service = FakeWorkflowRepoService(repo_workflows) + workflows_integrations_service = WorkflowsIntegrationsService( + FakeCoreRepoService(linked_integration_names) + ) + monkeypatch.setattr( + etl_handler_module, + "get_code_integration_service", + lambda: FakeCodeIntegrationService(), + ) + monkeypatch.setattr( + etl_handler_module, + "get_workflows_integrations_service", + lambda: workflows_integrations_service, + ) + handler = WorkflowETLHandler( + FakeCodeRepoService(org_repos), + workflow_repo_service, + None, + None, + None, + ) + return handler, workflow_repo_service + + +def test_a_jenkins_only_workspace_syncs_its_jenkins_workflow(monkeypatch): + org_id = uuid4_str() + org_repo = _org_repo(org_id) + jenkins_workflow = _repo_workflow(org_repo, RepoWorkflowProviders.JENKINS) + + handler, workflow_repo_service = _build_handler( + monkeypatch, + linked_integration_names=["github", "jenkins"], + org_repos=[org_repo], + repo_workflows=[jenkins_workflow], + ) + + active = handler._get_active_repo_workflows(org_id) + + # Before Jenkins was added to WORKFLOW_INTEGRATION_BUCKET this was []: + # get_org_providers returned nothing, so the function bailed out with + # "No workflow integrations found". + assert [workflow.id for _, workflow in active] == [jenkins_workflow.id] + assert active[0][0].id == org_repo.id + assert RepoWorkflowProviders.JENKINS in workflow_repo_service.providers_queried + + +def test_jenkins_survives_the_provider_filter_when_github_is_also_linked(monkeypatch): + org_id = uuid4_str() + github_repo = _org_repo(org_id) + jenkins_repo = _org_repo(org_id) + github_workflow = _repo_workflow(github_repo, RepoWorkflowProviders.GITHUB_ACTIONS) + jenkins_workflow = _repo_workflow(jenkins_repo, RepoWorkflowProviders.JENKINS) + + handler, workflow_repo_service = _build_handler( + monkeypatch, + linked_integration_names=["github", "jenkins"], + org_repos=[github_repo, jenkins_repo], + repo_workflows=[github_workflow, jenkins_workflow], + ) + + active = handler._get_active_repo_workflows(org_id) + + # Both providers must come back. Previously the Jenkins row was dropped by + # RepoWorkflow.provider.in_(providers) even with GitHub linked. + assert {workflow.id for _, workflow in active} == { + github_workflow.id, + jenkins_workflow.id, + } + assert set(workflow_repo_service.providers_queried) == { + RepoWorkflowProviders.GITHUB_ACTIONS, + RepoWorkflowProviders.JENKINS, + } + + +def test_get_org_providers_reports_jenkins_for_a_jenkins_only_workspace(): + service = WorkflowsIntegrationsService(FakeCoreRepoService(["jenkins"])) + + # This is the exact value sync_org_workflows checks before returning early. + assert service.get_org_providers(uuid4_str()) == ["jenkins"] + + +def test_an_unrelated_integration_is_still_not_a_workflow_provider(): + service = WorkflowsIntegrationsService(FakeCoreRepoService(["gitlab"])) + + assert service.get_org_providers(uuid4_str()) == [] + assert "gitlab" not in integration_module.WORKFLOW_INTEGRATION_BUCKET diff --git a/backend/analytics_server/tests/service/workflows/sync/test_etl_jenkins_handler.py b/backend/analytics_server/tests/service/workflows/sync/test_etl_jenkins_handler.py new file mode 100644 index 000000000..d0769d3b1 --- /dev/null +++ b/backend/analytics_server/tests/service/workflows/sync/test_etl_jenkins_handler.py @@ -0,0 +1,174 @@ +import re +from datetime import datetime + +import pytest +import pytz + +from mhq.service.workflows.sync.etl_jenkins_handler import JenkinsETLHandler +from mhq.store.models.code import RepoWorkflowRunsStatus +from mhq.utils.string import uuid4_str +from tests.factories.models.exapi.jenkins import ( + get_jenkins_build_dict, + get_jenkins_build_dict_without_git_plugin, +) + + +def _epoch_start() -> datetime: + return datetime.fromtimestamp(0, tz=pytz.UTC) + + +class FakeWorkflowRepoService: + def get_repo_workflow_run_by_provider_workflow_run_id(self, *args): + return None + + +def _handler(): + return JenkinsETLHandler(uuid4_str(), None, FakeWorkflowRepoService()) + + +def test_success_maps_to_success(): + build = get_jenkins_build_dict(result="SUCCESS") + assert ( + JenkinsETLHandler._get_repo_workflow_status(build) + == RepoWorkflowRunsStatus.SUCCESS + ) + + +def test_failure_maps_to_failure(): + build = get_jenkins_build_dict(result="FAILURE") + assert ( + JenkinsETLHandler._get_repo_workflow_status(build) + == RepoWorkflowRunsStatus.FAILURE + ) + + +def test_aborted_maps_to_cancelled(): + build = get_jenkins_build_dict(result="ABORTED") + assert ( + JenkinsETLHandler._get_repo_workflow_status(build) + == RepoWorkflowRunsStatus.CANCELLED + ) + + +def test_unstable_maps_to_failure_so_it_is_not_counted_as_a_deployment(): + build = get_jenkins_build_dict(result="UNSTABLE") + assert ( + JenkinsETLHandler._get_repo_workflow_status(build) + == RepoWorkflowRunsStatus.FAILURE + ) + + +def test_a_running_build_maps_to_pending(): + build = get_jenkins_build_dict(result=None, building=True) + assert ( + JenkinsETLHandler._get_repo_workflow_status(build) + == RepoWorkflowRunsStatus.PENDING + ) + + +def test_adapt_maps_every_field(): + build = get_jenkins_build_dict() + repo_workflow_id = uuid4_str() + + run = _handler()._adapt_jenkins_build_to_workflow_run(repo_workflow_id, build) + + assert run.repo_workflow_id == repo_workflow_id + assert run.provider_workflow_run_id == "42" + assert run.status == RepoWorkflowRunsStatus.SUCCESS + assert run.event_actor == "hamad" + # Jenkins reports "origin/main"; what gets stored has to be "main", or the + # prod-branch regex '^main$' excludes the run from every DORA metric. + assert run.head_branch == "main" + assert run.html_url == build["url"] + # Jenkins reports milliseconds; RepoWorkflowRuns.duration is seconds. + assert run.duration == 125 + assert run.conducted_at.isoformat() == "2025-08-10T12:00:00+00:00" + + +@pytest.mark.parametrize( + "reported,stored", + [ + # What the git plugin actually reports for a checkout of main. + ("origin/main", "main"), + ("refs/remotes/origin/main", "main"), + ("refs/heads/main", "main"), + ("main", "main"), + # A non-default remote name, still unambiguous after refs/remotes/. + ("refs/remotes/deploy-target/release", "release"), + ("upstream/main", "main"), + # Slash-containing branch names must survive untouched: the first + # segment here is part of the branch, not a remote. + ("feature/origin-story", "feature/origin-story"), + ("release/2.0", "release/2.0"), + ("refs/heads/feature/origin-story", "feature/origin-story"), + ("origin/feature/origin-story", "feature/origin-story"), + ], +) +def test_branch_is_normalised_to_what_the_prod_branch_regex_expects(reported, stored): + build = get_jenkins_build_dict(branch_name=reported) + + run = _handler()._adapt_jenkins_build_to_workflow_run(uuid4_str(), build) + + assert run.head_branch == stored + + +def test_the_stored_branch_matches_the_default_prod_branch_pattern(): + # The failure this guards: prod_branches defaults to ["^$"] + # (mhq/service/code/repository_service.py) and is applied as a Postgres + # regex against RepoWorkflowRuns.head_branch. Python's re uses the same + # syntax for this pattern, so it reproduces the exclusion faithfully. + prod_branch_pattern = "^main$" + build = get_jenkins_build_dict(branch_name="origin/main") + + run = _handler()._adapt_jenkins_build_to_workflow_run(uuid4_str(), build) + + assert re.search(prod_branch_pattern, "origin/main") is None + assert re.search(prod_branch_pattern, run.head_branch) is not None + + +def test_a_build_without_the_git_plugin_records_with_nulls_rather_than_raising(): + build = get_jenkins_build_dict_without_git_plugin() + + run = _handler()._adapt_jenkins_build_to_workflow_run(uuid4_str(), build) + + assert run.head_branch is None + assert run.event_actor is None + assert run.status == RepoWorkflowRunsStatus.SUCCESS + + +def test_a_malformed_build_is_skipped_without_losing_the_batch(): + class Repo: + def get_repo_workflow_run_by_provider_workflow_run_id(self, *args): + return None + + class Api: + @staticmethod + def get_builds(*args): + # Second build has no "number", so adaptation raises for it alone. + return [get_jenkins_build_dict(number=2), {"timestamp": 1754827200000}] + + class Workflow: + id = uuid4_str() + provider_workflow_id = "deploy-api" + + class OrgRepoStub: + repo_id = uuid4_str() + + handler = JenkinsETLHandler(uuid4_str(), Api(), Repo()) + runs, _ = handler.get_workflow_runs(OrgRepoStub(), Workflow(), _epoch_start()) + + assert [r.provider_workflow_run_id for r in runs] == ["2"] + + +def test_bookmark_rewinds_to_the_oldest_running_build(): + handler = _handler() + builds = [ + get_jenkins_build_dict(number=3, timestamp=3000, result=None, building=True), + get_jenkins_build_dict(number=2, timestamp=2000, result="SUCCESS"), + ] + + bookmark = handler._get_new_bookmark_time_stamp(builds) + + # The running build must be re-fetched next cycle, so the bookmark cannot + # advance past it. + assert bookmark == handler._get_datetime_from_epoch_ms(3000) diff --git a/backend/analytics_server/tests/service/workflows/sync/test_etl_workflows_factory.py b/backend/analytics_server/tests/service/workflows/sync/test_etl_workflows_factory.py new file mode 100644 index 000000000..3ef0b5168 --- /dev/null +++ b/backend/analytics_server/tests/service/workflows/sync/test_etl_workflows_factory.py @@ -0,0 +1,37 @@ +from unittest.mock import patch + +import pytest + +from mhq.service.workflows.sync.etl_workflows_factory import WorkflowETLFactory +from mhq.store.models.code import RepoWorkflowProviders +from mhq.store.repos.core import CoreRepoService + + +def test_jenkins_is_a_known_provider(): + assert RepoWorkflowProviders.JENKINS.value == "jenkins" + + +def test_factory_rejects_unknown_provider(): + factory = WorkflowETLFactory("org-id") + with pytest.raises(NotImplementedError): + factory("NOT_A_PROVIDER") + + +def test_factory_returns_a_jenkins_handler(): + from mhq.service.workflows.sync.etl_jenkins_handler import JenkinsETLHandler + + # CLUSTOX: get_jenkins_etl_handler resolves credentials through + # CoreRepoService, which needs a live Flask app context / DB session that + # this test suite does not provide (no conftest.py sets one up, and no + # other test in the suite calls CoreRepoService without mocking it either + # -- see tests/clustox_auth/test_sync_run.py). Mocking these two methods + # also exercises exactly the "unconfigured org" path described by the + # correctness requirement: the factory must still hand back a handler + # instead of raising when Jenkins isn't set up for this org. + with patch.object( + CoreRepoService, "get_access_token", return_value=None + ), patch.object(CoreRepoService, "get_org_integrations_for_names", return_value=[]): + factory = WorkflowETLFactory("org-id") + handler = factory(RepoWorkflowProviders.JENKINS.name) + + assert isinstance(handler, JenkinsETLHandler) diff --git a/backend/analytics_server/tests/store/__init__.py b/backend/analytics_server/tests/store/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/store/models/__init__.py b/backend/analytics_server/tests/store/models/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/store/models/code/__init__.py b/backend/analytics_server/tests/store/models/code/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/store/models/code/test_pr_filter_authors.py b/backend/analytics_server/tests/store/models/code/test_pr_filter_authors.py new file mode 100644 index 000000000..258638958 --- /dev/null +++ b/backend/analytics_server/tests/store/models/code/test_pr_filter_authors.py @@ -0,0 +1,34 @@ +from mhq.store.models.code.filter import PRFilter + + +def test_setting_authors_no_longer_raises(): + # Before this change the conditions dict had no "authors" key, so the + # filter_query comprehension raised KeyError as soon as the field was set. + conditions = PRFilter(authors=["hamad-clustox"]).filter_query + assert len(conditions) == 1 + + +def test_authors_produces_an_author_condition(): + conditions = PRFilter(authors=["hamad-clustox"]).filter_query + # Compiling to string is the cheapest way to assert the column and operator + # without a database. + rendered = str(conditions[0]) + assert "author" in rendered + assert "IN" in rendered.upper() + + +def test_no_authors_produces_no_author_condition(): + conditions = PRFilter(base_branches=["^main$"]).filter_query + assert all("author" not in str(c) for c in conditions) + + +def test_empty_author_list_is_ignored(): + # An empty list means "no filter selected", not "match nothing". + conditions = PRFilter(authors=[]).filter_query + assert conditions == [] + + +def test_existing_filters_are_unaffected(): + # Regression guard: the dashboards everyone already uses must not change. + conditions = PRFilter(base_branches=["^main$"], excluded_pr_ids=["a"]).filter_query + assert len(conditions) == 2 diff --git a/backend/analytics_server/tests/store/models/code/test_pull_requests.py b/backend/analytics_server/tests/store/models/code/test_pull_requests.py new file mode 100644 index 000000000..db824f883 --- /dev/null +++ b/backend/analytics_server/tests/store/models/code/test_pull_requests.py @@ -0,0 +1,25 @@ +from mhq.store.models.code import PullRequest + +# CLUSTOX: Jira integration -- ticket-PR matching needs the PR's body +# text provider-agnostically. See docs/JIRA_INTEGRATION_PROPOSAL.md and +# mhq/service/ticket_matching/service.py. + + +class TestPullRequestDescription: + def test_reads_body_for_a_github_pr(self): + pr = PullRequest(data={"body": "Closes PZDA-689"}) + assert pr.description == "Closes PZDA-689" + + def test_reads_description_for_a_gitlab_merge_request(self): + pr = PullRequest(data={"description": "Closes PZDA-689"}) + assert pr.description == "Closes PZDA-689" + + def test_prefers_body_when_both_keys_are_somehow_present(self): + pr = PullRequest(data={"body": "github body", "description": "gitlab desc"}) + assert pr.description == "github body" + + def test_returns_empty_string_when_data_is_none(self): + assert PullRequest(data=None).description == "" + + def test_returns_empty_string_when_neither_key_is_present(self): + assert PullRequest(data={"title": "x"}).description == "" diff --git a/backend/analytics_server/tests/store/models/code/workflows/__init__.py b/backend/analytics_server/tests/store/models/code/workflows/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/store/models/code/workflows/test_workflow_filter_actors.py b/backend/analytics_server/tests/store/models/code/workflows/test_workflow_filter_actors.py new file mode 100644 index 000000000..d06074e98 --- /dev/null +++ b/backend/analytics_server/tests/store/models/code/workflows/test_workflow_filter_actors.py @@ -0,0 +1,35 @@ +from mhq.service.workflows.workflow_filter import ParseWorkflowFilterProcessor +from mhq.store.models.code.workflows.filter import WorkflowFilter + + +def test_event_actors_produces_an_actor_condition(): + conditions = WorkflowFilter(event_actors=["hamad-clustox"]).filter_query + assert len(conditions) == 1 + rendered = str(conditions[0]) + assert "event_actor" in rendered + assert "IN" in rendered.upper() + + +def test_no_event_actors_produces_no_actor_condition(): + conditions = WorkflowFilter(head_branches=["^main$"]).filter_query + assert all("event_actor" not in str(c) for c in conditions) + + +def test_empty_actor_list_is_ignored(): + assert WorkflowFilter(event_actors=[]).filter_query == [] + + +def test_existing_workflow_filters_are_unaffected(): + # Regression guard for the deployment frequency everyone already sees. + conditions = WorkflowFilter(head_branches=["^main$"]).filter_query + assert len(conditions) == 1 + + +def test_parser_reads_event_actors_from_the_request(): + parsed = ParseWorkflowFilterProcessor().apply({"event_actors": ["hamad-clustox"]}) + assert parsed.event_actors == ["hamad-clustox"] + + +def test_parser_defaults_event_actors_to_none(): + parsed = ParseWorkflowFilterProcessor().apply({}) + assert parsed.event_actors is None diff --git a/backend/analytics_server/tests/store/models/code/workflows/test_workflow_filter_null_branches.py b/backend/analytics_server/tests/store/models/code/workflows/test_workflow_filter_null_branches.py new file mode 100644 index 000000000..b6c943877 --- /dev/null +++ b/backend/analytics_server/tests/store/models/code/workflows/test_workflow_filter_null_branches.py @@ -0,0 +1,42 @@ +""" +A repo with no prod_branches configured must not silently hide every +deployment. + +TeamRepos.prod_branches is nullable. The BFF flattens it into the workflow +filter's head_branches, so a null arrives here as [None]. `head_branch ~ NULL` +is NULL in Postgres -- never true -- so an unguarded filter dropped every row +and the dashboard reported "No Deployments" with no error anywhere. + +Observed live: a repo with 14 synced Jenkins runs returned 13 deployments when +queried without a filter and 0 through the dashboard. +""" + +from mhq.store.models.code.workflows.filter import WorkflowFilter + + +def test_a_single_none_branch_produces_no_condition(): + # The live failure: prod_branches was null, so head_branches was [None]. + assert WorkflowFilter(head_branches=[None]).filter_query == [] + + +def test_none_entries_are_dropped_but_real_ones_survive(): + conditions = WorkflowFilter(head_branches=[None, "^main$"]).filter_query + + assert len(conditions) == 1 + assert "head_branch" in str(conditions[0]) + + +def test_all_none_entries_produce_no_condition(): + assert WorkflowFilter(head_branches=[None, None]).filter_query == [] + + +def test_a_real_branch_still_filters(): + # Regression guard: the common case must be unchanged. + conditions = WorkflowFilter(head_branches=["^main$"]).filter_query + + assert len(conditions) == 1 + assert "head_branch" in str(conditions[0]) + + +def test_empty_list_is_still_ignored(): + assert WorkflowFilter(head_branches=[]).filter_query == [] diff --git a/backend/analytics_server/tests/store/repos/__init__.py b/backend/analytics_server/tests/store/repos/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/analytics_server/tests/store/repos/test_contributors_query.py b/backend/analytics_server/tests/store/repos/test_contributors_query.py new file mode 100644 index 000000000..13fd446ba --- /dev/null +++ b/backend/analytics_server/tests/store/repos/test_contributors_query.py @@ -0,0 +1,69 @@ +from datetime import datetime + +from mhq.store.models.code import PullRequestState +from mhq.store.repos.code import CodeRepoService + + +class FakeQuery: + """Records the conditions handed to .filter() and returns no rows. + + The suite has no database, so the only way to assert what + get_contributors_for_repos actually asks for is to capture the SQLAlchemy + expressions it builds. + """ + + def __init__(self): + self.conditions = [] + + def query(self, *_args): + return self + + def filter(self, *conditions): + self.conditions.extend(conditions) + return self + + def group_by(self, *_args): + return self + + def order_by(self, *_args): + return self + + def all(self): + return [] + + +class FakeDb: + def __init__(self, session): + self.session = session + + +def capture_contributor_query_conditions(): + fake_query = FakeQuery() + service = CodeRepoService() + service._db = FakeDb(fake_query) + + service.get_contributors_for_repos( + ["35737b5a-7f35-4fbd-b86b-5c6052f4e206"], + datetime(2024, 1, 1), + datetime(2024, 2, 1), + ) + + return fake_query.conditions + + +def test_only_merged_prs_are_counted(): + # The dropdown's counts have to agree with the metrics they filter, and + # every metric query pairs the window with a merged-state check (see + # CodeRepoService._filter_prs_merged_in_interval). Without it someone who + # opened and closed 15 PRs without merging any of them was listed as + # "15 PRs" and selecting them produced an empty Lead Time card. + conditions = capture_contributor_query_conditions() + + state_conditions = [ + condition + for condition in conditions + if "state" in str(condition) and "state_changed_at" not in str(condition) + ] + + assert len(state_conditions) == 1 + assert state_conditions[0].right.value == PullRequestState.MERGED diff --git a/backend/analytics_server/tests/store/repos/test_incidents_org_scoping.py b/backend/analytics_server/tests/store/repos/test_incidents_org_scoping.py new file mode 100644 index 000000000..9158c2d72 --- /dev/null +++ b/backend/analytics_server/tests/store/repos/test_incidents_org_scoping.py @@ -0,0 +1,60 @@ +from unittest.mock import MagicMock + +from mhq.store.models.incidents import IncidentProvider, IncidentType +from mhq.store.repos.incidents import IncidentsRepoService + +# CLUSTOX: cross-tenant incident collision fix. ticket.key ("PROJ-123") +# is site-local, not globally unique -- two orgs' Jira sites can land +# on the same key, so the lookup used to resolve/upsert a Jira-sourced +# incident must be scoped by org_id, not just (key, incident_type, +# provider). See get_incident_by_key_type_provider_and_org's own +# docstring and etl_jira_incidents_handler.py's _to_incident. + + +def _service_with_mock_db() -> (IncidentsRepoService, MagicMock): + db = MagicMock() + service = IncidentsRepoService() + service._db = db + return service, db + + +class TestGetIncidentByKeyTypeProviderAndOrg: + def test_joins_through_org_incident_service_before_filtering(self): + service, db = _service_with_mock_db() + query = db.session.query.return_value + joined_once = query.join.return_value + joined_twice = joined_once.join.return_value + + service.get_incident_by_key_type_provider_and_org( + "org-1", "PROJ-123", IncidentType.JIRA_ISSUE, IncidentProvider.JIRA + ) + + # Two joins -- Incident -> IncidentOrgIncidentServiceMap -> + # OrgIncidentService -- are what make an org_id filter possible + # at all, since Incident carries no org_id column of its own. + assert query.join.call_count == 1 + assert joined_once.join.call_count == 1 + joined_twice.filter.assert_called_once() + joined_twice.filter.return_value.one_or_none.assert_called_once() + + def test_two_orgs_with_the_same_ticket_key_resolve_independently(self): + # The actual regression: org A's incident for "PROJ-1" must not + # be returned when org B looks up its own "PROJ-1". + service, db = _service_with_mock_db() + org_a_incident = MagicMock() + org_b_incident = None # org B has no incident for this key yet + + db.session.query.return_value.join.return_value.join.return_value.filter.return_value.one_or_none.side_effect = [ + org_a_incident, + org_b_incident, + ] + + result_a = service.get_incident_by_key_type_provider_and_org( + "org-a", "PROJ-1", IncidentType.JIRA_ISSUE, IncidentProvider.JIRA + ) + result_b = service.get_incident_by_key_type_provider_and_org( + "org-b", "PROJ-1", IncidentType.JIRA_ISSUE, IncidentProvider.JIRA + ) + + assert result_a is org_a_incident + assert result_b is None diff --git a/backend/analytics_server/tests/store/repos/test_jira_connection.py b/backend/analytics_server/tests/store/repos/test_jira_connection.py new file mode 100644 index 000000000..96b0044c7 --- /dev/null +++ b/backend/analytics_server/tests/store/repos/test_jira_connection.py @@ -0,0 +1,194 @@ +from unittest.mock import MagicMock, patch + +import pytest + +from mhq.store.repos.jira_connection import ( + CHUNK_SIZE, + JiraConnectionInUseError, + JiraConnectionNotFoundError, + JiraConnectionRepoService, + _normalize_site_url, +) + + +def _service_with_mock_db() -> (JiraConnectionRepoService, MagicMock): + db = MagicMock() + with patch("mhq.store.repos.jira_connection.get_crypto_service") as mock_get_crypto: + service = JiraConnectionRepoService() + service._db = db + service._crypto = mock_get_crypto.return_value + return service, db + + +class TestCreateJiraConnection: + def test_encrypts_the_token_and_returns_the_saved_row(self): + service, db = _service_with_mock_db() + service._crypto.encrypt.return_value = ["chunk1", "chunk2"] + saved = MagicMock() + with patch.object(service, "get_jira_connection", return_value=saved): + result = service.create_jira_connection( + "org-1", + "acme.atlassian.net", + "person@acme.com", + "plain-text-token", + {"foo": "bar"}, + "user-1", + ) + + # The raw token is never persisted -- only encrypt()'s output goes + # onto the model, and it goes through the same CHUNK_SIZE the + # web-server side uses. + service._crypto.encrypt.assert_called_once_with("plain-text-token", CHUNK_SIZE) + added = db.session.add.call_args[0][0] + assert added.access_token_enc_chunks == ["chunk1", "chunk2"] + assert added.org_id == "org-1" + assert added.site_url == "acme.atlassian.net" + assert added.email == "person@acme.com" + db.session.commit.assert_called_once() + assert result is saved + + def test_normalizes_a_url_pasted_straight_from_the_browser(self): + # The regression this guards: a value like this stored verbatim + # used to build "https://https://acme.atlassian.net//..." at + # search/sync time -- unreachable, and the failure mode a real + # user actually hit. + service, db = _service_with_mock_db() + service._crypto.encrypt.return_value = ["chunk"] + with patch.object(service, "get_jira_connection", return_value=MagicMock()): + service.create_jira_connection( + "org-1", + "https://acme.atlassian.net/", + "person@acme.com", + "token", + {}, + None, + ) + + added = db.session.add.call_args[0][0] + assert added.site_url == "acme.atlassian.net" + + def test_normalizes_a_bare_host_to_itself(self): + service, db = _service_with_mock_db() + service._crypto.encrypt.return_value = ["chunk"] + with patch.object(service, "get_jira_connection", return_value=MagicMock()): + service.create_jira_connection( + "org-1", "acme.atlassian.net", "p@acme.com", "token", {}, None + ) + + added = db.session.add.call_args[0][0] + assert added.site_url == "acme.atlassian.net" + + +class TestNormalizeSiteUrl: + @pytest.mark.parametrize( + "raw,expected", + [ + ("acme.atlassian.net", "acme.atlassian.net"), + ("https://acme.atlassian.net", "acme.atlassian.net"), + ("https://acme.atlassian.net/", "acme.atlassian.net"), + ("HTTPS://ACME.atlassian.net/", "acme.atlassian.net"), + (" acme.atlassian.net ", "acme.atlassian.net"), + ("http://acme.atlassian.net/jira/", "acme.atlassian.net"), + ], + ) + def test_strips_scheme_case_whitespace_and_trailing_path(self, raw, expected): + assert _normalize_site_url(raw) == expected + + +class TestListJiraConnections: + def test_filters_by_org_id(self): + service, db = _service_with_mock_db() + + service.list_jira_connections("org-1") + + db.session.query.return_value.filter.assert_called_once() + db.session.query.return_value.filter.return_value.order_by.return_value.all.assert_called_once() + + +class TestGetJiraConnection: + def test_scopes_by_both_id_and_org_id(self): + service, db = _service_with_mock_db() + + service.get_jira_connection("org-1", "conn-1") + + filter_args = db.session.query.return_value.filter.call_args[0] + assert len(filter_args) == 2 + + +class TestDeleteJiraConnection: + def test_raises_not_found_when_connection_does_not_belong_to_org(self): + service, db = _service_with_mock_db() + with patch.object(service, "get_jira_connection", return_value=None): + with pytest.raises(JiraConnectionNotFoundError): + service.delete_jira_connection("org-1", "conn-1") + + db.session.delete.assert_not_called() + + def test_raises_in_use_when_an_org_project_connection_still_references_it(self): + service, db = _service_with_mock_db() + connection = MagicMock() + db.session.query.return_value.filter.return_value.first.return_value = ( + MagicMock() + ) + + with patch.object(service, "get_jira_connection", return_value=connection): + with pytest.raises(JiraConnectionInUseError): + service.delete_jira_connection("org-1", "conn-1") + + db.session.delete.assert_not_called() + db.session.commit.assert_not_called() + + def test_deletes_when_unreferenced(self): + service, db = _service_with_mock_db() + connection = MagicMock() + db.session.query.return_value.filter.return_value.first.return_value = None + + with patch.object(service, "get_jira_connection", return_value=connection): + service.delete_jira_connection("org-1", "conn-1") + + db.session.delete.assert_called_once_with(connection) + db.session.commit.assert_called_once() + + +class TestSetDefaultJiraConnection: + def test_raises_not_found_when_connection_does_not_belong_to_org(self): + service, db = _service_with_mock_db() + with patch.object(service, "get_jira_connection", return_value=None): + with pytest.raises(JiraConnectionNotFoundError): + service.set_default_jira_connection("org-1", "conn-1") + + db.session.commit.assert_not_called() + + def test_unsets_prior_defaults_before_setting_the_new_one(self): + # The regression this guards: reversing the order (set new True + # first, unset old second) would momentarily have two default rows + # in the same transaction and violate + # jira_connection_one_default_per_org's own invariant even before + # the DB gets a chance to enforce it. + service, db = _service_with_mock_db() + connection = MagicMock(is_default=False) + calls = [] + db.session.query.return_value.filter.return_value.update.side_effect = ( + lambda *a, **k: calls.append("unset") + ) + db.session.commit.side_effect = lambda: calls.append("commit") + + with patch.object(service, "get_jira_connection", return_value=connection): + service.set_default_jira_connection("org-1", "conn-1") + + assert calls == ["unset", "commit"] + assert connection.is_default is True + db.session.commit.assert_called_once() + + def test_excludes_the_target_connection_from_the_unset_filter(self): + service, db = _service_with_mock_db() + connection = MagicMock(is_default=False, id="conn-1") + + with patch.object(service, "get_jira_connection", return_value=connection): + service.set_default_jira_connection("org-1", "conn-1") + + filter_args = db.session.query.return_value.filter.call_args[0] + # org scoping, "not this row", and "currently default" -- all three, + # or an unscoped update could flip defaults across orgs or reset a + # connection that was never default in the first place. + assert len(filter_args) == 3 diff --git a/backend/analytics_server/tests/store/repos/test_projects_connection_scoping.py b/backend/analytics_server/tests/store/repos/test_projects_connection_scoping.py new file mode 100644 index 000000000..ff2613f4c --- /dev/null +++ b/backend/analytics_server/tests/store/repos/test_projects_connection_scoping.py @@ -0,0 +1,44 @@ +from unittest.mock import MagicMock + +from mhq.store.repos.projects import ProjectRepoService + +# CLUSTOX: Jira multi-account support. See docs/JIRA_MULTI_ACCOUNT_PLAN.md +# Task 4. get_active_org_projects_for_connection is the read side of the +# OrgProjectConnection join a connection-scoped JiraETLHandler depends on to +# know which projects are its own -- getting the join wrong here means a +# connection either syncs nothing or syncs another connection's projects. + + +def _service_with_mock_db() -> (ProjectRepoService, MagicMock): + db = MagicMock() + service = ProjectRepoService() + service._db = db + return service, db + + +class TestGetActiveOrgProjectsForConnection: + def test_joins_through_org_project_connection_before_filtering(self): + service, db = _service_with_mock_db() + query = db.session.query.return_value + joined = query.join.return_value + + service.get_active_org_projects_for_connection("conn-1") + + # The join is what makes filtering by jira_connection_id possible at + # all -- OrgProject itself carries no connection column. + assert query.join.call_count == 1 + joined.filter.assert_called_once() + joined.filter.return_value.all.assert_called_once() + + def test_filters_by_the_given_connection_and_active_projects_only(self): + service, db = _service_with_mock_db() + joined = db.session.query.return_value.join.return_value + + service.get_active_org_projects_for_connection("conn-1") + + filter_args = joined.filter.call_args[0] + # jira_connection_id == "conn-1" AND is_active -- both conditions, + # or an inactive/deselected project would keep syncing forever, or + # a bare join with no connection filter would return every + # connection's projects. + assert len(filter_args) == 2 diff --git a/backend/analytics_server/tests/store/repos/test_team_repo_project_mapping.py b/backend/analytics_server/tests/store/repos/test_team_repo_project_mapping.py new file mode 100644 index 000000000..20ccb7660 --- /dev/null +++ b/backend/analytics_server/tests/store/repos/test_team_repo_project_mapping.py @@ -0,0 +1,100 @@ +from unittest.mock import MagicMock +from uuid import UUID + +from mhq.store.repos.team_repo_project_mapping import TeamRepoProjectMappingRepoService + +# CLUSTOX: explicit, informational repo<->Jira-project pairing. See +# docs/JIRA_MULTI_ACCOUNT_PLAN.md's follow-up on Jira<->repo relationships. + +TEAM_ID = "11111111-1111-1111-1111-111111111111" +REPO_ID_1 = "22222222-2222-2222-2222-222222222222" +REPO_ID_2 = "33333333-3333-3333-3333-333333333333" +PROJECT_ID_1 = "44444444-4444-4444-4444-444444444444" + + +def _service_with_mock_db() -> (TeamRepoProjectMappingRepoService, MagicMock): + db = MagicMock() + service = TeamRepoProjectMappingRepoService() + service._db = db + return service, db + + +class TestGetMappingsForTeam: + def test_filters_by_team_id(self): + service, db = _service_with_mock_db() + + service.get_mappings_for_team(TEAM_ID) + + db.session.query.return_value.filter.assert_called_once() + db.session.query.return_value.filter.return_value.all.assert_called_once() + + +class TestSetMappingsForTeam: + def test_deletes_the_teams_existing_mappings_before_inserting_the_new_ones(self): + service, db = _service_with_mock_db() + calls = [] + db.session.query.return_value.filter.return_value.delete.side_effect = ( + lambda **kwargs: calls.append("delete") + ) + db.session.commit.side_effect = lambda: calls.append("commit") + + service.set_mappings_for_team(TEAM_ID, [(REPO_ID_1, PROJECT_ID_1)]) + + assert calls == ["delete", "commit"] + + def test_scopes_the_delete_to_this_team_only(self): + service, db = _service_with_mock_db() + + service.set_mappings_for_team(TEAM_ID, []) + + filter_args = db.session.query.return_value.filter.call_args[0] + assert len(filter_args) == 1 + + def test_adds_one_row_per_pair(self): + service, db = _service_with_mock_db() + + service.set_mappings_for_team( + TEAM_ID, [(REPO_ID_1, PROJECT_ID_1), (REPO_ID_2, PROJECT_ID_1)] + ) + + added = [call.args[0] for call in db.session.add.call_args_list] + assert len(added) == 2 + assert {(str(a.org_repo_id), str(a.org_project_id)) for a in added} == { + (REPO_ID_1, PROJECT_ID_1), + (REPO_ID_2, PROJECT_ID_1), + } + assert all(str(a.team_id) == TEAM_ID for a in added) + + def test_an_empty_set_of_pairs_still_clears_the_teams_existing_mappings(self): + # Saving "no pairings" (an admin unlinking everything) must not be + # a no-op -- it has to actually clear what was there before. + service, db = _service_with_mock_db() + + service.set_mappings_for_team(TEAM_ID, []) + + db.session.query.return_value.filter.return_value.delete.assert_called_once() + db.session.add.assert_not_called() + db.session.commit.assert_called_once() + + # CLUSTOX: regression test for a real bug -- team_id arrives as a real + # uuid.UUID (the caller in teams.py passes team.id, straight off the + # loaded Team ORM instance), while org_repo_id/org_project_id arrive as + # plain strings (uuid_validator in request_utils.py only validates the + # format, it never converts). Constructing every row's primary key from + # that mix -- UUID object for team_id, str for the other two columns -- + # bulk-inserted SQLAlchemy 2's insertmanyvalues into raising + # "Can't match sentinel values in result set to parameter sets", a 500 + # on every save. Every PK column must come out as the same Python type + # (uuid.UUID) regardless of what type each argument arrived as. + def test_normalizes_every_pk_column_to_uuid_regardless_of_input_type(self): + service, db = _service_with_mock_db() + + service.set_mappings_for_team( + UUID(TEAM_ID), # as the real caller passes it: team.id, already a UUID + [(REPO_ID_1, PROJECT_ID_1)], # as the real caller passes it: plain strings + ) + + added = db.session.add.call_args_list[0].args[0] + assert type(added.team_id) is UUID + assert type(added.org_repo_id) is UUID + assert type(added.org_project_id) is UUID diff --git a/backend/analytics_server/tests/store/repos/test_ticket_matching.py b/backend/analytics_server/tests/store/repos/test_ticket_matching.py new file mode 100644 index 000000000..0e72b6345 --- /dev/null +++ b/backend/analytics_server/tests/store/repos/test_ticket_matching.py @@ -0,0 +1,134 @@ +from unittest.mock import MagicMock + +from mhq.store.repos.ticket_matching import TicketMatchingRepoService + +# CLUSTOX: Jira multi-account support. See docs/JIRA_MULTI_ACCOUNT_PLAN.md +# Task 5. + + +def _service_with_mock_db() -> (TicketMatchingRepoService, MagicMock): + db = MagicMock() + service = TicketMatchingRepoService() + service._db = db + return service, db + + +def _stub_ticket_rows(db: MagicMock, rows): + """rows: list of (ticket_id, key, org_project_id) tuples.""" + db.session.query.return_value.join.return_value.filter.return_value.all.return_value = ( + rows + ) + + +class TestGetOrgTicketsKeyMap: + def test_maps_a_non_colliding_key_to_its_one_ticket(self): + service, db = _service_with_mock_db() + _stub_ticket_rows(db, [("ticket-1", "PZDA-543", "project-1")]) + + result = service.get_org_tickets_key_map("org-1") + + assert result == {"PZDA-543": "ticket-1"} + + def test_uppercases_the_key(self): + service, db = _service_with_mock_db() + _stub_ticket_rows(db, [("ticket-1", "pzda-543", "project-1")]) + + result = service.get_org_tickets_key_map("org-1") + + assert result == {"PZDA-543": "ticket-1"} + + def test_excludes_a_key_shared_by_two_tickets(self): + # Two connections' sites both producing a "PROJ" project can land + # two distinct tickets on the same key -- must not silently pick + # one (see get_colliding_ticket_key_candidates for how these are + # actually resolved). + service, db = _service_with_mock_db() + _stub_ticket_rows( + db, + [ + ("ticket-a", "PROJ-123", "project-a"), + ("ticket-b", "PROJ-123", "project-b"), + ], + ) + + result = service.get_org_tickets_key_map("org-1") + + assert result == {} + + def test_a_colliding_key_does_not_hide_an_unrelated_non_colliding_key(self): + service, db = _service_with_mock_db() + _stub_ticket_rows( + db, + [ + ("ticket-a", "PROJ-123", "project-a"), + ("ticket-b", "PROJ-123", "project-b"), + ("ticket-c", "PZDA-1", "project-c"), + ], + ) + + result = service.get_org_tickets_key_map("org-1") + + assert result == {"PZDA-1": "ticket-c"} + + +class TestGetCollidingTicketKeyCandidates: + def test_returns_every_candidate_for_a_shared_key(self): + service, db = _service_with_mock_db() + _stub_ticket_rows( + db, + [ + ("ticket-a", "PROJ-123", "project-a"), + ("ticket-b", "PROJ-123", "project-b"), + ], + ) + + result = service.get_colliding_ticket_key_candidates("org-1") + + assert result == { + "PROJ-123": [("ticket-a", "project-a"), ("ticket-b", "project-b")] + } + + def test_omits_keys_with_only_one_ticket(self): + service, db = _service_with_mock_db() + _stub_ticket_rows(db, [("ticket-1", "PZDA-543", "project-1")]) + + result = service.get_colliding_ticket_key_candidates("org-1") + + assert result == {} + + +class TestGetRelevantOrgProjectIdsForRepo: + def test_joins_team_projects_through_team_repos_before_filtering(self): + service, db = _service_with_mock_db() + query = db.session.query.return_value + joined = query.join.return_value + + service.get_relevant_org_project_ids_for_repo("repo-1") + + assert query.join.call_count == 1 + joined.filter.assert_called_once() + joined.filter.return_value.distinct.return_value.all.assert_called_once() + + def test_filters_by_repo_id_and_both_active_flags(self): + service, db = _service_with_mock_db() + joined = db.session.query.return_value.join.return_value + + service.get_relevant_org_project_ids_for_repo("repo-1") + + filter_args = joined.filter.call_args[0] + # org_repo_id == repo-1, TeamRepos.is_active, TeamProjects.is_active + # -- a deselected repo or project link must not disambiguate a + # collision. + assert len(filter_args) == 3 + + def test_returns_a_deduplicated_set_of_project_ids(self): + service, db = _service_with_mock_db() + chain = db.session.query.return_value.join.return_value.filter.return_value + chain.distinct.return_value.all.return_value = [ + ("project-a",), + ("project-b",), + ] + + result = service.get_relevant_org_project_ids_for_repo("repo-1") + + assert result == {"project-a", "project-b"} diff --git a/backend/analytics_server/tests/utils/time/test_fill_missing_week_buckets.py b/backend/analytics_server/tests/utils/time/test_fill_missing_week_buckets.py index 295078ff4..3044d176a 100644 --- a/backend/analytics_server/tests/utils/time/test_fill_missing_week_buckets.py +++ b/backend/analytics_server/tests/utils/time/test_fill_missing_week_buckets.py @@ -5,7 +5,6 @@ from mhq.utils.time import Interval, fill_missing_week_buckets - last_week_2022 = datetime(2022, 12, 26, 0, 0, 0, tzinfo=pytz.UTC) first_week_2023 = datetime(2023, 1, 2, 0, 0, 0, tzinfo=pytz.UTC) second_week_2023 = datetime(2023, 1, 9, 0, 0, 0, tzinfo=pytz.UTC) diff --git a/backend/requirements.txt b/backend/requirements.txt index 30060d2bb..a71510fef 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -12,3 +12,12 @@ psycopg2==2.9.3 python-dotenv==1.0.1 gunicorn==22.0.0 Flask-SQLAlchemy==3.1.1 +# CLUSTOX: mhq/exapi/jenkins.py imports requests directly. It resolved only +# transitively through PyGithub before, so an upstream bump that dropped it +# would have broken the Jenkins client. +requests==2.32.5 +# CLUSTOX: mhq/exapi/jenkins.py enforces its wall-clock ceiling with +# response.raw.read1(), which urllib3 only grew in 2.0. requests itself accepts +# anything <3, so without this pin an environment could resolve urllib3 1.26 +# and the Jenkins client would fail with AttributeError on every call. +urllib3>=2.0,<3 diff --git a/database-docker/db/migrations/20260805150000_clustox_auth.sql b/database-docker/db/migrations/20260805150000_clustox_auth.sql new file mode 100644 index 000000000..c7e2b3bdd --- /dev/null +++ b/database-docker/db/migrations/20260805150000_clustox_auth.sql @@ -0,0 +1,23 @@ +-- migrate:up +-- CLUSTOX: authentication and role-based access tables. Deliberately separate +-- from upstream "Users" so upstream migrations can never collide with ours. +CREATE TABLE "ClustoxUserAuth" ( + user_id uuid PRIMARY KEY REFERENCES "Users"(id) ON DELETE CASCADE, + password_hash text NOT NULL, + role varchar NOT NULL CHECK (role IN ('SUPERADMIN','ADMIN')), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE "ClustoxUserTeamAccess" ( + user_id uuid NOT NULL REFERENCES "Users"(id) ON DELETE CASCADE, + team_id uuid NOT NULL REFERENCES "Team"(id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, team_id) +); + +CREATE INDEX idx_clustox_user_team_access_user ON "ClustoxUserTeamAccess"(user_id); + +-- migrate:down +DROP TABLE "ClustoxUserTeamAccess"; +DROP TABLE "ClustoxUserAuth"; diff --git a/database-docker/db/migrations/20260806120000_clustox_multitenancy.sql b/database-docker/db/migrations/20260806120000_clustox_multitenancy.sql new file mode 100644 index 000000000..39cc69100 --- /dev/null +++ b/database-docker/db/migrations/20260806120000_clustox_multitenancy.sql @@ -0,0 +1,27 @@ +-- migrate:up +-- CLUSTOX: multitenancy. Each ADMIN owns a workspace (an Organization); +-- SUPERADMIN sits above all of them and owns none. +-- +-- No schema change is needed: upstream's "Users".org_id already exists and is +-- already nullable, and every other table already carries org_id. Upstream +-- simply only ever created one Organization. This migration only corrects the +-- data left behind by the single-workspace bootstrap. + +-- A superadmin must not own a workspace, or their view would be scoped to it. +UPDATE "ClustoxUserAuth" a +SET updated_at = now() +FROM "Users" u +WHERE u.id = a.user_id AND a.role = 'SUPERADMIN'; + +UPDATE "Users" u +SET org_id = NULL +FROM "ClustoxUserAuth" a +WHERE a.user_id = u.id AND a.role = 'SUPERADMIN'; + +-- migrate:down +-- Reattach any orphaned superadmin to the oldest workspace, which is what the +-- single-workspace bootstrap would have done. +UPDATE "Users" u +SET org_id = (SELECT id FROM "Organization" ORDER BY created_at ASC LIMIT 1) +FROM "ClustoxUserAuth" a +WHERE a.user_id = u.id AND a.role = 'SUPERADMIN' AND u.org_id IS NULL; diff --git a/database-docker/db/migrations/20260806160000_clustox_sync_runs.sql b/database-docker/db/migrations/20260806160000_clustox_sync_runs.sql new file mode 100644 index 000000000..29bbead10 --- /dev/null +++ b/database-docker/db/migrations/20260806160000_clustox_sync_runs.sql @@ -0,0 +1,27 @@ +-- migrate:up +-- CLUSTOX: per-workspace sync outcomes. +-- +-- Upstream's POST /sync returns 200 whether every repo synced or every repo +-- failed, and the only record of what happened is a log file capped at 512KB +-- with no rotation history. "Sync succeeded but the numbers are wrong" was the +-- expected failure signature, and diagnosing it meant reading logs before they +-- rotated away. +-- +-- With one workspace that was merely awkward. With one sync per workspace it +-- becomes untenable: a single failing workspace is invisible among the ones +-- that worked. +CREATE TABLE "ClustoxSyncRun" ( + id uuid PRIMARY KEY DEFAULT extensions.uuid_generate_v4(), + org_id uuid NOT NULL REFERENCES "Organization"(id) ON DELETE CASCADE, + started_at timestamptz NOT NULL DEFAULT now(), + finished_at timestamptz, + status varchar NOT NULL CHECK (status IN ('RUNNING','SUCCESS','FAILED','SKIPPED')), + -- Why a run was skipped, or how it failed. Null on success. + detail text +); + +CREATE INDEX idx_clustox_sync_run_org_started + ON "ClustoxSyncRun"(org_id, started_at DESC); + +-- migrate:down +DROP TABLE "ClustoxSyncRun"; diff --git a/database-docker/db/migrations/20260806180000_clustox_invites.sql b/database-docker/db/migrations/20260806180000_clustox_invites.sql new file mode 100644 index 000000000..bba1d4d84 --- /dev/null +++ b/database-docker/db/migrations/20260806180000_clustox_invites.sql @@ -0,0 +1,36 @@ +-- migrate:up +-- CLUSTOX: single-use invite links. +-- +-- A superadmin generates a link and sends it over Slack; the invitee sets +-- their own password. This avoids a superadmin choosing and transmitting +-- someone else's password, which is what the current create-user flow +-- requires. +-- +-- token_hash, not token: the link is a bearer credential, so anyone reading +-- this table should not be able to accept invitations. SHA-256 rather than +-- bcrypt is deliberate -- the token is 32 random bytes, so it is not +-- brute-forceable and a fast hash keeps acceptance a single indexed lookup +-- instead of a scan over every pending row. +CREATE TABLE "ClustoxInvite" ( + id uuid PRIMARY KEY DEFAULT extensions.uuid_generate_v4(), + token_hash varchar NOT NULL UNIQUE, + email text NOT NULL, + name varchar NOT NULL, + role varchar NOT NULL CHECK (role IN ('SUPERADMIN','ADMIN')), + -- Adopt an existing workspace instead of provisioning a new one. + org_id uuid REFERENCES "Organization"(id) ON DELETE SET NULL, + created_by uuid REFERENCES "Users"(id) ON DELETE SET NULL, + created_at timestamptz NOT NULL DEFAULT now(), + expires_at timestamptz NOT NULL, + -- Set on acceptance. Non-null means the link is spent. + accepted_at timestamptz, + accepted_by uuid REFERENCES "Users"(id) ON DELETE SET NULL, + revoked_at timestamptz +); + +CREATE INDEX idx_clustox_invite_pending + ON "ClustoxInvite"(expires_at) + WHERE accepted_at IS NULL AND revoked_at IS NULL; + +-- migrate:down +DROP TABLE "ClustoxInvite"; diff --git a/database-docker/db/migrations/20260807200000_clustox_invite_emailed.sql b/database-docker/db/migrations/20260807200000_clustox_invite_emailed.sql new file mode 100644 index 000000000..91e69ab3d --- /dev/null +++ b/database-docker/db/migrations/20260807200000_clustox_invite_emailed.sql @@ -0,0 +1,10 @@ +-- migrate:up +-- CLUSTOX: track whether an invite's email actually went out, separately +-- from the invite existing. SMTP is optional and best-effort (see +-- src/utils/mailer.ts) -- a superadmin can always fall back to copying the +-- link manually, but the "Pending invitations" list should be able to say +-- whether that fallback is actually necessary for a given invite. +ALTER TABLE "ClustoxInvite" ADD COLUMN emailed_at timestamptz; + +-- migrate:down +ALTER TABLE "ClustoxInvite" DROP COLUMN emailed_at; diff --git a/database-docker/db/migrations/20260811120000_clustox_jira_projects.sql b/database-docker/db/migrations/20260811120000_clustox_jira_projects.sql new file mode 100644 index 000000000..f562ca422 --- /dev/null +++ b/database-docker/db/migrations/20260811120000_clustox_jira_projects.sql @@ -0,0 +1,54 @@ +-- migrate:up +-- CLUSTOX: Jira integration, Phase 2 (project selection) -- see +-- docs/JIRA_INTEGRATION_PROPOSAL.md. Mirrors "OrgRepo"/"TeamRepos" exactly +-- (org-level catalog + team join table), minus the code-specific columns +-- (default_branch, language, contributors, deployment_type, prod_branches) +-- that don't apply to a project-tracking tool. +-- +-- idempotency_key is NOT the bare Jira project id the way OrgRepo's is the +-- bare GitLab project id -- GitLab project ids are unique across the one +-- shared gitlab.com instance, but every org here links its own independent +-- Jira site, and Jira Cloud project ids are small site-local integers +-- (starting at 10000) that two different orgs' sites can easily collide +-- on. Scoping it as "::" keeps it globally +-- unique the same way OrgRepo's happens to be, rather than assuming a +-- property Jira doesn't actually have. +CREATE TABLE "OrgProject" ( + id uuid PRIMARY KEY DEFAULT extensions.uuid_generate_v4(), + org_id uuid REFERENCES "Organization"(id), + key varchar NOT NULL, + name varchar NOT NULL, + provider varchar NOT NULL, + idempotency_key varchar NOT NULL, + is_active boolean NOT NULL DEFAULT true, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE UNIQUE INDEX orgproject_unique_idempotency_key + ON "OrgProject"(idempotency_key); + +CREATE INDEX org_project_fetch_active_index + ON "OrgProject" USING btree (org_id, is_active); + +CREATE INDEX org_project_search_index + ON "OrgProject" USING btree (org_id, name); + +CREATE TABLE "TeamProjects" ( + team_id uuid NOT NULL REFERENCES "Team"(id), + org_project_id uuid NOT NULL REFERENCES "OrgProject"(id), + is_active boolean NOT NULL DEFAULT true, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (team_id, org_project_id) +); + +CREATE INDEX team_projects_fetch_index + ON "TeamProjects" USING btree (team_id, is_active); + +CREATE INDEX team_projects_org_project_id_index + ON "TeamProjects" USING btree (org_project_id); + +-- migrate:down +DROP TABLE "TeamProjects"; +DROP TABLE "OrgProject"; diff --git a/database-docker/db/migrations/20260811130000_clustox_jira_tickets.sql b/database-docker/db/migrations/20260811130000_clustox_jira_tickets.sql new file mode 100644 index 000000000..6b604282d --- /dev/null +++ b/database-docker/db/migrations/20260811130000_clustox_jira_tickets.sql @@ -0,0 +1,77 @@ +-- migrate:up +-- CLUSTOX: Jira integration, Phase 2 (issue sync) -- see +-- docs/JIRA_INTEGRATION_PROPOSAL.md. "Ticket" (current state) + +-- "TicketState" (append-only status-transition history) mirror +-- "PullRequest"/"PullRequestEvent" -- a current-state row plus a +-- separate event-history table, rather than trying to cram history into +-- the current-state row. +-- +-- Deliberately no rigid column per Jira field (summary, assignee, +-- reporter, issue type, ...): those live in the "data" JSONB payload, +-- the same choice PullRequest/PullRequestEvent already made for their +-- own provider payloads ("data"/"meta"). Jira custom fields vary per +-- instance (e.g. story points is a differently-numbered customfield_* +-- per site) -- a fixed column per field would mean a migration every +-- time a new field is wanted, where JSONB just needs a new @property. +-- status/status_category are still real columns because cycle-time +-- queries need to filter/group on them directly. +CREATE TABLE "Ticket" ( + id uuid PRIMARY KEY DEFAULT extensions.uuid_generate_v4(), + org_project_id uuid REFERENCES "OrgProject"(id), + key varchar NOT NULL, + provider varchar NOT NULL, + status varchar NOT NULL, + status_category varchar, + idempotency_key varchar NOT NULL, + data jsonb, + -- Jira's own created/updated timestamps -- NOT this row's own + -- bookkeeping (see created_in_db_at/updated_in_db_at below). updated_at + -- is the incremental-sync bookmark cursor (see ProjectIssuesBookmark). + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + created_in_db_at timestamptz NOT NULL DEFAULT now(), + updated_in_db_at timestamptz NOT NULL DEFAULT now() +); + +CREATE UNIQUE INDEX ticket_unique_idempotency_key + ON "Ticket"(idempotency_key); + +CREATE INDEX ticket_org_project_fetch_index + ON "Ticket" USING btree (org_project_id, updated_at); + +CREATE TABLE "TicketState" ( + id uuid PRIMARY KEY DEFAULT extensions.uuid_generate_v4(), + ticket_id uuid REFERENCES "Ticket"(id), + from_status varchar, + to_status varchar NOT NULL, + changed_at timestamptz NOT NULL, + idempotency_key varchar NOT NULL, + data jsonb, + created_in_db_at timestamptz NOT NULL DEFAULT now(), + updated_in_db_at timestamptz NOT NULL DEFAULT now() +); + +CREATE UNIQUE INDEX ticket_state_unique_idempotency_key + ON "TicketState"(idempotency_key); + +CREATE INDEX ticket_state_ticket_fetch_index + ON "TicketState" USING btree (ticket_id); + +-- Incremental-sync watermark, one per (project, provider) -- mirrors +-- "Bookmark" (org repos) and "RepoWorkflowRunsBookmark", just scoped to +-- OrgProject instead of OrgRepo. Not reusing "Bookmark" itself: its +-- repo_id column has a real FK to "OrgRepo" at the DB level, so an +-- OrgProject id would be rejected by that constraint. +CREATE TABLE "ProjectIssuesBookmark" ( + org_project_id uuid REFERENCES "OrgProject"(id), + provider varchar NOT NULL, + bookmark varchar, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (org_project_id, provider) +); + +-- migrate:down +DROP TABLE "ProjectIssuesBookmark"; +DROP TABLE "TicketState"; +DROP TABLE "Ticket"; diff --git a/database-docker/db/migrations/20260811140000_clustox_pr_ticket_matching.sql b/database-docker/db/migrations/20260811140000_clustox_pr_ticket_matching.sql new file mode 100644 index 000000000..1c2811e56 --- /dev/null +++ b/database-docker/db/migrations/20260811140000_clustox_pr_ticket_matching.sql @@ -0,0 +1,19 @@ +-- migrate:up +-- CLUSTOX: Jira integration, Phase 3 (ticket-PR matching) -- see +-- docs/JIRA_INTEGRATION_PROPOSAL.md step 4. A PR can reference more than +-- one ticket (real example found in this org's own PR history: a title +-- referencing "PZDA-544/546"), so this is a many-to-many join table, not +-- a single ticket_id column on PullRequest. +CREATE TABLE "PullRequestTicketMapping" ( + pr_id uuid REFERENCES "PullRequest"(id), + ticket_id uuid REFERENCES "Ticket"(id), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (pr_id, ticket_id) +); + +CREATE INDEX pr_ticket_mapping_ticket_id_index + ON "PullRequestTicketMapping" USING btree (ticket_id); + +-- migrate:down +DROP TABLE "PullRequestTicketMapping"; diff --git a/database-docker/db/migrations/20260812150000_clustox_jira_sprints.sql b/database-docker/db/migrations/20260812150000_clustox_jira_sprints.sql new file mode 100644 index 000000000..928908934 --- /dev/null +++ b/database-docker/db/migrations/20260812150000_clustox_jira_sprints.sql @@ -0,0 +1,40 @@ +-- migrate:up +-- CLUSTOX: Jira integration -- Sprint rollup chart (docs/ +-- JIRA_INTEGRATION_PROPOSAL.md §6D). "Sprint" mirrors "OrgProject"'s +-- shape (org-scoped catalog row with an idempotency_key), but stores +-- planned/completed issue COUNTS directly rather than a per-ticket +-- sprint-membership join table -- the rollup chart only ever needs +-- "how many were planned vs. shipped, per sprint," never per-ticket +-- detail, so a join table here would be speculative structure with no +-- current consumer (YAGNI). +-- +-- No bookmark table for this, unlike ProjectIssuesBookmark: a project +-- typically has a small, bounded number of sprints (re-fetching all of +-- them each cycle is cheap), and a sprint's completed count can keep +-- changing after it closes (a straggler ticket resolved late) -- +-- unlike a ticket's own append-only status history, there's no safe +-- "only look at what changed since X" watermark here. +CREATE TABLE "Sprint" ( + id uuid PRIMARY KEY DEFAULT extensions.uuid_generate_v4(), + org_project_id uuid REFERENCES "OrgProject"(id), + provider varchar NOT NULL, + external_id varchar NOT NULL, + name varchar NOT NULL, + state varchar NOT NULL, + start_date timestamptz, + end_date timestamptz, + planned_count integer NOT NULL DEFAULT 0, + completed_count integer NOT NULL DEFAULT 0, + idempotency_key varchar NOT NULL, + created_in_db_at timestamptz NOT NULL DEFAULT now(), + updated_in_db_at timestamptz NOT NULL DEFAULT now() +); + +CREATE UNIQUE INDEX sprint_unique_idempotency_key + ON "Sprint"(idempotency_key); + +CREATE INDEX sprint_org_project_fetch_index + ON "Sprint" USING btree (org_project_id, start_date); + +-- migrate:down +DROP TABLE "Sprint"; diff --git a/database-docker/db/migrations/20260813120000_clustox_jira_incident_source_unconditional.sql b/database-docker/db/migrations/20260813120000_clustox_jira_incident_source_unconditional.sql new file mode 100644 index 000000000..1d11ab3a7 --- /dev/null +++ b/database-docker/db/migrations/20260813120000_clustox_jira_incident_source_unconditional.sql @@ -0,0 +1,45 @@ +-- migrate:up +-- CLUSTOX: JIRA_ISSUE moved from an opt-in IncidentSource to an +-- unconditional one -- see default_settings_data.py's +-- INCIDENT_SOURCES_SETTING default for the reasoning (a reopened ticket +-- is the same kind of "the fix didn't hold" signal a revert PR already +-- is for GIT_REPO, which has never had an opt-out toggle either). +-- +-- That default is only ever consulted the first time an org's +-- INCIDENT_SOURCES_SETTING is read (get_or_create, see +-- IncidentsIntegrationService._get_or_create_incident_source_setting) -- +-- every org that had already read it before this change has its own +-- persisted row, missing JIRA_ISSUE, that the code-level default no +-- longer touches. This backfills those existing rows so the new +-- unconditional default actually reaches every org, not just new ones. +UPDATE "Settings" +SET data = jsonb_set( + data, + '{incident_sources}', + (COALESCE(data->'incident_sources', '[]'::jsonb) || '["JIRA_ISSUE"]'::jsonb) +), +updated_at = now() +WHERE setting_type = 'INCIDENT_SOURCES_SETTING' + AND NOT (COALESCE(data->'incident_sources', '[]'::jsonb) @> '["JIRA_ISSUE"]'::jsonb); + +-- migrate:down +-- Best-effort only: removes JIRA_ISSUE from every row this migration +-- could have touched, including any org that had genuinely opted in on +-- its own before this change shipped -- that distinction isn't +-- recoverable from data alone once merged. +UPDATE "Settings" +SET data = jsonb_set( + data, + '{incident_sources}', + COALESCE( + ( + SELECT jsonb_agg(elem) + FROM jsonb_array_elements(data->'incident_sources') elem + WHERE elem <> '"JIRA_ISSUE"' + ), + '[]'::jsonb + ) +), +updated_at = now() +WHERE setting_type = 'INCIDENT_SOURCES_SETTING' + AND (COALESCE(data->'incident_sources', '[]'::jsonb) @> '["JIRA_ISSUE"]'::jsonb); diff --git a/database-docker/db/migrations/20260902090000_org_scoped_idempotency_key.sql b/database-docker/db/migrations/20260902090000_org_scoped_idempotency_key.sql new file mode 100644 index 000000000..38e448076 --- /dev/null +++ b/database-docker/db/migrations/20260902090000_org_scoped_idempotency_key.sql @@ -0,0 +1,46 @@ +-- migrate:up +-- CLUSTOX: OrgRepo carried TWO unique constraints without org_id, and either +-- one makes two workspaces tracking the same repo collide: +-- +-- orgrepo_unique_idempotency_key UNIQUE (idempotency_key) +-- one_repo_per_provider UNIQUE (org_name, name, provider) +-- +-- The idempotency key is the provider's repo id (GitHub numeric id, GitLab +-- project id, Bitbucket uuid) and (org_name, name, provider) is the repo's +-- provider-side identity -- both are the same values in every workspace that +-- links the repo. The second workspace's team save died on the service-level +-- cross-org check ("Data integrity error, matching idempotency key across +-- orgs"), and with that check gone it would die on one_repo_per_provider +-- instead (caught live, locally, before this shipped). Scoping BOTH per org +-- lets each workspace own its row, which is what org_id on the table always +-- implied. +-- +-- Both new constraints are strictly weaker than the old ones, so existing +-- data always satisfies them -- no data fix-up needed before applying. +ALTER TABLE "OrgRepo" + DROP CONSTRAINT orgrepo_unique_idempotency_key; + +ALTER TABLE "OrgRepo" + ADD CONSTRAINT orgrepo_unique_org_idempotency_key UNIQUE (org_id, idempotency_key); + +ALTER TABLE "OrgRepo" + DROP CONSTRAINT one_repo_per_provider; + +ALTER TABLE "OrgRepo" + ADD CONSTRAINT one_repo_per_provider_per_org UNIQUE (org_id, org_name, name, provider); + +-- migrate:down +-- NOTE: restoring the global constraints FAILS if two orgs have since linked +-- the same repo -- exactly the state the up-migration exists to allow. To +-- roll back after that, one org's duplicate rows must be deleted first. +ALTER TABLE "OrgRepo" + DROP CONSTRAINT one_repo_per_provider_per_org; + +ALTER TABLE "OrgRepo" + ADD CONSTRAINT one_repo_per_provider UNIQUE (org_name, name, provider); + +ALTER TABLE "OrgRepo" + DROP CONSTRAINT orgrepo_unique_org_idempotency_key; + +ALTER TABLE "OrgRepo" + ADD CONSTRAINT orgrepo_unique_idempotency_key UNIQUE (idempotency_key); diff --git a/database-docker/db/migrations/20260909120000_clustox_jira_multi_account.sql b/database-docker/db/migrations/20260909120000_clustox_jira_multi_account.sql new file mode 100644 index 000000000..04212f33c --- /dev/null +++ b/database-docker/db/migrations/20260909120000_clustox_jira_multi_account.sql @@ -0,0 +1,61 @@ +-- migrate:up +-- CLUSTOX: Jira multi-account support -- see docs/JIRA_MULTI_ACCOUNT_PLAN.md. +-- +-- Deliberately NOT touching "Integration": that table's PK is +-- (org_id, name), i.e. one row per provider per org, which is exactly the +-- single-Jira-account limitation this feature lifts. Rather than widen an +-- existing, shared-across-providers table (GitHub/GitLab/Jenkins all rely +-- on that same singleton shape), "JiraConnection" is a new, Jira-only, +-- surrogate-keyed table: many rows per org, one per connected Jira site+ +-- account. The legacy "Integration" row (name='jira') is left completely +-- alone and keeps working for orgs that haven't linked a JiraConnection -- +-- there is deliberately no migration/sync between the two tables. +CREATE TABLE "JiraConnection" ( + id uuid PRIMARY KEY DEFAULT extensions.uuid_generate_v4(), + org_id uuid NOT NULL REFERENCES "Organization"(id), + -- Normalized host, e.g. "acme.atlassian.net" -- no scheme/trailing slash. + site_url varchar NOT NULL, + email varchar NOT NULL, + access_token_enc_chunks varchar[], + provider_meta jsonb, + is_default boolean NOT NULL DEFAULT false, + generated_by uuid REFERENCES "Users"(id), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +-- (org_id, site_url) alone would block a legitimate case: two different +-- Jira accounts (different emails) both linking the same site. Including +-- email keeps that possible while still preventing the same account being +-- linked twice. +CREATE UNIQUE INDEX jira_connection_unique_account + ON "JiraConnection"(org_id, site_url, email); + +-- Enforced at the DB level, not just app code -- app-level races (two +-- concurrent "set as default" requests) would otherwise be able to leave +-- an org with zero or two default connections. +CREATE UNIQUE INDEX jira_connection_one_default_per_org + ON "JiraConnection"(org_id) WHERE is_default; + +CREATE INDEX jira_connection_org_fetch_index + ON "JiraConnection" USING btree (org_id); + +-- Records which JiraConnection a given synced OrgProject came from. A new +-- join table rather than a column on "OrgProject" itself, so "OrgProject" +-- (and everything that already reads it -- TeamProjects, Ticket, sync +-- bookmarks) stays completely unchanged. A row's absence here means the +-- project was synced by the legacy single-Jira-account flow (Integration), +-- not a JiraConnection -- callers should treat "no row" as "the org's +-- legacy/default Jira", not as an error. +CREATE TABLE "OrgProjectConnection" ( + org_project_id uuid PRIMARY KEY REFERENCES "OrgProject"(id), + jira_connection_id uuid NOT NULL REFERENCES "JiraConnection"(id), + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX org_project_connection_connection_index + ON "OrgProjectConnection" USING btree (jira_connection_id); + +-- migrate:down +DROP TABLE "OrgProjectConnection"; +DROP TABLE "JiraConnection"; diff --git a/database-docker/db/migrations/20260910100000_clustox_team_repo_project_mapping.sql b/database-docker/db/migrations/20260910100000_clustox_team_repo_project_mapping.sql new file mode 100644 index 000000000..7ef09ac04 --- /dev/null +++ b/database-docker/db/migrations/20260910100000_clustox_team_repo_project_mapping.sql @@ -0,0 +1,40 @@ +-- migrate:up +-- CLUSTOX: explicit, informational repo<->Jira-project pairing within a +-- team -- see docs/JIRA_MULTI_ACCOUNT_PLAN.md's follow-up on Jira<->repo +-- relationships. A team's TeamRepos and TeamProjects are two independent +-- flat lists today (a team just "tracks" some repos and some projects, +-- with no link between which repo goes with which project); once a team +-- has more than one of each, there is no way to say "this repo's tickets +-- come from this project" anywhere in the UI or data model. +-- +-- Deliberately informational only, not a matching constraint: PR<->ticket +-- association (TicketMatchingService) keeps resolving purely from the +-- Jira issue key found in the PR title, exactly as already implemented +-- and tested -- this table has no foreign key from anywhere in that path +-- and nothing reads it during matching or sync. It exists so an admin can +-- record and see the pairing for organization/reporting purposes, without +-- touching the already-verified matching behavior. +-- +-- team_id is included even though org_repo_id/org_project_id could in +-- principle resolve their own org unambiguously, because both TeamRepos +-- and TeamProjects are themselves team-scoped -- the same repo can be +-- tracked by two different teams, each potentially pairing it with a +-- different project, and this mirrors that shape rather than introducing +-- a single global repo->project pairing that would conflict with it. +CREATE TABLE "TeamRepoProjectMapping" ( + team_id uuid NOT NULL REFERENCES "Team"(id), + org_repo_id uuid NOT NULL REFERENCES "OrgRepo"(id), + org_project_id uuid NOT NULL REFERENCES "OrgProject"(id), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (team_id, org_repo_id, org_project_id) +); + +CREATE INDEX team_repo_project_mapping_repo_index + ON "TeamRepoProjectMapping" USING btree (team_id, org_repo_id); + +CREATE INDEX team_repo_project_mapping_project_index + ON "TeamRepoProjectMapping" USING btree (team_id, org_project_id); + +-- migrate:down +DROP TABLE "TeamRepoProjectMapping"; diff --git a/database-docker/db/schema.sql b/database-docker/db/schema.sql index a700a0cd5..d9f248fb9 100644 --- a/database-docker/db/schema.sql +++ b/database-docker/db/schema.sql @@ -71,6 +71,67 @@ CREATE TABLE public."BookmarkPullRequestRevertPRMapping" ( ); +-- +-- Name: ClustoxInvite; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."ClustoxInvite" ( + id uuid DEFAULT extensions.uuid_generate_v4() NOT NULL, + token_hash character varying NOT NULL, + email text NOT NULL, + name character varying NOT NULL, + role character varying NOT NULL, + org_id uuid, + created_by uuid, + created_at timestamp with time zone DEFAULT now() NOT NULL, + expires_at timestamp with time zone NOT NULL, + accepted_at timestamp with time zone, + accepted_by uuid, + revoked_at timestamp with time zone, + CONSTRAINT "ClustoxInvite_role_check" CHECK (((role)::text = ANY ((ARRAY['SUPERADMIN'::character varying, 'ADMIN'::character varying])::text[]))) +); + + +-- +-- Name: ClustoxSyncRun; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."ClustoxSyncRun" ( + id uuid DEFAULT extensions.uuid_generate_v4() NOT NULL, + org_id uuid NOT NULL, + started_at timestamp with time zone DEFAULT now() NOT NULL, + finished_at timestamp with time zone, + status character varying NOT NULL, + detail text, + CONSTRAINT "ClustoxSyncRun_status_check" CHECK (((status)::text = ANY ((ARRAY['RUNNING'::character varying, 'SUCCESS'::character varying, 'FAILED'::character varying, 'SKIPPED'::character varying])::text[]))) +); + + +-- +-- Name: ClustoxUserAuth; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."ClustoxUserAuth" ( + user_id uuid NOT NULL, + password_hash text NOT NULL, + role character varying NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "ClustoxUserAuth_role_check" CHECK (((role)::text = ANY ((ARRAY['SUPERADMIN'::character varying, 'ADMIN'::character varying])::text[]))) +); + + +-- +-- Name: ClustoxUserTeamAccess; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."ClustoxUserTeamAccess" ( + user_id uuid NOT NULL, + team_id uuid NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + -- -- Name: Incident; Type: TABLE; Schema: public; Owner: - -- @@ -482,6 +543,46 @@ CREATE TABLE public.schema_migrations ( ); +-- +-- Name: ClustoxInvite ClustoxInvite_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."ClustoxInvite" + ADD CONSTRAINT "ClustoxInvite_pkey" PRIMARY KEY (id); + + +-- +-- Name: ClustoxInvite ClustoxInvite_token_hash_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."ClustoxInvite" + ADD CONSTRAINT "ClustoxInvite_token_hash_key" UNIQUE (token_hash); + + +-- +-- Name: ClustoxSyncRun ClustoxSyncRun_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."ClustoxSyncRun" + ADD CONSTRAINT "ClustoxSyncRun_pkey" PRIMARY KEY (id); + + +-- +-- Name: ClustoxUserAuth ClustoxUserAuth_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."ClustoxUserAuth" + ADD CONSTRAINT "ClustoxUserAuth_pkey" PRIMARY KEY (user_id); + + +-- +-- Name: ClustoxUserTeamAccess ClustoxUserTeamAccess_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."ClustoxUserTeamAccess" + ADD CONSTRAINT "ClustoxUserTeamAccess_pkey" PRIMARY KEY (user_id, team_id); + + -- -- Name: IncidentOrgIncidentServiceMap IncidentOrgIncidentServiceMap_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -804,6 +905,27 @@ CREATE INDEX "TeamIncidentService_team_id_index" ON public."TeamIncidentService" CREATE INDEX "Team_org_idx" ON public."Team" USING btree (org_id); +-- +-- Name: idx_clustox_invite_pending; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_clustox_invite_pending ON public."ClustoxInvite" USING btree (expires_at) WHERE ((accepted_at IS NULL) AND (revoked_at IS NULL)); + + +-- +-- Name: idx_clustox_sync_run_org_started; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_clustox_sync_run_org_started ON public."ClustoxSyncRun" USING btree (org_id, started_at DESC); + + +-- +-- Name: idx_clustox_user_team_access_user; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_clustox_user_team_access_user ON public."ClustoxUserTeamAccess" USING btree (user_id); + + -- -- Name: incident_resolved_date_index; Type: INDEX; Schema: public; Owner: - -- @@ -1107,6 +1229,62 @@ ALTER TABLE ONLY public."Bookmark" ADD CONSTRAINT "Bookmark_repo_id_fkey" FOREIGN KEY (repo_id) REFERENCES public."OrgRepo"(id); +-- +-- Name: ClustoxInvite ClustoxInvite_accepted_by_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."ClustoxInvite" + ADD CONSTRAINT "ClustoxInvite_accepted_by_fkey" FOREIGN KEY (accepted_by) REFERENCES public."Users"(id) ON DELETE SET NULL; + + +-- +-- Name: ClustoxInvite ClustoxInvite_created_by_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."ClustoxInvite" + ADD CONSTRAINT "ClustoxInvite_created_by_fkey" FOREIGN KEY (created_by) REFERENCES public."Users"(id) ON DELETE SET NULL; + + +-- +-- Name: ClustoxInvite ClustoxInvite_org_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."ClustoxInvite" + ADD CONSTRAINT "ClustoxInvite_org_id_fkey" FOREIGN KEY (org_id) REFERENCES public."Organization"(id) ON DELETE SET NULL; + + +-- +-- Name: ClustoxSyncRun ClustoxSyncRun_org_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."ClustoxSyncRun" + ADD CONSTRAINT "ClustoxSyncRun_org_id_fkey" FOREIGN KEY (org_id) REFERENCES public."Organization"(id) ON DELETE CASCADE; + + +-- +-- Name: ClustoxUserAuth ClustoxUserAuth_user_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."ClustoxUserAuth" + ADD CONSTRAINT "ClustoxUserAuth_user_id_fkey" FOREIGN KEY (user_id) REFERENCES public."Users"(id) ON DELETE CASCADE; + + +-- +-- Name: ClustoxUserTeamAccess ClustoxUserTeamAccess_team_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."ClustoxUserTeamAccess" + ADD CONSTRAINT "ClustoxUserTeamAccess_team_id_fkey" FOREIGN KEY (team_id) REFERENCES public."Team"(id) ON DELETE CASCADE; + + +-- +-- Name: ClustoxUserTeamAccess ClustoxUserTeamAccess_user_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."ClustoxUserTeamAccess" + ADD CONSTRAINT "ClustoxUserTeamAccess_user_id_fkey" FOREIGN KEY (user_id) REFERENCES public."Users"(id) ON DELETE CASCADE; + + -- -- Name: IncidentOrgIncidentServiceMap IncidentOrgIncidentServiceMap_incident_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -1360,4 +1538,8 @@ INSERT INTO public.schema_migrations (version) VALUES ('20240404142732'), ('20240430142502'), ('20240503060203'), - ('20240503073715'); + ('20240503073715'), + ('20260805150000'), + ('20260806120000'), + ('20260806160000'), + ('20260806180000'); diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 000000000..414918035 --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,76 @@ +# CLUSTOX: production deployment. +# +# docker-compose.yml is upstream's *local development* file: it builds +# Dockerfile.dev, which never runs `yarn build`, so the container serves the +# app with `next dev` and compiles every route on first request. That is +# correct for a laptop with hot reload and wrong for a server, where it costs +# 7-22s on each page's first visit and holds the webpack compiler in memory. +# +# This file builds the production Dockerfile instead, which runs `yarn build` +# and serves a compiled bundle. +# +# Deploy with: +# docker compose -f docker-compose.prod.yml up -d --build +# +# Two things here are deliberate and load-bearing: +# +# 1. The volume names match docker-compose.yml exactly. Compose namespaces +# volumes by project (directory) name, so `dev_postgres_data` resolves to +# the same `middleware_dev_postgres_data` that the running deployment +# already uses. Renaming it would silently start against an empty database +# and lose every synced metric and integration. +# +# 2. ENVIRONMENT is pinned under `environment:` rather than left to `.env`. +# `environment:` takes precedence over `env_file:`, and the server's .env +# still says ENVIRONMENT=dev. Without this pin the prod image would build +# correctly and then start `next dev` anyway, which is the exact bug this +# file exists to fix. + +volumes: + dev_postgres_data: + driver: local + dev_keys: + driver: local + +services: + # Service and container names match docker-compose.yml so that `up -d` + # replaces the existing container rather than starting a second one that + # fights it for port 3333. + middleware-dev: + container_name: middleware-dev + build: + context: ./ + dockerfile: Dockerfile + args: + ENVIRONMENT: prod + POSTGRES_DB_ENABLED: ${POSTGRES_DB_ENABLED:-true} + DB_INIT_ENABLED: ${DB_INIT_ENABLED:-true} + REDIS_ENABLED: ${REDIS_ENABLED:-true} + BACKEND_ENABLED: ${BACKEND_ENABLED:-true} + FRONTEND_ENABLED: ${FRONTEND_ENABLED:-true} + CRON_ENABLED: ${CRON_ENABLED:-true} + DB_PORT: ${DB_PORT:-5434} + + env_file: + - .env + + environment: + # See note 2 above -- these must win over .env. + ENVIRONMENT: prod + NEXT_PUBLIC_APP_ENVIRONMENT: prod + + restart: unless-stopped + + ports: + - "127.0.0.1:${ANALYTICS_SERVER_PORT:-9696}:${ANALYTICS_SERVER_PORT:-9696}" + - "127.0.0.1:${SYNC_SERVER_PORT:-9697}:${SYNC_SERVER_PORT:-9697}" + - "127.0.0.1:${PORT:-3333}:${PORT:-3333}" + - "127.0.0.1:${DB_PORT:-5434}:${DB_PORT:-5434}" + - "127.0.0.1:${REDIS_PORT:-6385}:${REDIS_PORT:-6385}" + + extra_hosts: + - "host.docker.internal:host-gateway" + + volumes: + - dev_postgres_data:/var/lib/postgresql/15/main + - dev_keys:/app/backend/analytics_server/mhq/config diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 000000000..a50386040 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,1133 @@ +# Software Architecture Document — Middleware (Clustox fork) + +| | | +|---|---| +| **System** | Middleware — open-source DORA metrics platform | +| **Upstream** | [middlewarehq/middleware](https://github.com/middlewarehq/middleware) (Apache-2.0) | +| **This fork** | [Clustox/middleware](https://github.com/Clustox/middleware) | +| **Baseline commit** | `844eb42` (`main`) | +| **Status** | Living document — update in the same PR as any architectural change | +| **Audience** | Engineers onboarding to the fork; reviewers of proposed changes | + +> **Scope.** This describes the system **as it exists in the code at the baseline commit**, not as it is +> aspirationally described in the upstream README. Where the implementation diverges from the marketing +> description, the implementation is documented and the gap is flagged in +> [§10 Constraints, risks and gaps](#10-constraints-risks-and-gaps). +> +> Every claim here is traceable to a file path. If you change the code and the claim stops being true, +> the doc is now a bug — fix it in the same PR. + +--- + +## Table of contents + +1. [Purpose and context](#1-purpose-and-context) +2. [System context (C4 L1)](#2-system-context-c4-l1) +3. [Container view (C4 L2)](#3-container-view-c4-l2) +4. [Runtime and process model](#4-runtime-and-process-model) +5. [Backend component view (C4 L3)](#5-backend-component-view-c4-l3) +6. [Data model](#6-data-model) +7. [The sync pipeline — the heart of the system](#7-the-sync-pipeline--the-heart-of-the-system) +8. [How the four DORA metrics are derived](#8-how-the-four-dora-metrics-are-derived) +9. [Frontend architecture](#9-frontend-architecture) +10. [Cross-cutting concerns](#10-cross-cutting-concerns) +11. [Constraints, risks and gaps](#11-constraints-risks-and-gaps) +12. [Extension points](#12-extension-points) +13. [Glossary](#13-glossary) + +--- + +## 1. Purpose and context + +Middleware ingests engineering activity from Git providers and CI systems, stores it in a normalised +relational model, and computes the four [DORA metrics](https://dora.dev) per **team** over a +configurable time interval: + +| Metric | Meaning | +|---|---| +| Deployment Frequency | How often code reaches production | +| Lead Time for Changes | First commit → deployed | +| Mean Time to Restore (MTTR) | Incident created → resolved | +| Change Failure Rate (CFR) | % of deployments that caused an incident | + +**Design intent (inferred from the code, not stated upstream):** this is a *single-tenant, self-hosted* +appliance. One Postgres, one implicit organisation, no user-facing tenancy. Understanding that intent +explains most of the choices that otherwise look surprising — no auth on the analytics API, one container +running six processes, a hardcoded `default` org. + +The bootstrap of that single org is explicit in +[`initialise_db.py`](../backend/analytics_server/mhq/store/initialise_db.py): on first boot the app +creates exactly one `Organization` named `default`, and every request resolves back to it via +`get_query_validator().get_default_org()`. + +--- + +## 2. System context (C4 L1) + +```mermaid +graph TB + subgraph external["External systems"] + GH["GitHub
REST + GraphQL"] + GL["GitLab
REST"] + GHA["GitHub Actions
workflow runs"] + LLM["LLM provider
optional, for AI summaries"] + end + + subgraph mw["Middleware (self-hosted appliance)"] + SYS["DORA analytics
ingest · store · compute · visualise"] + end + + EM["Engineering manager /
team lead"] + ADMIN["Operator
installs, holds the PAT"] + + ADMIN -->|"configures integration, PAT,
selects repos & teams"| SYS + EM -->|"views dashboards"| SYS + + SYS -->|"pulls repos, PRs, reviews,
commits, workflow runs"| GH + SYS -->|"pulls projects, MRs, notes"| GL + SYS -->|"pulls run status & duration"| GHA + SYS -.->|"sends aggregated metrics
for narrative summary"| LLM + + style SYS fill:#4a6fa5,stroke:#2c4a6f,color:#fff + style LLM stroke-dasharray: 4 4 +``` + +**Integration direction is exclusively outbound-pull.** There are no inbound webhooks anywhere in the +codebase. Everything arrives because a cron-triggered sync went and fetched it. Two consequences worth +internalising: + +- Data is **stale by up to 30 minutes** by design (the cron interval). +- The system needs **no ingress from the internet** — which is the main reason its lack of API auth has + not been catastrophic upstream. + +--- + +## 3. Container view (C4 L2) + +```mermaid +graph TB + USER["Browser"] + + subgraph container["Single Docker container — middleware-dev"] + direction TB + + FE["Next.js web server
:3333
pages · SSR · BFF API routes
React 18 · MUI 5 · Redux Toolkit"] + AN["Flask analytics server
:9696
app.py — 9 blueprints
read-mostly: metrics & queries"] + SY["Flask sync server
:9697
sync_app.py — 1 endpoint
write-heavy: the ETL"] + CRON["cron
*/30 * * * *
POST :9697/sync"] + PG[("PostgreSQL 15
:5434
db: mhq-oss")] + RD[("Redis
:6385
distributed locks only")] + SUP["supervisord — PID 1, starts all of the above by priority"] + end + + EXT["GitHub / GitLab APIs"] + + USER -->|":3333 HTTP"| FE + FE -->|"axios · INTERNAL_API_BASE_URL"| AN + FE -->|"knex · direct SQL ⚠️"| PG + FE -->|"direct provider calls
for integration setup"| EXT + CRON --> SY + AN --> PG + AN --> RD + SY --> PG + SY --> RD + SY --> EXT + + style FE fill:#3d5a80,color:#fff + style AN fill:#4a6fa5,color:#fff + style SY fill:#5c8001,color:#fff + style PG fill:#336791,color:#fff + style RD fill:#a41e11,color:#fff + style SUP fill:#555,color:#fff +``` + +### 3.1 Container responsibilities + +| Container | Tech | Entry point | Responsibility | +|---|---|---|---| +| Web server | Next.js (pages router), TS, MUI 5, Redux Toolkit | [`web-server/`](../web-server/) | UI, SSR, and a BFF layer under `pages/api/**` | +| Analytics server | Flask 3.0, SQLAlchemy 2.0, gunicorn | [`app.py`](../backend/analytics_server/app.py) | Serves computed metrics. Read-mostly. | +| Sync server | Flask 3.0, same ORM | [`sync_app.py`](../backend/analytics_server/sync_app.py) | Runs the ETL. Write-heavy. Deliberately a separate process. | +| Database | PostgreSQL 15 | [`database-docker/db/`](../database-docker/db/) | Single source of truth. Schema via `dbmate`. | +| Redis | Redis (no persistence configured) | — | **Distributed locks only.** Not a cache. | +| Process manager | supervisord | [`supervisord.conf`](../setup_utils/supervisord.conf) | PID 1; ordered startup, restart policy | +| CLI | Node 22, Ink (React for terminals) | [`cli/`](../cli/) | Wraps `docker compose` for a friendly local install | + +### 3.2 ⚠️ The most important architectural fact in this document + +**There are two independent write paths to the same database, and they do not know about each other.** + +```mermaid +graph LR + subgraph paths["Two paths to one schema"] + P1["Flask backend
SQLAlchemy models
mhq/store/models/*.py"] + P2["Next.js BFF
knex query builder
web-server/src/utils/db.ts"] + end + PG[("PostgreSQL
mhq-oss")] + P1 -->|"ORM"| PG + P2 -->|"raw SQL builder"| PG + + style P1 fill:#4a6fa5,color:#fff + style P2 fill:#3d5a80,color:#fff + style PG fill:#336791,color:#fff +``` + +The Next.js API routes do **not** proxy everything to Flask. **23 files** under `web-server/` import +`@/utils/db` and talk to Postgres directly through knex with its own connection pool +([`web-server/src/utils/db.ts`](../web-server/src/utils/db.ts)). Table and column names are +re-declared by hand in TypeScript in `web-server/src/constants/db.ts`. + +Reproduce the list yourself: + +```bash +grep -rl "from '@/utils/db'" web-server/pages/api web-server/src +``` + +Examples of direct-SQL writes, not proxied through the API: +[`team_repos.ts`](../web-server/pages/api/resources/team_repos.ts) deactivates rows in `TeamRepos` with +a knex `UPDATE`; [`onboarding.ts`](../web-server/pages/api/resources/orgs/%5Borg_id%5D/onboarding.ts) and +[`integration.ts`](../web-server/pages/api/resources/orgs/%5Borg_id%5D/integration.ts) do likewise. + +**Why this matters for every future change we make:** + +1. **A schema change is a three-place change.** The `dbmate` migration, the SQLAlchemy model, *and* the + TypeScript `Columns`/`Table` declarations. Miss the third and you get a runtime failure in the BFF + that no Python test will catch. +2. **Business logic can be bypassed.** Validation living in a Flask service does not protect a write + that arrives via knex. +3. **Two connection pools** contend for the same Postgres: SQLAlchemy `pool_size=10, max_overflow=5` + per Flask process ([`store/__init__.py`](../backend/analytics_server/mhq/store/__init__.py)) and + knex `max=7` in non-dev ([`db.ts`](../web-server/src/utils/db.ts)). + +This is not a defect to fix casually — it is load-bearing, and unpicking it is a project of its own. +It is documented here so nobody discovers it the hard way. See +[§12 Extension points](#12-extension-points) for the practical checklist. + +--- + +## 4. Runtime and process model + +Everything runs in **one container** under supervisord. Postgres, Redis, two Python servers, Next.js, +and cron are siblings in a single failure domain. + +```mermaid +graph TB + S["supervisord (PID 1, nodaemon)"] + S -->|"priority 1"| P1["postgres
autorestart=false ⚠️"] + S -->|"priority 2"| P2["initialize_db
dbmate up · autorestart=false"] + S -->|"priority 3"| P3["redis
autorestart=true"] + S -->|"priority 4"| P4["backend :9696
autorestart=true"] + S -->|"priority 5"| P5["backend_sync :9697
autorestart=true"] + S -->|"no priority"| P6["frontend :3333
autorestart=true"] + S -->|"no priority"| P7["cron
autorestart=true"] + + style P1 fill:#8b0000,color:#fff + style P2 fill:#a0522d,color:#fff +``` + +Read from [`setup_utils/supervisord.conf`](../setup_utils/supervisord.conf). Notes that matter +operationally: + +- **`postgres` has `autorestart=false`.** If Postgres dies, supervisord will *not* bring it back, and + the container keeps running with every other process failing its queries. The container looks alive + and the app is dead. This is the first thing to check when someone reports "it's up but broken". +- **Each process is individually switchable** via env flags (`POSTGRES_DB_ENABLED`, `REDIS_ENABLED`, + `BACKEND_ENABLED`, `FRONTEND_ENABLED`, `CRON_ENABLED`). This is the seam to exploit when splitting + the container for production — set the flag false and point the env var at an external service. +- **Every log is capped at 512 KB with `logfile_backups=0`** — rotation discards the old file. There is + no log history beyond half a megabyte per process. Anything resembling production needs log shipping. +- Logs live at `/var/log/{postgres,init_db,redis,apiserver,sync_server,web-server,cron}/`. The UI + surfaces them at `/system-logs` ([`pages/system-logs.tsx`](../web-server/pages/system-logs.tsx)). + +### 4.1 Ports + +| Port | Process | Bound to | +|---|---|---| +| 3333 | Next.js | `127.0.0.1` | +| 9696 | Flask analytics | `127.0.0.1` | +| 9697 | Flask sync | `127.0.0.1` | +| 5434 | Postgres | `127.0.0.1` | +| 6385 | Redis | `127.0.0.1` | + +All five are bound to loopback on the host by +[`docker-compose.yml`](../docker-compose.yml) (`"127.0.0.1:${PORT}:${PORT}"`). **This loopback binding +is the only thing standing between the unauthenticated analytics API and the network.** See +[§11.1](#111-no-authentication-on-the-backend-apis). + +### 4.2 Schema migrations + +`dbmate`, not Alembic — despite the backend being SQLAlchemy. +[`init_db.sh`](../setup_utils/init_db.sh) waits for Postgres, creates the database if absent, then runs +`dbmate -u "$DB_URL" up` against +[`database-docker/db/migrations/`](../database-docker/db/migrations/). + +There are only **4** migration files, the earliest named `20240404142732_init.sql`. The full schema +lives in [`database-docker/db/schema.sql`](../database-docker/db/schema.sql). Two implications: + +- SQLAlchemy models are **hand-kept in sync** with SQL migrations. Nothing auto-generates or verifies + the correspondence — no `alembic revision --autogenerate`, no drift check in CI. A model/schema + mismatch is caught only at runtime. +- Adding a migration means writing SQL by hand and updating the model by hand. Adding a **drift check** + to CI is one of the highest-value, lowest-risk hardening changes available to us. + +--- + +## 5. Backend component view (C4 L3) + +The backend is cleanly layered, and this consistency is its best quality — it makes the codebase far +more tractable than its size suggests. + +```mermaid +graph TB + subgraph api["mhq/api/ — HTTP boundary"] + A1["pull_requests · incidents
deployment_analytics
teams · settings · integrations
bookmark · sync · ai/dora_ai"] + A2["request_utils.py
voluptuous schema validation"] + A3["resources/
ORM → JSON adapters"] + end + + subgraph svc["mhq/service/ — business logic"] + S1["code/
lead_time · pr_analytics
sync/ ETL handlers"] + S2["deployments/
factory · analytics
pr_ vs workflow_ strategies"] + S3["incidents/
MTTR · CFR"] + S4["workflows/
CI run ingest"] + S5["merge_to_deploy_broker/
joins merge → deploy"] + S6["settings/ · bookmark/ · ai/"] + S7["query_validator.py
resolves default org & team"] + end + + subgraph store["mhq/store/ — persistence"] + R1["repos/
code · core · incidents
integrations · settings · workflows"] + M1["models/
SQLAlchemy declarative"] + end + + subgraph ex["mhq/exapi/ — external clients"] + E1["github.py — PyGithub + raw GraphQL"] + E2["gitlab.py"] + E3["git_incidents.py"] + end + + subgraph util["mhq/utils/"] + U1["cryptography.py — RSA
lock.py — Redis locks
time.py — Interval, buckets
log.py · dict.py · regex.py"] + end + + api --> svc --> store + svc --> ex + svc --> util + store --> util + + style api fill:#4a6fa5,color:#fff + style svc fill:#5c8001,color:#fff + style store fill:#336791,color:#fff + style ex fill:#8b5a2b,color:#fff +``` + +**The layering rule, stated plainly:** `api → service → store/repos → store/models`, with `exapi` +reachable only from `service`. No layer reaches backwards. Every future change should preserve this — +it is the property that makes the code navigable. + +### 5.1 Dependency injection convention + +Services take their collaborators as constructor arguments and are assembled by module-level +`get_*_service()` factory functions: + +```python +# mhq/service/code/sync/etl_handler.py +code_etl_handler = CodeETLHandler( + CodeRepoService(), + etl_factory(provider), # provider-specific strategy + get_merge_to_deploy_broker_utils_service(), + get_bookmark_service(), + get_settings_service(), +) +``` + +Follow this pattern in new code. It is why the test suite can substitute fakes without a DI framework, +and it is what makes the provider-strategy pattern in §12.1 work. + +### 5.2 Blueprint registration + +Nine blueprints on the analytics server, two on the sync server +([`app.py`](../backend/analytics_server/app.py), [`sync_app.py`](../backend/analytics_server/sync_app.py)). +`hello` is registered on both as a health endpoint. Adding an API surface means creating a blueprint +module and registering it in the relevant entry point — there is no auto-discovery. + +### 5.3 Selected endpoints + +Generate the current list with: + +```bash +grep -rn -A1 '@app.route' backend/analytics_server/mhq/api/ +``` + +| Method | Path | Purpose | +|---|---|---| +| GET | `/teams//lead_time` | Lead time, aggregate | +| GET | `/teams//lead_time/trends` | Lead time, weekly buckets | +| GET | `/teams//lead_time/prs` | Contributing PRs (drill-down) | +| GET | `/teams//deployment_frequency` | Deployment frequency | +| GET | `/teams//deployment_analytics` | Deployments + related PRs | +| GET | `/teams//mean_time_to_recovery` | MTTR | +| GET | `/teams//change_failure_rate` | CFR | +| GET | `/teams//deployments_with_related_incidents` | Deploy↔incident join | +| GET/PUT | `/teams//settings` | Team settings | +| GET/PUT | `/orgs//settings` | Org settings | +| GET | `/orgs//integrations/{github,gitlab}/...` | Repo/org discovery | +| PUT | `/orgs//bookmark/reset` | **Force full re-sync** | +| POST | `/ai/dora_score`, `/ai/*_trends` | LLM narrative summaries | +| POST | `/sync` | Trigger ETL *(sync server, :9697)* | + +Note the shape: almost every analytics route is `GET /teams//` with an interval and +filter in the query string, and most have a paired `/trends` variant. New metrics should follow it. + +--- + +## 6. Data model + +Tables use `PascalCase` — unusual for Postgres, and it means **every raw SQL reference needs double +quotes** (`SELECT * FROM "PullRequest"`). Expect to be bitten by this once. + +```mermaid +erDiagram + Organization ||--o{ Users : has + Organization ||--o{ Team : has + Organization ||--o{ OrgRepo : has + Organization ||--o{ Integration : has + Organization ||--o{ OrgIncidentService : has + + Team ||--o{ TeamRepos : selects + Team ||--o{ TeamIncidentService : watches + OrgRepo ||--o{ TeamRepos : "mapped to" + OrgRepo ||--o{ PullRequest : contains + OrgRepo ||--o{ RepoWorkflow : has + OrgRepo ||--o{ RepoSyncLogs : logs + + PullRequest ||--o{ PullRequestEvent : "reviews, comments" + PullRequest ||--o{ PullRequestCommit : contains + PullRequest ||--o{ PullRequestRevertPRMapping : "reverted by" + + RepoWorkflow ||--o{ RepoWorkflowRuns : "runs" + RepoWorkflow ||--o{ RepoWorkflowRunsBookmark : "sync cursor" + + OrgIncidentService ||--o{ IncidentOrgIncidentServiceMap : maps + Incident ||--o{ IncidentOrgIncidentServiceMap : maps + + Organization { + UUID id PK + string name "always 'default' in OSS" + string domain + } + Team { + UUID id PK + UUID org_id FK + UUID_array member_ids "array column, not a join table" + boolean is_deleted "soft delete" + } + OrgRepo { + UUID id PK + string provider "github or gitlab" + string org_name + string default_branch + jsonb contributors + boolean is_active + } + TeamRepos { + UUID team_id PK,FK + UUID org_repo_id PK,FK + string_array prod_branches "regex — defines production" + enum deployment_type "PR_MERGE or WORKFLOW" + } + PullRequest { + UUID id PK + UUID repo_id FK + enum state "OPEN, CLOSED or MERGED" + int first_commit_to_open "precomputed segment" + int first_response_time "precomputed segment" + int rework_time "precomputed segment" + int merge_time "precomputed segment" + int merge_to_deploy "backfilled by MTD broker" + int cycle_time + int rework_cycles + jsonb meta "code_stats, user_profile" + } + RepoWorkflowRuns { + UUID id PK + enum status "SUCCESS, FAILURE, etc" + datetime conducted_at "equals deployment time" + int duration + string head_branch + } + Incident { + UUID id PK + string key "dedup key" + datetime creation_date + datetime resolved_date + enum incident_type + } + Integration { + UUID org_id PK,FK + string name PK + string_array access_token_enc_chunks "RSA-encrypted chunks" + jsonb provider_meta + } +``` + +### 6.1 Three modelling decisions worth understanding + +**1. Lead-time segments are precomputed and stored on the row.** +`PullRequest.first_commit_to_open`, `first_response_time`, `rework_time`, `merge_time`, and +`merge_to_deploy` are integer **seconds**, written during ETL rather than derived at query time +([`models/code/pull_requests.py`](../backend/analytics_server/mhq/store/models/code/pull_requests.py)). + +This makes reads fast and writes authoritative — and it means **changing a metric definition does not +change historical data**. Old rows keep their old numbers. Any change to how a segment is computed +requires a backfill or a bookmark reset (`PUT /orgs//bookmark/reset`) to be visible in +history. This is the single most common way to get a confusing result after a "simple" metric change. + +**2. `TeamRepos.prod_branches` is where "production" is defined.** +A `string[]` of branch regexes, per team, per repo. There is no environment concept, no deploy target +model. "Production" means "matched one of these branch patterns". Everything downstream — +deployment frequency, lead time, CFR — inherits this definition. + +**3. `TeamRepos.deployment_type` selects the deployment strategy per repo.** +`PR_MERGE` (a merge to a prod branch *is* the deployment) or `WORKFLOW` (a successful CI run is the +deployment). This enum is the switch that picks a strategy class at query time — see §8.1. + +### 6.2 Incremental sync cursors ("bookmarks") + +Four separate cursor tables, so a failure in one stream does not rewind the others: + +| Table | Cursor for | +|---|---| +| `Bookmark` | PR sync, keyed `(repo_id, type)` | +| `BookmarkMergeToDeployBroker` | Merge-to-deploy backfill position | +| `RepoWorkflowRunsBookmark` | CI workflow run sync | +| `IncidentsBookmark` | Incident sync | + +Cursors are stored as **ISO-8601 strings**, not timestamps. On first sync there is no cursor, so the +window falls back to the `DEFAULT_SYNC_DAYS` setting (31 in [`env.example`](../env.example)) — +resolved via `SettingType.DEFAULT_SYNC_DAYS_SETTING`, so the DB setting wins over the env default. + +--- + +## 7. The sync pipeline — the heart of the system + +If you read only one section, read this one. Almost every bug and every feature request lands here. + +### 7.1 Orchestration + +[`mhq/service/sync_data.py`](../backend/analytics_server/mhq/service/sync_data.py) is the whole +orchestrator — four functions run in a fixed order: + +```python +sync_sequence = [ + sync_code_repos, # 1. repos, PRs, reviews, commits + sync_org_workflows, # 2. CI workflow runs + process_merge_to_deploy_cache, # 3. join merges to deploys + sync_org_incidents, # 4. incidents +] +``` + +**The order is a real dependency, not a preference.** Step 3 needs both PRs (step 1) and workflow runs +(step 2) already in the database. Step 4's PR-derived incidents need step 1. Reordering silently +produces incomplete metrics rather than an error. + +### 7.2 Sequence of a sync run + +```mermaid +sequenceDiagram + participant C as cron (*/30) + participant S as sync server :9697 + participant R as Redis + participant E as ETL handlers + participant G as GitHub / GitLab + participant D as Postgres + + C->>S: POST /sync + S->>S: resolve default org + S->>R: acquire lock {org}::data_sync + Note over R: serialises concurrent syncs + + rect rgb(238,246,238) + Note over E,D: 1. sync_code_repos + S->>E: per code provider + E->>G: check_pat_validity() + E->>D: read active OrgRepos + E->>G: fetch repo metadata + E->>D: read Bookmark (else now - DEFAULT_SYNC_DAYS) + E->>G: fetch PRs updated since cursor + E->>E: compute lead-time segments + E->>D: upsert PR + commits + events + E->>D: advance Bookmark to max(updated_at) + E->>D: save revert-PR mappings + end + + rect rgb(238,242,250) + Note over E,D: 2. sync_org_workflows + E->>G: fetch workflow runs since cursor + E->>D: upsert RepoWorkflowRuns + end + + rect rgb(252,246,232) + Note over E,D: 3. merge_to_deploy broker + S->>R: lock {org_repo}::merge_to_deploy_broker + E->>D: successful runs after cursor (max 500) + E->>D: PRs merged before run, merge_to_deploy IS NULL + E->>E: merge_to_deploy = conducted_at - state_changed_at + E->>D: UPDATE PullRequest + end + + rect rgb(250,240,240) + Note over E,D: 4. sync_org_incidents + E->>G: fetch incidents + E->>D: upsert Incident + service map + end + + S->>R: release lock + S-->>C: {"message": "sync started"} +``` + +### 7.3 The merge-to-deploy broker, explained + +This component confuses everyone at first, so: **a PR does not know when it was deployed at the moment +it is merged.** The deployment happens later. So `merge_to_deploy` is written as `NULL` at PR-sync time +and **backfilled on a later sync run**, once a successful workflow run exists that contains the merge +commit. + +From [`mtd_handler.py`](../backend/analytics_server/mhq/service/merge_to_deploy_broker/mtd_handler.py): + +```python +pr.merge_to_deploy = int((conducted_at - pr.state_changed_at).total_seconds()) +``` + +Two operational consequences: + +- **Recently merged PRs legitimately have incomplete lead time.** Their `merge_to_deploy` is `NULL` + until a deploy catches them. A dashboard showing "lead time dropped this week" often just means the + last few PRs have not been backfilled yet. +- **`DEPLOYMENTS_TO_PROCESS = 500`** caps how many runs are processed per repo per pass. A repo with a + large backlog needs several sync cycles to catch up. There is no warning when the cap is hit. + +### 7.4 Error handling philosophy: isolate and continue + +The same `try/except/LOG.error/continue` pattern nests at four levels — org sync, provider, repo, and +PR batch. A failure in one repo does not abort the run; the rest still sync. + +**The cost:** `POST /sync` returns `200 {"message": "sync started"}` even if every single repo failed. +Failures are visible **only in the logs**, which are capped at 512 KB with no rotation history and no +alerting. "Sync succeeded but the numbers look wrong" is the expected failure signature, and the only +way to diagnose it today is reading `/var/log/sync_server/sync_server.log` before it rotates away. + +Adding real sync-run status (per-repo outcome persisted and surfaced) is a strong candidate for our +first substantial contribution — it is genuinely useful, self-contained, and touches the layering in +an instructive way. + +--- + +### 7.5 The bookmark trap — read this before debugging "no data" + +The single most confusing behaviour in the system, and the first thing to check whenever a repo shows +no metrics. + +**A newly added repo only syncs history back to its bookmark, and the bookmark starts at "now minus +`DEFAULT_SYNC_DAYS`" — or, if a sync already ran, at whenever that sync last got to.** The GitHub ETL +then discards any PR that closed *before* the bookmark: + +```python +# mhq/service/code/sync/etl_github_handler.py +state_changed_at = pr.merged_at if pr.merged_at else pr.closed_at +if pr.state.upper() != PullRequestState.OPEN.value and state_changed_at < bookmark: + continue +``` + +So a repo can sit at zero PRs while the sync log cheerfully reports success: + +``` +[INFO] Starting data sync for org +Nothing to process 🎉 +[INFO] Data sync for sync_code_repos completed successfully +``` + +Meanwhile the UI shows **"Calculating Dora — we're processing your data, it usually takes ~5 mins"** +forever. That banner is not a progress indicator. It is what +`web-server/src/content/DoraMetrics/DoraMetricsBody.tsx` renders when there is no data to draw, and +it never resolves on its own. + +**Diagnosing it.** Compare bookmarks against PR counts per repo: + +```sql +SELECT r.name, b.bookmark, count(p.id) AS prs +FROM "OrgRepo" r +LEFT JOIN "Bookmark" b ON b.repo_id = r.id +LEFT JOIN "PullRequest" p ON p.repo_id = r.id +GROUP BY r.name, b.bookmark; +``` + +- **No bookmark row + 0 PRs** → the repo has never synced. Run a sync; cron will also pick it up + within 30 minutes. +- **Bookmark ≈ the time the repo was added + 0 PRs** → history is being filtered out. Rewind it. + +**Rewinding.** `PUT /orgs//bookmark/reset` sets every bookmark in the org back to +`now - DEFAULT_SYNC_DAYS`, or to an explicit timestamp: + +```bash +curl -X PUT "http://localhost:9696/orgs//bookmark/reset?bookmark_timestamp=2026-05-07T00:00:00" +curl -X POST http://localhost:9697/sync +``` + +**The ordering catch:** `reset_org_bookmarks` iterates *existing* bookmark rows, so it cannot rewind a +repo that has never synced — there is no row yet. For a brand-new repo the sequence must be +**sync → reset → sync**: the first sync creates the bookmark, the reset rewinds it, the second sync +pulls the older history. + +Related: §11.5 on silent sync failures. A sync that finds nothing and a sync that failed for every +repo produce the same HTTP 200 and the same green log line. + +## 8. How the four DORA metrics are derived + +### 8.1 Deployment Frequency + +A "deployment" is polymorphic, resolved per repo by `TeamRepos.deployment_type` through +[`deployments/factory.py`](../backend/analytics_server/mhq/service/deployments/factory.py): + +```mermaid +graph TB + Q["Query: deployments for team + interval"] --> F{"TeamRepos
.deployment_type"} + F -->|PR_MERGE| A["PRDeploymentsService
merged PR into a prod branch
= one deployment"] + F -->|WORKFLOW| B["WorkflowDeploymentsService
successful RepoWorkflowRun
= one deployment"] + A --> N["Normalised Deployment
via DeploymentsAdaptorFactory"] + B --> N + N --> C["Count / bucket by week"] + + style A fill:#5c8001,color:#fff + style B fill:#4a6fa5,color:#fff +``` + +Both strategies implement the `DeploymentsFactoryService` ABC and are normalised to a common +`Deployment` model by an adaptor — so downstream analytics never branch on deployment type. **This is +the cleanest extension seam in the codebase**; a third deployment source is a new subclass plus an +enum value, and nothing downstream changes. + +Deployment IDs are composite strings, `"|"`, parsed by +`get_deployment_type_and_entity_id_from_deployment_id`. Do not assume a deployment ID is a bare UUID. + +### 8.2 Lead Time for Changes + +Lead time is the sum of five stored segments +([`service/code/models/lead_time.py`](../backend/analytics_server/mhq/service/code/models/lead_time.py)): + +```mermaid +graph LR + A["first commit"] -->|first_commit_to_open| B["PR opened"] + B -->|first_response_time| C["first review"] + C -->|rework_time| D["review approved"] + D -->|merge_time| E["merged"] + E -->|merge_to_deploy| F["deployed"] + + style A fill:#3d5a80,color:#fff + style F fill:#5c8001,color:#fff +``` + +```python +lead_time = first_commit_to_open + first_response_time + rework_time + merge_time + merge_to_deploy +cycle_time = first_response_time + rework_time + merge_time + merge_to_deploy +``` + +`cycle_time` is lead time minus the pre-PR authoring period — the part a review process can influence. + +Team aggregates use a **weighted average by PR count**, not a mean of per-repo means +(`_get_weighted_avg_lead_time_metrics` in +[`lead_time.py`](../backend/analytics_server/mhq/service/code/lead_time.py)). Trends bucket weekly via +`generate_expanded_buckets`, then `fill_missing_week_buckets` inserts zeroed weeks so the chart has no +gaps. When comparing a computed number against the UI, replicate the weighting — a plain average will +not match. + +### 8.3 Mean Time to Recovery + +Per incident, `resolved_date - creation_date`, averaged over incidents resolved in the interval; +`MeanTimeToRecoveryMetrics` carries `mean_time_to_recovery` and `incident_count` +([`models/mean_time_to_recovery.py`](../backend/analytics_server/mhq/service/incidents/models/mean_time_to_recovery.py)). + +Incidents come from two sources, unioned and **deduplicated by `incident.key`** +([`incidents.py`](../backend/analytics_server/mhq/service/incidents/incidents.py)): + +1. A configured incident service (`OrgIncidentService`). +2. **PRs matched by regex** — `IncidentPRsSetting.filters` lets a team declare that PRs matching a + pattern (e.g. `^hotfix/`, `revert`) *are* incidents. This is how a team with no PagerDuty still gets + MTTR and CFR. `mhq/utils/regex.py` guards the patterns. + +### 8.4 Change Failure Rate + +```python +change_failure_rate = len(failed_deployments) / len(total_deployments) * 100 +``` + +Both are **`Set[Deployment]`**, so a deployment linked to three incidents counts once +([`ChangeFailureRateMetrics`](../backend/analytics_server/mhq/service/incidents/models/mean_time_to_recovery.py)). +A deployment is "failed" if an incident is attributed to it — the attribution join lives in +`deployments_with_related_incidents`. Empty `total_deployments` returns `0`, not a division error. + +--- + +## 9. Frontend architecture + +```mermaid +graph TB + subgraph browser["Browser"] + UI["React 18 + MUI 5
pages/dora-metrics · teams
integrations · settings · system-logs"] + RX["Redux Toolkit
slices: dora_metrics · team · org
auth · app · service"] + end + + subgraph nextjs["Next.js server :3333"] + MW["middleware.ts
injects feature_flags into every
/api/{auth,integrations,internal,resources}/*"] + BFF["pages/api/**
yup-validated Endpoint handlers"] + end + + FL["Flask :9696"] + PG[("Postgres")] + GH["GitHub / GitLab"] + + UI <--> RX + RX -->|"axios baseURL=/api"| MW --> BFF + BFF -->|"handleRequest → axios"| FL + BFF -->|"knex ⚠️"| PG + BFF -->|"integration setup"| GH + + style UI fill:#3d5a80,color:#fff + style BFF fill:#4a6fa5,color:#fff + style PG fill:#336791,color:#fff +``` + +Conventions to follow when touching the frontend: + +- **BFF handlers** use an `Endpoint` abstraction with **yup** schemas per method + (`endpoint.handle.GET(schema, fn)`), giving runtime request validation — see + [`team_repos.ts`](../web-server/pages/api/resources/team_repos.ts). +- **Feature flags** are merged from `defaultFlags` plus request overrides in + [`middleware.ts`](../web-server/middleware.ts) and appended as a `feature_flags` query param to every + matched API route. A `use_mock_data` flag makes handlers return fixtures from `src/mocks/` — + useful for UI work without a synced database. +- **Pagination** is client-driven: `paginatedRequest` and `batchPaginatedRequest` in + [`internal.ts`](../web-server/src/api-helpers/internal.ts) walk pages recursively, the latter + fetching 5 pages concurrently. +- **There is no login.** `next-auth` is in `package.json` but is **never configured** — there is no + `[...nextauth]` API route and no sign-in page. Its only two imports are a `Session` *type* in + `ThirdPartyAuthContext.tsx` and a `signIn` call in `AppErrors.tsx` that has nothing to call. + `/api/auth/session` is a hand-rolled `Endpoint` handler that returns org details and integration + state to any caller. See §11.1. +- **Tests:** Jest (`jest.config.js`) plus Playwright (`playwright.config.ts`). + +--- + +## 10. Cross-cutting concerns + +### 10.1 Secrets and token encryption + +Provider PATs are **RSA-2048 / PKCS1-OAEP encrypted at rest**, chunked across a Postgres `string[]` +because RSA cannot encrypt arbitrarily long input +([`utils/cryptography.py`](../backend/analytics_server/mhq/utils/cryptography.py), `Integration.access_token_enc_chunks`). + +The keypair is generated **once per install** by +[`generate_config_ini.sh`](../setup_utils/generate_config_ini.sh) into +`backend/analytics_server/mhq/config/config.ini`, base64-encoded, and persisted in the `dev_keys` +Docker volume. + +Three things to know before touching this: + +- **Losing the volume makes every stored token unrecoverable.** Not a corruption — a permanent loss. + Any backup plan must include `dev_keys`, not just the Postgres volume. +- The script also **appends both keys to `~/.bashrc`** as exported env vars — meaning the private key + lands in a shell profile in plaintext. Worth flagging in any production hardening review. +- `config.ini` must never be committed. Confirm it is git-ignored before any commit that touches + `mhq/config/`. + +### 10.2 Concurrency control + +Redis is used **exclusively for distributed locks** (`python-redis-lock`), never as a cache: + +| Lock key | Guards | +|---|---| +| `{org}::data_sync` | Whole sync run — prevents overlapping syncs | +| `{org_repo}::merge_to_deploy_broker` | Per-repo MTD backfill | +| `initialize_database` | First-boot org creation race | + +The `{...}` braces are Redis Cluster hash-tag syntax, keeping related keys on one slot. Redis has no +persistence configured — correct, since locks should not survive a restart. + +### 10.3 Configuration + +Single `.env` at the repo root, loaded by [`env.py`](../backend/analytics_server/env.py) — +`.env.prod` when `FLASK_ENV=production`, otherwise `../../.env`. The same file feeds compose, both +Flask apps, and Next.js. Copy from [`env.example`](../env.example). + +Two dangerous defaults to fix before anything leaves a laptop: `DB_PASS=postgres`, and +`pg_hba.conf` extended with `host all all 0.0.0.0/0 md5` in +[`Dockerfile.dev`](../Dockerfile.dev) — Postgres accepts password auth from any address inside the +container network. + +### 10.4 Observability + +| Concern | Current state | +|---|---| +| Logs | Files under `/var/log/*`, 512 KB cap, `backups=0`, surfaced at `/system-logs` | +| Metrics | None — no Prometheus, no StatsD | +| Tracing | None | +| Health | `GET /` on both Flask apps; `/api/status`, `/api/db_status`, `/api/internal_status` on Next.js | +| Alerting | None | +| Sync outcome | **Not persisted** — logs only (see §7.4) | + +### 10.5 CI and tests + +Five workflows in [`.github/workflows/`](../.github/workflows/): `build.yml`, `build-base.yml`, +`pytest.yml`, `black.yml`, `pre-commit.yml`. Python is formatted with **black** and linted with +**flake8**; hooks in [`.pre-commit-config.yaml`](../.pre-commit-config.yaml). + +**20 Python test files** under `backend/analytics_server/tests/`, concentrated on service-layer logic +(`code`, `deployments`, `incidents`, `workflows`) with model factories in `tests/factories/`. This is +a thin suite for a codebase this size — the metric computations are covered, the ETL edge cases and +the entire BFF/knex path largely are not. Treat "tests pass" as a weak signal here and rely on it +least where the two write paths meet. + +--- + +## 11. Constraints, risks and gaps + +Ordered by significance to Clustox. Nothing here is a criticism of upstream — most are rational +choices for a self-hosted single-tenant appliance, and they simply stop holding if we change the +deployment or tenancy assumptions. + +### 11.1 No authentication anywhere in the stack + +There is **no authentication or authorization of any kind** — not on the backend, and not on the +frontend either. Verify both: + +```bash +# Backend API — zero matches +grep -rniE "login_required|authenticat|jwt|api_key|bearer" backend/analytics_server/mhq/api/ + +# Frontend — no NextAuth route, no sign-in page +find web-server/pages -path '*nextauth*' +ls web-server/pages | grep -iE 'login|signin' +``` + +`next-auth` appears in `web-server/package.json`, which makes it look like there is a login. There +is not: no `[...nextauth]` API route exists and no sign-in page exists. Its only two imports are a +`Session` *type* in `ThirdPartyAuthContext.tsx` and a `signIn` call in `AppErrors.tsx` that has +nothing to call. `/api/auth/session` is a hand-rolled handler returning org details to any caller. +Loading the app goes straight to the dashboard. + +So anything that can reach `:3333` or `:9696` can read every metric and mutate settings, teams and +repos. **The only control is the loopback port binding** in `docker-compose.yml`. + +*Impact if we change anything about deployment topology:* the moment either port is reachable beyond +localhost — a shared host, Kubernetes, a reverse proxy misconfigured to forward `:9696` — the entire +system is open. Any hosting or multi-user work must add an auth layer as a prerequisite, not a +follow-up. + +*Impact for an internal tool specifically:* DORA metrics are data about named engineers' work. With +no auth and no access-control model, anyone who can reach the URL sees every team's numbers. That is +a people question as much as a security one, and worth settling before this is hosted anywhere shared. + +### 11.2 Single container, single failure domain + +Six processes including the database share one container, one restart policy, one resource limit, and +one crash. Combined with `autorestart=false` on Postgres (§4), a database blip yields a container that +passes a liveness check while serving errors. Fine for evaluation, unsuitable for production. + +*Mitigation already available:* the per-process `*_ENABLED` env flags are designed exactly for pulling +components out. Splitting is configuration work, not a rewrite. + +### 11.3 Two write paths to one schema + +See [§3.2](#32-️-the-most-important-architectural-fact-in-this-document). Highest-probability source +of subtle bugs in any change we make. + +### 11.4 No schema/model drift detection + +Hand-maintained SQLAlchemy models against hand-written `dbmate` SQL, with a third hand-maintained +TypeScript copy in the BFF, and nothing checking that the three agree. + +### 11.5 Sync failures are silent + +`POST /sync` returns success regardless of outcome; per-repo failures live only in a 512 KB log with no +history. There is no way to answer "did last night's sync actually work?" without reading logs in time. + +### 11.6 Insecure-by-default configuration + +`DB_PASS=postgres`; `host all all 0.0.0.0/0 md5` appended to `pg_hba.conf`; RSA private key written to +`~/.bashrc`. All acceptable for a laptop; all must be changed before a shared environment. + +### 11.7 Feature gaps versus the upstream README + +The README advertises "integration with various CI/CD tools" and "integration with popular project +management platforms". What is actually implemented: + +| Advertised | Implemented | +|---|---| +| Git providers | GitHub, GitLab (`mhq/exapi/`) — **no Bitbucket, no Azure DevOps** | +| CI/CD | GitHub Actions only (`etl_github_actions_handler.py`) — **no Jenkins, CircleCI, GitLab CI** | +| Incidents | Git-derived + incident-service model — **no PagerDuty/Opsgenie client in `exapi/`** | +| Project management | **None found** | + +Confirm before planning any feature that assumes an integration exists: + +```bash +ls backend/analytics_server/mhq/exapi/ +``` + +### 11.8 Fork maintenance + +Upstream is actively developed and this fork carries no upstream remote and no divergence yet — the +best possible moment to establish a strategy. Covered in [FORK_STRATEGY.md](./FORK_STRATEGY.md). + +--- + +## 12. Extension points + +A map from "we want to change X" to the files involved. Paths are relative to the repo root. + +### 12.1 Add a Git provider + +The strategy pattern makes this the well-trodden path — follow `gitlab` as the reference +implementation, since it was itself added after `github`. + +1. `backend/analytics_server/mhq/store/models/code/enums.py` — add to `CodeProvider` +2. `backend/analytics_server/mhq/exapi/.py` — API client +3. `backend/analytics_server/mhq/exapi/models/.py` — response DTOs +4. `backend/analytics_server/mhq/service/code/sync/etl__handler.py` — implement `CodeProviderETLHandler` +5. `backend/analytics_server/mhq/service/code/sync/etl_code_factory.py` — register in `CodeETLFactory.__call__` +6. `backend/analytics_server/mhq/api/integrations.py` — discovery endpoints +7. `web-server/src/constants/integrations.ts` + `web-server/pages/api/integrations/` — UI wiring +8. `backend/analytics_server/tests/` — tests + +### 12.2 Add a CI/CD provider (for deployments) + +1. `mhq/store/models/code/workflows/enums.py` — add to `RepoWorkflowProviders` +2. `mhq/service/workflows/sync/etl__handler.py` — implement the provider ETL interface +3. `mhq/service/workflows/sync/etl_workflows_factory.py` — register it +4. `mhq/exapi/.py` — client + +### 12.3 Add a deployment source (beyond PR-merge / workflow) + +1. `mhq/service/deployments/models/models.py` — add to `DeploymentType` +2. `mhq/service/deployments/_deployments_service.py` — subclass `DeploymentsFactoryService` +3. `mhq/service/deployments/models/adapter.py` — adaptor to the common `Deployment` +4. `mhq/service/deployments/factory.py` — register in `get_deployments_factory` +5. `mhq/store/models/code/enums.py` — extend `TeamReposDeploymentType` if repo-selectable + +Downstream analytics need no changes — that is the point of the adaptor. + +### 12.4 Change a metric definition + +1. Locate the computation: `mhq/service/code/lead_time.py`, `mhq/service/deployments/analytics.py`, or `mhq/service/incidents/incidents.py` +2. If a **stored** PR segment changes, also change the ETL that writes it (`mhq/service/code/sync/etl_code_analytics.py`) +3. **Plan the backfill.** Historical rows keep old values (§6.1). Either backfill or reset bookmarks via `PUT /orgs//bookmark/reset` +4. Update `tests/service/…` +5. Update the UI's copy and any tooltip that states the definition + +### 12.5 Add a database column + +**Four** places — miss any one and it breaks at runtime, not build time: + +1. `database-docker/db/migrations/_.sql` — hand-written SQL +2. `database-docker/db/schema.sql` — keep the dumped schema current +3. `backend/analytics_server/mhq/store/models/**` — SQLAlchemy model +4. `web-server/src/constants/db.ts` — TypeScript `Table`/`Columns` for the knex path + +### 12.6 Add an API endpoint + +1. Blueprint module in `mhq/api/` (or a new one registered in `app.py`) +2. `voluptuous` validation via `mhq/api/request_utils.py` +3. Service method — keep logic out of the API layer +4. Serializer in `mhq/api/resources/` +5. BFF route in `web-server/pages/api/` with a `yup` schema +6. Redux slice + hook in `web-server/src/slices/` + +### 12.7 Change sync behaviour + +- Order or steps: `mhq/service/sync_data.py` (`sync_sequence` — mind the dependencies in §7.1) +- Frequency: `setup_utils/cronjob.txt` +- Default lookback: `DEFAULT_SYNC_DAYS` in `.env` and `SettingType.DEFAULT_SYNC_DAYS_SETTING` +- MTD batch size: `DEPLOYMENTS_TO_PROCESS` in `mhq/service/merge_to_deploy_broker/mtd_handler.py` + +### 12.8 Split the container for production + +Start from the `*_ENABLED` flags in `docker-compose.yml` / `Dockerfile.dev` and `setup_utils/supervisord.conf`; +point `DB_HOST`/`REDIS_HOST` at managed services. **Add authentication first** (§11.1) — splitting the +container is what makes the unauthenticated API reachable. + +--- + +## 13. Glossary + +| Term | Meaning | +|---|---| +| **Bookmark** | Incremental-sync cursor; ISO-8601 timestamp string per stream | +| **MTD broker** | Merge-to-deploy broker; backfills `PullRequest.merge_to_deploy` on later syncs | +| **`mhq`** | MiddlewareHQ — the backend Python package root | +| **OrgRepo** | A repository known to the system | +| **TeamRepos** | Team↔repo mapping; carries `prod_branches` and `deployment_type` | +| **Prod branch** | Regex in `TeamRepos.prod_branches`; the system's only definition of "production" | +| **Deployment type** | `PR_MERGE` or `WORKFLOW` — selects the deployment strategy per repo | +| **PAT** | Personal access token; RSA-encrypted in `Integration.access_token_enc_chunks` | +| **PR incident** | An incident inferred from a PR matching a configured regex | +| **Lead time segments** | The five stored integer-second spans summing to lead time | +| **BFF** | Backend-for-frontend; the `web-server/pages/api/**` layer | +| **dbmate** | The SQL migration tool (not Alembic, despite SQLAlchemy) | + +--- + +## Appendix A — Verifying this document + +Every structural claim above is reproducible. Re-run these after pulling upstream; a changed count is +a signal the doc needs revisiting. + +```bash +# Flask routes +grep -rn -A1 '@app.route' backend/analytics_server/mhq/api/ + +# Auth on the backend API (expected: no output) +grep -rniE "login_required|authenticat|jwt|api_key|bearer" backend/analytics_server/mhq/api/ + +# Files bypassing Flask to hit Postgres directly (baseline: 23) +grep -rl "from '@/utils/db'" web-server/pages/api web-server/src | wc -l + +# Implemented external providers +ls backend/analytics_server/mhq/exapi/ + +# Migrations (baseline: 4) +ls database-docker/db/migrations | wc -l + +# Python test files (baseline: 20) +find backend/analytics_server/tests -name 'test_*.py' | wc -l + +# Sync order +sed -n '/sync_sequence/,/]/p' backend/analytics_server/mhq/service/sync_data.py +``` + +## Appendix B — Local run quick reference + +Full instructions are in the upstream [README](../README.md); this is the short path. + +```bash +cp env.example .env +docker compose up --build +``` + +| URL | What | +|---|---| +| http://localhost:3333 | Web UI | +| http://localhost:9696 | Analytics API | +| http://localhost:9697 | Sync API | +| http://localhost:3333/system-logs | Logs in the UI | + +Force a sync instead of waiting up to 30 minutes for cron: + +```bash +curl -X POST http://localhost:9697/sync +``` + +Re-ingest history from scratch after changing a metric definition: + +```bash +curl -X PUT http://localhost:9696/orgs//bookmark/reset +``` diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md new file mode 100644 index 000000000..a8c6e566b --- /dev/null +++ b/docs/BENCHMARKS.md @@ -0,0 +1,313 @@ +# Per-team DORA benchmarks — design + +**Status:** implemented +**Date:** 2026-08-11 +**Estimate:** 3–4 days + +Each team can set its own target for the four DORA metrics. A team without a +target inherits a baseline the superadmin sets. Each metric card shows its +target as a line on the trend it already draws, with a caption saying how far +off it is and which benchmark applied. + +--- + +## Decisions + +Settled before design. Changing one of these changes the design, not the +implementation. + +| Decision | Choice | Consequence | +|---|---|---| +| Scope unit | **Team** | Teams are already what the dashboard filters by, so a benchmark per team matches what is on screen. `Settings` already stores per-team rows, so no new pattern. | +| Benchmark shape | **One target value per metric** | Four numbers, not twelve band thresholds. Simple to enter, simple to explain, and the graph becomes obvious: actual against a target line. | +| Relationship to the industry pills | **Additive** | Elite/High/Medium/Low stay exactly as they are, as an external reference. The benchmark is your own goal. A team can be "High" by industry standards and still missing its own target — usually the more useful conversation. | +| Graph | **Trend with a target line** | The Lead Time and Deployment Frequency cards already render a trend behind the number. Adding a line makes an existing decorative graphic informative. | +| Storage | **Reuse `Settings`** | No migration. `data` is JSONB, so the LOC target arrives later for free. | +| Resolution | **Backend** | The API returns the resolved target and its source. The browser draws a line; it never implements inheritance. | + +## Scope + +This spec covers benchmarks and the graphs. **Lines of code is a separate +feature** with its own spec — it is a new metric, not part of this one, and +depends on nothing here. Worth noting for whoever picks it up: the data is +already synced. All 429 pull requests in the current database have +`meta.code_stats` populated, and `PullRequest` already exposes `additions`, +`deletions` and `changed_files` as properties. LOC is aggregation and a card, +not an integration. + +--- + +## Data model + +Four numbers stored as JSONB under a new `SettingType.BENCHMARK_SETTING`: + +```json +{ + "lead_time": 86400, + "deployment_frequency": 5, + "change_failure_rate": 15, + "mean_time_to_recovery": 3600 +} +``` + +Units match what the metrics API already returns — **seconds** for lead time +and MTTR, **deployments per week** for frequency, **percent** for CFR. No +conversion layer: the API compares like with like and the UI formats targets +exactly as it already formats actual values. + +Every field is optional. A team can set a lead-time target and inherit the +rest — **fallback is per metric, not all-or-nothing**. Teams have opinions +about one or two metrics, rarely all four. + +### Where rows live + +| Scope | Row | Who sets it | +|---|---|---| +| Team | `(team_id, TEAM, BENCHMARK_SETTING)` | Workspace admin, own teams only | +| Global baseline | `(sentinel-uuid, GLOBAL, BENCHMARK_SETTING)` | Superadmin only | + +`EntityType.GLOBAL` is new. `Settings.entity_type` is `character varying` and +the schema declares no native enum types, so this is a code change — +**no migration anywhere in this feature**. + +`entity_id` is `NOT NULL`, so the global row uses a fixed sentinel UUID. That +is a documented wart, not a clever trick. + +### Resolution + +Per metric, in the backend: + +``` +team value set? → use it, source: "team" +otherwise → global baseline, source: "global" +neither → no target, source: null +``` + +The metrics response carries the result: + +```json +"benchmarks": { + "lead_time": { "target": 86400, "source": "team" }, + "deployment_frequency": { "target": 5, "source": "global" }, + "change_failure_rate": { "target": null, "source": null } +} +``` + +Three reasons this is server-side. Every consumer would otherwise reimplement +the fallback and eventually disagree. The API can report *which* benchmark +applied, so an admin who thinks they set one can see that they did not. And +when LOC arrives it is a fifth key here, not a fifth reimplementation. + +--- + +## API surface + +**Team benchmark** reuses the existing settings routes. No new Flask code: + +``` +GET /teams//settings?setting_type=BENCHMARK_SETTING +PUT /teams//settings +``` + +Workspace scoping comes free: `Endpoint.serve()` already asserts team access +on any payload carrying a `team_id`. + +**Global baseline** is the one new route, because settings routes today are +team- and org-scoped and this is neither: + +``` +GET /settings/global?setting_type=BENCHMARK_SETTING +PUT /settings/global +``` + +**`PUT` is superadmin only**, enforced with `assertRole(session, 'SUPERADMIN')` +at the BFF. The Flask layer has no notion of who is calling — it trusts the +internal token — so the check cannot live there. + +**`GET` requires only an authenticated session.** Gating the read on superadmin +was the original design and it was wrong: the team configuration form shows the +inherited value as a placeholder, and a workspace admin loading that form would +get a 403 and see a generic "not set" instead. The baseline is four numbers that +already appear on that admin's own dashboard as target lines, resolved +server-side — hiding them from the form that explains where a target came from +protects nothing and makes the inheritance invisible. Writing stays locked, +because one superadmin's numbers become every unset team's targets. + +**Resolved benchmarks ride on the existing metrics response** rather than +getting their own endpoint. The dashboard already makes one call for all four +metrics; a second call for targets would let the numbers and the lines drawn +against them disagree, and would add a round trip to a page that is already +the slowest in the product. + +In Flask this is a second route: + +``` +GET /teams//benchmarks +``` + +but the BFF folds it into the existing `Promise.all` behind +`/api/internal/team//dora_metrics`, so the browser still makes one +call and the targets still arrive with the numbers they are drawn against. +The route that serves them has to be on this path and no other: the cards read +`metrics_summary.benchmarks`, and that redux slice is written **only** by the +`dora_metrics` response. Benchmarks attached to any other endpoint land in a +slice no card reads and render nothing. + +### Already exists — not being built + +`SettingsService.get_settings`, `save_settings`, `get_settings_map`; `GET`/`PUT` +on `/teams//settings`; and `get_default_setting_data`. + +**There is no shipped global baseline.** `get_default_setting_data` returns all +four benchmark keys as `None`, and no `GET` at any scope creates a row. This is +what makes the zero-config state below reachable: a default with numbers in it +gave every card in every workspace a target line and a "the default benchmark" +caption the moment any admin opened any settings form, and gave a team that had +never set a benchmark `source: "team"` on all four metrics. A superadmin types +the baseline in; the code does not guess it. + +--- + +## The graph + +Each card gains a dashed target line on its existing trend, plus a caption: + +> **6h 49m under target** · your team's benchmark + +Three things in a few words: the gap, the direction, and which benchmark +applied. `your team's benchmark` versus `the default benchmark` is how an +admin discovers their setting did not save. + +**Direction is per metric.** Lower is better for lead time, CFR and MTTR; +higher is better for deployment frequency. So "under target" is good on three +cards and bad on the fourth. The caption states the fact — `above target` / +`below target` — and colour carries the judgement: green on the good side, +amber on the other. **Never red.** A missed internal goal is not an error, and +colouring it like one makes the dashboard punitive. + +**No benchmark at any level** means no line, no caption, and the card renders +exactly as it does today. That is the state every card is in before anyone +configures anything, so it must look deliberate rather than broken. + +### Configuration screen + +One form per scope: four labelled number inputs with units shown, each with a +placeholder showing the inherited value — an admin sees `24h (default)` greyed +into the lead-time field and understands what clearing it does. + +Team benchmarks live in the existing team settings area. The global baseline +gets a section on the superadmin's Workspaces page, already the superadmin-only +surface. + +Clearing a field removes that key and falls back. That is the only way to +*un*-set a target, so "empty means inherit" is stated in the UI, not just +implied by the code. + +--- + +## Validation + +Server-side on `PUT`. The browser is not a trust boundary and these numbers +feed a shared dashboard. + +Server-side on **both** the team and global `PUT`. The team route is the one +every workspace admin's form actually writes to, so validating only the global +route validates the route nobody uses. + +| Rule | Why | +|---|---| +| Non-negative numbers only | A negative target is unachievable | +| CFR is 0–100 | It is a percentage | +| Unknown keys rejected | A typo like `leadtime` would store silently and inherit forever | +| Absent is not zero | Omitting a key means inherit; `0` is deliberate and must be preserved | + +The last rule is the trap. A form sending `0` for empty fields would turn +"inherit" into "target zero" on every save, and the dashboard would show every +team failing everything. The UI omits empty fields rather than sending zero, +and the API distinguishes the two. + +`0` is therefore **accepted**, not rejected — an earlier draft of the first +rule said "positive numbers only… a zero deployment target is meaningless", +which contradicted the last rule. A team targeting zero failed deployments has +said something meaningful and must not be silently switched back to inheriting. + +**A benchmark `PUT` is a full-state write.** Validation normalises the payload +to all four keys, `None` for the ones the form left empty, because the form +omits empty fields and clearing every field would otherwise post `{}` — +which `save_settings` treats as "no data supplied" and replaces with the +defaults. Without the normalisation, "clear everything to go back to +inheriting" saves targets. + +## Edge cases + +**A team spanning very different repos** gets one benchmark, because the +dashboard aggregates to team level. A real limitation, stated rather than +solved — per-repo targets would need a rule for combining them into the single +number on the card. + +**A deleted team's benchmark** is orphaned in `Settings`. Harmless and +invisible; cleaning it up means touching team deletion for no user-facing gain. + +**A superadmin viewing another workspace** sees that workspace's team +benchmarks. Resolution keys on the team being viewed, not on who is looking. + +**Saving a team benchmark refetches the dashboard**, so the new target line +appears immediately. This was originally specified as "does not appear until +reload — the same as every other filter on that page", which was wrong on both +counts: the form is *on* that page, so the admin saves and watches nothing +happen, and the two sibling modals in the same menu already refetch on save. + +**A benchmark changed elsewhere** — another admin, another tab, or the global +baseline — still does not appear until reload. + +**Zero deployments against a frequency target** still shows the line and +`5/week below target`, which is more useful than hiding the line because the +actual value happens to be zero. + +--- + +## Testing + +Resolution is where the bugs will be, so that is where the tests concentrate: + +- Team value set → used, `source: "team"` +- Team value absent → global used, `source: "global"` +- Neither set → `target: null`, card renders without a line +- **Per-metric mixing** — team sets only lead time; the other three inherit and + report `source: "global"` in the same response +- **`0` is preserved, not treated as absent** + +**Validation:** negatives rejected, CFR above 100 rejected, unknown keys +rejected, booleans rejected, `0` preserved, and an empty payload normalised to +four explicit `None`s rather than falling through to the defaults. + +**The seam, not just the function.** Resolution and validation are pure +functions with thorough unit tests, and every defect this feature shipped was +still in the seam between one of them and its caller — targets attached to the +wrong response, a `GET` that wrote a row, a cleared form that saved defaults. +So the benchmark routes are tested through Flask's test client, asserting the +body a caller actually receives. + +**Isolation, request-level e2e:** an admin cannot read or write another +workspace's team benchmark → 403. A non-superadmin cannot `PUT` the global +baseline → 403. + +**Regression, and the most important test here:** with no benchmarks +configured anywhere, every existing metric response is byte-identical to +today. This feature is additive, and the four cards people already rely on +must not shift because a `benchmarks` key appeared. + +**Frontend:** direction handled per metric, so "under target" reads as good on +lead time and bad on deployment frequency; the caption names the source; no +line at all when `target` is null. + +--- + +## Deliberately excluded + +- **No per-workspace tier.** Two levels were specified; a third needs its own + precedence rules for no stated benefit. +- **No custom band thresholds.** The industry pills stay as they are. +- **No history of benchmark changes.** Who changed a target and when is a + reasonable thing to want, and not this feature. +- **No LOC.** Separate spec, separate PR. diff --git a/docs/BENCHMARKS_PLAN.md b/docs/BENCHMARKS_PLAN.md new file mode 100644 index 000000000..79657f59e --- /dev/null +++ b/docs/BENCHMARKS_PLAN.md @@ -0,0 +1,849 @@ +# Per-team DORA Benchmarks Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Each team can set its own target for the four DORA metrics, inheriting a superadmin baseline when unset, with the target drawn on each metric card. + +**Architecture:** Benchmarks are a new `SettingType` stored in the existing `Settings` table — team rows keyed `(team_id, TEAM)`, the global baseline keyed `(sentinel-uuid, GLOBAL)`. The backend resolves team-then-global **per metric** and returns the resolved target plus its source on the existing metrics response. The frontend draws a line and a caption; it never implements inheritance. + +**Tech Stack:** Python 3.9, Flask, SQLAlchemy 2, pytest (backend); Next.js 15 pages router, TypeScript, MUI 5 (frontend). + +**Design spec:** `docs/BENCHMARKS.md`. Read it before starting. + +## Global Constraints + +- **No database migration, no new tables.** `Settings.entity_type` is `character varying` and the schema declares no native enum types, so `EntityType.GLOBAL` is a code change. `Settings.data` is JSONB. +- **Units match the metrics API exactly** — seconds for `lead_time` and `mean_time_to_recovery`, deployments per week for `deployment_frequency`, percent for `change_failure_rate`. No conversion layer. +- **Absent is not zero.** Omitting a key means inherit; `0` is a deliberate value and must be preserved. A form sending `0` for empty fields would turn inherit into target-zero on every save and show every team failing everything. +- **Direction is per metric.** Lower is better for lead time, CFR and MTTR; higher is better for deployment frequency. +- **Additive only.** With no benchmarks configured anywhere, every existing metric response must be byte-identical to today and the four cards must not shift. +- Backend style: `black` formatted, `flake8` clean. Run from `backend/analytics_server` using `./venv/bin/python`. Baseline: **269 passing**. +- Frontend must be `tsc` clean. Local Node is 20 and the project needs 22 — verify inside the `middleware-dev` container. +- Every Clustox-authored change carries a `# CLUSTOX:` or `// CLUSTOX:` comment explaining **why**, per `docs/FORK_STRATEGY.md`. + +## The eleven extension points — read before Task 1 + +Adding a `SettingType` to this codebase means touching eleven places. Upstream +marks each with a comment. **Missing one is a silent failure, not a crash** — +the setting saves but reads back wrong, or validates but never persists. + +| # | File | Marker | +|---|---|---| +| 1 | `mhq/store/models/settings/configuration_settings.py:22` | `ADD NEW SETTING TYPE ENUM HERE` | +| 2 | `mhq/service/settings/models.py:60` | `ADD NEW SETTING CLASS HERE` | +| 3 | `mhq/service/settings/default_settings_data.py:38` | `ADD NEW DEFAULT SETTING HERE` | +| 4 | `mhq/service/settings/configuration_settings.py:76` | `ADD NEW DICT TO DATACLASS ADAPTERS HERE` | +| 5 | `mhq/service/settings/configuration_settings.py:101` | `ADD NEW HANDLE FROM DB SETTINGS HERE` | +| 6 | `mhq/service/settings/configuration_settings.py:201` | `ADD NEW DICT TO API ADAPTERS HERE` | +| 7 | `mhq/service/settings/configuration_settings.py:226` | `ADD NEW HANDLE FROM JSON DATA HERE` | +| 8 | `mhq/service/settings/configuration_settings.py:272` | `ADD NEW DATACLASS TO JSON DATA ADAPTERS HERE` | +| 9 | `mhq/service/settings/configuration_settings.py:308` | `ADD NEW HANDLE TO DB SETTINGS HERE` | +| 10 | `mhq/service/settings/setting_type_validator.py:25` | `ADD NEW VALIDATOR HERE` | +| 11 | `mhq/api/resources/settings_resource.py` | `ADD NEW API ADAPTER HERE` | + +Line numbers drift — search for the marker text, not the number. + +**Point 11 was missed when this plan was first written**, because the search +that found the others covered only `mhq/service/settings/` and +`mhq/store/models/settings/`. Without it, `adapt_configuration_settings_response` +never populates `response["setting"]` for a benchmark, so `GET` returns only +timestamps — invisible until something tries to read a saved value. Search the +whole of `mhq/` for the marker text, not a subtree: + +``` +grep -rn "ADD NEW" mhq/ +``` + +--- + +## File Structure + +**Backend — modify:** +- `mhq/store/models/settings/configuration_settings.py` — `SettingType.BENCHMARK_SETTING` +- `mhq/store/models/settings/enums.py` — `EntityType.GLOBAL` +- `mhq/service/settings/models.py` — `BenchmarkSetting` dataclass +- `mhq/service/settings/default_settings_data.py` — shipped baseline +- `mhq/service/settings/configuration_settings.py` — six adapter/dispatch points +- `mhq/service/settings/setting_type_validator.py` — accept the new type +- `mhq/api/settings.py` — global baseline routes +- `mhq/api/deployment_analytics.py` — attach resolved benchmarks (see Task 3 note) + +**Backend — create:** +- `mhq/service/settings/benchmarks.py` — resolution and validation +- `tests/service/settings/test_benchmarks.py` +- `tests/service/settings/test_benchmark_setting_type.py` + +**Frontend — create:** +- `web-server/pages/api/clustox/benchmarks/global.ts` +- `web-server/src/components/BenchmarkTargetLine.tsx` +- `web-server/src/components/BenchmarkSettingsForm.tsx` +- `web-server/src/utils/benchmarks.ts` + +**Frontend — modify:** +- the four cards under `web-server/src/content/DoraMetrics/DoraCards/` +- `web-server/pages/workspaces.tsx` — global baseline section + +--- + +### Task 1: Register BENCHMARK_SETTING across the extension points + +**Files:** +- Modify: `backend/analytics_server/mhq/store/models/settings/configuration_settings.py` +- Modify: `backend/analytics_server/mhq/store/models/settings/enums.py` +- Modify: `backend/analytics_server/mhq/service/settings/models.py` +- Modify: `backend/analytics_server/mhq/service/settings/default_settings_data.py` +- Modify: `backend/analytics_server/mhq/service/settings/configuration_settings.py` +- Modify: `backend/analytics_server/mhq/service/settings/setting_type_validator.py` +- Test: `backend/analytics_server/tests/service/settings/test_benchmark_setting_type.py` (create) + +**Interfaces:** +- Consumes: nothing. +- Produces: + - `SettingType.BENCHMARK_SETTING` (value `"BENCHMARK_SETTING"`) + - `EntityType.GLOBAL` (value `"GLOBAL"`) + - `BenchmarkSetting(lead_time, deployment_frequency, change_failure_rate, mean_time_to_recovery)` — every field `Optional[float]`, defaulting to `None` + - `settings_type_validator("BENCHMARK_SETTING")` returns the enum member + +- [ ] **Step 1: Write the failing tests** + +Create `backend/analytics_server/tests/service/settings/test_benchmark_setting_type.py`: + +```python +from mhq.service.settings.default_settings_data import get_default_setting_data +from mhq.service.settings.models import BenchmarkSetting +from mhq.service.settings.setting_type_validator import settings_type_validator +from mhq.store.models.settings import SettingType +from mhq.store.models.settings.enums import EntityType + + +def test_benchmark_setting_type_exists(): + assert SettingType.BENCHMARK_SETTING.value == "BENCHMARK_SETTING" + + +def test_global_entity_type_exists(): + # The superadmin baseline belongs to no team and no workspace. + assert EntityType.GLOBAL.value == "GLOBAL" + + +def test_validator_accepts_the_new_type(): + assert settings_type_validator("BENCHMARK_SETTING") == SettingType.BENCHMARK_SETTING + + +def test_default_baseline_has_all_four_metrics(): + data = get_default_setting_data(SettingType.BENCHMARK_SETTING) + + assert set(data.keys()) == { + "lead_time", + "deployment_frequency", + "change_failure_rate", + "mean_time_to_recovery", + } + + +def test_benchmark_setting_fields_default_to_none(): + # Every field is optional: a team may set one target and inherit the rest. + setting = BenchmarkSetting() + + assert setting.lead_time is None + assert setting.deployment_frequency is None + assert setting.change_failure_rate is None + assert setting.mean_time_to_recovery is None +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd backend/analytics_server && ./venv/bin/python -m pytest tests/service/settings/test_benchmark_setting_type.py -v` +Expected: FAIL with `AttributeError: BENCHMARK_SETTING` + +- [ ] **Step 3: Add the enums** + +In `mhq/store/models/settings/configuration_settings.py`, at the `ADD NEW SETTING TYPE ENUM HERE` marker: + +```python + # CLUSTOX: per-team DORA targets. Stored here rather than in a new table + # because Settings already has the entity scoping, service and API this + # needs, and its JSONB data column means adding a fifth metric later costs + # no migration. + BENCHMARK_SETTING = "BENCHMARK_SETTING" +``` + +In `mhq/store/models/settings/enums.py`: + +```python +class EntityType(Enum): + USER = "USER" + TEAM = "TEAM" + ORG = "ORG" + # CLUSTOX: the superadmin's baseline benchmark belongs to no team and no + # workspace. entity_type is character varying with no native enum, so this + # needs no migration. + GLOBAL = "GLOBAL" +``` + +- [ ] **Step 4: Add the dataclass** + +In `mhq/service/settings/models.py`, at the `ADD NEW SETTING CLASS HERE` marker: + +```python +# CLUSTOX: every field is optional so fallback is per metric, not +# all-or-nothing -- a team may set a lead-time target and inherit the rest. +@dataclass +class BenchmarkSetting(BaseSetting): + lead_time: Optional[float] = None + deployment_frequency: Optional[float] = None + change_failure_rate: Optional[float] = None + mean_time_to_recovery: Optional[float] = None +``` + +Check that file's existing imports for `dataclass` and `Optional` before adding them. + +- [ ] **Step 5: Add the shipped baseline** + +In `mhq/service/settings/default_settings_data.py`, at the `ADD NEW DEFAULT SETTING HERE` marker: + +```python + # CLUSTOX: the baseline a workspace sees before a superadmin edits it. + # Values are the DORA "High" band, in the same units the metrics API + # returns: seconds, deployments per week, percent. + if setting_type == SettingType.BENCHMARK_SETTING: + return { + "lead_time": 86400, + "deployment_frequency": 5, + "change_failure_rate": 15, + "mean_time_to_recovery": 3600, + } +``` + +- [ ] **Step 6: Add the six adapters and dispatch branches** + +In `mhq/service/settings/configuration_settings.py`, add an adapter at the `ADD NEW DICT TO DATACLASS ADAPTERS HERE` marker: + +```python + # CLUSTOX: `data.get(key)` rather than `data.get(key, 0)` -- absent means + # inherit, and 0 is a deliberate target. Collapsing them would make every + # unset metric read as a target of zero. + def _adapt_benchmark_setting_from_setting_data(self, data: Dict[str, any]): + return BenchmarkSetting( + lead_time=data.get("lead_time"), + deployment_frequency=data.get("deployment_frequency"), + change_failure_rate=data.get("change_failure_rate"), + mean_time_to_recovery=data.get("mean_time_to_recovery"), + ) +``` + +At `ADD NEW HANDLE FROM DB SETTINGS HERE`: + +```python + if setting_type == SettingType.BENCHMARK_SETTING: + return self._adapt_benchmark_setting_from_setting_data(setting_data) +``` + +At `ADD NEW DICT TO API ADAPTERS HERE`, and at `ADD NEW DATACLASS TO JSON DATA ADAPTERS HERE`, add adapters mirroring the two `default_sync_days` adapters immediately above each marker — read those first and follow their shape exactly, substituting the four benchmark keys. At `ADD NEW HANDLE FROM JSON DATA HERE` and `ADD NEW HANDLE TO DB SETTINGS HERE`, add the matching `if setting_type == SettingType.BENCHMARK_SETTING:` branches. + +All four keys must round-trip through every adapter. A key missing from one of them saves and then reads back as `None`. + +- [ ] **Step 7: Add the validator branch** + +In `mhq/service/settings/setting_type_validator.py`, at the `ADD NEW VALIDATOR HERE` marker: + +```python + if setting_type == SettingType.BENCHMARK_SETTING.value: + return SettingType.BENCHMARK_SETTING +``` + +- [ ] **Step 8: Run tests to verify they pass** + +Run: `cd backend/analytics_server && ./venv/bin/python -m pytest tests/service/settings/test_benchmark_setting_type.py -v` +Expected: PASS (5 tests) + +- [ ] **Step 9: Run the full suite, format, lint, commit** + +```bash +cd backend/analytics_server +./venv/bin/python -m pytest tests -q # expect 274 passed +./venv/bin/python -m black mhq/store/models/settings/ mhq/service/settings/ tests/service/settings/ +./venv/bin/python -m flake8 mhq/store/models/settings/ mhq/service/settings/ +cd ../.. +git add backend/analytics_server/mhq backend/analytics_server/tests +git commit -m "feat(benchmarks): register the benchmark setting type" +``` + +--- + +### Task 2: Resolution and validation + +**Files:** +- Create: `backend/analytics_server/mhq/service/settings/benchmarks.py` +- Test: `backend/analytics_server/tests/service/settings/test_benchmarks.py` (create) + +**Interfaces:** +- Consumes: `BenchmarkSetting`, `SettingType.BENCHMARK_SETTING`, `EntityType.GLOBAL` (Task 1). +- Produces: + - `GLOBAL_BENCHMARK_ENTITY_ID: str` — the sentinel UUID for the global row + - `BENCHMARK_METRICS: List[str]` — the four metric keys + - `resolve_benchmarks(team_setting: Optional[BenchmarkSetting], global_setting: Optional[BenchmarkSetting]) -> Dict[str, Dict]` returning `{metric: {"target": float|None, "source": "team"|"global"|None}}` + - `validate_benchmark_payload(data: Dict) -> Dict` — returns the cleaned dict, raises `BadRequest` on invalid input + +- [ ] **Step 1: Write the failing tests** + +Create `backend/analytics_server/tests/service/settings/test_benchmarks.py`: + +```python +import pytest +from werkzeug.exceptions import BadRequest + +from mhq.service.settings.benchmarks import ( + BENCHMARK_METRICS, + resolve_benchmarks, + validate_benchmark_payload, +) +from mhq.service.settings.models import BenchmarkSetting + + +def test_team_value_wins_and_reports_its_source(): + team = BenchmarkSetting(lead_time=3600) + glob = BenchmarkSetting(lead_time=86400) + + resolved = resolve_benchmarks(team, glob) + + assert resolved["lead_time"] == {"target": 3600, "source": "team"} + + +def test_absent_team_value_falls_back_to_global(): + team = BenchmarkSetting(lead_time=3600) + glob = BenchmarkSetting(lead_time=86400, deployment_frequency=5) + + resolved = resolve_benchmarks(team, glob) + + assert resolved["deployment_frequency"] == {"target": 5, "source": "global"} + + +def test_neither_set_yields_no_target(): + resolved = resolve_benchmarks(BenchmarkSetting(), BenchmarkSetting()) + + assert resolved["change_failure_rate"] == {"target": None, "source": None} + + +def test_metrics_mix_sources_within_one_response(): + # The case that makes per-metric fallback worth having. + team = BenchmarkSetting(lead_time=3600) + glob = BenchmarkSetting(lead_time=86400, deployment_frequency=5) + + resolved = resolve_benchmarks(team, glob) + + assert resolved["lead_time"]["source"] == "team" + assert resolved["deployment_frequency"]["source"] == "global" + assert resolved["mean_time_to_recovery"]["source"] is None + + +def test_zero_is_a_real_target_not_an_absent_one(): + # A team targeting zero failures must not silently inherit 15%. + team = BenchmarkSetting(change_failure_rate=0) + glob = BenchmarkSetting(change_failure_rate=15) + + resolved = resolve_benchmarks(team, glob) + + assert resolved["change_failure_rate"] == {"target": 0, "source": "team"} + + +def test_missing_settings_are_treated_as_empty(): + resolved = resolve_benchmarks(None, None) + + assert set(resolved.keys()) == set(BENCHMARK_METRICS) + assert all(v["target"] is None for v in resolved.values()) + + +def test_validation_rejects_negatives(): + with pytest.raises(BadRequest): + validate_benchmark_payload({"lead_time": -1}) + + +def test_validation_rejects_change_failure_rate_above_100(): + with pytest.raises(BadRequest): + validate_benchmark_payload({"change_failure_rate": 101}) + + +def test_validation_rejects_unknown_keys(): + # A typo would otherwise store silently and inherit forever. + with pytest.raises(BadRequest): + validate_benchmark_payload({"leadtime": 3600}) + + +def test_validation_keeps_zero_and_omits_nothing_else(): + cleaned = validate_benchmark_payload({"change_failure_rate": 0}) + + assert cleaned == {"change_failure_rate": 0} + + +def test_validation_accepts_a_partial_payload(): + cleaned = validate_benchmark_payload({"lead_time": 3600}) + + assert cleaned == {"lead_time": 3600} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd backend/analytics_server && ./venv/bin/python -m pytest tests/service/settings/test_benchmarks.py -v` +Expected: FAIL with `ModuleNotFoundError: No module named 'mhq.service.settings.benchmarks'` + +- [ ] **Step 3: Write the module** + +Create `backend/analytics_server/mhq/service/settings/benchmarks.py`: + +```python +# CLUSTOX: benchmark resolution and validation. +# +# Resolution lives in the backend rather than the browser for three reasons: +# every consumer would otherwise reimplement the fallback and eventually +# disagree; the response can report WHICH benchmark applied, so an admin who +# thinks they set a target can see that they did not; and a fifth metric later +# is one more key here rather than one more reimplementation. +from typing import Dict, List, Optional + +from werkzeug.exceptions import BadRequest + +from mhq.service.settings.models import BenchmarkSetting + +# The global baseline belongs to no team and no workspace, but Settings. +# entity_id is NOT NULL. A fixed sentinel is the documented cost of reusing +# that table instead of adding one. +GLOBAL_BENCHMARK_ENTITY_ID = "00000000-0000-4000-8000-000000000001" + +BENCHMARK_METRICS: List[str] = [ + "lead_time", + "deployment_frequency", + "change_failure_rate", + "mean_time_to_recovery", +] + +# Percent, so it has an upper bound the others do not. +_BOUNDED_METRICS = {"change_failure_rate": 100} + + +def resolve_benchmarks( + team_setting: Optional[BenchmarkSetting], + global_setting: Optional[BenchmarkSetting], +) -> Dict[str, Dict]: + resolved = {} + + for metric in BENCHMARK_METRICS: + team_value = getattr(team_setting, metric, None) if team_setting else None + global_value = getattr(global_setting, metric, None) if global_setting else None + + # `is not None` rather than truthiness: 0 is a deliberate target. + if team_value is not None: + resolved[metric] = {"target": team_value, "source": "team"} + elif global_value is not None: + resolved[metric] = {"target": global_value, "source": "global"} + else: + resolved[metric] = {"target": None, "source": None} + + return resolved + + +def validate_benchmark_payload(data: Dict) -> Dict: + unknown = set(data.keys()) - set(BENCHMARK_METRICS) + if unknown: + raise BadRequest(f"Unknown benchmark metrics: {', '.join(sorted(unknown))}") + + cleaned = {} + for metric, value in data.items(): + if value is None: + continue + + if not isinstance(value, (int, float)) or isinstance(value, bool): + raise BadRequest(f"{metric} must be a number") + + if value < 0: + raise BadRequest(f"{metric} must not be negative") + + upper = _BOUNDED_METRICS.get(metric) + if upper is not None and value > upper: + raise BadRequest(f"{metric} must be between 0 and {upper}") + + cleaned[metric] = value + + return cleaned +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd backend/analytics_server && ./venv/bin/python -m pytest tests/service/settings/test_benchmarks.py -v` +Expected: PASS (11 tests) + +- [ ] **Step 5: Run the full suite, format, lint, commit** + +```bash +cd backend/analytics_server +./venv/bin/python -m pytest tests -q # expect 285 passed +./venv/bin/python -m black mhq/service/settings/benchmarks.py tests/service/settings/test_benchmarks.py +./venv/bin/python -m flake8 mhq/service/settings/benchmarks.py +cd ../.. +git add backend/analytics_server/mhq/service/settings/benchmarks.py \ + backend/analytics_server/tests/service/settings/test_benchmarks.py +git commit -m "feat(benchmarks): resolve team targets against the global baseline" +``` + +--- + +### Task 3: Attach resolved benchmarks to the metrics response + +**Files:** +- Modify: `backend/analytics_server/mhq/api/deployment_analytics.py` +- Test: extend `backend/analytics_server/tests/service/settings/test_benchmarks.py` + +**Interfaces:** +- Consumes: `resolve_benchmarks`, `GLOBAL_BENCHMARK_ENTITY_ID` (Task 2); `SettingsService.get_settings` (existing). +- Produces: `get_resolved_benchmarks_for_team(team_id: str) -> Dict[str, Dict]`, and a `"benchmarks"` key on the team DORA metrics response. + +- [ ] **Step 1: Write the failing test** + +Append to `backend/analytics_server/tests/service/settings/test_benchmarks.py`: + +```python +def test_resolution_asks_for_the_team_row_and_the_global_row(): + from mhq.service.settings.benchmarks import GLOBAL_BENCHMARK_ENTITY_ID + from mhq.store.models.settings import SettingType + from mhq.store.models.settings.enums import EntityType + + asked = [] + + class FakeSettingsService: + def get_settings(self, setting_type, entity_type, entity_id): + asked.append((setting_type, entity_type, entity_id)) + return None + + from mhq.api.deployment_analytics import get_resolved_benchmarks_for_team + + get_resolved_benchmarks_for_team("team-1", settings_service=FakeSettingsService()) + + assert ( + SettingType.BENCHMARK_SETTING, + EntityType.TEAM, + "team-1", + ) in asked + assert ( + SettingType.BENCHMARK_SETTING, + EntityType.GLOBAL, + GLOBAL_BENCHMARK_ENTITY_ID, + ) in asked +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd backend/analytics_server && ./venv/bin/python -m pytest tests/service/settings/test_benchmarks.py::test_resolution_asks_for_the_team_row_and_the_global_row -v` +Expected: FAIL with `ImportError: cannot import name 'get_resolved_benchmarks_for_team'` + +- [ ] **Step 3: Add the helper** + +Add to `backend/analytics_server/mhq/api/deployment_analytics.py`: + +```python +# CLUSTOX: resolve a team's benchmarks for the metrics response. Injectable +# settings_service so the resolution order can be tested without a database. +def get_resolved_benchmarks_for_team(team_id: str, settings_service=None): + from mhq.service.settings.benchmarks import ( + GLOBAL_BENCHMARK_ENTITY_ID, + resolve_benchmarks, + ) + from mhq.service.settings.configuration_settings import get_settings_service + from mhq.store.models.settings import SettingType + from mhq.store.models.settings.enums import EntityType + + settings_service = settings_service or get_settings_service() + + team_setting = settings_service.get_settings( + SettingType.BENCHMARK_SETTING, EntityType.TEAM, team_id + ) + global_setting = settings_service.get_settings( + SettingType.BENCHMARK_SETTING, EntityType.GLOBAL, GLOBAL_BENCHMARK_ENTITY_ID + ) + + return resolve_benchmarks( + getattr(team_setting, "specific_settings", None), + getattr(global_setting, "specific_settings", None), + ) +``` + +- [ ] **Step 4: Attach it to the response** + +Find the route in `mhq/api/deployment_analytics.py` that serves the team's DORA metrics (`/teams//deployment_analytics`). Add `"benchmarks": get_resolved_benchmarks_for_team(team_id)` to the dict it returns. + +Read the existing return statement before editing. Do not reorder or rename any existing key — the regression guard in Step 5 exists because the four cards already rely on this response. + +- [ ] **Step 5: Run the full suite** + +Run: `cd backend/analytics_server && ./venv/bin/python -m pytest tests -q` +Expected: 286 passed, with no pre-existing test failing. If an existing test breaks, you changed the response shape rather than adding to it. + +- [ ] **Step 6: Format, lint, commit** + +```bash +cd backend/analytics_server +./venv/bin/python -m black mhq/api/deployment_analytics.py tests/service/settings/test_benchmarks.py +./venv/bin/python -m flake8 mhq/api/deployment_analytics.py +cd ../.. +git add backend/analytics_server/mhq/api/deployment_analytics.py \ + backend/analytics_server/tests/service/settings/test_benchmarks.py +git commit -m "feat(benchmarks): return resolved targets with team metrics" +``` + +--- + +### Task 4: Global baseline routes + +**Files:** +- Modify: `backend/analytics_server/mhq/api/settings.py` +- Create: `web-server/pages/api/clustox/benchmarks/global.ts` + +**Interfaces:** +- Consumes: `GLOBAL_BENCHMARK_ENTITY_ID`, `validate_benchmark_payload` (Task 2). +- Produces: + - Flask: `GET /settings/global?setting_type=BENCHMARK_SETTING`, `PUT /settings/global` + - BFF: `GET`/`PUT` on `/api/clustox/benchmarks/global`, superadmin only + +- [ ] **Step 1: Add the Flask routes** + +In `backend/analytics_server/mhq/api/settings.py`, read `get_team_settings` and `put_team_settings` first and mirror their decorator stack, schema style and response adapter exactly. Add: + +```python +# CLUSTOX: the superadmin's baseline belongs to no team and no workspace, so +# it cannot use the team- or org-scoped routes above. Authorisation is enforced +# at the BFF -- this layer only knows the internal token, not who is calling. +@app.route("/settings/global", methods={"GET"}) +``` + +and the matching `PUT`, both operating on `entity_type=EntityType.GLOBAL` and `entity_id=GLOBAL_BENCHMARK_ENTITY_ID`. The `PUT` must pass its payload through `validate_benchmark_payload` before saving. + +- [ ] **Step 2: Add the BFF route** + +Create `web-server/pages/api/clustox/benchmarks/global.ts`: + +```ts +import * as yup from 'yup'; + +import { Endpoint, nullSchema } from '@/api-helpers/global'; +import { internal } from '@/api-helpers/axios'; +import { assertRole } from '@/auth/guard'; + +// CLUSTOX: the global baseline is superadmin-only. The Flask layer trusts the +// internal token and has no notion of who is calling, so the role check has to +// live here. +const putSchema = yup.object().shape({ + lead_time: yup.number().min(0).nullable().optional(), + deployment_frequency: yup.number().min(0).nullable().optional(), + change_failure_rate: yup.number().min(0).max(100).nullable().optional(), + mean_time_to_recovery: yup.number().min(0).nullable().optional() +}); + +const endpoint = new Endpoint(nullSchema); + +endpoint.handle.GET(nullSchema, async (req, res) => { + assertRole((req as any).session, 'SUPERADMIN'); + + const result = await internal.get('/settings/global', { + params: { setting_type: 'BENCHMARK_SETTING' } + }); + res.send(result.data); +}); + +endpoint.handle.PUT(putSchema, async (req, res) => { + assertRole((req as any).session, 'SUPERADMIN'); + + const result = await internal.put('/settings/global', { + setting_type: 'BENCHMARK_SETTING', + setting_data: req.payload + }); + res.send(result.data); +}); + +export default endpoint.serve(); +``` + +Note the constructor receives `nullSchema`, not `putSchema`. `Endpoint.serve()` validates the constructor's schema against every method before dispatch, so a method-specific schema there makes `GET` a permanent 400. That exact bug has been found in this codebase twice. + +- [ ] **Step 3: Verify** + +```bash +cd backend/analytics_server && ./venv/bin/python -m pytest tests -q +docker cp web-server/pages/api/clustox/benchmarks/global.ts middleware-dev:/app/web-server/pages/api/clustox/benchmarks/global.ts +docker exec middleware-dev sh -c 'cd /app/web-server && ./node_modules/.bin/tsc --noEmit 2>&1 | grep -c "error TS"' +``` +Expected: suite unchanged at 286; tsc reports `0`. + +- [ ] **Step 4: Commit** + +```bash +git add backend/analytics_server/mhq/api/settings.py \ + web-server/pages/api/clustox/benchmarks/global.ts +git commit -m "feat(benchmarks): expose the superadmin baseline" +``` + +--- + +### Task 5: Target line and caption on the metric cards + +**Files:** +- Create: `web-server/src/utils/benchmarks.ts` +- Create: `web-server/src/components/BenchmarkTargetLine.tsx` +- Modify: the four cards in `web-server/src/content/DoraMetrics/DoraCards/` + +**Interfaces:** +- Consumes: the `benchmarks` key on the metrics response (Task 3). +- Produces: + - `benchmarkCaption(metric, actual, target, source) -> { text: string; tone: 'good' | 'warn' }` + - `` + +- [ ] **Step 1: Write the failing test** + +Create `web-server/src/utils/__tests__/benchmarks.test.ts`: + +```ts +import { benchmarkCaption } from '../benchmarks'; + +describe('benchmarkCaption', () => { + it('treats being under target as good for lead time', () => { + const r = benchmarkCaption('lead_time', 3600, 7200, 'team'); + expect(r.tone).toBe('good'); + expect(r.text).toContain('under target'); + }); + + it('treats being under target as bad for deployment frequency', () => { + // Higher is better here -- the same direction means the opposite thing. + const r = benchmarkCaption('deployment_frequency', 2, 5, 'team'); + expect(r.tone).toBe('warn'); + expect(r.text).toContain('below target'); + }); + + it('names the source so a team can see its setting did not save', () => { + expect(benchmarkCaption('lead_time', 3600, 7200, 'global').text).toContain( + 'default' + ); + expect(benchmarkCaption('lead_time', 3600, 7200, 'team').text).toContain( + 'team' + ); + }); + + it('returns null when there is no target', () => { + expect(benchmarkCaption('lead_time', 3600, null, null)).toBeNull(); + }); +}); +``` + +- [ ] **Step 2: Run it to verify it fails** + +Run: `docker exec middleware-dev sh -c 'cd /app/web-server && yarn test --testPathPattern=benchmarks'` +Expected: FAIL — module not found + +- [ ] **Step 3: Write the helper** + +Create `web-server/src/utils/benchmarks.ts` exporting `benchmarkCaption`. Encode direction per metric — `lead_time`, `change_failure_rate` and `mean_time_to_recovery` are better when lower; `deployment_frequency` is better when higher. Return `tone: 'good'` on the favourable side and `'warn'` on the other. **Never red**: a missed internal goal is not an error, and colouring it like one makes the dashboard punitive. Return `null` when `target` is null so cards render exactly as they do today. + +- [ ] **Step 4: Add the line and caption to the cards** + +Create `BenchmarkTargetLine.tsx` — a dashed horizontal rule positioned on the card's existing trend chart — and render it plus the caption in each of `ChangeTimeCard`, `WeeklyDeliveryVolumeCard`, `ChangeFailureRateCard` and `MeanTimeToRestoreCard`. Read one card fully before editing; follow its existing layout rather than introducing a new one. + +When `benchmarks[metric].target` is null, render neither. That is the state every card is in before anyone configures anything, so it must look deliberate. + +- [ ] **Step 5: Verify and commit** + +```bash +docker exec middleware-dev sh -c 'cd /app/web-server && yarn test --testPathPattern=benchmarks' +docker exec middleware-dev sh -c 'cd /app/web-server && ./node_modules/.bin/tsc --noEmit 2>&1 | grep -c "error TS"' +git add web-server/src/utils/benchmarks.ts web-server/src/utils/__tests__/benchmarks.test.ts \ + web-server/src/components/BenchmarkTargetLine.tsx \ + web-server/src/content/DoraMetrics/DoraCards/ +git commit -m "feat(benchmarks): draw the target on each metric card" +``` + +--- + +### Task 6: Configuration forms + +**Files:** +- Create: `web-server/src/components/BenchmarkSettingsForm.tsx` +- Modify: `web-server/pages/workspaces.tsx` + +**Interfaces:** +- Consumes: `/api/clustox/benchmarks/global` (Task 4) and the existing team settings routes. +- Produces: nothing consumed by later tasks. + +> **Resolution note.** Tasks 1–4 give exact code. This task specifies behaviour +> and contracts rather than full MUI bodies, because the markup is long and +> mechanical and is better written against the live components. Treat the +> acceptance criteria as the requirement. + +- [ ] **Step 1: Build the form** + +`BenchmarkSettingsForm.tsx` — four labelled number inputs with units shown (`hours`, `per week`, `%`, `hours`), taking a `scope` prop of `'team'` or `'global'` and posting to the corresponding endpoint. + +Required behaviours: + +- **An empty input means inherit, and must be omitted from the payload — never sent as `0`.** Sending `0` would turn inherit into target-zero on every save and show every team failing everything. +- Each input's placeholder shows the inherited value, e.g. `24h (default)`, so an admin can see what clearing it does. +- Hours are entered as hours and converted to seconds on submit; the API stores seconds. +- Validation errors from the server are shown inline against the field named in the message, not as a toast. + +- [ ] **Step 2: Mount both scopes** + +Team benchmarks go in the existing team settings area. The global baseline gets a section on `pages/workspaces.tsx`, already the superadmin-only surface, rendered only when `isSuperadmin`. + +- [ ] **Step 3: Verify** + +```bash +docker exec middleware-dev sh -c 'cd /app/web-server && ./node_modules/.bin/tsc --noEmit 2>&1 | grep -c "error TS"' +docker exec middleware-dev sh -c 'cd /app/web-server && yarn lint' +``` + +Then by hand: set a team lead-time target, confirm the card shows a line and `your team's benchmark`; clear it, confirm it reverts to `the default benchmark`; set a target of `0` for change failure rate and confirm it persists as 0 rather than inheriting. + +- [ ] **Step 4: Commit** + +```bash +git add web-server/src/components/BenchmarkSettingsForm.tsx web-server/pages/workspaces.tsx +git commit -m "feat(benchmarks): add the benchmark configuration forms" +``` + +--- + +### Task 7: Cross-workspace isolation tests + +**Files:** +- Create: `web-server/e2e/benchmarks.spec.ts` + +**Interfaces:** +- Consumes: the endpoints from Tasks 4 and 6. +- Produces: nothing. + +- [ ] **Step 1: Write the isolation tests** + +Create `web-server/e2e/benchmarks.spec.ts` following the structure of the existing `web-server/e2e/multitenancy.spec.ts` — read it first and match its sign-in helper, admin provisioning and `afterAll` cleanup exactly. Cover: + +- An admin cannot read another workspace's team benchmark → **403** +- An admin cannot write another workspace's team benchmark → **403** +- A non-superadmin `PUT` to `/api/clustox/benchmarks/global` → **403** +- An unauthenticated request to either → **401** +- A superadmin can read and write the global baseline → **200** + +Use RFC-4122 valid UUIDs for any placeholder ids. `yup.string().uuid()` enforces the version and variant nibbles, so `00000000-0000-0000-0000-000000000001` is rejected at schema validation and the test would assert 400-as-403 — a bug already found once in this codebase. + +Use `00000000-0000-4000-8000-0000000000ff` for placeholders. Do **not** reuse `00000000-0000-4000-8000-000000000001`: that is `GLOBAL_BENCHMARK_ENTITY_ID`, the real global baseline row, and a test using it as a "nonexistent team" would be reasoning about live data. + +- [ ] **Step 2: Run and commit** + +```bash +cd web-server && SUPERADMIN_PASSWORD='' yarn playwright test e2e/benchmarks.spec.ts +git add web-server/e2e/benchmarks.spec.ts +git commit -m "test(benchmarks): cross-workspace isolation" +``` + +--- + +## Final verification + +- [ ] `cd backend/analytics_server && ./venv/bin/python -m pytest tests -q` — expect 286 passed +- [ ] `./venv/bin/python -m black --check mhq && ./venv/bin/python -m flake8 mhq` on changed files — clean +- [ ] `docker exec middleware-dev sh -c 'cd /app/web-server && ./node_modules/.bin/tsc --noEmit'` — 0 errors +- [ ] `cd web-server && yarn playwright test e2e/` — all suites pass +- [ ] **With no benchmarks configured anywhere, the four cards render exactly as they do today.** This is the highest-value check in the plan: the feature is additive, and the dashboards people already rely on must not shift because a `benchmarks` key appeared. +- [ ] Manual: set a team target, confirm the line and `your team's benchmark`; clear it, confirm `the default benchmark`; set `0` and confirm it persists. diff --git a/docs/BENCHMARK_GRAPHS_AND_LOC.md b/docs/BENCHMARK_GRAPHS_AND_LOC.md new file mode 100644 index 000000000..f18a7cc32 --- /dev/null +++ b/docs/BENCHMARK_GRAPHS_AND_LOC.md @@ -0,0 +1,221 @@ +# Benchmark graphs and the LOC metric — design + +**Status:** approved, not implemented +**Date:** 2026-08-12 +**Estimate:** 2–3 days +**Follows:** `docs/BENCHMARKS.md` (merged as `feat/benchmarks`) + +Two changes to the same four cards, done in one pass so the layout is only +reworked once: + +1. Replace the dashed target rule with a **shaded target band** on each card's + trend — the region on the good side of the target is tinted, so the whole + graph is coloured by whether the target is being met. +2. Add **lines of code** as a fifth metric, turning the 2×2 card grid into 2×3. + +--- + +## Why the current visual is being replaced + +The shipped version draws a 1px dashed `
` positioned by percentage over +the card's trend. Two problems, both visible on a real dashboard: + +- **It pins to an edge.** The line's height is `target / max(target, ...values)`. + A deployment-frequency target of 1/week against a series topping 35 sits at + 3% — indistinguishable from the card's bottom border. A lead-time target + larger than every plotted value pins to the top. Correct arithmetic, + unreadable result. +- **Two of four cards render nothing.** Change Failure Rate and MTTR gate both + the caption and the line on `canShowIncidentsData` / `canShowMTRData`. With + no incidents — the common case on a healthy team — half the dashboard shows + no benchmark at all, despite a target being set. + +## Decisions + +| Decision | Choice | Consequence | +|---|---|---| +| Visual | **Shaded target band on the existing trend** | The graph itself carries the verdict; no extra vertical space on a dense page. | +| Drawn by | **`chartjs-plugin-annotation`** | Already a dependency and already registered in `InternalChart2.tsx`, unused. Annotations are positioned in *data* space, so the band tracks the y-scale instead of being hand-positioned in percent. Deletes `BenchmarkTargetLine.tsx` and its clamping arithmetic. | +| LOC placement | **2×3 grid** | LOC is a peer metric, not an addendum. Costs a reflow of all four existing cards and leaves one empty slot. | +| LOC source | **Merged PRs only** | Matches the contributor dropdown, which already counts merged PRs only. An abandoned 5,000-line PR is not delivered work. | +| LOC benchmark | **Average lines changed per merged PR** | See below — this is the one genuinely contentious choice. | + +### The LOC benchmark targets PR size, not weekly volume + +The obvious reading of "a LOC benchmark" is a target for lines changed per +week. It is the wrong thing to benchmark, because **it has no good direction.** +Higher weekly LOC might mean a productive week or a bloated one; lower might +mean focus or a stall. A caption saying `above target` would be neither praise +nor warning, and the band would tint a region that carries no meaning. + +Average lines per merged PR does have a direction: **lower is better.** Small +PRs review faster and merge sooner, which is why it is worth putting next to +lead time — the two move together, and a team missing its lead-time target can +look one card across and see PR size as a likely cause. + +So the card shows **both**: weekly volume as the headline (that is what people +mean by "lines of code"), and average PR size as the benchmarked figure with +the band drawn against it. + +If this is wrong for your team, it is one line — `LOWER_IS_BETTER` and which +series the band keys off. Nothing else changes. + +--- + +## The band + +Each card's trend gains two annotations: + +- a **line** at the target, solid, low contrast +- a **box** from the target to the good-side edge of the scale, tinted + +Colour follows the shipped rule: `success` when the current value is on the +good side, `warning` when not. **Never red** — a missed internal goal is not an +error. + +Direction per metric, unchanged from `docs/BENCHMARKS.md`: + +| Metric | Lower is better | Band covers | +|---|---|---| +| Lead time | yes | target → 0 | +| Change failure rate | yes | target → 0 | +| MTTR | yes | target → 0 | +| Deployment frequency | no | target → top | +| **Average PR size** | **yes** | target → 0 | + +**The y-scale must include the target.** Chart2's grids `beginAtZero` with +`grace: '10%'` on the max. If the target exceeds the plotted max the band would +be clipped off-canvas, so the scale's `suggestedMax` becomes +`max(dataMax, target)`. This is the fix for the pinning problem: the target is +always on-canvas, in proportion. + +### The two empty-incident cards + +This is the known cost of choosing a band over a bar, and it is handled per +metric rather than left blank: + +**Change Failure Rate with no incidents** is not missing data — it is a +**genuine 0%**, which beats any target. The card currently suppresses the +caption entirely. It will instead render the band and +`0% is under target (15%) — the default benchmark`. This is a strict +improvement: the best possible result stops being displayed as nothing. + +**MTTR with no incidents** has genuinely no value — there is no recovery time +to measure. No band, no comparison. The card keeps *"No incidents reported"* +and adds one grey line: `target 1h — the default benchmark`, so an admin can +still see the target exists without being shown a comparison against nothing. + +The distinction matters: conflating "zero failures" with "no data" would +either hide a perfect score or invent a fake one. + +--- + +## LOC data + +**Nothing needs syncing.** Verified in the live database: 431 of 431 pull +requests have `meta.code_stats` populated, and `PullRequest` already exposes +`additions`, `deletions` and `changed_files` as properties. This is +aggregation and a card. + +```json +{"commits": 1, "comments": 1, "additions": 23, "deletions": 2, "changed_files": 4} +``` + +### Backend + +New service `mhq/service/code/loc.py`, and two routes on +`mhq/api/pull_requests.py` mirroring the lead-time pair exactly: + +``` +GET /teams//loc -> current + previous aggregate +GET /teams//loc/trends -> weekly series +``` + +Both take the existing `PRFilter`, so **the contributor filter, branch mode and +the excluded-PRs setting all apply with no extra work** — LOC per person comes +along for free. + +Response shape, matching the other stats blocks: + +```json +"loc_stats": { + "current": {"additions": 12043, "deletions": 4110, "total": 16153, "avg_pr_size": 214}, + "previous": {"additions": 9800, "deletions": 3050, "total": 12850, "avg_pr_size": 187} +} +``` + +`avg_pr_size` is `(additions + deletions) / merged PR count`, `0` when there +are no merged PRs — never a division by zero, and never `null`, so the card +does not need a third empty state. + +`total` is `additions + deletions`, the card's headline figure. It is +derivable, and is sent anyway so that the headline and the benchmarked +`avg_pr_size` come from one place — a browser recomputing it would eventually +disagree with the trend series drawn beside it. + +### Frontend + +`fetchLocStats` in `cockpitMetricUtils.ts`, added to the `Promise.all` in the +`dora_metrics` BFF route and sent in the same response — the same path the +benchmarks now take, and for the same reason: one call, one consistent set of +numbers. + +**It must be `.catch(() => undefined)`** like `fetchTeamBenchmarks`. A new +route failing inside that `Promise.all` would blank all five cards. + +--- + +## Benchmarks for LOC + +`lines_of_code` becomes a fifth key. The storage decision in +`docs/BENCHMARKS.md` was made for exactly this: + +- `BENCHMARK_METRICS` gains one entry — **no migration**, `data` is JSONB +- `validate_benchmark_payload` picks it up from that list automatically +- both forms gain one field, in **lines** (no unit conversion) +- `LOWER_IS_BETTER` gains it + +Unit is average lines per PR, matching what the band is drawn against. + +--- + +## Layout + +The four cards move from a 2×2 to a 2×3 grid, LOC in slot five, slot six +empty. The empty slot is deliberate and will not be filled with a placeholder +— an empty cell reads as "room for more", a "coming soon" tile reads as +unfinished. + +At `md` and below the grid is already single-column, so LOC simply appends and +nothing reflows. + +--- + +## Testing + +**The band, per metric:** target above the plotted max is on-canvas and not +clipped; target below the min likewise; colour flips with the direction table +above; deployment frequency's band covers the opposite side from the other +four. + +**The two empty-incident paths:** CFR with zero incidents renders a band and a +`0% is under target` caption; MTTR with zero incidents renders neither, plus +the grey target line. + +**LOC aggregation:** unmerged PRs excluded; `avg_pr_size` is 0 rather than a +crash when no PRs merged; the contributor filter narrows it; branch mode +narrows it. + +**Regression, and the most important test again:** with no benchmarks +configured anywhere, all five cards render exactly as they do today — no band, +no caption. And with LOC absent from the response entirely (an old backend +behind a new frontend), the other four cards must still render. + +--- + +## Deliberately excluded + +- **No per-repo LOC breakdown.** The dashboard aggregates to team level. +- **No language or file-type split.** `code_stats` does not carry it. +- **No LOC in the DORA score.** The score is the four DORA metrics; adding a + fifth input would change every historical number on every dashboard. diff --git a/docs/BENCHMARK_GRAPHS_AND_LOC_PLAN.md b/docs/BENCHMARK_GRAPHS_AND_LOC_PLAN.md new file mode 100644 index 000000000..dd5327565 --- /dev/null +++ b/docs/BENCHMARK_GRAPHS_AND_LOC_PLAN.md @@ -0,0 +1,524 @@ +# Benchmark Graphs and LOC Metric Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the dashed benchmark rule with a shaded target band drawn inside each card's trend, and add lines of code as a fifth metric in a 2×3 grid. + +**Architecture:** LOC is aggregated in Flask from `PullRequest.meta.code_stats` (already synced) through the existing `PRFilter`, served on two routes mirroring the lead-time pair, and delivered on the existing `dora_metrics` BFF response. The band is drawn by `chartjs-plugin-annotation`, already a dependency and already registered but unused, so it positions in data space instead of by CSS percentage. + +**Tech Stack:** Flask 3, SQLAlchemy 2, Postgres 15, Next.js 15 (pages router), MUI 5, chart.js 3.9 + chartjs-plugin-annotation 2.0. + +**Spec:** `docs/BENCHMARK_GRAPHS_AND_LOC.md`. **Predecessor:** `docs/BENCHMARKS.md`. + +## Global Constraints + +- **No database migration and no new tables.** `Settings.data` is JSONB and every enum column is `character varying`. +- Backend tests must pass. Baseline is **303**. Run `cd backend/analytics_server && ./venv/bin/python -m pytest` — that venv is the only working one; the repo-root venv lacks Flask. +- Frontend must typecheck: `npx tsc --noEmit`. Host `node_modules` is not installed — run inside the `middleware-dev` container. +- `black` and `flake8` clean on changed Python files. +- `# CLUSTOX:` / `// CLUSTOX:` comments explain **why**, never what. +- **Never run any `docker compose` command.** Copying files into the container and reloading a process is fine; rebuilding the image is not authorised. +- LOC counts **merged PRs only**. +- Units: lead time and MTTR **seconds**, deployment frequency **per week**, CFR **percent**, LOC benchmark **lines per PR**. +- **`0` is a deliberate target, never "absent".** Every check must be `is not None` / `!= null`, never truthiness. +- Commit in logical chunks. Do not amend or force-push. + +--- + +### Task 1: LOC aggregation service and Flask routes + +**Files:** +- Create: `backend/analytics_server/mhq/service/code/loc.py` +- Modify: `backend/analytics_server/mhq/api/pull_requests.py` (add routes after the `lead_time/trends` route ending line 175) +- Test: `backend/analytics_server/tests/service/code/test_loc.py` + +**Interfaces:** +- Consumes: `PRFilter` (`mhq/service/code/pr_filter.py`), `PullRequest.additions` / `.deletions` / `.changed_files` (`mhq/store/models/code/pull_requests.py:59-68`), `get_query_validator()`. +- Produces: `LOCMetrics` dataclass and `get_team_loc_metrics(team_id, interval, pr_filter) -> LOCMetrics`, plus `get_team_loc_trends(...) -> Dict[datetime, LOCMetrics]`. Task 2 consumes the two routes. + +Model the service and both routes on the lead-time pair in the same files — `/teams//lead_time` (line 98) and `/teams//lead_time/trends` (line 135). Read them first and match their filter parsing, `interval` handling and decorators exactly. + +- [ ] **Step 1: Write the failing test** + +```python +# tests/service/code/test_loc.py +from mhq.service.code.loc import LOCMetrics, aggregate_loc + + +def _pr(additions, deletions, state="MERGED"): + class FakePR: + pass + pr = FakePR() + pr.additions = additions + pr.deletions = deletions + pr.state = state + return pr + + +def test_aggregate_sums_additions_and_deletions(): + result = aggregate_loc([_pr(10, 5), _pr(20, 1)]) + assert result.additions == 30 + assert result.deletions == 6 + + +def test_avg_pr_size_is_gross_lines_over_pr_count(): + # (10+5 + 20+1) / 2 == 18 + result = aggregate_loc([_pr(10, 5), _pr(20, 1)]) + assert result.avg_pr_size == 18 + + +def test_avg_pr_size_is_zero_not_a_crash_when_no_prs(): + # A team with no merged PRs in range must not divide by zero, and must not + # report None -- the card would then need a third empty state. + result = aggregate_loc([]) + assert result == LOCMetrics(additions=0, deletions=0, avg_pr_size=0) +``` + +- [ ] **Step 2: Run it and watch it fail** + +Run: `cd backend/analytics_server && ./venv/bin/python -m pytest tests/service/code/test_loc.py -v` +Expected: FAIL, `ModuleNotFoundError: No module named 'mhq.service.code.loc'` + +- [ ] **Step 3: Write the service** + +```python +# mhq/service/code/loc.py +from dataclasses import dataclass +from datetime import datetime +from typing import Dict, List, Optional + +from mhq.service.code.pr_filter import PRFilter +from mhq.store.models.code.pull_requests import PullRequest +from mhq.store.repos.code import CodeRepoService +from mhq.utils.time import Interval + + +@dataclass +class LOCMetrics: + additions: int = 0 + deletions: int = 0 + # CLUSTOX: gross lines per merged PR, not net. A 2,000-line refactor that + # nets to zero is still 2,000 lines to review, and PR size is benchmarked + # precisely because it predicts review latency. + avg_pr_size: int = 0 + + +def aggregate_loc(prs: List[PullRequest]) -> LOCMetrics: + if not prs: + # Zero, never None: the card would otherwise need a third empty state + # on top of "no data" and "no target". + return LOCMetrics() + + additions = sum(pr.additions for pr in prs) + deletions = sum(pr.deletions for pr in prs) + return LOCMetrics( + additions=additions, + deletions=deletions, + avg_pr_size=round((additions + deletions) / len(prs)), + ) +``` + +Then add `get_team_loc_metrics` and `get_team_loc_trends` alongside, fetching merged PRs through `CodeRepoService` with the supplied `PRFilter` — mirror how `mhq/service/code/lead_time.py` obtains its PRs rather than writing a new query. + +- [ ] **Step 4: Run the tests** + +Run: `cd backend/analytics_server && ./venv/bin/python -m pytest tests/service/code/test_loc.py -v` +Expected: PASS, 3 tests. + +- [ ] **Step 5: Add the two routes** + +In `mhq/api/pull_requests.py`, after the trends route: + +```python +@app.route("/teams//loc", methods={"GET"}) +@queryschema( + Schema( + { + Required("from_time"): All(str, Coerce(datetime.fromisoformat)), + Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + Optional("pr_filter"): All(str, Coerce(json.loads)), + } + ), +) +def get_team_loc(team_id: str, from_time: datetime, to_time: datetime, pr_filter: Dict = None): + ... +``` + +Match the surrounding routes' validator and filter-parsing calls exactly; do not invent a new pattern. + +- [ ] **Step 6: Verify over real HTTP, not only in tests** + +``` +docker exec middleware-dev sh -lc 'curl -s -H "X-Internal-Token: $INTERNAL_API_TOKEN" "http://localhost:9696/teams/754e3764-cb0a-4bc8-9e63-ff887fd33ebd/loc?from_time=2026-05-20T00:00:00&to_time=2026-08-12T00:00:00"' +``` + +Copy the changed `.py` files in with `docker cp` and reload gunicorn first (`kill -HUP` the master on :9696). Paste the actual output into your report. The database has 431 PRs with `code_stats`, so a zero result means something is wrong. + +- [ ] **Step 7: Commit** + +```bash +git add backend/analytics_server/mhq/service/code/loc.py backend/analytics_server/mhq/api/pull_requests.py backend/analytics_server/tests/service/code/test_loc.py +git commit -m "feat(loc): aggregate lines of code from merged pull requests" +``` + +--- + +### Task 2: Deliver LOC on the metrics response + +**Files:** +- Modify: `web-server/src/utils/cockpitMetricUtils.ts` (add beside `fetchLeadTimeStats`, line 43) +- Modify: `web-server/pages/api/internal/team/[team_id]/dora_metrics.ts` (the `Promise.all` at line ~65 and the `res.send` at ~217) +- Modify: `web-server/src/types/resources.ts` +- Test: `web-server/pages/api/internal/team/[team_id]/__tests__/dora_metrics.test.ts` (extend the existing shape test) + +**Interfaces:** +- Consumes: Task 1's two routes. +- Produces: `loc_stats` on the `dora_metrics` response, reaching `state.doraMetrics.metrics_summary.loc_stats`. Task 6 renders it. + +- [ ] **Step 1: Add the fetcher** + +```ts +export const fetchLocStats = async (params: { + teamId: ID; + from_date: DateString; + to_date: DateString; + pr_filter?: PRFilter; +}) => { /* mirror fetchLeadTimeStats exactly, including its previous-period call */ }; +``` + +- [ ] **Step 2: Wire it into the existing Promise.all** + +```ts +const [branchAndRepoFilters, unsyncedRepos, benchmarks, locStats] = await Promise.all([ + // ...existing three... + // CLUSTOX: same soft-failure contract as fetchTeamBenchmarks above -- a + // rejection here would blank all five cards over one optional metric. + fetchLocStats({ teamId, from_date, to_date }).catch(() => undefined) +]); +``` + +and add `loc_stats: locStats` to the `res.send({...})` object. + +- [ ] **Step 3: Pin the contract in a test** + +Assert the response carries `loc_stats` **and** that the four existing `*_stats` keys are still present and unchanged. A test that only checks the new key would not catch the regression that matters. + +- [ ] **Step 4: Typecheck and commit** + +```bash +git add web-server/src/utils/cockpitMetricUtils.ts "web-server/pages/api/internal/team/[team_id]/dora_metrics.ts" web-server/src/types/resources.ts +git commit -m "feat(loc): send loc_stats on the dora metrics response" +``` + +--- + +### Task 3: Register `lines_of_code` as a fifth benchmark metric + +**Files — every site that names a metric explicitly. All seven backend, all six frontend:** + +Backend: +- `mhq/service/settings/models.py:67` — add `lines_of_code: Optional[float] = None` +- `mhq/service/settings/benchmarks.py:26` — add to `BENCHMARK_METRICS` +- `mhq/service/settings/configuration_settings.py:85` and `:224` — both `data.get(...)` adapters +- `mhq/service/settings/configuration_settings.py:308` — the to-dict adapter +- `mhq/api/resources/settings_resource.py:82` — the API response adapter +- `mhq/service/settings/default_settings_data.py:51` — add `"lines_of_code": None` + +Frontend: +- `src/utils/benchmarks.ts:16` — the `BenchmarkMetric` union +- `src/utils/benchmarks.ts:42` — `LOWER_IS_BETTER` (LOC **is** lower-is-better) +- `src/utils/benchmarks.ts:66` — `formatBenchmarkValue` (`${value} lines`) +- `src/components/BenchmarkSettingsForm.tsx:37,44,51,58,68,113` — `METRICS`, `DURATION_METRICS` (LOC is **not** one), label `Average PR size`, unit `lines`, empty value, validation switch +- `pages/api/clustox/benchmarks/global.ts:15` — `lines_of_code: yup.number().min(0).nullable().optional()` + +**Interfaces:** Produces `benchmarks.lines_of_code` with the same `{target, source}` shape as the other four. Task 5 draws its band. + +- [ ] **Step 1: Write the failing test** + +```python +def test_lines_of_code_resolves_like_every_other_metric(): + resolved = resolve_benchmarks( + BenchmarkSetting(lines_of_code=200), + BenchmarkSetting(lines_of_code=400, lead_time=86400), + ) + assert resolved["lines_of_code"] == {"target": 200, "source": "team"} + assert resolved["lead_time"] == {"target": 86400, "source": "global"} + + +def test_zero_lines_of_code_is_a_real_target(): + resolved = resolve_benchmarks(BenchmarkSetting(lines_of_code=0), None) + assert resolved["lines_of_code"] == {"target": 0, "source": "team"} +``` + +- [ ] **Step 2: Run it, watch it fail** + +Expected: `KeyError: 'lines_of_code'`. + +- [ ] **Step 3: Make the change at all thirteen sites, then run the whole suite** + +Run: `cd backend/analytics_server && ./venv/bin/python -m pytest` +Expected: 303 + your new tests, zero failures. A failure elsewhere means you missed an adapter. + +- [ ] **Step 4: Verify the round trip over HTTP** + +`PUT` a team benchmark containing only `lines_of_code`, then `GET /teams//benchmarks` and confirm it comes back `source: "team"` while the other four report `global` or `null`. Paste the output into your report. + +- [ ] **Step 5: Commit** + +```bash +git commit -am "feat(benchmarks): add lines_of_code as a fifth benchmark metric" +``` + +--- + +### Task 4: The target band + +**Files:** +- Create: `web-server/src/utils/benchmarkBand.ts` +- Delete: `web-server/src/components/BenchmarkTargetLine.tsx` +- Test: `web-server/src/utils/__tests__/benchmarkBand.test.ts` + +**Interfaces:** +- Consumes: `LOWER_IS_BETTER` from `src/utils/benchmarks.ts`. +- Produces: `benchmarkBandOptions({metric, target, actual, values, theme})` returning a partial chart.js options object — the `annotation` plugin block plus the `y.suggestedMax` override. Task 5 merges it into each card's `chartOptions`. + +`chartjs-plugin-annotation` is imported and registered at `src/components/Chart2/InternalChart2.tsx:9,33` and used nowhere. Cards hide both axes (`display: false`), which does not affect annotations — they position in data space regardless. + +- [ ] **Step 1: Write the failing test** + +```ts +describe('benchmarkBandOptions', () => { + it('extends the y scale so a target above the data is not clipped', () => { + const opts = benchmarkBandOptions({ + metric: 'lead_time', target: 100, actual: 20, values: [10, 20, 30] + }); + // Without this the band is drawn off-canvas and the card looks unchanged. + expect(opts.scales.y.suggestedMax).toBeGreaterThanOrEqual(100); + }); + + it('refuses to stretch the axis more than 2x past the data', () => { + // A fat-fingered target must not flatten the real series into a hairline. + const opts = benchmarkBandOptions({ + metric: 'lines_of_code', target: 20000, actual: 200, values: [180, 200, 220] + }); + expect(opts.scales.y.suggestedMax).toBeLessThan(1000); + }); + + it('covers target down to zero when lower is better', () => { + const { yMin, yMax } = boxOf(benchmarkBandOptions({ + metric: 'lead_time', target: 100, actual: 20, values: [10, 20, 30] + })); + expect(yMin).toBe(0); + expect(yMax).toBe(100); + }); + + it('covers target upward for deployment frequency', () => { + const { yMin } = boxOf(benchmarkBandOptions({ + metric: 'deployment_frequency', target: 5, actual: 9, values: [4, 9] + })); + expect(yMin).toBe(5); + }); + + it('tints success when the actual value is on the good side', () => { + const opts = benchmarkBandOptions({ + metric: 'lead_time', target: 100, actual: 20, values: [20] + }); + expect(colorOf(opts)).toMatch(/success/); + }); + + it('never returns red', () => { + const opts = benchmarkBandOptions({ + metric: 'lead_time', target: 10, actual: 900, values: [900] + }); + expect(colorOf(opts)).not.toMatch(/error|red/); + }); +}); +``` + +- [ ] **Step 2: Run it, watch it fail** + +Run: `docker exec middleware-dev sh -lc 'cd /app/web-server && npx jest src/utils/__tests__/benchmarkBand.test.ts'` + +- [ ] **Step 3: Implement** + +```ts +// CLUSTOX: replaces BenchmarkTargetLine, a CSS-positioned div whose height was +// target / max(target, ...values). A deployment-frequency target of 1 against +// a series topping 35 landed at 3% -- visually identical to the card's bottom +// border. An annotation is positioned in data space by chart.js itself, so it +// cannot disagree with the series it is drawn over. +export const benchmarkBandOptions = ({ metric, target, actual, values, theme }) => { + const lowerIsBetter = LOWER_IS_BETTER.has(metric); + const dataMax = Math.max(0, ...values.filter(Number.isFinite)); + // CLUSTOX: two opposite failures to avoid at once. Clamp the axis to the + // data and a target above it is clipped off-canvas, so the card looks + // exactly as it did before and the feature appears not to work. Extend the + // axis to the target unconditionally and a typo'd 20000-line target against + // a 200-line actual flattens the real series into a hairline at the bottom. + // So: include the target, but never stretch more than 4x past the data. + // Beyond that the band runs to the edge and the caption carries the number, + // because a scale nobody can read is worse than one that admits it is + // truncated. + // + // 4x, not 2x: a target 3.3x the current value is a perfectly ordinary target + // that a team is comfortably beating, and clipping it would hide real good + // news. A 100x target is a typo. 4x sits between them and leaves the series + // about a quarter of the plot height. (An earlier draft said 2x while its + // own first test demanded 3.33x -- the two could not both be satisfied.) + const AXIS_STRETCH_LIMIT = 4; + const suggestedMax = + Math.min(Math.max(dataMax, target), Math.max(dataMax, 1) * AXIS_STRETCH_LIMIT) * 1.1; + const targetIsOffScale = target > suggestedMax; + const onGoodSide = lowerIsBetter ? actual <= target : actual >= target; + // Never red: a missed internal goal is not an error (spec, "The graph"). + const tone = onGoodSide ? theme.colors.success : theme.colors.warning; + return { + scales: { y: { suggestedMax } }, + plugins: { + annotation: { + annotations: { + band: { + type: 'box', + yMin: lowerIsBetter ? 0 : target, + yMax: lowerIsBetter ? target : suggestedMax, + backgroundColor: alpha(tone, 0.12), + borderWidth: 0 + }, + targetLine: { + type: 'line', + yMin: target, + yMax: target, + borderColor: alpha(tone, 0.6), + borderWidth: 1 + } + } + } + } + }; +}; +``` + +- [ ] **Step 4: Run the tests, then delete the old component** + +Confirm no imports of `BenchmarkTargetLine` remain: `grep -rn BenchmarkTargetLine web-server/src` must return nothing. + +- [ ] **Step 5: Commit** + +```bash +git commit -am "feat(benchmarks): draw the target as a shaded band in data space" +``` + +--- + +### Task 5: Apply the band to the four existing cards + +**Files:** +- Modify: `src/content/DoraMetrics/DoraCards/ChangeTimeCard.tsx` (chartOptions line 36, chart line 322) +- Modify: `src/content/DoraMetrics/DoraCards/WeeklyDeliveryVolumeCard.tsx` +- Modify: `src/content/DoraMetrics/DoraCards/ChangeFailureRateCard.tsx` (caption gate line 124, line render line 238) +- Modify: `src/content/DoraMetrics/DoraCards/MeanTimeToRestoreCard.tsx` (caption gate line 96, line render line 206) + +Each card currently builds a module-level constant `chartOptions`. The band depends on props, so it must move into a `useMemo` merged with the constant via ramda's `mergeDeepRight` (already used in `InternalChart2.tsx`). + +**The two empty-incident paths are the point of this task** — do not treat them as edge cases. + +- [ ] **Step 1: Change Failure Rate — zero incidents is a real 0%, not missing data** + +The caption is gated on `canShowIncidentsData && isCfrDataAvailable`. With no incidents the team has a **0% failure rate**, which beats any target, and today that renders as nothing at all. Gate the band and caption on the **target** being set rather than on incidents existing, and pass `actual = 0`: + +```tsx +// CLUSTOX: zero incidents is a genuine 0% change failure rate -- the best +// possible result -- not absent data. Gating this on canShowIncidentsData +// meant a perfect score displayed as an empty card. +const cfrActual = isCfrDataAvailable ? changeFailureRateProps.count : 0; +``` + +- [ ] **Step 2: MTTR — zero incidents genuinely has no value** + +There is no recovery time to measure, so no band and no comparison. Render only a grey target line beneath *"No incidents reported"*: + +```tsx +{!canShowMTRData && meanTimeToRecoveryBenchmark?.target != null && ( + + target {formatBenchmarkValue('mean_time_to_recovery', meanTimeToRecoveryBenchmark.target)} + {' — '}{sourceLabel(meanTimeToRecoveryBenchmark.source)} + +)} +``` + +- [ ] **Step 3: Lead Time and Deployment Frequency** + +Merge `benchmarkBandOptions` into each card's options. Deployment frequency is the one metric where the band covers *upward* — confirm visually that its band is on the opposite side from the other three. + +- [ ] **Step 4: Typecheck, run the frontend tests, commit** + +```bash +git commit -am "feat(benchmarks): render the target band on all four DORA cards" +``` + +--- + +### Task 6: The LOC card and the 2×3 grid + +**Files:** +- Create: `src/content/DoraMetrics/DoraCards/LinesOfCodeCard.tsx` +- Modify: `src/content/DoraMetrics/DoraMetricsBody.tsx:143-157` (the ``) + +- [ ] **Step 1: Build the card** + +Copy the structure of `ChangeTimeCard.tsx` — the same `FlexBox` shell, the same `Chart2` trend behind the number, the same caption slot. Headline is **weekly lines changed**, with `+additions / −deletions` beneath it and the benchmarked **average PR size** as the figure the band is drawn against. + +- [ ] **Step 2: Add the sixth grid slot** + +```tsx + + + +``` + +Leave slot six empty — no placeholder tile. An empty cell reads as room for more; a "coming soon" tile reads as unfinished. + +- [ ] **Step 3: Handle the old-backend case** + +With `loc_stats` absent from the response the other four cards must still render. Verify by deleting the key from the response in the browser devtools, or by pointing the frontend at a backend without Task 1. + +- [ ] **Step 4: Commit** + +```bash +git commit -am "feat(loc): add the lines of code card in a 2x3 grid" +``` + +--- + +### Task 7: Regression and end-to-end verification + +**Files:** +- Modify: `web-server/e2e/benchmarks.spec.ts` +- Test: `backend/analytics_server/tests/service/code/test_loc.py` (filter coverage) + +- [ ] **Step 1: LOC respects every existing filter** + +Assert in the backend tests that `get_team_loc_metrics` narrows with a contributor `authors` filter and with `branch_mode = PROD`, and that unmerged PRs are excluded. LOC inherits these through `PRFilter`; a test proves it stayed inherited. + +- [ ] **Step 2: Extend the isolation suite** + +Add `lines_of_code` to the payloads in `e2e/benchmarks.spec.ts` and assert it round-trips. The suite honours `E2E_BASE_URL`, so it can run against a second server: + +``` +docker exec -e SUPERADMIN_PASSWORD='' -e E2E_BASE_URL='http://localhost:3334' \ + middleware-dev sh -lc 'cd /app/web-server && npx playwright test e2e/benchmarks.spec.ts --reporter=list' +``` + +Playwright's browsers are **not** installed in the image — `APIRequestContext` tests run fine, page-level ones do not. Do not try to install them. + +- [ ] **Step 3: The regression guarantee, restated** + +With no benchmark rows at all: all five cards render with no band and no caption, and every existing `*_stats` value is unchanged. This is the single most important test in the plan — the feature is additive and the four cards people already rely on must not move. + +- [ ] **Step 4: Full suites, then commit** + +Backend `pytest`, frontend `tsc` and `jest`, and the isolation suite. Report the actual numbers. + +```bash +git commit -am "test(loc): cover filter inheritance and the five-metric round trip" +``` diff --git a/docs/BITBUCKET.md b/docs/BITBUCKET.md new file mode 100644 index 000000000..fc70f4392 --- /dev/null +++ b/docs/BITBUCKET.md @@ -0,0 +1,179 @@ +# Bitbucket Cloud integration — design + +**Status:** phase 1 implemented and live-verified; phase 2 (Pipelines) pending +**Date:** 2026-08-25 +**Delivery:** two sequential PRs from one design — code provider first, Pipelines second +**Estimate:** 4–5 days (phase 1), 2–3 days (phase 2) + +Bitbucket Cloud joins GitHub and GitLab as a code provider, and Bitbucket +Pipelines joins GitHub Actions and Jenkins as a deployment source. Phase 1 +lights up lead time, LOC, the contributor filter and revert-PR incidents for +Bitbucket repos; phase 2 adds deployment frequency from Pipelines. A team on +Bitbucket + Jenkins is fully served after phase 1 alone. + +--- + +## Decisions + +| Decision | Choice | Consequence | +|---|---|---| +| Target | **Bitbucket Cloud only** (v2 REST) | Server/Data Center is a different API; nothing here reuses for it, and nothing here blocks adding it later. | +| Auth | **Atlassian API token**, Basic auth `email:token` | Fits the existing token-modal pattern. App passwords are being phased out by Atlassian; OAuth is a different shape from every integration in this app. The modal gains one extra field (email) vs the PAT modals. | +| Token validation | **Server-side in the BFF** | Bitbucket's API does not serve CORS for Basic auth from arbitrary origins, so the browser-side check GitHub uses is impossible. GitLab's modal already validates server-side — same pattern. | +| Delivery | **Two sequential PRs** | Phase 1 ships value alone; a Pipelines problem cannot hold code metrics hostage. Same playbook as Jenkins + contributor filter. | +| Scope unit | **Workspace** = org | Bitbucket's workspace is the org-level container. Repo listing and sync iterate the workspaces the token can see. | +| Revert detection | **`revert-pr-` branch pattern** | Bitbucket's own Revert button creates this branch — a structured link to the original PR. See the phase-1 section for why the title heuristic was dropped. | +| Rate limits | **429 = pause + resume, never fail** | ~1,000 req/hr on Cloud. First sync of a large repo will hit it; the bookmark makes stopping safe. | + +**No database migration anywhere.** `CodeProvider.BITBUCKET` and +`RepoWorkflowProviders.BITBUCKET_PIPELINES` are enum values persisted into +`character varying` columns, like every provider before them. + +--- + +## Phase 1 — code provider + +### New pieces, each mirroring a GitLab counterpart + +| New | Mirrors | Job | +|---|---|---| +| `mhq/exapi/bitbucket.py` | `exapi/gitlab.py` | v2 REST client: user, workspaces, repos, PRs, activity, diffstat | +| `mhq/service/code/sync/etl_bitbucket_handler.py` | `etl_gitlab_handler.py` | The four-method `CodeProviderETLHandler` contract | +| `mhq/service/code/sync/revert_prs_bitbucket_sync.py` | `revert_pr_gitlab_sync.py` | Revert-PR mapping for CFR incidents | +| `ConfigureBitbucketModalBody.tsx` | GitLab modal | Email + API token entry, server-side validation | +| Bitbucket branch in `git_org_repos.ts` | GitLab GraphQL branch | Repo listing per workspace | + +Plus one-line registrations: `CodeProvider.BITBUCKET`, a third branch in +`CodeETLFactory`, the frontend's existing `Integration.BITBUCKET` stub wired +to the new modal, and the integrations card on the dashboard. + +### API endpoints used + +``` +GET /2.0/user token validity +GET /2.0/user/permissions/workspaces org discovery (/2.0/workspaces is dead: 410 CHANGE-2770; scoped tokens cannot enumerate at all) +GET /2.0/repositories/{workspace} repo listing (pagelen=50) +GET /2.0/repositories/{ws}/{slug}/pullrequests PRs, ?q=updated_on>bookmark +GET /2.0/repositories/{ws}/{slug}/pullrequests/{id}/activity reviews/approvals +GET /2.0/repositories/{ws}/{slug}/pullrequests/{id}/diffstat additions/deletions +``` + +### Sync flow + +Incremental via the existing `CodeBookmarkType` mechanism, exactly as GitHub +and GitLab sync: per repo, fetch PRs with `updated_on` after the bookmark, +newest last; per PR, one activity call (first response time, approvals) and +one diffstat call (`code_stats` — the LOC feed); advance the bookmark only +past PRs fully adapted and stored. + +**Rate limit maths, stated up front:** a first-ever sync of a 500-PR repo +costs roughly 3 calls per PR page + activity + diffstat ≈ 1,100–1,500 +requests against a ~1,000/hr ceiling. It WILL 429. On 429 the handler stops +the batch, keeps what it has, leaves the bookmark at the last stored PR, logs +one warning with the reset time, and the next scheduled sync resumes. After +the first sync, incremental cost is trivial. + +### Field mapping (the boundary table — check both sides) + +| Ours | Bitbucket v2 | Notes | +|---|---|---| +| `state` MERGED / OPEN / CLOSED | `state` MERGED / OPEN / DECLINED (+ SUPERSEDED) | DECLINED and SUPERSEDED → CLOSED | +| `author` | `author.nickname`, uuid fallback (uuid kept in `data`) | The contributor dropdown lists `author` strings verbatim and the schema has no display indirection, so a stored uuid would surface as `{a1b2…}` in the UI. Nickname renames split history under two handles — the same accepted cost GitHub logins carry today. | +| `base_branch` / `head_branch` | `destination.branch.name` / `source.branch.name` | | +| `merge_commit_sha` | `merge_commit.hash` | **Null on unmerged PRs** — adapt per PR, never assume | +| `state_changed_at` (merge time) | `updated_on` when state=MERGED | Bitbucket has no separate merged_at; `updated_on` at merge is the closest truth and is what lead time keys on | +| `first_response_time` | earliest of comment / approval / changes-requested in `activity` | The activity feed mixes event kinds in one stream; parse defensively | +| `code_stats.additions/deletions/changed_files` | sum over `diffstat` values (`lines_added`, `lines_removed`, entry count) | Separate request; on failure the PR syncs with `code_stats` absent — LOC undercounts honestly rather than the PR vanishing from four metrics | +| reviewers | `participants` with `role: REVIEWER` | | + +### Revert detection (CFR) + +Bitbucket's own "Revert" button creates the branch `revert-pr-` — a +structured link to the original PR's number. Detection matches that branch +pattern in both directions (a revert syncing after its original, and an +original syncing after its revert), mirroring how the GitLab handler matches +`revert-` branches. This is STRONGER than the title-prefix heuristic +this spec originally proposed: a title alone identifies a PR as "a revert" +but names no target, and a mapping without both ends is useless to CFR — so +the title heuristic was dropped during implementation, not weakened. +A manual revert from a hand-named branch goes undetected; accepted and +recorded. + +--- + +## Phase 2 — Pipelines as a deployment source + +`RepoWorkflowProviders.BITBUCKET_PIPELINES = "bitbucket_pipelines"`, an +`etl_bitbucket_pipelines_handler.py` implementing the two-method +`WorkflowProviderETLHandler` contract (`check_pat_validity`, +`get_workflow_runs`), and a third branch in the workflows factory. + +``` +GET /2.0/repositories/{ws}/{slug}/pipelines/?sort=-created_on +``` + +| Ours | Pipelines | Notes | +|---|---|---| +| deployment | completed pipeline on the repo's production branch | Same definition Jenkins uses | +| `conducted_at` | `completed_on` | | +| `status` SUCCESS / FAILURE | `state.result.name` SUCCESSFUL / FAILED / STOPPED | STOPPED → FAILURE; anything unrecognised skips that run with a warning, never the batch | +| `event_actor` | `creator.nickname` (fallback `trigger` account) | Contributor filter works on day one | +| run URL | `links.html.href` or built from build_number | Deep link from the deployments drill-down | + +The repo→pipeline mapping reuses the workflow-selection UI built for Jenkins. +Auth reuses the phase-1 integration row — one Bitbucket link serves both +phases; phase 2 adds no second token. + +--- + +## Error handling + +Fail loud at the boundary, degrade visibly, never render a plausible wrong +answer: + +- **Token invalid/revoked**: caught at link time and at every sync start; + sync record fails with the provider's error body, visible in the existing + sync-status UI. The message says "revoked or expired" — the API cannot + distinguish them. +- **429**: pause + resume as above. Tested in both failure directions: + bookmark never advances past unfetched PRs (data loss) and never re-covers + stored ones (double count). +- **Malformed objects**: one bad PR or pipeline run skips that item with a + warning, never the batch — the Jenkins lesson, applied from day one. +- **Diffstat failure**: PR syncs without `code_stats`; LOC undercounts and + claims nothing false. Dropping the PR would corrupt four metrics to + protect one. +- **Nothing downstream changes.** Metrics read provider-agnostic tables; a + Bitbucket defect is a sync defect, visible in sync status — never a silent + number shift. + +## Testing + +- **Contract tests on recorded payloads**: real captured Bitbucket JSON (PR + page, activity, diffstat, pipeline run) as fixtures, asserting the adapted + models. Real vendor JSON is what stops a test from encoding a broken shape + back to itself — this project has shipped that bug twice. +- **The 429 bookmark test**, both directions. +- **Handler tests** mirroring `test_etl_gitlab_handler.py`: PR states, review + timestamps, null `merge_commit` on open PRs, revert-title matching with a + negative case. +- **Provider-isolation regression**: Bitbucket linked + zero Bitbucket repos + selected ⇒ every existing GitHub/GitLab number byte-identical. +- **Live verification before any PR opens**: a real scratch workspace with a + few PRs, linked through the real modal, synced by the real scheduler, + numbers on the real dashboard. Every serious bug in the last three + features was invisible until exactly this step. + +## Deliberately excluded + +- **No Bitbucket Server / Data Center.** Different API, own effort, nothing + here blocks it. +- **No OAuth flow.** Token-based like every other integration here; OAuth is + a product decision for all providers at once, not a rider on this one. +- **No cross-provider identity merging.** A person on GitHub and Bitbucket + appears twice — already true for GitHub+GitLab, unchanged. +- **No Bitbucket Deployments-API environments.** Pipelines runs on the prod + branch are the deployment signal, matching how Jenkins is modelled; the + richer environments API can layer on later without schema change. +- **No incident provider.** Bitbucket has no issues-based incident source + worth modelling; incidents remain reverts + the existing providers. diff --git a/docs/BITBUCKET_PLAN.md b/docs/BITBUCKET_PLAN.md new file mode 100644 index 000000000..d1c131cb3 --- /dev/null +++ b/docs/BITBUCKET_PLAN.md @@ -0,0 +1,217 @@ +# Bitbucket Code Provider Implementation Plan (Phase 1) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Bitbucket Cloud as a third code provider — repos, PRs, reviews and code stats feeding lead time, LOC, the contributor filter and revert-PR incidents. + +**Architecture:** A v2 REST client (`exapi/bitbucket.py`) and an ETL handler implementing the existing four-method `CodeProviderETLHandler` contract, registered in `CodeETLFactory`. Frontend: an email+token modal validated through a small BFF endpoint (Bitbucket serves no CORS for Basic auth, unlike GitLab), and a workspace-based repo listing branch in `git_org_repos.ts`. Phase 2 (Pipelines) is a separate plan. + +**Tech Stack:** Flask 3 + SQLAlchemy 2, requests, Next.js 15 pages router, MUI 5, yup. + +**Spec:** `docs/BITBUCKET.md` — its **field-mapping table is the contract**; every adapter task below points at it. + +## Global Constraints + +- **No database migration, no new tables.** `CodeProvider.BITBUCKET` and `UserIdentityProvider.BITBUCKET` persist into `character varying` columns. +- Backend tests must pass. **Record the baseline count before your first change** (`cd backend/analytics_server && ./venv/bin/python -m pytest`) and report old → new. That venv is the only working one — the repo-root venv lacks Flask. +- Frontend must typecheck: `docker exec middleware-dev sh -lc 'cd /app/web-server && npx tsc --noEmit'`. Host `node_modules` is not installed; copy files in with `docker cp`. +- `black` + `flake8` clean on changed Python; `prettier` + `eslint` clean on changed TS. +- `# CLUSTOX:` / `// CLUSTOX:` comments explain WHY, never what. +- **The token never appears in a log, an error message, a test fixture, or a commit.** The email is not a secret; the token always is. +- **Never run `docker compose`.** Copying files into the running container is fine; rebuilding is not. +- Adapt per item: one malformed PR/activity entry skips that item with a warning, never the batch. +- Commit in logical chunks. Do not amend or force-push. + +--- + +### Task 1: The Bitbucket API client + +**Files:** +- Create: `backend/analytics_server/mhq/exapi/bitbucket.py` +- Create: `backend/analytics_server/mhq/exapi/models/bitbucket.py` +- Test: `backend/analytics_server/tests/exapi/test_bitbucket_api.py` (create `tests/exapi/__init__.py` if absent) + +**Interfaces:** +- Consumes: nothing internal — pure HTTP client. +- Produces (Task 2 consumes these exact signatures): + +```python +class BitbucketRateLimitExceeded(Exception): ... + +class BitbucketApiService: + def __init__(self, email: str, api_token: str): ... # Basic auth pair + def check_pat(self) -> bool: ... # GET /2.0/user + def get_workspaces(self) -> List[Dict]: ... + def get_workspace_repos(self, workspace: str) -> List[Dict]: ... + def get_repo_pull_requests(self, workspace: str, repo_slug: str, + updated_since: datetime) -> List[Dict]: ... + def get_pr_activity(self, workspace: str, repo_slug: str, pr_id: int) -> List[Dict]: ... + def get_pr_diffstat(self, workspace: str, repo_slug: str, pr_id: int) -> List[Dict]: ... +``` + +Base URL `https://api.bitbucket.org/2.0`. Auth via `requests.Session` with `session.auth = (email, api_token)` — never a hand-built header, and never interpolated into a URL. + +**Pagination is by `next` link, not page numbers.** Every list endpoint returns `{"values": [...], "next": "https://..."}`; follow `next` until absent, `pagelen=50`. PR listing filters server-side: `params={"q": f'updated_on > "{updated_since.isoformat()}"', "sort": "updated_on", "state": "MERGED,OPEN,DECLINED,SUPERSEDED"}` — without the explicit `state` list Bitbucket returns only OPEN PRs, which would silently hide every merged PR from lead time. + +**429 raises `BitbucketRateLimitExceeded`** carrying the `Retry-After` header when present. The handler (Task 3) decides policy; the client only reports. + +- [ ] **Step 1: Write the failing contract tests against realistic fixtures** + +Fixtures are dicts in the test file shaped exactly like Bitbucket's documented payloads — not hand-simplified. The minimum PR fixture: + +```python +BB_PR = { + "id": 42, + "title": "feat: add rate limiter", + "state": "MERGED", + "author": {"uuid": "{a1b2c3d4-0000-4000-8000-000000000001}", "nickname": "hamadr"}, + "source": {"branch": {"name": "feat/rate-limiter"}}, + "destination": {"branch": {"name": "main"}}, + "created_on": "2026-08-20T10:00:00+00:00", + "updated_on": "2026-08-21T15:30:00+00:00", + "merge_commit": {"hash": "abc123def456"}, + "participants": [ + {"role": "REVIEWER", "approved": True, + "user": {"uuid": "{b2c3d4e5-0000-4000-8000-000000000002}", "nickname": "muzz"}, + "participated_on": "2026-08-21T12:00:00+00:00"} + ], + "links": {"html": {"href": "https://bitbucket.org/ws/repo/pull-requests/42"}}, +} +``` + +Tests, each written to fail first: +- `test_pagination_follows_next_links` — two pages via mocked session, values concatenated, stops when `next` absent. +- `test_pr_listing_requests_all_states` — assert the `state` param includes MERGED; this is the silent-hide trap. +- `test_429_raises_rate_limit_with_retry_after` — mocked 429 with `Retry-After: 1800`. +- `test_auth_is_basic_session_auth` — `session.auth == (email, token)`; the token never appears in any URL the mock saw. + +- [ ] **Step 2: Run, watch them fail** (`ModuleNotFoundError`), **implement, run again** + +Model the error handling on `GitlabApiService._handle_error` (`mhq/exapi/gitlab.py:35`). + +- [ ] **Step 3: black + flake8, commit** + +```bash +git add backend/analytics_server/mhq/exapi backend/analytics_server/tests/exapi +git commit -m "feat(bitbucket): Bitbucket Cloud v2 API client" +``` + +--- + +### Task 2: The ETL handler — repos and PR adaptation + +**Files:** +- Create: `backend/analytics_server/mhq/service/code/sync/etl_bitbucket_handler.py` +- Modify: `backend/analytics_server/mhq/store/models/code/enums.py` (add `BITBUCKET = "bitbucket"` to `CodeProvider`) +- Modify: `backend/analytics_server/mhq/store/models/integrations/enums.py:4` (`UserIdentityProvider`) — add `BITBUCKET = "bitbucket"` +- Test: `backend/analytics_server/tests/service/code/sync/test_etl_bitbucket_handler.py` + +**Interfaces:** +- Consumes: Task 1's client, verbatim. +- Produces: `BitbucketETLHandler(CodeProviderETLHandler)` and `get_bitbucket_etl_handler(org_id) -> BitbucketETLHandler`. Task 4 registers it; Task 3 extends it. + +Mirror `GitlabETLHandler` (`etl_gitlab_handler.py:42`) method-for-method, and its factory function (`:367`) for credentials: token via `CoreRepoService().get_access_token(org_id, UserIdentityProvider.BITBUCKET)`, **email via `provider_meta["email"]`** from `get_org_integrations_for_names` — the same pattern GitLab uses for `custom_domain`. + +**The spec's field-mapping table is the requirement set.** The load-bearing rows: + +- `state`: MERGED→MERGED, OPEN→OPEN, **DECLINED and SUPERSEDED→CLOSED**. Unknown state: skip that PR with a warning, never guess. +- `state_changed_at` for merged PRs = `updated_on` — Bitbucket has no `merged_at`; write the CLUSTOX comment on why. +- `merge_commit` is **null on unmerged PRs** — `_get_merge_commit_sha` must be `None`-safe, per PR. +- `author` = `author.uuid`, display name from `nickname`. +- `url` from `links.html.href`. + +- [ ] **Step 1: Failing tests** — mirror `test_etl_gitlab_handler.py`'s shape (7 tests there; record what each covers before writing). Minimum: the four state mappings including an unknown state skipping only that PR; the null `merge_commit` case; `updated_on`-as-merge-time; author uuid landing in `PullRequest.author`. +- [ ] **Step 2: Implement, tests green** +- [ ] **Step 3: Full suite** — a failure in another provider's tests means a shared model was disturbed; fix before proceeding. +- [ ] **Step 4: Commit** (`feat(bitbucket): ETL handler for repos and pull requests`) + +--- + +### Task 3: Activity, diffstat, and the 429 bookmark contract + +**Files:** +- Modify: `etl_bitbucket_handler.py` +- Test: extend `test_etl_bitbucket_handler.py` + +**Interfaces:** completes `get_repo_pull_requests_data(org_repo, bookmark) -> Tuple[List[PullRequest], List[PullRequestCommit], List[PullRequestEvent]]` — the exact contract signature at `etl_provider_handler.py:31-33`. All three lists matter: events carry the review timestamps that first_response_time reads. + +Three behaviours, each with its own test written to fail first: + +1. **Activity → review events.** From the mixed activity stream, an entry with `approval` or `comment` becomes a review event; earliest timestamp drives `first_response_time` (see how `_to_pr_events` / `_get_event_state` do it for GitLab, `etl_gitlab_handler.py:326,357`). Entries with unrecognised shape are skipped individually. +2. **Diffstat → `code_stats`,** summing `lines_added` / `lines_removed` over entries, `changed_files` = entry count. **A diffstat failure syncs the PR with `code_stats` absent** — assert the PR is present AND `additions` falls back to 0 via the model property; the LOC feed undercounts honestly, the PR never vanishes from lead time. +3. **429 = pause + resume.** When the client raises `BitbucketRateLimitExceeded` mid-repo, the handler returns everything fully processed so far; the bookmark (see how GitLab's caller advances it) ends at the last **stored** PR. Two tests, two failure directions: + - `test_429_keeps_already_processed_prs_and_stops` — PRs before the 429 are returned, none after. + - `test_429_does_not_advance_past_unfetched_prs` — the returned set's max `updated_on` is the resume point; a PR after the 429 is absent and NOT covered by it. + +- [ ] Steps: failing tests → implement → full suite → commit (`feat(bitbucket): reviews, code stats, and rate-limit-safe sync`) + +--- + +### Task 4: Revert detection and registration + +**Files:** +- Create: `backend/analytics_server/mhq/service/code/sync/revert_prs_bitbucket_sync.py` +- Modify: `backend/analytics_server/mhq/service/code/sync/etl_code_factory.py` (third branch) +- Test: `backend/analytics_server/tests/service/code/sync/test_revert_pr_bitbucket_sync.py` + +Mirror `revert_pr_gitlab_sync.py`. Detection: title starts with `revert:` or `revert "` (case-insensitive — Bitbucket's UI writes `Revert: `); link to the original by branch name when the revert branch encodes it, title suffix match otherwise. + +- [ ] **Tests must include the negative case**: a PR titled `Revert the decision to use tabs` (prose, not a revert) maps to nothing. Title heuristics without a negative test are how false CFR incidents ship. +- [ ] Factory branch + `test_every_provider_reaches_the_factory` asserting all three `CodeProvider` values dispatch without `NotImplementedError`. +- [ ] Full suite, black/flake8, commit (`feat(bitbucket): revert-PR incidents and factory registration`) + +--- + +### Task 5: BFF — token check, linking, repo listing + +**Files:** +- Create: `web-server/pages/api/internal/bitbucket/token-check.ts` +- Modify: `web-server/pages/api/internal/[org_id]/git_org_repos.ts` (Bitbucket branch beside the GitLab GraphQL branch at `:156`) +- Modify: `web-server/src/utils/auth.ts` (add `checkBitbucketValidity(email, token)` calling the BFF endpoint) +- Test: `web-server/src/utils/__tests__/bitbucketAuth.test.ts` + +**Why a BFF endpoint where GitHub/GitLab validate in the browser:** Bitbucket's API sends no CORS headers for Basic auth from foreign origins — a browser call dies in preflight. The check endpoint takes `{email, token}`, calls `GET https://api.bitbucket.org/2.0/user` server-side with Basic auth, and returns only `{valid: boolean, nickname?: string}`. + +- **The token is not logged, not echoed in the error body, and not persisted by this endpoint.** Linking still goes through the existing `linkProvider` → `/api/resources/orgs/<id>/integration` path with `meta_data: {email}` — email in `provider_meta`, token encrypted in `access_token_enc_chunks`, exactly like GitLab's `custom_domain` pattern. +- Repo listing: workspaces → repos per workspace, following `next` pagination, mapped to the same shape the GitLab branch returns (compare in-file). Fields: `id` = repo `uuid`, `name` = `slug`, `slug`, `web_url` from `links.html.href`, `branch` = `mainbranch.name`. + +- [ ] Failing jest tests for the repo-shape adapter (fixture → the exact object `git_org_repos.ts` returns for GitLab — same keys) +- [ ] Implement; `tsc` clean; eslint/prettier clean +- [ ] Commit (`feat(bitbucket): server-side token check and workspace repo listing`) + +--- + +### Task 6: The modal and wiring + +**Files:** +- Create: `web-server/src/content/Dashboards/ConfigureBitbucketModalBody.tsx` +- Modify: `web-server/src/content/Dashboards/useIntegrationHandlers.tsx` (link + unlink entries beside GitLab's at `:29,:41`) +- Test: `web-server/src/content/Dashboards/__tests__/ConfigureBitbucketModalBody.test.tsx` + +Mirror `ConfigureGitlabModalBody.tsx` structure. Differences, and only these: +- **Two fields**: Atlassian account email + API token. Email validated with a plain regex before any network call; the submit is disabled until both fields are non-empty. +- Validation via `checkBitbucketValidity` (Task 5) instead of a direct provider call. +- `linkProvider(token, orgId, Integration.BITBUCKET, { email })`. +- Error copy: invalid credentials → "Invalid email or API token" (the API cannot distinguish); missing-scope detection is not possible for Atlassian tokens, so the help link points at the token-creation page with the required scopes named in text: `account`, `repository`, `pullrequest`. + +The `Integration.BITBUCKET` enum value already exists (`src/constants/integrations.ts:6`) — this task wires it, adds no enum. + +- [ ] Render test mirroring `__tests__/ConfigureJiraModalBody.test.tsx`'s harness: renders both fields, submit disabled until filled, a failed check shows the error inline and does NOT call `linkProvider` (assert the mock), a passed check calls it with `{email}` meta. +- [ ] `tsc` clean, scoped jest green, commit (`feat(bitbucket): configuration modal`) + +--- + +### Task 7: Provider-isolation regression and live verification + +**Files:** +- Test: extend `backend/analytics_server/tests/service/code/sync/test_etl_bitbucket_handler.py` +- Test: `web-server/src/content/Dashboards/__tests__/integrations.test.tsx` (only if no equivalent exists — check first) + +- [ ] **Isolation, backend:** with a Bitbucket integration row present and zero Bitbucket repos selected, the GitHub and GitLab sync paths produce identical output to a run without the row — assert on the factory dispatch and one representative GitHub handler test parameterised over "bitbucket row present/absent". +- [ ] **Isolation, frontend:** the integrations card shows Bitbucket as linkable without altering the GitHub/GitLab cards' state logic. +- [ ] **Live verification — the gate for opening the PR.** Not a subagent step; coordinate with the controller: + 1. Link a scratch Bitbucket workspace (2–3 PRs, at least one merged, one open, one with a review) through the real modal on `localhost:3333`. + 2. Trigger sync; watch `/app/backend/analytics_server` logs for the Bitbucket handler. + 3. Verify over real HTTP: the team's `/lead_time`, `/loc`, and `/contributors` responses include the Bitbucket repo's PRs with non-zero `code_stats`. + 4. Verify the dashboard renders them and the contributor dropdown lists the Bitbucket author. +- [ ] Full backend suite + full frontend scoped suites, report all numbers, commit (`test(bitbucket): provider isolation`) diff --git a/docs/CONTRIBUTOR_FILTER_PLAN.md b/docs/CONTRIBUTOR_FILTER_PLAN.md new file mode 100644 index 000000000..41769ef31 --- /dev/null +++ b/docs/CONTRIBUTOR_FILTER_PLAN.md @@ -0,0 +1,753 @@ +# Contributor Filter Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Filter the DORA metrics page by contributor — Lead Time by pull-request author, Deployment Frequency by deploy actor. + +**Architecture:** The dropdown lists distinct git usernames found in the selected team's synced pull requests, bots excluded. Lead Time reuses `PRFilter.authors`, a field upstream declared and parsed but never applied. Deployment Frequency needs a genuinely new `event_actors` filter on `RepoWorkflowRuns`. No new tables, no identity resolution. + +**Tech Stack:** Python 3.9, Flask, SQLAlchemy 2, pytest (backend); Next.js 15 pages router, TypeScript, MUI 5 (frontend). + +## Decisions already made — do not revisit + +| Decision | Choice | +|---|---| +| Identity | Raw git usernames as synced. No mapping table, no merging of handles. A person using two handles appears twice; that is accepted. | +| Metric scope | Lead Time and Deployment Frequency only. **CFR and MTTR stay team-level** — per-person versions wait for the future Jira integration, because GitHub-issue assignees are too weak a foundation. | +| Involvement | **Authored only.** Reviewers are not included. | +| List source | Distinct authors from the selected team's PRs within the active date range, so the dropdown can never offer someone with no data on screen. | +| Bots | Excluded by default, no UI toggle. | + +## Global Constraints + +- **No database migration, no new tables.** +- Backend style: `black` formatted, `flake8` clean. Run from `backend/analytics_server` using `./venv/bin/python`. +- Frontend must be `tsc` clean. Local Node is 20 and the project needs 22 — verify inside the `middleware-dev` container, which has Node 22.16. +- Every Clustox-authored change carries a `# CLUSTOX:` or `// CLUSTOX:` comment explaining **why**, per `docs/FORK_STRATEGY.md`. +- Backend suite baseline is **159 passing** (this branch is cut from `main`, which does not include the Jenkins work). Do not break any. +- Bot matching is on the exact `[bot]` suffix plus an explicit list — never a substring match on `bot`, which would exclude a real user called `robotnik`. + +## The trap both filter classes share — read before Task 1 or 2 + +`PRFilter.filter_query` and `WorkflowFilter.filter_query` both end with: + +```python +return [ + conditions[x] + for x in self.__dict__.keys() + if getattr(self, x) is not None and conditions[x] is not None +] +``` + +It iterates **every** dataclass field and indexes `conditions` by name. A field with no matching `conditions` key raises `KeyError` the moment that field is set. + +This is not hypothetical. `PRFilter.authors` exists today with no `conditions` entry, so sending `authors` in a filter payload raises `KeyError: 'authors'` — verified: + +``` +PRFilter(authors=['hamad']).filter_query → KeyError: 'authors' +``` + +**Consequence for this plan:** adding a field without adding its `conditions` entry in the same commit reproduces the bug. Both tasks below add both halves together. + +--- + +## File Structure + +**Backend — modify:** +- `mhq/store/models/code/filter.py` — apply `authors` in `PRFilter.filter_query`. +- `mhq/store/models/code/workflows/filter.py` — add `event_actors` to `WorkflowFilter`. +- `mhq/service/workflows/workflow_filter.py` — parse `event_actors` from the request. +- `mhq/store/repos/code.py` — contributor query. +- `mhq/api/pull_requests.py` — contributors route. + +**Backend — create:** +- `tests/store/models/code/test_pr_filter_authors.py` +- `tests/store/models/code/workflows/test_workflow_filter_actors.py` +- `tests/service/code/test_contributors.py` + +**Frontend — create:** +- `web-server/pages/api/internal/team/[team_id]/contributors.ts` +- `web-server/src/components/ContributorFilter.tsx` + +**Frontend — modify:** +- `web-server/src/api-helpers/team.ts` — thread `authors` into `pr_filter`. +- `web-server/pages/api/internal/team/[team_id]/dora_metrics.ts` — thread `event_actors`. +- `web-server/pages/dora-metrics/index.tsx` — render the filter. + +--- + +### Task 1: Apply the author filter (and fix the KeyError) + +**Files:** +- Modify: `backend/analytics_server/mhq/store/models/code/filter.py` +- Test: `backend/analytics_server/tests/store/models/code/test_pr_filter_authors.py` (create) + +**Interfaces:** +- Consumes: nothing. +- Produces: `PRFilter(authors=[...])` yields a `PullRequest.author.in_(authors)` condition. `ParsePRFilterProcessor` already populates the field — no change needed there. + +- [ ] **Step 1: Write the failing tests** + +Create `backend/analytics_server/tests/store/models/code/test_pr_filter_authors.py`: + +```python +from mhq.store.models.code.filter import PRFilter + + +def test_setting_authors_no_longer_raises(): + # Before this change the conditions dict had no "authors" key, so the + # filter_query comprehension raised KeyError as soon as the field was set. + conditions = PRFilter(authors=["hamad-clustox"]).filter_query + assert len(conditions) == 1 + + +def test_authors_produces_an_author_condition(): + conditions = PRFilter(authors=["hamad-clustox"]).filter_query + # Compiling to string is the cheapest way to assert the column and operator + # without a database. + rendered = str(conditions[0]) + assert "author" in rendered + assert "IN" in rendered.upper() + + +def test_no_authors_produces_no_author_condition(): + conditions = PRFilter(base_branches=["^main$"]).filter_query + assert all("author" not in str(c) for c in conditions) + + +def test_empty_author_list_is_ignored(): + # An empty list means "no filter selected", not "match nothing". + conditions = PRFilter(authors=[]).filter_query + assert conditions == [] + + +def test_existing_filters_are_unaffected(): + # Regression guard: the dashboards everyone already uses must not change. + conditions = PRFilter(base_branches=["^main$"], excluded_pr_ids=["a"]).filter_query + assert len(conditions) == 2 +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd backend/analytics_server && ./venv/bin/python -m pytest tests/store/models/code/test_pr_filter_authors.py -v` +Expected: FAIL — the first two with `KeyError: 'authors'` + +- [ ] **Step 3: Add the condition** + +In `mhq/store/models/code/filter.py`, inside `filter_query`, add this alongside the other inner functions: + +```python + # CLUSTOX: authors was declared on PRFilter and parsed from the request + # payload upstream, but never added to the conditions dict below -- so + # sending it raised KeyError rather than filtering. This applies it. + def _authors_query(): + if not self.authors: + return None + + return PullRequest.author.in_(self.authors) +``` + +Then add the entry to the `conditions` dict, as the first key: + +```python + conditions = { + "authors": _authors_query(), + "base_branches": _base_branch_query(), + "repo_filters": _repo_filters_query(), + "excluded_pr_ids": _excluded_pr_ids_query(), + "max_cycle_time": _include_prs_below_max_cycle_time(), + "incident_pr_filters": _incident_pr_filters_query(), + } +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd backend/analytics_server && ./venv/bin/python -m pytest tests/store/models/code/test_pr_filter_authors.py -v` +Expected: PASS (5 tests) + +- [ ] **Step 5: Run the full suite** + +Run: `cd backend/analytics_server && ./venv/bin/python -m pytest tests -q` +Expected: 164 passed (159 baseline + 5) + +- [ ] **Step 6: Format, lint, commit** + +```bash +cd backend/analytics_server +./venv/bin/python -m black mhq/store/models/code/filter.py tests/store/models/code/test_pr_filter_authors.py +./venv/bin/python -m flake8 mhq/store/models/code/filter.py +cd ../.. +git add backend/analytics_server/mhq/store/models/code/filter.py \ + backend/analytics_server/tests/store/models/code/test_pr_filter_authors.py +git commit -m "fix(filters): apply the pull request author filter" +``` + +--- + +### Task 2: Add the deploy-actor filter + +**Files:** +- Modify: `backend/analytics_server/mhq/store/models/code/workflows/filter.py` +- Modify: `backend/analytics_server/mhq/service/workflows/workflow_filter.py` +- Test: `backend/analytics_server/tests/store/models/code/workflows/test_workflow_filter_actors.py` (create) + +**Interfaces:** +- Consumes: nothing from Task 1. +- Produces: `WorkflowFilter(event_actors=[...])` yields a `RepoWorkflowRuns.event_actor.in_(...)` condition; `ParseWorkflowFilterProcessor` reads `event_actors` from the request dict. + +- [ ] **Step 1: Write the failing tests** + +Create `backend/analytics_server/tests/store/models/code/workflows/test_workflow_filter_actors.py`: + +```python +from mhq.service.workflows.workflow_filter import ParseWorkflowFilterProcessor +from mhq.store.models.code.workflows.filter import WorkflowFilter + + +def test_event_actors_produces_an_actor_condition(): + conditions = WorkflowFilter(event_actors=["hamad-clustox"]).filter_query + assert len(conditions) == 1 + rendered = str(conditions[0]) + assert "event_actor" in rendered + assert "IN" in rendered.upper() + + +def test_no_event_actors_produces_no_actor_condition(): + conditions = WorkflowFilter(head_branches=["^main$"]).filter_query + assert all("event_actor" not in str(c) for c in conditions) + + +def test_empty_actor_list_is_ignored(): + assert WorkflowFilter(event_actors=[]).filter_query == [] + + +def test_existing_workflow_filters_are_unaffected(): + # Regression guard for the deployment frequency everyone already sees. + conditions = WorkflowFilter(head_branches=["^main$"]).filter_query + assert len(conditions) == 1 + + +def test_parser_reads_event_actors_from_the_request(): + parsed = ParseWorkflowFilterProcessor().apply({"event_actors": ["hamad-clustox"]}) + assert parsed.event_actors == ["hamad-clustox"] + + +def test_parser_defaults_event_actors_to_none(): + parsed = ParseWorkflowFilterProcessor().apply({}) + assert parsed.event_actors is None +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd backend/analytics_server && ./venv/bin/python -m pytest tests/store/models/code/workflows/test_workflow_filter_actors.py -v` +Expected: FAIL — `TypeError: __init__() got an unexpected keyword argument 'event_actors'` + +- [ ] **Step 3: Add the field and its condition together** + +In `mhq/store/models/code/workflows/filter.py`, change the `WorkflowFilter` dataclass. Add the field **and** the conditions entry in this same step — adding one without the other reproduces the `KeyError` described at the top of this plan: + +```python +@dataclass +class WorkflowFilter: + head_branches: List[str] = None + repo_filters: Dict[str, Dict] = None + # CLUSTOX: who triggered the deployment. Lead Time filters on the pull + # request author; deployments have no author, so the nearest equivalent is + # the actor who ran the build. + event_actors: List[str] = None + + @property + def filter_query(self) -> List: + def _head_branches_query(): + if not self.head_branches: + return None + + return or_( + RepoWorkflowRuns.head_branch.op("~")(term) + for term in self.head_branches + ) + + def _repo_filters_query(): + if not self.repo_filters: + return None + + return or_( + RepoWorkflowFilter(repo_id, repo_filters).filter_query + for repo_id, repo_filters in self.repo_filters.items() + if repo_filters + ) + + # CLUSTOX: runs with a null event_actor -- Jenkins builds without the + # git plugin, for instance -- are excluded by IN, so per-contributor + # deployment counts will not sum to the team total. That is intended: + # an unattributable deploy belongs to nobody. + def _event_actors_query(): + if not self.event_actors: + return None + + return RepoWorkflowRuns.event_actor.in_(self.event_actors) + + conditions = { + "head_branches": _head_branches_query(), + "repo_filters": _repo_filters_query(), + "event_actors": _event_actors_query(), + } + return [ + conditions[x] + for x in self.__dict__.keys() + if getattr(self, x) is not None and conditions[x] is not None + ] +``` + +- [ ] **Step 4: Parse it from the request** + +In `mhq/service/workflows/workflow_filter.py`, extend `ParseWorkflowFilterProcessor.apply`: + +```python + def apply(self, workflow_filter: Dict = None) -> WorkflowFilter: + workflow_filter = workflow_filter or {} + head_branches: List[str] = self._parse_head_branches(workflow_filter) + repo_filters: Dict[str, Dict] = self._parse_repo_filters(workflow_filter) + # CLUSTOX: contributor filter for deployment frequency. + event_actors: List[str] = workflow_filter.get("event_actors") + + return WorkflowFilter( + head_branches=head_branches, + repo_filters=repo_filters, + event_actors=event_actors, + ) +``` + +Note the added `workflow_filter = workflow_filter or {}` guard — the signature defaults to `None` and the existing body would raise `AttributeError` on a bare `.get`. + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `cd backend/analytics_server && ./venv/bin/python -m pytest tests/store/models/code/workflows/test_workflow_filter_actors.py -v` +Expected: PASS (6 tests) + +- [ ] **Step 6: Run the full suite, format, lint, commit** + +```bash +cd backend/analytics_server +./venv/bin/python -m pytest tests -q # expect 170 passed +./venv/bin/python -m black mhq/store/models/code/workflows/filter.py mhq/service/workflows/workflow_filter.py tests/store/models/code/workflows/test_workflow_filter_actors.py +./venv/bin/python -m flake8 mhq/store/models/code/workflows/filter.py mhq/service/workflows/workflow_filter.py +cd ../.. +git add backend/analytics_server/mhq/store/models/code/workflows/filter.py \ + backend/analytics_server/mhq/service/workflows/workflow_filter.py \ + backend/analytics_server/tests/store/models/code/workflows/test_workflow_filter_actors.py +git commit -m "feat(filters): filter deployments by the actor who triggered them" +``` + +--- + +### Task 3: Contributor list query and route + +**Files:** +- Modify: `backend/analytics_server/mhq/store/repos/code.py` +- Modify: `backend/analytics_server/mhq/api/pull_requests.py` +- Test: `backend/analytics_server/tests/service/code/test_contributors.py` (create) + +**Interfaces:** +- Consumes: nothing from Tasks 1-2. +- Produces: + - `is_bot_author(username: str) -> bool` in `mhq/utils/string.py` + - `CodeRepoService.get_contributors_for_repos(repo_ids: List[str], from_time, to_time) -> List[Tuple[str, int]]` returning `(author, pr_count)` ordered by count descending + - `GET /teams/<team_id>/contributors?from_time=&to_time=` returning `[{"username": str, "pr_count": int}]` + +- [ ] **Step 1: Write the failing bot-exclusion tests** + +Create `backend/analytics_server/tests/service/code/test_contributors.py`: + +```python +from mhq.utils.string import is_bot_author + + +def test_bracket_bot_suffix_is_a_bot(): + assert is_bot_author("dependabot[bot]") is True + assert is_bot_author("renovate[bot]") is True + + +def test_known_bot_names_are_bots(): + assert is_bot_author("dependabot") is True + assert is_bot_author("github-actions") is True + + +def test_a_human_whose_name_contains_bot_is_not_a_bot(): + # Substring matching on "bot" would wrongly exclude these people. + assert is_bot_author("robotnik") is False + assert is_bot_author("botond") is False + assert is_bot_author("abbott") is False + + +def test_none_and_empty_are_not_bots(): + assert is_bot_author(None) is False + assert is_bot_author("") is False + + +def test_matching_is_case_insensitive(): + assert is_bot_author("Dependabot[Bot]") is True +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd backend/analytics_server && ./venv/bin/python -m pytest tests/service/code/test_contributors.py -v` +Expected: FAIL — `ImportError: cannot import name 'is_bot_author'` + +- [ ] **Step 3: Write the bot predicate** + +Append to `backend/analytics_server/mhq/utils/string.py`: + +```python +# CLUSTOX: bot accounts author real pull requests, so they appear in the +# contributor list unless excluded. Matching is on the exact "[bot]" suffix +# plus an explicit list -- a substring match on "bot" would exclude real +# people called robotnik or abbott. +KNOWN_BOT_AUTHORS = frozenset( + { + "dependabot", + "renovate", + "github-actions", + "snyk-bot", + "greenkeeper", + "codecov", + "imgbot", + } +) + + +def is_bot_author(username: Optional[str]) -> bool: + if not username: + return False + + lowered = username.lower() + return lowered.endswith("[bot]") or lowered in KNOWN_BOT_AUTHORS +``` + +Add `from typing import Optional` to that file's imports if it is not already present. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd backend/analytics_server && ./venv/bin/python -m pytest tests/service/code/test_contributors.py -v` +Expected: PASS (5 tests) + +- [ ] **Step 5: Add the repository query** + +Append to `CodeRepoService` in `backend/analytics_server/mhq/store/repos/code.py`: + +```python + # CLUSTOX: distinct authors for the contributor filter, scoped to the + # repos and window currently on screen so the dropdown can never offer + # someone with no data in view. + def get_contributors_for_repos( + self, repo_ids: List[str], from_time: datetime, to_time: datetime + ) -> List[Tuple[str, int]]: + if not repo_ids: + return [] + + rows = ( + self._db.session.query( + PullRequest.author, func.count(PullRequest.id).label("pr_count") + ) + .filter( + PullRequest.repo_id.in_(repo_ids), + PullRequest.author.isnot(None), + PullRequest.state_changed_at.between(from_time, to_time), + ) + .group_by(PullRequest.author) + .order_by(func.count(PullRequest.id).desc()) + .all() + ) + + return [(author, count) for author, count in rows if not is_bot_author(author)] +``` + +Add whatever of `datetime`, `Tuple`, `func`, and `is_bot_author` that file does not already import. Check its existing imports first rather than adding duplicates. + +- [ ] **Step 6: Add the Flask route** + +Append to `backend/analytics_server/mhq/api/pull_requests.py`. The decorator stack below matches `get_lead_time_prs` in the same file (line 60) — `@queryschema` with a voluptuous `Schema`, coercing the ISO timestamps: + +```python +# CLUSTOX: contributor list backing the DORA metrics filter. Scoped to the +# team's repos and the window on screen, so the dropdown cannot offer someone +# with no data in view. +@app.route("/teams/<team_id>/contributors", methods={"GET"}) +@queryschema( + Schema( + { + Required("from_time"): All(str, Coerce(datetime.fromisoformat)), + Required("to_time"): All(str, Coerce(datetime.fromisoformat)), + } + ), +) +def get_team_contributors(team_id: str, from_time: datetime, to_time: datetime): + query_validator = get_query_validator() + team: Team = query_validator.team_validator(team_id) + + repos = CodeRepoService().get_team_repos(team) + repo_ids = [str(repo.id) for repo in repos] + + contributors = CodeRepoService().get_contributors_for_repos( + repo_ids, from_time, to_time + ) + return [ + {"username": username, "pr_count": pr_count} + for username, pr_count in contributors + ] +``` + +`Schema`, `Required`, `All`, `Coerce`, `queryschema`, `get_query_validator`, `Team` and `CodeRepoService` are all already imported by that file — confirm rather than re-import. + +- [ ] **Step 7: Run the full suite, format, lint, commit** + +```bash +cd backend/analytics_server +./venv/bin/python -m pytest tests -q # expect 175 passed +./venv/bin/python -m black mhq/utils/string.py mhq/store/repos/code.py mhq/api/pull_requests.py tests/service/code/test_contributors.py +./venv/bin/python -m flake8 mhq/utils/string.py mhq/store/repos/code.py mhq/api/pull_requests.py +cd ../.. +git add backend/analytics_server/mhq/utils/string.py \ + backend/analytics_server/mhq/store/repos/code.py \ + backend/analytics_server/mhq/api/pull_requests.py \ + backend/analytics_server/tests/service/code/test_contributors.py +git commit -m "feat(contributors): list a team's pull request authors, bots excluded" +``` + +--- + +### Task 4: Thread the filter through the BFF + +**Files:** +- Create: `web-server/pages/api/internal/team/[team_id]/contributors.ts` +- Modify: `web-server/src/api-helpers/team.ts` +- Modify: `web-server/pages/api/internal/team/[team_id]/dora_metrics.ts` + +**Interfaces:** +- Consumes: the Flask route from Task 3. +- Produces: + - `GET /api/internal/team/{team_id}/contributors?from_date=&to_date=` → `[{username, pr_count}]` + - `updatePrFilterParams(teamId, params, filters)` accepts `filters.authors?: string[]` and emits it inside `pr_filter` + - `dora_metrics.ts` forwards `event_actors` into the workflow filter payload + +- [ ] **Step 1: Create the contributors BFF route** + +Create `web-server/pages/api/internal/team/[team_id]/contributors.ts`: + +```ts +import * as yup from 'yup'; + +import { Endpoint } from '@/api-helpers/global'; +import { internal } from '@/api-helpers/axios'; + +// CLUSTOX: backs the contributor filter on the DORA metrics page. team_id is +// validated by Endpoint.serve(), which asserts the caller may read that team. +const pathSchema = yup.object().shape({ + team_id: yup.string().uuid().required() +}); + +const getSchema = yup.object().shape({ + team_id: yup.string().uuid().required(), + from_date: yup.date().required(), + to_date: yup.date().required() +}); + +const endpoint = new Endpoint(pathSchema); + +endpoint.handle.GET(getSchema, async (req, res) => { + const { team_id, from_date, to_date } = req.payload; + + const contributors = await internal.get(`/teams/${team_id}/contributors`, { + params: { from_time: from_date, to_time: to_date } + }); + + res.send(contributors.data); +}); + +export default endpoint.serve(); +``` + +Note `pathSchema` contains only `team_id` — the same construction bug fixed during the Jenkins work, where passing a method-specific schema as the path schema made every other method a permanent 400. + +- [ ] **Step 2: Accept authors in the PR filter builder** + +In `web-server/src/api-helpers/team.ts`, change `updatePrFilterParams`: + +```ts +export const updatePrFilterParams = async <T extends {} = {}>( + _teamId: ID, + params: T, + filters?: Partial<{ + branches: string; + repo_filters: RepoFilterConfig; + // CLUSTOX: contributor filter -- git usernames, as synced. + authors: string[]; + }> +) => { + const updatedParams = { + base_branches: filters?.branches?.split(','), + repo_filters: filters?.repo_filters, + authors: filters?.authors?.length ? filters.authors : undefined + }; + const reducedParams = reject(isNil, updatedParams); + const pr_filter = equals({}, reducedParams) ? null : reducedParams; + + return reject(isNil, { + ...params, + pr_filter + }) as T & { pr_filter?: Partial<typeof updatedParams> }; +}; +``` + +The `?.length ? ... : undefined` matters: an empty array must become `undefined` so `reject(isNil, ...)` drops it. Sending `authors: []` would filter to nobody. + +- [ ] **Step 3: Accept event actors in the workflow filter builder** + +In `web-server/src/utils/filterUtils.ts`, extend `getWorkFlowFiltersAsPayloadForSingleTeam` (line 84). The new parameter is optional, so the other caller — `pages/api/internal/team/[team_id]/deployment_analytics.ts` — is unaffected: + +```ts +export const getWorkFlowFiltersAsPayloadForSingleTeam = async (params: { + orgId: ID; + teamId: ID; + // CLUSTOX: contributor filter. Deployments have no author, so the nearest + // equivalent is the actor who triggered the run. + eventActors?: string[]; +}) => { + const { orgId, teamId, eventActors } = params; + const teamProdBranchesMap = + await getAllTeamsReposProdBranchesForOrgAsMap(orgId); + const filter = Object.fromEntries( + Object.entries(workFlowFiltersFromTeamProdBranches(teamProdBranchesMap)) + )[teamId]; + + if (!eventActors?.length) return filter; + return { ...filter, event_actors: eventActors }; +}; +``` + +Add a case to the existing suite at `web-server/src/utils/__tests__/filterUtils.test.ts` (its `getWorkFlowFiltersAsPayloadForSingleTeam` describe block starts at line 264) asserting that passing `eventActors` adds `event_actors` to the payload, and that omitting it leaves the payload unchanged. + +- [ ] **Step 4: Thread the selection through the DORA route** + +In `web-server/pages/api/internal/team/[team_id]/dora_metrics.ts`, add `authors: yup.array().of(yup.string()).optional()` to the request schema, destructure it alongside the other params, then change the `Promise.all` block at line 66: + +```ts + const [prFilters, workflowFilters] = await Promise.all([ + updatePrFilterParams(teamId, {}, { ...branchAndRepoFilters, authors }).then( + ({ pr_filter }) => ({ + pr_filter + }) + ), + getWorkFlowFiltersAsPayloadForSingleTeam({ + orgId: org_id, + teamId: teamId, + eventActors: authors + }) + ]); +``` + +The same `authors` value feeds both builders — that is what makes one control drive both metrics. + +- [ ] **Step 5: Run the frontend unit tests** + +Run: `docker exec middleware-dev sh -c 'cd /app/web-server && yarn test --testPathPattern=filterUtils'` +Expected: PASS, including the two cases added in Step 3 + +- [ ] **Step 6: Verify types compile** + +```bash +docker cp web-server/pages/api/internal/team/\[team_id\]/contributors.ts middleware-dev:/app/web-server/pages/api/internal/team/\[team_id\]/contributors.ts +docker cp web-server/src/api-helpers/team.ts middleware-dev:/app/web-server/src/api-helpers/team.ts +docker cp web-server/src/utils/filterUtils.ts middleware-dev:/app/web-server/src/utils/filterUtils.ts +docker cp web-server/pages/api/internal/team/\[team_id\]/dora_metrics.ts middleware-dev:/app/web-server/pages/api/internal/team/\[team_id\]/dora_metrics.ts +docker exec middleware-dev sh -c 'cd /app/web-server && ./node_modules/.bin/tsc --noEmit 2>&1 | grep -c "error TS"' +``` +Expected: `0` + +The container now runs this branch, so these files overwrite their own current versions — no restore step is needed, but re-copy from git if you abandon a change. + +- [ ] **Step 7: Commit** + +```bash +git add web-server/pages/api/internal/team/\[team_id\]/contributors.ts \ + web-server/src/api-helpers/team.ts \ + web-server/src/utils/filterUtils.ts \ + web-server/src/utils/__tests__/filterUtils.test.ts \ + web-server/pages/api/internal/team/\[team_id\]/dora_metrics.ts +git commit -m "feat(contributors): expose the contributor list and thread the filter" +``` + +--- + +### Task 5: Contributor dropdown + +**Files:** +- Create: `web-server/src/components/ContributorFilter.tsx` +- Modify: `web-server/pages/dora-metrics/index.tsx` + +**Interfaces:** +- Consumes: `GET /api/internal/team/{team_id}/contributors` from Task 4. +- Produces: nothing consumed by later tasks. + +- [ ] **Step 1: Build the dropdown** + +Create `web-server/src/components/ContributorFilter.tsx`: an MUI `Autocomplete` (multiple, `size="small"`) fetching from `/api/internal/team/${teamId}/contributors` whenever `teamId`, `fromDate` or `toDate` change. Each option shows the username with its PR count as secondary text. Follow the styling conventions in `web-server/src/components/ClustoxWorkspaceMetrics.tsx`. + +Required behaviours: + +- Selection is held in the URL query string, so the filter survives a refresh and can be shared. +- When the fetched list no longer contains a selected username — because the team or date range changed — drop that selection and show a brief inline notice. Do not silently clear it; the numbers would move with no visible cause. +- While loading, disable the control rather than showing an empty list, which reads as "no contributors". +- On fetch failure, disable the control and show the error inline. The dashboard must still render unfiltered — the filter is an enhancement, and losing it should not cost the page. +- When the list is empty, disable the control with the text "No contributors in this range". + +- [ ] **Step 2: Render it and label the metrics** + +In `web-server/pages/dora-metrics/index.tsx`, render `ContributorFilter` alongside the existing selectors and pass the selection into the metrics requests. + +When a contributor is selected, each affected card gains a qualifier: + +- Lead Time: `authored by <username>` +- Deployment Frequency: `deployed by <username>` + +These wordings are deliberate and load-bearing. One control drives two different relationships — the PR author for Lead Time, the deploy actor for Deployment Frequency — and those are often different people. Without the qualifiers, a user seeing the two numbers disagree will reasonably conclude the page is broken. + +Change Failure Rate and MTTR are **not** filtered. When a contributor is selected, show on both: `team-wide — per-contributor arrives with Jira`. + +- [ ] **Step 3: Verify types and lint** + +```bash +docker cp web-server/src/components/ContributorFilter.tsx middleware-dev:/app/web-server/src/components/ContributorFilter.tsx +docker cp web-server/pages/dora-metrics/index.tsx middleware-dev:/app/web-server/pages/dora-metrics/index.tsx +docker exec middleware-dev sh -c 'cd /app/web-server && ./node_modules/.bin/tsc --noEmit 2>&1 | grep -c "error TS"' +``` +Expected: `0` + +- [ ] **Step 4: Verify by hand** + +The container runs this branch, so the running app at `http://localhost:3333/dora-metrics` picks these up. Confirm: + +1. The dropdown lists contributors with PR counts, no `[bot]` entries. +2. Selecting one changes Lead Time and Deployment Frequency; CFR and MTTR do not move. +3. Both filtered cards show their qualifier. +4. Refreshing preserves the selection. +5. Narrowing the date range until the selected contributor has no PRs clears the selection with a notice. + +- [ ] **Step 5: Commit** + +```bash +git add web-server/src/components/ContributorFilter.tsx \ + web-server/pages/dora-metrics/index.tsx +git commit -m "feat(contributors): add the contributor filter to dora metrics" +``` + +--- + +## Final verification + +- [ ] `cd backend/analytics_server && ./venv/bin/python -m pytest tests -q` — expect 175 passed +- [ ] `./venv/bin/python -m black --check mhq && ./venv/bin/python -m flake8 mhq` — clean +- [ ] `docker exec middleware-dev sh -c 'cd /app/web-server && ./node_modules/.bin/tsc --noEmit'` — 0 errors +- [ ] Unfiltered dashboards are byte-identical to before this work — the regression tests in Tasks 1 and 2 cover the query layer, but confirm in the running app too + +**The test that matters most** is the one asserting the numbers *change*. The defect being fixed here is a filter that was accepted and never applied; a test that only checks the request succeeds would have passed against the broken behaviour. diff --git a/docs/FORK_STRATEGY.md b/docs/FORK_STRATEGY.md new file mode 100644 index 000000000..7338d84bf --- /dev/null +++ b/docs/FORK_STRATEGY.md @@ -0,0 +1,448 @@ +# Fork & Upstream Maintenance Strategy + +| | | +|---|---| +| **Fork** | [Clustox/middleware](https://github.com/Clustox/middleware) | +| **Upstream** | [middlewarehq/middleware](https://github.com/middlewarehq/middleware) | +| **Upstream licence** | Apache License 2.0 | +| **Fork point** | `844eb42` on `main` | +| **Divergence today** | **Zero commits.** Clean fork. | +| **Status** | Proposed — needs sign-off before the first customisation lands | + +--- + +## Why this document exists, and why now + +Upstream is **actively developed** — the fork point sits on a merge commit from a PR numbered #699, +with fixes landing days apart. Every commit they make from here is one we either take or consciously +decline. + +Right now this fork has **zero divergence**. That is the cheapest moment in the entire life of a fork +to decide how it will be maintained. The cost of deciding later compounds: after twenty ad-hoc +customisations scattered through upstream files, "just pull upstream" becomes a multi-day merge with +real risk of silently reverting our own work. + +The failure mode is specific and common: six months in, an upstream security fix lands, the merge +conflicts in fifteen files, someone resolves it under time pressure, and a Clustox customisation +quietly disappears. Nobody notices until a customer does. + +Verify the current state: + +```bash +git log --oneline -1 && git remote -v && git status --short +``` + +--- + +## 1. Decide the fork's posture first + +Everything downstream follows from one question. **This needs a decision from the CTO before the first +customisation is written**, because it determines how much isolation discipline is worth paying for. + +| Posture | What it means | Upstream merge cost | Right when | +|---|---|---|---| +| **A. Track closely** | Stay near-identical to upstream. Customisations are thin and few. Contribute fixes back. | Low, ongoing | We mainly want the product, plus small tweaks | +| **B. Soft fork** | Meaningful Clustox features, but deliberately isolated so upstream files stay mostly untouched. Periodic upstream merges. | Moderate, manageable | **Most likely fit** — we want their engine plus our differentiation | +| **C. Hard fork** | Take the code and go. Upstream becomes a one-time donor. | N/A — we stop merging | Our direction diverges fundamentally | + +**Recommendation: B, with the discipline of A.** + +Rationale: the upstream codebase is cleanly layered and full of factory/strategy seams (see +[ARCHITECTURE.md §12](./ARCHITECTURE.md#12-extension-points)) that were clearly designed for extension. +Most of what a client would plausibly ask for — a new Git provider, a new deployment source, a new +metric — can be added by *registering* a new implementation rather than editing existing files. That is +exactly the shape that makes posture B cheap. Choosing C forfeits upstream's ongoing work for very +little gain. + +The rest of this document assumes B. If the CTO picks A or C, sections 4 and 5 change; the rest holds. + +> **Open question for the CTO — worth asking early:** is this fork intended to stay internal, or ship +> to clients? It changes the licence obligations in §7 and the auth work in +> [ARCHITECTURE.md §11.1](./ARCHITECTURE.md#111-no-authentication-on-the-backend-apis) from +> "nice to have" to "blocking". + +--- + +## 2. Remote setup + +The fork currently has **only `origin`**. There is no way to fetch upstream at all. First fix: + +```bash +git remote add upstream https://github.com/middlewarehq/middleware.git +git remote set-url --push upstream DISABLED +git fetch upstream +``` + +The `--push DISABLED` line is deliberate: it makes an accidental `git push upstream` fail loudly +instead of attempting to write to a repository we do not own. Cheap insurance. + +Verify: + +```bash +git remote -v +# origin https://github.com/Clustox/middleware.git (fetch) +# origin https://github.com/Clustox/middleware.git (push) +# upstream https://github.com/middlewarehq/middleware.git (fetch) +# upstream DISABLED (push) +``` + +Every engineer on the project runs this once. Add it to the onboarding checklist — a teammate without +the upstream remote cannot review a sync PR properly. + +--- + +## 3. Branch model + +```mermaid +gitGraph + commit id: "844eb42 fork point" + commit id: "clustox docs" + commit id: "clustox feature A" + branch upstream-sync + commit id: "merge upstream" type: HIGHLIGHT + commit id: "resolve conflicts" + checkout main + merge upstream-sync + commit id: "clustox feature B" +``` + +`upstream-sync` is cut from `main`, absorbs the upstream merge and all conflict resolution, then goes +back into `main` through a reviewed PR and is deleted. `main` never takes a raw upstream merge directly. + +| Branch | Role | +|---|---| +| `main` | **Clustox's product.** Protected. Upstream + our changes. Deployable. | +| `upstream-sync` | Short-lived, one per sync. Where upstream merges and conflict resolution happen. Deleted after merge. | +| `feature/*` | Normal feature work off `main`. | +| `fix/*` | Bug fixes off `main`. | +| `contrib/*` | Work intended for an upstream PR. Kept free of Clustox-specific code. | + +**We do not keep a branch that mirrors upstream `main` verbatim** — `upstream/main` already is that +branch, fetched on demand. An extra local mirror is one more thing to forget to update. + +### Branch protection to enable on `main` + +Configure in GitHub repo settings (needs org admin): + +- Require a pull request before merging, minimum 1 approval +- Require status checks: `build`, `pytest`, `black`, `pre-commit` +- Dismiss stale approvals on new commits +- No force-push, no deletion + +Rationale: the entire strategy below depends on **sync merges being reviewed**. Without protection, +one `git push --force` to `main` undoes it. + +--- + +## 4. Where Clustox changes are allowed to live + +This is the core discipline of the whole document. **Merge pain is almost perfectly proportional to +how many upstream files we edit.** Adding files is nearly free; editing shared files is where conflicts +come from. + +### The three-tier rule + +```mermaid +graph TB + G["🟢 GREEN — new files<br/>Conflict risk: ~none<br/>No approval needed"] + Y["🟡 YELLOW — registration points<br/>Conflict risk: small, mechanical<br/>Note it in the PR"] + R["🔴 RED — editing upstream logic<br/>Conflict risk: high, semantic<br/>Justify in the PR"] + G --> Y --> R + style G fill:#2d6a4f,color:#fff + style Y fill:#b7791f,color:#fff + style R fill:#9b2226,color:#fff +``` + +**🟢 Green — prefer this. Always ask "can this be a new file?" first.** + +- New provider clients: `mhq/exapi/<provider>.py` +- New ETL handlers: `mhq/service/*/sync/etl_<provider>_handler.py` +- New deployment strategies: `mhq/service/deployments/<name>_deployments_service.py` +- New API blueprints: `mhq/api/<name>.py` +- New UI pages/components under new paths +- Anything under `docs/` or `clustox/` + +**🟡 Yellow — unavoidable, keep surgical.** Upstream's extension seams are *designed* to be edited, but +they are also the lines upstream touches when they add their own providers. Conflicts here are +one-line and mechanical. + +- Factories: `etl_code_factory.py`, `etl_workflows_factory.py`, `deployments/factory.py` +- Enums: `CodeProvider`, `RepoWorkflowProviders`, `DeploymentType` +- Blueprint registration in `app.py` / `sync_app.py` +- `sync_sequence` in `sync_data.py` + +Keep the diff to the minimum lines. Add at the **end** of enums and lists, never in the middle — a +trailing addition conflicts far less often than an insertion. + +**🔴 Red — requires explicit justification in the PR description.** Editing existing upstream logic. +State what you changed, why a green/yellow approach did not work, and what breaks if an upstream merge +silently reverts it. + +- Metric computation: `lead_time.py`, `analytics.py`, `incidents.py` +- The MTD broker: `mtd_handler.py` +- Existing models in `mhq/store/models/**` +- `web-server/src/utils/db.ts`, `constants/db.ts` +- Anything in `setup_utils/`, `Dockerfile*`, `docker-compose.yml` + +### Marking red-zone edits so a merge cannot silently eat them + +Every edit to an upstream file gets a sentinel comment: + +```python +# CLUSTOX: <what and why> — <ticket> +... our change ... +# END CLUSTOX +``` + +```typescript +// CLUSTOX: <what and why> — <ticket> +``` + +```sql +-- CLUSTOX: <what and why> — <ticket> +``` + +This is not decoration. It gives us a **one-command audit** of our entire divergence footprint, which +becomes the review checklist during every upstream sync: + +```bash +grep -rn "CLUSTOX:" --include='*.py' --include='*.ts' --include='*.tsx' --include='*.sql' . \ + | grep -v node_modules +``` + +After a sync merge, this count should not drop. If it does, the merge ate one of our changes. + +### The database exception + +Custom columns and tables need their own migration files (green — new files), but they mutate a schema +upstream also migrates. Two rules: + +- **Prefix custom migrations** `<timestamp>_clustox_<name>.sql` so they are identifiable at a glance. +- **Never edit an upstream migration that has already run.** `dbmate` tracks applied migrations; editing + one that has run is a silent no-op locally and a divergence in every other environment. +- Remember custom columns are a **four-place change** — see + [ARCHITECTURE.md §12.5](./ARCHITECTURE.md#125-add-a-database-column). + +--- + +## 5. The upstream sync procedure + +**Cadence: monthly, plus immediately for any upstream security fix.** Monthly is frequent enough that +each merge is small, infrequent enough that it is not a tax. Put it on someone's calendar — syncs that +depend on someone remembering do not happen. + +### Step 1 — See what is coming before committing to it + +```bash +git fetch upstream +git log --oneline main..upstream/main | wc -l # how many commits behind +git log --oneline main..upstream/main # what they are +git diff --stat main..upstream/main # blast radius +``` + +Then check the overlap between what upstream changed and what we have touched: + +```bash +# Files upstream changed +git diff --name-only main..upstream/main | sort > /tmp/upstream-changed.txt +# Files we changed since the fork point +git diff --name-only 844eb42..main | sort > /tmp/clustox-changed.txt +# The intersection is exactly where conflicts will be +comm -12 /tmp/upstream-changed.txt /tmp/clustox-changed.txt +``` + +That last command is the whole risk assessment in one line. Empty output means a trivial merge. + +### Step 2 — Merge on a branch, never on `main` + +```bash +git checkout main && git pull origin main +git checkout -b upstream-sync/$(date +%Y-%m) +git merge upstream/main +``` + +**Merge, not rebase.** `main` is shared and protected; rebasing rewrites published history. A merge +commit also leaves an honest record of exactly when each upstream batch arrived. + +### Step 3 — Resolve, with the sentinel audit as the checklist + +For each conflict, the question is always: *does upstream's change subsume ours, or must both survive?* + +```bash +git status --short | grep '^UU' # conflicted files +git checkout --theirs <path> # take upstream wholesale +git checkout --ours <path> # keep ours wholesale +# — but usually: edit by hand and keep both +``` + +Then confirm nothing of ours vanished: + +```bash +grep -rn "CLUSTOX:" --include='*.py' --include='*.ts' --include='*.tsx' --include='*.sql' . \ + | grep -v node_modules | wc -l +``` + +Compare against the pre-merge count. A drop means a resolution dropped one of our changes — go find it. + +### Step 4 — Verify + +```bash +# Backend +cd backend/analytics_server && python -m pytest tests/ -q && cd ../.. +black --check backend/ +flake8 backend/ + +# Frontend +cd web-server && yarn jest && cd .. + +# Migrations still apply, app still boots +docker compose down -v && docker compose up --build +``` + +Then a manual smoke pass, because the automated suite is thin (20 Python test files, and the BFF/knex +path is largely uncovered — see +[ARCHITECTURE.md §10.5](./ARCHITECTURE.md#105-ci-and-tests)): + +- [ ] All six supervisord processes running (`docker compose logs`) +- [ ] UI loads at :3333 +- [ ] Integration settings page renders and the existing PAT still decrypts +- [ ] `curl -X POST http://localhost:9697/sync` completes; check `/system-logs` for per-repo errors +- [ ] All four DORA metrics render with data +- [ ] Every Clustox customisation still behaves + +**The PAT-decrypt check matters more than it looks.** Token encryption depends on the RSA keypair in +the `dev_keys` volume; a `docker compose down -v` destroys it. If upstream changes anything about +`utils/cryptography.py` or the config bootstrap, this is where it surfaces. + +### Step 5 — PR into `main` + +Open `upstream-sync/YYYY-MM` → `main`, with a description covering: + +- Commit count and date range pulled +- Notable upstream changes affecting us +- Every conflict and how it was resolved +- Sentinel count before/after +- Smoke-test results + +Reviewed by someone who did not do the merge. This is the control that catches a silently reverted +customisation, and it is the reason `main` is protected. + +### Step 6 — Record it + +Append to `docs/UPSTREAM_SYNC_LOG.md`: + +| Date | Upstream ref | Commits | Conflicts | Notes | +|---|---|---|---|---| +| _(first sync)_ | | | | | + +--- + +## 6. Contributing back to upstream + +Worth doing deliberately, for two self-interested reasons: **anything upstream accepts is code we no +longer maintain**, and a visible Clustox presence in an active OSS project is good for the company. + +**Send upstream:** genuine bug fixes, provider integrations of general use, performance work, doc fixes. +**Keep private:** Clustox branding, client-specific logic, anything commercially differentiating. + +Work on a `contrib/*` branch cut from `upstream/main` (not from `main`), so the PR contains no Clustox +code and no sentinel comments: + +```bash +git fetch upstream +git checkout -b contrib/<short-name> upstream/main +# ... make the change, no CLUSTOX: markers ... +git push origin contrib/<short-name> +# open PR: Clustox/middleware contrib/<name> → middlewarehq/middleware main +``` + +Read [`CONTRIBUTING.md`](../CONTRIBUTING.md) and [`CODE_OF_CONDUCT.md`](../CODE_OF_CONDUCT.md) first; +upstream has an active community and its own review conventions. + +> **Check before the first external contribution:** whether Clustox requires anything of employees +> contributing to third-party OSS, and whether upstream asks for a CLA. Better to ask than to retract. + +--- + +## 7. Licence obligations (Apache 2.0) + +Not legal advice — flag anything commercially significant to whoever owns legal at Clustox. But the +mechanics are simple and we are currently not meeting one of them. + +**What Apache 2.0 permits:** commercial use, modification, private use, distribution, sublicensing. +We can build a product on this and sell it. We are **not** required to open-source our changes. + +**What it requires when we distribute:** + +| Requirement | Status | +|---|---| +| Include the Apache 2.0 licence text | ✅ [`LICENSE`](../LICENSE) is present | +| Preserve copyright/attribution notices | ⚠️ **Gap** — see below | +| State significant changes made to the files | ⚠️ Not yet — the `CLUSTOX:` sentinels are the mechanism | +| Include the `NOTICE` file if one exists | ⚠️ **Gap — no `NOTICE` file exists** | + +### The gap, and the fix + +Apache 2.0 §4(d) requires preserving attribution notices in derivative works. This fork has **no +`NOTICE` file** — verify with `ls NOTICE`. Upstream did not ship one, so strictly we are not yet +failing to propagate theirs; but once we distribute a modified version, clean attribution is both an +obligation and simply good practice. + +A [`NOTICE`](../NOTICE) file has been added in this change set recording upstream authorship and +Clustox's modifications. Keep it current as the fork diverges. + +**Also do not:** use the Middleware name or logo to brand a Clustox product (trademarks are not granted +by Apache 2.0 — §6), or imply upstream endorsement. If we ship this to clients under a Clustox name, +`media_files/` and README branding need review. + +--- + +## 8. Immediate actions + +Ordered. Items 1–4 are the ones to do before any customisation lands. + +| # | Action | Owner | Status | +|---|---|---|---| +| 1 | **CTO decides the fork posture** (§1) — blocks the rest | CTO | ⬜ | +| 2 | Add the `upstream` remote, push-disabled (§2) | Every engineer | ⬜ | +| 3 | Enable branch protection on `main` (§3) | Repo admin | ⬜ | +| 4 | Agree the three-tier rule and `CLUSTOX:` sentinel (§4) | Team | ⬜ | +| 5 | Add `NOTICE`, keep it current (§7) | — | ✅ added | +| 6 | Confirm CI is green on the fork, record the baseline | — | ⬜ | +| 7 | Create `docs/UPSTREAM_SYNC_LOG.md`, do a first sync to prove the runbook | — | ⬜ | +| 8 | Put the monthly sync on a calendar with a named owner | Team lead | ⬜ | + +Two notes on sequencing. **Item 6 before any code changes** — if upstream CI is already failing on our +fork, that is far better discovered now than blamed on our first feature. And **item 7's first sync +should happen while divergence is still zero**, precisely because it is guaranteed trivial: it proves +the runbook works before we need it under pressure. + +--- + +## Appendix — Command reference + +```bash +# One-time setup +git remote add upstream https://github.com/middlewarehq/middleware.git +git remote set-url --push upstream DISABLED + +# How far behind are we? +git fetch upstream && git log --oneline main..upstream/main | wc -l + +# Where will conflicts happen? +comm -12 \ + <(git diff --name-only main..upstream/main | sort) \ + <(git diff --name-only 844eb42..main | sort) + +# Our full divergence footprint +grep -rn "CLUSTOX:" --include='*.py' --include='*.ts' --include='*.tsx' --include='*.sql' . \ + | grep -v node_modules + +# Start a sync +git checkout main && git pull origin main +git checkout -b upstream-sync/$(date +%Y-%m) +git merge upstream/main + +# Start an upstream contribution (clean of Clustox code) +git checkout -b contrib/<name> upstream/main +``` diff --git a/docs/JENKINS_INTEGRATION.md b/docs/JENKINS_INTEGRATION.md new file mode 100644 index 000000000..b6274e22f --- /dev/null +++ b/docs/JENKINS_INTEGRATION.md @@ -0,0 +1,228 @@ +# Jenkins integration — design + +**Status:** approved, not implemented +**Date:** 2026-08-10 +**Estimate:** 5–6 days + +Jenkins builds become deployments, so Deployment Frequency and Lead Time +reflect how teams actually ship rather than only what GitHub Actions does. + +--- + +## Decisions + +Four decisions shape everything below. They were settled before design, and +changing any of them changes the design rather than the implementation. + +| Decision | Choice | Consequence | +|---|---|---| +| Network direction | **Pull** — Middleware calls Jenkins | Jenkins is reachable from the Middleware host, so this fits the existing ETL contract with no new ingest endpoint. | +| Job → repo mapping | **Manual, in the UI** | Explicit and accurate. Mapping a job is also what designates it a deployment, so PR builds, lint jobs, and nightlies are ignored by default — and staging jobs are excluded simply by not mapping them. | +| Repo with two CI systems | **One deployment source per repo** | Mapping Jenkins deactivates that repo's active deployment workflows, after a UI warning. Prevents silent double-counting of Deployment Frequency. Unmapping restores exactly the rows that mapping switched off — recorded on the Jenkins row's `meta` — and nothing else: a repo can hold deployment workflows the admin deselected in the team config, which Jenkins never touched. | +| Delivery shape | **Single slice** | One PR rather than a thin proof first. Simpler to plan; the Jenkins API risk (below) surfaces later than it otherwise would. | + +## The risk worth naming up front + +Jenkins' REST API varies by version, by job type (freestyle, pipeline, +multibranch), and by installed plugins. This design is written against the +documented API. Whether it matches *your* Jenkins is not knowable until it +talks to one. + +Everything else here is predictable. This is not. Under the single-slice +approach it is verified near the end, so budget for follow-up work. + +--- + +## Architecture + +Nothing downstream of the sync changes. `sync_org_workflows` already iterates +active `RepoWorkflow` rows and asks a factory for a handler per provider. +Jenkins is a third branch in that factory. Deployment frequency, merge-to-deploy +and lead time read `RepoWorkflowRuns` and never ask where a run came from. + +### Components + +**`JenkinsETLHandler`** (new) implements `WorkflowProviderETLHandler`, whose +whole contract is two methods: + +- `check_pat_validity()` — `GET {base_url}/api/json`, expect 200. +- `get_workflow_runs(org_repo, repo_workflow, bookmark)` — `GET + {base_url}/job/{job_path}/api/json` with a `tree=` selector; keep builds + newer than the bookmark; return them and the new high-water mark. + +**`WorkflowETLFactory`** (changed) — one branch returning the Jenkins handler. + +**`RepoWorkflowProviders`** (changed) — add `JENKINS = "jenkins"`. No migration +required: despite the SQLAlchemy `ENUM()` wrapper, `RepoWorkflow.provider` is +persisted as `character varying` and the schema declares no Postgres enum +types, so a new provider is a code change only. + +**Job mapping API and UI** (new) — list the workspace's Jenkins jobs; create a +mapping. A mapping writes a `RepoWorkflow` row with `provider=JENKINS`, +`org_repo_id` set to the chosen repo, and `provider_workflow_id` set to the +job's full path form, so folders and multibranch jobs address correctly. + +**One-source enforcement** (new) — creating a Jenkins mapping sets +`is_active=False` on that repo's `GITHUB_ACTIONS` workflows. Reversible: the +rows survive. + +### Credentials + +No schema change. `Integration` is keyed `(org_id, name)`, giving one Jenkins +connection per workspace, exactly like GitHub. Jenkins needs three values and +they split across existing columns: + +| Value | Column | +|---|---| +| API token | `access_token_enc_chunks` (encrypted, as GitHub's PAT) | +| Base URL, username | `provider_meta` (JSONB) | + +Jenkins uses HTTP basic auth as `username:api_token`. The username is a +non-secret companion to the token, so it belongs in metadata. + +### Deliberately excluded + +No new tables. No change to metric computation. No abstraction layer over +providers — the contract is two methods, and there is nothing worth +generalising until a third real provider shows what actually varies. + +--- + +## Data flow + +``` +cron (30 min) → POST /sync → per workspace → sync_org_workflows(org_id) + → for each active RepoWorkflow + → WorkflowETLFactory(provider) → JenkinsETLHandler + → bookmark = BookmarkService.get_bookmark(...) + → GET {base}/job/{path}/api/json?tree=builds[...] + → keep builds newer than bookmark → [RepoWorkflowRuns] + → persist, then advance bookmark to newest conducted_at + → process_merge_to_deploy_cache → lead time, deployment frequency +``` + +The bookmark advances **only after a successful persist**, so a mid-sync +failure re-fetches its window rather than silently skipping it. + +Backfill needs no special handling: `BookmarkService` already defaults a first +run to `DEFAULT_SYNC_DAYS` (31, overridable per workspace) rather than to the +present moment. This is the setting behind the earlier "no history after +connecting a repo" behaviour; Jenkins inherits the fix. + +### Build → `RepoWorkflowRuns` + +| Jenkins field | Column | Note | +|---|---|---| +| `number` | `provider_workflow_run_id` | as string | +| `result` | `status` | mapped below | +| `timestamp` | `conducted_at` | epoch ms → UTC datetime | +| `duration` | `duration` | ms → seconds, matching the Actions handler | +| `url` | `html_url` | | +| `actions[].lastBuiltRevision.branch` | `head_branch` | git plugin | +| `actions[].causes[].userId` | `event_actor` | `timer` / `scm` when auto-triggered | +| commit SHA | `meta` | unused for now; enables commit-level de-duplication later without a migration | + +### Status mapping + +``` +SUCCESS → SUCCESS +FAILURE → FAILURE +ABORTED → CANCELLED +null → PENDING (still running) +UNSTABLE → FAILURE +``` + +`UNSTABLE` means the build finished but something — usually tests — failed. +Mapping it to `FAILURE` keeps it out of Deployment Frequency. A team that ships +unstable builds anyway would be undercounted; that is the safer direction than +counting a red build as a successful deploy. Flip it if your Jenkins uses +`UNSTABLE` differently. + +### Git plugin dependency + +`head_branch`, `event_actor` and the commit SHA all come from the git plugin's +contribution to `actions[]`. A job without the git plugin, or a freestyle job +with no SCM, returns none of it. Those builds still record as deployments with +correct timing, but with a null branch, so branch filtering will not apply to +them. + +This is graceful degradation, not failure — but it will show up as nulls in the +data and should not be mistaken for a bug. + +--- + +## Error handling + +Every case degrades to "this workspace's sync records FAILED with a reason, +other workspaces continue" — the isolation the multi-workspace sync already +provides. Jenkins-specific handling: + +| Failure | Handling | +|---|---| +| Token revoked or wrong | `check_pat_validity` fails before any job is fetched. Recorded with reason, visible on the Workspaces page. | +| Jenkins unreachable | Bounded connect and read timeouts. Recorded; not retried within the run. | +| Job deleted or renamed | 404 for that job. Record the detail, **keep the mapping**, continue to the next workflow. | +| Build JSON missing fields | Skip that build, log it, keep the rest of the batch. | +| Any of the above | Bookmark not advanced; the window is re-fetched next cycle. | + +Keeping the mapping on a 404 is deliberate. A renamed job and a briefly +unreachable Jenkins are indistinguishable over the API, and silently unmapping +a repo on a transient error is worse than surfacing a failure an admin can act +on. + +### Timeouts are load-bearing + +The workspace sync loop is sequential. Without an explicit timeout, one hanging +Jenkins stalls every workspace queued behind it, turning a workspace-level +problem into an instance-level outage. The client therefore sets bounded +connect and read timeouts and a per-job ceiling. + +Jenkins is the first integration likely to actually trigger the sequential-loop +limitation. If it becomes a real constraint, the fix is parallelising the +workspace loop — out of scope here. + +### TLS verification + +Internal Jenkins instances frequently use self-signed certificates. + +**Decision: verify certificates, with no skip option in v1.** Add an escape +hatch only if a real Jenkins requires it. A skip-verification flag is easy to +add, hard to remove once relied upon, and silently accepts man-in-the-middle on +a connection carrying an API token. + +--- + +## Testing + +**pytest, against recorded Jenkins JSON fixtures:** + +- Status mapping across all five `result` values, including `null` and `UNSTABLE` +- Epoch-ms → UTC, and ms → seconds conversions +- Bookmark advances to newest build on success; unchanged on failure +- Builds with no git plugin data record with a null branch rather than raising +- Malformed build is skipped without failing the batch +- Factory returns the Jenkins handler for `JENKINS` + +**Request-level e2e, matching the existing isolation suites:** + +- An admin cannot list or map another workspace's Jenkins jobs → 403 +- Mapping endpoints reject unauthenticated requests → 401 +- Mapping a Jenkins job deactivates that repo's GitHub Actions workflows + +**Known gap:** none of this proves the integration works against a specific +Jenkins. Fixtures are recorded from a real instance and therefore encode +assumptions that may not hold for another version or plugin set. That +verification is manual, must happen against the target server, and is the most +likely source of follow-up work. + +--- + +## Out of scope + +- Jenkins as a source of **incidents** — `IncidentProvider` supports only + github and gitlab; separate work. +- Build **logs** or artefacts. +- Triggering builds from Middleware. Read-only by design: the integration + observes deployments, it does not perform them. +- Auto-discovering job → repo mappings from SCM configuration. +- Parallelising the workspace sync loop. diff --git a/docs/JENKINS_INTEGRATION_PLAN.md b/docs/JENKINS_INTEGRATION_PLAN.md new file mode 100644 index 000000000..f87c88c61 --- /dev/null +++ b/docs/JENKINS_INTEGRATION_PLAN.md @@ -0,0 +1,1269 @@ +# Jenkins Integration Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Jenkins builds are recorded as deployments so Deployment Frequency and Lead Time reflect how teams actually ship. + +**Architecture:** A pull adapter implementing the existing two-method `WorkflowProviderETLHandler` contract. Nothing downstream changes — metric computation reads `RepoWorkflowRuns` and never asks which provider produced a run. Credentials reuse the `Integration` table; job-to-repo mapping is manual and doubles as designating a job a deployment. + +**Tech Stack:** Python 3.9, Flask, SQLAlchemy 2, pytest (backend); Next.js 15 pages router, TypeScript, MUI 5, Playwright (frontend). + +**Design spec:** `docs/JENKINS_INTEGRATION.md`. Read it before starting. + +## Global Constraints + +- **No database migration.** `RepoWorkflow.provider` is `character varying`; the schema declares no Postgres enum types. Adding a provider is a code change only. +- **No new tables.** Credentials go in `Integration` (`access_token_enc_chunks` + `provider_meta`); mappings are `RepoWorkflow` rows. +- **TLS verification is always on.** No skip-verify option. Do not add one. +- **All HTTP calls to Jenkins carry explicit timeouts** — `(5, 30)` connect/read. The workspace sync loop is sequential; an untimed request stalls every workspace behind it. +- **Bookmark advances only after a successful persist.** On any error, return the incoming bookmark unchanged. +- Backend style: `black` formatted, `flake8` clean. Frontend: `tsc` clean. +- Every Clustox-authored change carries a `# CLUSTOX:` or `// CLUSTOX:` comment explaining why, per `docs/FORK_STRATEGY.md`. + +--- + +## File Structure + +**Backend — create:** +- `mhq/exapi/jenkins.py` — HTTP client. Knows Jenkins' REST shape, knows nothing about our models. +- `mhq/service/workflows/sync/etl_jenkins_handler.py` — ETL handler. Maps Jenkins JSON to `RepoWorkflowRuns`. +- `mhq/utils/jenkins.py` — reads base URL and username out of `Integration.provider_meta`. +- `tests/factories/models/exapi/jenkins.py` — build fixtures. +- `tests/service/workflows/sync/test_etl_jenkins_handler.py` — handler tests. +- `tests/exapi/test_jenkins.py` — client tests. + +**Backend — modify:** +- `mhq/store/models/code/workflows/enums.py` — add `JENKINS`. +- `mhq/store/models/integrations/enums.py` — add `JENKINS` to `UserIdentityProvider`. +- `mhq/service/workflows/sync/etl_workflows_factory.py` — register the handler. + +**Frontend — create:** +- `web-server/pages/api/clustox/jenkins/jobs.ts` — list jobs. +- `web-server/pages/api/clustox/jenkins/mappings.ts` — create/delete mappings. +- `web-server/src/components/ClustoxJenkinsSetup.tsx` — credentials form. +- `web-server/src/components/ClustoxJenkinsMapping.tsx` — job-to-repo table. +- `web-server/e2e/jenkins.spec.ts` — isolation tests. + +**Frontend — modify:** +- `web-server/src/constants/integrations.ts` — add `JENKINS`. +- `web-server/pages/integrations.tsx` — render the Jenkins card. + +--- + +### Task 1: Register Jenkins as a provider + +**Files:** +- Modify: `backend/analytics_server/mhq/store/models/code/workflows/enums.py` +- Modify: `backend/analytics_server/mhq/store/models/integrations/enums.py` +- Test: `backend/analytics_server/tests/service/workflows/sync/test_etl_workflows_factory.py` (create) + +**Interfaces:** +- Consumes: nothing. +- Produces: `RepoWorkflowProviders.JENKINS` (value `"jenkins"`), `UserIdentityProvider.JENKINS` (value `"jenkins"`). + +- [ ] **Step 1: Write the failing test** + +Create `backend/analytics_server/tests/service/workflows/sync/test_etl_workflows_factory.py`: + +```python +import pytest + +from mhq.service.workflows.sync.etl_workflows_factory import WorkflowETLFactory +from mhq.store.models.code import RepoWorkflowProviders + + +def test_jenkins_is_a_known_provider(): + assert RepoWorkflowProviders.JENKINS.value == "jenkins" + + +def test_factory_rejects_unknown_provider(): + factory = WorkflowETLFactory("org-id") + with pytest.raises(NotImplementedError): + factory("NOT_A_PROVIDER") +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd backend/analytics_server && python -m pytest tests/service/workflows/sync/test_etl_workflows_factory.py -v` +Expected: FAIL with `AttributeError: JENKINS` + +- [ ] **Step 3: Add the enum values** + +In `mhq/store/models/code/workflows/enums.py`, inside `RepoWorkflowProviders`: + +```python +class RepoWorkflowProviders(Enum): + GITHUB_ACTIONS = "github" + CIRCLE_CI = "circle_ci" + # CLUSTOX: Jenkins as a deployment source. Persisted as varchar, so no + # migration is required despite the ENUM() wrapper on the column. + JENKINS = "jenkins" +``` + +In `mhq/store/models/integrations/enums.py`, inside `UserIdentityProvider`: + +```python +class UserIdentityProvider(Enum): + GITHUB = "github" + GITLAB = "gitlab" + # CLUSTOX: Jenkins credentials live in Integration like any other provider. + JENKINS = "jenkins" +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd backend/analytics_server && python -m pytest tests/service/workflows/sync/test_etl_workflows_factory.py -v` +Expected: PASS (both tests) + +- [ ] **Step 5: Commit** + +```bash +git add backend/analytics_server/mhq/store/models/code/workflows/enums.py \ + backend/analytics_server/mhq/store/models/integrations/enums.py \ + backend/analytics_server/tests/service/workflows/sync/test_etl_workflows_factory.py +git commit -m "feat(jenkins): register jenkins as a workflow and identity provider" +``` + +--- + +### Task 2: Jenkins API client + +**Files:** +- Create: `backend/analytics_server/mhq/exapi/jenkins.py` +- Test: `backend/analytics_server/tests/exapi/test_jenkins.py` + +**Interfaces:** +- Consumes: nothing from earlier tasks. +- Produces: + - `JenkinsApiService(base_url: str, username: str, api_token: str, timeout: Tuple[int, int] = (5, 30))` + - `.check_pat() -> bool` + - `.get_jobs() -> List[Dict]` — each `{"name": str, "full_name": str, "url": str}` + - `.get_builds(job_full_name: str, bookmark: datetime) -> List[Dict]` — raw Jenkins build dicts, newest first, only those with `timestamp` after `bookmark` + - `job_path(job_full_name: str) -> str` — module-level helper + +- [ ] **Step 1: Write the failing test** + +Create `backend/analytics_server/tests/exapi/test_jenkins.py`: + +```python +from datetime import datetime + +import pytz + +from mhq.exapi.jenkins import JenkinsApiService, job_path + + +def test_job_path_encodes_a_top_level_job(): + assert job_path("deploy-api") == "job/deploy-api" + + +def test_job_path_encodes_a_folder_job(): + # Jenkins addresses nested jobs by repeating /job/ per segment. + assert job_path("platform/deploy-api") == "job/platform/job/deploy-api" + + +def test_get_builds_filters_out_builds_at_or_before_the_bookmark(): + class FakeResponse: + status_code = 200 + + @staticmethod + def json(): + return { + "builds": [ + {"number": 3, "timestamp": 3000, "result": "SUCCESS"}, + {"number": 2, "timestamp": 2000, "result": "SUCCESS"}, + {"number": 1, "timestamp": 1000, "result": "SUCCESS"}, + ] + } + + @staticmethod + def raise_for_status(): + return None + + service = JenkinsApiService("https://jenkins.example.com", "user", "token") + service._get = lambda path: FakeResponse() + + bookmark = datetime.fromtimestamp(2, tz=pytz.UTC) + builds = service.get_builds("deploy-api", bookmark) + + assert [b["number"] for b in builds] == [3] +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd backend/analytics_server && python -m pytest tests/exapi/test_jenkins.py -v` +Expected: FAIL with `ModuleNotFoundError: No module named 'mhq.exapi.jenkins'` + +- [ ] **Step 3: Write the client** + +Create `backend/analytics_server/mhq/exapi/jenkins.py`: + +```python +# CLUSTOX: Jenkins REST client. Knows Jenkins' API shape and nothing about our +# models -- adaptation to RepoWorkflowRuns lives in the ETL handler. +from datetime import datetime +from typing import Dict, List, Tuple + +import pytz +import requests + +# Connect and read timeouts. The workspace sync loop is sequential, so an +# untimed request against a hung Jenkins stalls every workspace behind it. +DEFAULT_TIMEOUT: Tuple[int, int] = (5, 30) + +# Fetched per build. Explicit rather than a wildcard so a Jenkins with many +# plugins does not return megabytes of action data per build. +BUILD_TREE = ( + "builds[number,result,timestamp,duration,url,building," + "actions[causes[userId,userName,shortDescription]," + "lastBuiltRevision[SHA1,branch[name]]]]" +) + +JOB_TREE = "jobs[name,fullName,url]" + + +def job_path(job_full_name: str) -> str: + """Jenkins addresses nested jobs by repeating /job/ for each segment.""" + return "/".join(f"job/{segment}" for segment in job_full_name.split("/")) + + +class JenkinsApiService: + def __init__( + self, + base_url: str, + username: str, + api_token: str, + timeout: Tuple[int, int] = DEFAULT_TIMEOUT, + ): + self._base_url = base_url.rstrip("/") + self._auth = (username, api_token) + self._timeout = timeout + + def _get(self, path: str): + # TLS verification is deliberately not configurable. A skip-verify + # option is easy to add, hard to remove, and this connection carries an + # API token. + return requests.get( + f"{self._base_url}/{path}", + auth=self._auth, + timeout=self._timeout, + verify=True, + ) + + def check_pat(self) -> bool: + response = self._get("api/json") + return response.status_code == 200 + + def get_jobs(self) -> List[Dict]: + response = self._get(f"api/json?tree={JOB_TREE}") + response.raise_for_status() + return [ + { + "name": job.get("name"), + "full_name": job.get("fullName") or job.get("name"), + "url": job.get("url"), + } + for job in response.json().get("jobs", []) + ] + + def get_builds(self, job_full_name: str, bookmark: datetime) -> List[Dict]: + response = self._get(f"{job_path(job_full_name)}/api/json?tree={BUILD_TREE}") + response.raise_for_status() + builds = response.json().get("builds", []) + return [ + build + for build in builds + if self._build_time(build) and self._build_time(build) > bookmark + ] + + @staticmethod + def _build_time(build: Dict): + timestamp = build.get("timestamp") + if not timestamp: + return None + return datetime.fromtimestamp(timestamp / 1000, tz=pytz.UTC) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd backend/analytics_server && python -m pytest tests/exapi/test_jenkins.py -v` +Expected: PASS (3 tests) + +- [ ] **Step 5: Commit** + +```bash +git add backend/analytics_server/mhq/exapi/jenkins.py \ + backend/analytics_server/tests/exapi/test_jenkins.py +git commit -m "feat(jenkins): add REST client with bounded timeouts" +``` + +--- + +### Task 3: Build fixtures + +**Files:** +- Create: `backend/analytics_server/tests/factories/models/exapi/jenkins.py` + +**Interfaces:** +- Consumes: nothing. +- Produces: `get_jenkins_build_dict(**kwargs) -> Dict` — a realistic successful build with git plugin data; every field overridable by keyword. + +- [ ] **Step 1: Write the fixture factory** + +Create `backend/analytics_server/tests/factories/models/exapi/jenkins.py`: + +```python +# CLUSTOX: recorded from a real Jenkins pipeline build. Shape matters more than +# values -- these encode the assumptions the handler makes about the API. +from typing import Dict + + +def get_jenkins_build_dict( + number: int = 42, + result: str = "SUCCESS", + timestamp: int = 1754827200000, # 2025-08-10T12:00:00Z + duration: int = 125000, + url: str = "https://jenkins.example.com/job/deploy-api/42/", + building: bool = False, + user_id: str = "hamad", + branch_name: str = "origin/main", + sha: str = "a1b2c3d4e5f6", +) -> Dict: + return { + "number": number, + "result": result, + "timestamp": timestamp, + "duration": duration, + "url": url, + "building": building, + "actions": [ + {"causes": [{"userId": user_id, "shortDescription": "Started by user"}]}, + {"lastBuiltRevision": {"SHA1": sha, "branch": [{"name": branch_name}]}}, + ], + } + + +def get_jenkins_build_dict_without_git_plugin(**kwargs) -> Dict: + """A freestyle job with no SCM: no branch, no revision, no user cause.""" + build = get_jenkins_build_dict(**kwargs) + build["actions"] = [{}] + return build +``` + +- [ ] **Step 2: Verify it imports** + +Run: `cd backend/analytics_server && python -c "from tests.factories.models.exapi.jenkins import get_jenkins_build_dict; print(get_jenkins_build_dict()['number'])"` +Expected: prints `42` + +- [ ] **Step 3: Commit** + +```bash +git add backend/analytics_server/tests/factories/models/exapi/jenkins.py +git commit -m "test(jenkins): add build fixtures including the no-git-plugin case" +``` + +--- + +### Task 4: Jenkins ETL handler + +**Files:** +- Create: `backend/analytics_server/mhq/service/workflows/sync/etl_jenkins_handler.py` +- Test: `backend/analytics_server/tests/service/workflows/sync/test_etl_jenkins_handler.py` + +**Interfaces:** +- Consumes: `JenkinsApiService` (Task 2), `get_jenkins_build_dict` (Task 3), `RepoWorkflowProviders.JENKINS` (Task 1). +- Produces: + - `JenkinsETLHandler(org_id: str, jenkins_api_service, workflow_repo_service)` + - `._adapt_jenkins_build_to_workflow_run(repo_workflow_id: str, build: Dict) -> RepoWorkflowRuns` + - `._get_repo_workflow_status(build: Dict) -> RepoWorkflowRunsStatus` (static) + - `._get_datetime_from_epoch_ms(ms: int) -> datetime` (static) + - `._get_branch(build: Dict) -> Optional[str]` (static) + - `._get_actor(build: Dict) -> Optional[str]` (static) + +- [ ] **Step 1: Write the failing tests** + +Create `backend/analytics_server/tests/service/workflows/sync/test_etl_jenkins_handler.py`: + +```python +from mhq.service.workflows.sync.etl_jenkins_handler import JenkinsETLHandler +from mhq.store.models.code import RepoWorkflowRunsStatus +from mhq.utils.string import uuid4_str +from tests.factories.models.exapi.jenkins import ( + get_jenkins_build_dict, + get_jenkins_build_dict_without_git_plugin, +) + + +class FakeWorkflowRepoService: + def get_repo_workflow_run_by_provider_workflow_run_id(self, *args): + return None + + +def _handler(): + return JenkinsETLHandler(uuid4_str(), None, FakeWorkflowRepoService()) + + +def test_success_maps_to_success(): + build = get_jenkins_build_dict(result="SUCCESS") + assert ( + JenkinsETLHandler._get_repo_workflow_status(build) + == RepoWorkflowRunsStatus.SUCCESS + ) + + +def test_failure_maps_to_failure(): + build = get_jenkins_build_dict(result="FAILURE") + assert ( + JenkinsETLHandler._get_repo_workflow_status(build) + == RepoWorkflowRunsStatus.FAILURE + ) + + +def test_aborted_maps_to_cancelled(): + build = get_jenkins_build_dict(result="ABORTED") + assert ( + JenkinsETLHandler._get_repo_workflow_status(build) + == RepoWorkflowRunsStatus.CANCELLED + ) + + +def test_unstable_maps_to_failure_so_it_is_not_counted_as_a_deployment(): + build = get_jenkins_build_dict(result="UNSTABLE") + assert ( + JenkinsETLHandler._get_repo_workflow_status(build) + == RepoWorkflowRunsStatus.FAILURE + ) + + +def test_a_running_build_maps_to_pending(): + build = get_jenkins_build_dict(result=None, building=True) + assert ( + JenkinsETLHandler._get_repo_workflow_status(build) + == RepoWorkflowRunsStatus.PENDING + ) + + +def test_adapt_maps_every_field(): + build = get_jenkins_build_dict() + repo_workflow_id = uuid4_str() + + run = _handler()._adapt_jenkins_build_to_workflow_run(repo_workflow_id, build) + + assert run.repo_workflow_id == repo_workflow_id + assert run.provider_workflow_run_id == "42" + assert run.status == RepoWorkflowRunsStatus.SUCCESS + assert run.event_actor == "hamad" + assert run.head_branch == "origin/main" + assert run.html_url == build["url"] + # Jenkins reports milliseconds; RepoWorkflowRuns.duration is seconds. + assert run.duration == 125 + assert run.conducted_at.isoformat() == "2025-08-10T12:00:00+00:00" + + +def test_a_build_without_the_git_plugin_records_with_nulls_rather_than_raising(): + build = get_jenkins_build_dict_without_git_plugin() + + run = _handler()._adapt_jenkins_build_to_workflow_run(uuid4_str(), build) + + assert run.head_branch is None + assert run.event_actor is None + assert run.status == RepoWorkflowRunsStatus.SUCCESS + + +def test_a_malformed_build_is_skipped_without_losing_the_batch(): + class Repo: + def get_repo_workflow_run_by_provider_workflow_run_id(self, *args): + return None + + class Api: + @staticmethod + def get_builds(*args): + # Second build has no "number", so adaptation raises for it alone. + return [get_jenkins_build_dict(number=2), {"timestamp": 1754827200000}] + + class Workflow: + id = uuid4_str() + provider_workflow_id = "deploy-api" + + class OrgRepoStub: + repo_id = uuid4_str() + + handler = JenkinsETLHandler(uuid4_str(), Api(), Repo()) + runs, _ = handler.get_workflow_runs( + OrgRepoStub(), Workflow(), _epoch_start() + ) + + assert [r.provider_workflow_run_id for r in runs] == ["2"] + + +def test_bookmark_rewinds_to_the_oldest_running_build(): + handler = _handler() + builds = [ + get_jenkins_build_dict(number=3, timestamp=3000, result=None, building=True), + get_jenkins_build_dict(number=2, timestamp=2000, result="SUCCESS"), + ] + + bookmark = handler._get_new_bookmark_time_stamp(builds) + + # The running build must be re-fetched next cycle, so the bookmark cannot + # advance past it. + assert bookmark == handler._get_datetime_from_epoch_ms(3000) +``` + +Add this helper near the top of the file, below the imports: + +```python +import pytz +from datetime import datetime + + +def _epoch_start() -> datetime: + return datetime.fromtimestamp(0, tz=pytz.UTC) +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd backend/analytics_server && python -m pytest tests/service/workflows/sync/test_etl_jenkins_handler.py -v` +Expected: FAIL with `ModuleNotFoundError: No module named 'mhq.service.workflows.sync.etl_jenkins_handler'` + +- [ ] **Step 3: Write the handler** + +Create `backend/analytics_server/mhq/service/workflows/sync/etl_jenkins_handler.py`: + +```python +# CLUSTOX: Jenkins deployment detection. Implements the same two-method +# contract as GitHub Actions, so nothing downstream of the sync changes. +from datetime import datetime +from typing import Dict, List, Optional, Tuple +from uuid import uuid4 + +import pytz + +from mhq.service.workflows.sync.etl_provider_handler import WorkflowProviderETLHandler +from mhq.store.models.code import ( + OrgRepo, + RepoWorkflow, + RepoWorkflowProviders, + RepoWorkflowRuns, + RepoWorkflowRunsStatus, +) +from mhq.utils.log import LOG +from mhq.utils.time import time_now + + +class JenkinsETLHandler(WorkflowProviderETLHandler): + def __init__(self, org_id: str, jenkins_api_service, workflow_repo_service): + self.org_id = org_id + self._api = jenkins_api_service + self._workflow_repo_service = workflow_repo_service + self._provider = RepoWorkflowProviders.JENKINS.value + + def check_pat_validity(self) -> bool: + if not self._api.check_pat(): + raise Exception("Jenkins credentials are invalid or Jenkins is unreachable") + return True + + def get_workflow_runs( + self, + org_repo: OrgRepo, + repo_workflow: RepoWorkflow, + bookmark: datetime, + ) -> Tuple[List[RepoWorkflowRuns], datetime]: + try: + builds = self._api.get_builds( + repo_workflow.provider_workflow_id, bookmark + ) + except Exception as e: + # Bookmark is returned unchanged by raising: the caller does not + # advance it, so this window is re-fetched next cycle. + raise Exception( + f"[Jenkins Sync Repo Workflow Worker] Error fetching job " + f"{repo_workflow.provider_workflow_id} for repo " + f"{str(org_repo.repo_id)}: {str(e)}" + ) + + if not builds: + LOG.info( + f"[Jenkins Sync Repo Workflow Worker] No builds found for job " + f"{repo_workflow.provider_workflow_id}. Org: {self.org_id}" + ) + return [], bookmark + + runs = [] + for build in builds: + try: + runs.append( + self._adapt_jenkins_build_to_workflow_run( + str(repo_workflow.id), build + ) + ) + except Exception as e: + # One malformed build must not lose the rest of the batch. + LOG.warn( + f"[Jenkins Sync Repo Workflow Worker] Skipping build " + f"{build.get('number')}: {str(e)}" + ) + + return runs, self._get_new_bookmark_time_stamp(builds) + + def _get_new_bookmark_time_stamp(self, builds: List[Dict]) -> datetime: + """ + Rewind to the oldest still-running build so it is re-fetched once it + finishes. Mirrors the GitHub Actions handler. + """ + pending = [ + self._get_datetime_from_epoch_ms(build["timestamp"]) + for build in builds + if build.get("building") or build.get("result") is None + ] + return min(pending) if pending else time_now() + + def _adapt_jenkins_build_to_workflow_run( + self, repo_workflow_id: str, build: Dict + ) -> RepoWorkflowRuns: + existing = self._workflow_repo_service.get_repo_workflow_run_by_provider_workflow_run_id( + repo_workflow_id, str(build["number"]) + ) + run_id = existing.id if existing else uuid4() + + duration = build.get("duration") + return RepoWorkflowRuns( + id=run_id, + repo_workflow_id=repo_workflow_id, + provider_workflow_run_id=str(build["number"]), + event_actor=self._get_actor(build), + head_branch=self._get_branch(build), + status=self._get_repo_workflow_status(build), + created_at=time_now(), + updated_at=time_now(), + conducted_at=self._get_datetime_from_epoch_ms(build["timestamp"]), + duration=int(duration / 1000) if duration else None, + meta=build, + html_url=build.get("url"), + ) + + @staticmethod + def _get_repo_workflow_status(build: Dict) -> RepoWorkflowRunsStatus: + result = build.get("result") + if build.get("building") or result is None: + return RepoWorkflowRunsStatus.PENDING + if result == "SUCCESS": + return RepoWorkflowRunsStatus.SUCCESS + if result == "ABORTED": + return RepoWorkflowRunsStatus.CANCELLED + # FAILURE and UNSTABLE. UNSTABLE means the build finished but something + # -- usually tests -- failed; counting it as a deployment would report a + # red build as a successful ship. + return RepoWorkflowRunsStatus.FAILURE + + @staticmethod + def _get_datetime_from_epoch_ms(ms: int) -> datetime: + return datetime.fromtimestamp(ms / 1000, tz=pytz.UTC) + + @staticmethod + def _get_branch(build: Dict) -> Optional[str]: + # Contributed by the git plugin. A freestyle job with no SCM has none. + for action in build.get("actions", []): + revision = (action or {}).get("lastBuiltRevision") + if revision and revision.get("branch"): + return revision["branch"][0].get("name") + return None + + @staticmethod + def _get_actor(build: Dict) -> Optional[str]: + for action in build.get("actions", []): + causes = (action or {}).get("causes") + if causes: + return causes[0].get("userId") or causes[0].get("userName") + return None +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd backend/analytics_server && python -m pytest tests/service/workflows/sync/test_etl_jenkins_handler.py -v` +Expected: PASS (9 tests) + +- [ ] **Step 5: Format and lint** + +Run: `cd backend/analytics_server && black mhq/service/workflows/sync/etl_jenkins_handler.py mhq/exapi/jenkins.py && flake8 mhq/service/workflows/sync/etl_jenkins_handler.py mhq/exapi/jenkins.py` +Expected: reformatted if needed, no flake8 output + +- [ ] **Step 6: Commit** + +```bash +git add backend/analytics_server/mhq/service/workflows/sync/etl_jenkins_handler.py \ + backend/analytics_server/tests/service/workflows/sync/test_etl_jenkins_handler.py +git commit -m "feat(jenkins): map builds to workflow runs" +``` + +--- + +### Task 5: Wire the handler into the sync + +**Files:** +- Create: `backend/analytics_server/mhq/utils/jenkins.py` +- Modify: `backend/analytics_server/mhq/service/workflows/sync/etl_workflows_factory.py` +- Modify: `backend/analytics_server/mhq/service/workflows/sync/etl_jenkins_handler.py` (append factory function) +- Test: `backend/analytics_server/tests/service/workflows/sync/test_etl_workflows_factory.py` (extend) + +**Interfaces:** +- Consumes: `JenkinsETLHandler` (Task 4), `JenkinsApiService` (Task 2). +- Produces: + - `get_jenkins_config(org_id: str) -> Tuple[Optional[str], Optional[str]]` returning `(base_url, username)` + - `get_jenkins_etl_handler(org_id: str) -> JenkinsETLHandler` + +- [ ] **Step 1: Write the failing test** + +Append to `backend/analytics_server/tests/service/workflows/sync/test_etl_workflows_factory.py`: + +```python +def test_factory_returns_a_jenkins_handler(): + from mhq.service.workflows.sync.etl_jenkins_handler import JenkinsETLHandler + + factory = WorkflowETLFactory("org-id") + handler = factory(RepoWorkflowProviders.JENKINS.name) + + assert isinstance(handler, JenkinsETLHandler) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd backend/analytics_server && python -m pytest tests/service/workflows/sync/test_etl_workflows_factory.py::test_factory_returns_a_jenkins_handler -v` +Expected: FAIL with `NotImplementedError: Unknown provider - JENKINS` + +- [ ] **Step 3: Add the config reader** + +Create `backend/analytics_server/mhq/utils/jenkins.py`: + +```python +# CLUSTOX: Jenkins needs a base URL and username alongside its API token. +# The token is a secret and lives encrypted in access_token_enc_chunks; these +# two are not, and follow the existing provider_meta precedent used for +# GitHub's custom domain. +from typing import Optional, Tuple + +from mhq.store.models import Integration, UserIdentityProvider +from mhq.store.repos.core import CoreRepoService + + +def get_jenkins_config(org_id: str) -> Tuple[Optional[str], Optional[str]]: + core_repo_service = CoreRepoService() + integrations = core_repo_service.get_org_integrations_for_names( + org_id, [UserIdentityProvider.JENKINS.value] + ) + if not integrations or not integrations[0].provider_meta: + return None, None + + meta = integrations[0].provider_meta + return meta.get("base_url"), meta.get("username") +``` + +- [ ] **Step 4: Add the handler factory function** + +Append to `backend/analytics_server/mhq/service/workflows/sync/etl_jenkins_handler.py`: + +```python +def get_jenkins_etl_handler(org_id: str) -> JenkinsETLHandler: + from mhq.exapi.jenkins import JenkinsApiService + from mhq.store.models import UserIdentityProvider + from mhq.store.repos.core import CoreRepoService + from mhq.store.repos.workflows import WorkflowRepoService + from mhq.utils.jenkins import get_jenkins_config + + api_token = CoreRepoService().get_access_token( + org_id, UserIdentityProvider.JENKINS + ) + base_url, username = get_jenkins_config(org_id) + + if not (api_token and base_url and username): + LOG.error( + f"Jenkins is not fully configured for org {org_id}: " + f"base_url={'set' if base_url else 'missing'}, " + f"username={'set' if username else 'missing'}, " + f"token={'set' if api_token else 'missing'}" + ) + + return JenkinsETLHandler( + org_id, + JenkinsApiService(base_url or "", username or "", api_token or ""), + WorkflowRepoService(), + ) +``` + +- [ ] **Step 5: Register in the factory** + +Replace `backend/analytics_server/mhq/service/workflows/sync/etl_workflows_factory.py` with: + +```python +from mhq.service.workflows.sync.etl_github_actions_handler import ( + get_github_actions_etl_handler, +) + +# CLUSTOX: Jenkins as a third deployment source. +from mhq.service.workflows.sync.etl_jenkins_handler import get_jenkins_etl_handler +from mhq.service.workflows.sync.etl_provider_handler import WorkflowProviderETLHandler +from mhq.store.models.code import RepoWorkflowProviders + + +class WorkflowETLFactory: + def __init__(self, org_id: str): + self.org_id = org_id + + def __call__(self, provider: str) -> WorkflowProviderETLHandler: + if provider == RepoWorkflowProviders.GITHUB_ACTIONS.name: + return get_github_actions_etl_handler(self.org_id) + if provider == RepoWorkflowProviders.JENKINS.name: + return get_jenkins_etl_handler(self.org_id) + raise NotImplementedError(f"Unknown provider - {provider}") +``` + +- [ ] **Step 6: Run the full backend suite** + +Run: `cd backend/analytics_server && python -m pytest tests -q` +Expected: all pass, including the pre-existing 159 + +- [ ] **Step 7: Commit** + +```bash +git add backend/analytics_server/mhq/utils/jenkins.py \ + backend/analytics_server/mhq/service/workflows/sync/etl_workflows_factory.py \ + backend/analytics_server/mhq/service/workflows/sync/etl_jenkins_handler.py \ + backend/analytics_server/tests/service/workflows/sync/test_etl_workflows_factory.py +git commit -m "feat(jenkins): resolve credentials and register the handler" +``` + +--- + +### Task 6: Flask routes for jobs and mappings + +**Files:** +- Modify: `backend/analytics_server/mhq/api/integrations.py` +- Test: `backend/analytics_server/tests/api/test_jenkins_routes.py` (create) + +**Interfaces:** +- Consumes: `JenkinsApiService` (Task 2), `get_jenkins_config` (Task 5), `RepoWorkflowProviders.JENKINS` (Task 1). +- Produces: + - `GET /internal/<org_id>/integrations/jenkins/jobs` → `[{name, full_name, url}]` + - `POST /internal/<org_id>/integrations/jenkins/mappings` body `{org_repo_id, job_full_name}` → `{ok: true, deactivated_github_workflows: int}` + - `DELETE /internal/<org_id>/integrations/jenkins/mappings` body `{repo_workflow_id}` → `{ok: true}` + +- [ ] **Step 1: Write the failing test for one-source enforcement** + +Create `backend/analytics_server/tests/api/test_jenkins_routes.py`: + +```python +from mhq.api.integrations import deactivate_github_actions_workflows_for_repo +from mhq.store.models.code import RepoWorkflowProviders + + +class FakeWorkflow: + def __init__(self, provider, is_active=True): + self.provider = provider + self.is_active = is_active + + +def test_deactivates_only_github_actions_workflows(): + workflows = [ + FakeWorkflow(RepoWorkflowProviders.GITHUB_ACTIONS), + FakeWorkflow(RepoWorkflowProviders.GITHUB_ACTIONS), + FakeWorkflow(RepoWorkflowProviders.JENKINS), + ] + + count = deactivate_github_actions_workflows_for_repo(workflows) + + assert count == 2 + assert [w.is_active for w in workflows] == [False, False, True] + + +def test_deactivating_is_idempotent(): + workflows = [FakeWorkflow(RepoWorkflowProviders.GITHUB_ACTIONS, is_active=False)] + + count = deactivate_github_actions_workflows_for_repo(workflows) + + # Already inactive, so nothing changed and nothing is reported. + assert count == 0 +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd backend/analytics_server && python -m pytest tests/api/test_jenkins_routes.py -v` +Expected: FAIL with `ImportError: cannot import name 'deactivate_github_actions_workflows_for_repo'` + +- [ ] **Step 3: Write the enforcement helper** + +Add to `backend/analytics_server/mhq/api/integrations.py`: + +```python +# CLUSTOX: one deployment source per repo. A repo tracked through both GitHub +# Actions and Jenkins would count every deploy twice, doubling Deployment +# Frequency with nothing visibly wrong. Deactivation is reversible: the rows +# survive, so removing the Jenkins mapping can restore them. +from typing import List + +from mhq.store.models.code import RepoWorkflowProviders + + +def deactivate_github_actions_workflows_for_repo(workflows: List) -> int: + deactivated = 0 + for workflow in workflows: + if ( + workflow.provider == RepoWorkflowProviders.GITHUB_ACTIONS + and workflow.is_active + ): + workflow.is_active = False + deactivated += 1 + return deactivated +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd backend/analytics_server && python -m pytest tests/api/test_jenkins_routes.py -v` +Expected: PASS (2 tests) + +- [ ] **Step 5: Add the three routes** + +Add to `backend/analytics_server/mhq/api/integrations.py`, following the blueprint and `@queryschema` patterns already in that file. The mapping route must insert the Jenkins `RepoWorkflow` row and call `deactivate_github_actions_workflows_for_repo` **inside the same transaction**, so a failure cannot leave a repo with two active deployment sources: + +```python +@app.route("/internal/<org_id>/integrations/jenkins/jobs", methods={"GET"}) +def get_jenkins_jobs(org_id: str): + from mhq.exapi.jenkins import JenkinsApiService + from mhq.store.models import UserIdentityProvider + from mhq.store.repos.core import CoreRepoService + from mhq.utils.jenkins import get_jenkins_config + + api_token = CoreRepoService().get_access_token( + org_id, UserIdentityProvider.JENKINS + ) + base_url, username = get_jenkins_config(org_id) + if not (api_token and base_url and username): + return {"error": "Jenkins is not configured for this workspace"}, 400 + + return JenkinsApiService(base_url, username, api_token).get_jobs() +``` + +Write the two `mappings` routes in the same file, using `WorkflowRepoService` to create and soft-delete `RepoWorkflow` rows with `provider=RepoWorkflowProviders.JENKINS` and `type=RepoWorkflowType.DEPLOYMENT`. + +- [ ] **Step 6: Run the full backend suite** + +Run: `cd backend/analytics_server && python -m pytest tests -q && black --check mhq && flake8 mhq` +Expected: all pass, formatting clean + +- [ ] **Step 7: Commit** + +```bash +git add backend/analytics_server/mhq/api/integrations.py \ + backend/analytics_server/tests/api/test_jenkins_routes.py +git commit -m "feat(jenkins): expose job listing and mapping routes" +``` + +--- + +### Task 7: Job listing and mapping BFF endpoints + +**Files:** +- Create: `web-server/pages/api/clustox/jenkins/jobs.ts` +- Create: `web-server/pages/api/clustox/jenkins/mappings.ts` + +**Interfaces:** +- Consumes: the Flask routes from Task 6; `Endpoint` from `@/api-helpers/global`; the `internal` axios instance exported from `web-server/src/api-helpers/axios.ts:26`. Workspace scoping is applied centrally by `Endpoint.serve()` on the validated `org_id`, so these handlers need no explicit guard call. +- Produces: + - `GET /api/clustox/jenkins/jobs?org_id=<uuid>` → `{ jobs: Array<{name, full_name, url}> }` + - `POST /api/clustox/jenkins/mappings` body `{ org_id, org_repo_id, job_full_name }` → `{ ok: true, deactivated_github_workflows: number }` + - `DELETE /api/clustox/jenkins/mappings` body `{ org_id, repo_workflow_id }` → `{ ok: true }` + +- [ ] **Step 1: Write the endpoint that lists jobs** + +Create `web-server/pages/api/clustox/jenkins/jobs.ts`: + +```ts +import * as yup from 'yup'; + +import { Endpoint } from '@/api-helpers/global'; +import { internal } from '@/api-helpers/axios'; + +// CLUSTOX: proxies the workspace's Jenkins job list. org_id is validated by +// Endpoint.serve(), which asserts the caller may act on that workspace. +const getSchema = yup.object().shape({ + org_id: yup.string().uuid().required() +}); + +const endpoint = new Endpoint(getSchema); + +endpoint.handle.GET(getSchema, async (req, res) => { + const jobs = await internal.get( + `/internal/${req.payload.org_id}/integrations/jenkins/jobs` + ); + res.send({ jobs: jobs.data }); +}); + +export default endpoint.serve(); +``` + +- [ ] **Step 2: Write the mapping endpoint** + +Create `web-server/pages/api/clustox/jenkins/mappings.ts`: + +```ts +import * as yup from 'yup'; + +import { Endpoint } from '@/api-helpers/global'; +import { internal } from '@/api-helpers/axios'; + +const postSchema = yup.object().shape({ + org_id: yup.string().uuid().required(), + org_repo_id: yup.string().uuid().required(), + job_full_name: yup.string().required() +}); + +const deleteSchema = yup.object().shape({ + org_id: yup.string().uuid().required(), + repo_workflow_id: yup.string().uuid().required() +}); + +const endpoint = new Endpoint(postSchema); + +endpoint.handle.POST(postSchema, async (req, res) => { + // One deployment source per repo: creating this mapping deactivates the + // repo's GitHub Actions workflows so deployments are not counted twice. + const result = await internal.post( + `/internal/${req.payload.org_id}/integrations/jenkins/mappings`, + { + org_repo_id: req.payload.org_repo_id, + job_full_name: req.payload.job_full_name + } + ); + res.send(result.data); +}); + +endpoint.handle.DELETE(deleteSchema, async (req, res) => { + const result = await internal.delete( + `/internal/${req.payload.org_id}/integrations/jenkins/mappings`, + { data: { repo_workflow_id: req.payload.repo_workflow_id } } + ); + res.send(result.data); +}); + +export default endpoint.serve(); +``` + +- [ ] **Step 3: Verify types compile** + +Run: `cd web-server && yarn tsc --noEmit` +Expected: no errors + +- [ ] **Step 4: Commit** + +```bash +git add web-server/pages/api/clustox/jenkins/ +git commit -m "feat(jenkins): add job listing and mapping endpoints" +``` + +--- + +### Task 8: Cross-workspace isolation tests + +**Files:** +- Create: `web-server/e2e/jenkins.spec.ts` + +**Interfaces:** +- Consumes: the BFF endpoints from Task 7. +- Produces: nothing consumed by later tasks. + +- [ ] **Step 1: Write the isolation tests** + +Create `web-server/e2e/jenkins.spec.ts`: + +```ts +/** + * Jenkins endpoints must obey the same workspace boundary as everything else. + * + * Running: + * docker compose up -d + * cd web-server && yarn playwright test e2e/jenkins.spec.ts + */ +import { APIRequestContext, expect, request, test } from '@playwright/test'; + +const APP = 'http://localhost:3333'; + +const SUPERADMIN = { + email: process.env.SUPERADMIN_EMAIL || 'admin@clustox.com', + password: process.env.SUPERADMIN_PASSWORD || '' +}; + +const signIn = async (email: string, password: string) => { + const ctx = await request.newContext({ baseURL: APP }); + const csrf = await (await ctx.get('/api/auth/csrf')).json(); + const res = await ctx.post('/api/auth/callback/credentials', { + form: { csrfToken: csrf.csrfToken, email, password, json: 'true' }, + failOnStatusCode: false + }); + expect(res.status(), `sign-in failed for ${email}`).toBe(200); + return ctx; +}; + +const unique = (p: string) => + `${p}.${Date.now()}${Math.floor(Math.random() * 1000)}@clustox.com`; + +const createAdmin = async (su: APIRequestContext, name: string) => { + const email = unique(name.toLowerCase().replace(/\s+/g, '.')); + const password = 'E2eJenkinsPass123'; + const res = await su.post('/api/clustox/users', { + data: { name, email, password, role: 'ADMIN', team_ids: [] } + }); + expect(res.status()).toBe(200); + const { user_id, org_id } = await res.json(); + return { userId: user_id, orgId: org_id as string, email, password }; +}; + +test.describe('jenkins workspace isolation', () => { + test.skip(!SUPERADMIN.password, 'SUPERADMIN_PASSWORD not set'); + + let su: APIRequestContext; + let alpha: Awaited<ReturnType<typeof createAdmin>>; + let beta: Awaited<ReturnType<typeof createAdmin>>; + let alphaCtx: APIRequestContext; + + test.beforeAll(async () => { + su = await signIn(SUPERADMIN.email, SUPERADMIN.password); + alpha = await createAdmin(su, 'Jenkins Alpha'); + beta = await createAdmin(su, 'Jenkins Beta'); + alphaCtx = await signIn(alpha.email, alpha.password); + }); + + test.afterAll(async () => { + for (const id of [alpha.userId, beta.userId]) { + await su.fetch(`/api/clustox/users/${id}`, { + method: 'DELETE', + failOnStatusCode: false + }); + } + }); + + test('an admin cannot list another workspace jenkins jobs', async () => { + const res = await alphaCtx.get( + `/api/clustox/jenkins/jobs?org_id=${beta.orgId}`, + { failOnStatusCode: false } + ); + expect(res.status()).toBe(403); + }); + + test('an admin cannot map a job into another workspace', async () => { + const res = await alphaCtx.post('/api/clustox/jenkins/mappings', { + data: { + org_id: beta.orgId, + org_repo_id: '00000000-0000-4000-8000-000000000001', + job_full_name: 'deploy-api' + }, + failOnStatusCode: false + }); + expect(res.status()).toBe(403); + }); + + test('unauthenticated requests are rejected', async ({ request: req }) => { + const res = await req.get( + `${APP}/api/clustox/jenkins/jobs?org_id=${alpha.orgId}`, + { failOnStatusCode: false, maxRedirects: 0 } + ); + expect(res.status()).toBe(401); + }); + + test('a malformed org_id is a 400, not a 403', async () => { + // Schema validation runs before the access check, so bad input is not + // reported as a permission problem. + const res = await alphaCtx.get('/api/clustox/jenkins/jobs?org_id=nonsense', { + failOnStatusCode: false + }); + expect(res.status()).toBe(400); + }); +}); +``` + +- [ ] **Step 2: Run the tests** + +Run: `cd web-server && SUPERADMIN_PASSWORD=<the value from .env> yarn playwright test e2e/jenkins.spec.ts` +Expected: PASS (4 tests) + +- [ ] **Step 3: Commit** + +```bash +git add web-server/e2e/jenkins.spec.ts +git commit -m "test(jenkins): cross-workspace isolation for jenkins endpoints" +``` + +--- + +### Task 9: Integrations UI + +> **Resolution warning.** Tasks 1–8 give exact code. This one specifies +> behaviour and contracts but not full component bodies, because the MUI markup +> is long, mechanical, and better written against the live components than +> transcribed from a plan. Treat the acceptance criteria in each step as the +> requirement and follow the existing patterns in `pages/integrations.tsx`. If +> you want this at the same resolution as the backend tasks, ask before +> starting rather than improvising. + +**Files:** +- Modify: `web-server/src/constants/integrations.ts` +- Modify: `web-server/pages/integrations.tsx` +- Create: `web-server/src/components/ClustoxJenkinsSetup.tsx` +- Create: `web-server/src/components/ClustoxJenkinsMapping.tsx` + +**Interfaces:** +- Consumes: BFF endpoints from Task 7. +- Produces: nothing consumed by later tasks. + +- [ ] **Step 1: Add Jenkins to the integration enum** + +In `web-server/src/constants/integrations.ts`, add to the `Integration` enum: + +```ts +// CLUSTOX: Jenkins is a deployment provider, not a code provider -- it is +// deliberately absent from CODE_PROVIDER_INTEGRATIONS_MAP. +JENKINS = 'jenkins' +``` + +- [ ] **Step 2: Build the credentials form** + +Create `web-server/src/components/ClustoxJenkinsSetup.tsx` with a form collecting **base URL**, **username**, and **API token**, posting to `/api/resources/orgs/{orgId}/integration` with `provider: 'jenkins'`, `the_good_stuff: <api token>`, and `meta_data: { base_url, username }`. Validate the connection server-side before saving and surface a failure inline rather than as a toast, so a wrong URL is visible next to the field that caused it. + +- [ ] **Step 3: Build the mapping table** + +Create `web-server/src/components/ClustoxJenkinsMapping.tsx`: a table of the workspace's repos, each with a dropdown of Jenkins jobs from `GET /api/clustox/jenkins/jobs`. Selecting a job POSTs to `/api/clustox/jenkins/mappings`. + +Before submitting, if the repo has active GitHub Actions workflows, show a confirmation dialog: + +> Mapping a Jenkins job will stop counting this repo's GitHub Actions runs as deployments, so they are not counted twice. You can undo this by removing the mapping. + +- [ ] **Step 4: Render the Jenkins card** + +In `web-server/pages/integrations.tsx`, add a Jenkins card alongside GitHub and GitLab that opens `ClustoxJenkinsSetup` when unlinked and `ClustoxJenkinsMapping` when linked. + +- [ ] **Step 5: Verify** + +Run: `cd web-server && yarn tsc --noEmit && yarn lint` +Expected: no errors + +Then manually: link Jenkins, confirm jobs list, map one to a repo, confirm the GitHub Actions warning appears, and confirm the mapping persists across a refresh. + +- [ ] **Step 6: Commit** + +```bash +git add web-server/src/constants/integrations.ts \ + web-server/pages/integrations.tsx \ + web-server/src/components/ClustoxJenkinsSetup.tsx \ + web-server/src/components/ClustoxJenkinsMapping.tsx +git commit -m "feat(jenkins): connect jenkins and map jobs to repositories" +``` + +--- + +## Final verification + +- [ ] `cd backend/analytics_server && python -m pytest tests -q` — all pass +- [ ] `cd backend/analytics_server && black --check mhq && flake8 mhq` — clean +- [ ] `cd web-server && yarn tsc --noEmit && yarn test` — clean +- [ ] `cd web-server && yarn playwright test e2e/` — all suites pass +- [ ] Manual: against a real Jenkins, link it, map a job, trigger a sync, confirm the build appears as a deployment and Deployment Frequency increases + +**The last item is the one that matters most and cannot be automated.** Fixtures encode assumptions about a Jenkins API that varies by version and plugin set. Until this passes against the target server, the integration is unverified regardless of how green the suite is. diff --git a/docs/JIRA_MULTI_ACCOUNT_PLAN.md b/docs/JIRA_MULTI_ACCOUNT_PLAN.md new file mode 100644 index 000000000..f9d96c978 --- /dev/null +++ b/docs/JIRA_MULTI_ACCOUNT_PLAN.md @@ -0,0 +1,175 @@ +# Jira Multi-Account Support — Implementation Plan + +**Goal:** Let one org connect more than one Jira site/account (today: exactly one, via `Integration(name='jira')`), map different teams' projects to different connections, and sync/match tickets correctly per connection. + +**Architecture:** A new, Jira-only, surrogate-keyed table `JiraConnection` (many rows per org) sits alongside the existing `Integration` table, which is left completely untouched and keeps working for orgs that never link a `JiraConnection`. A new join table `OrgProjectConnection` records which connection a given `OrgProject` came from, so `OrgProject`/`TeamProjects`/`Ticket` need zero schema changes — a team already maps to Jira projects via the existing `TeamProjects` table; which *connection* backs a given project is derived by joining through `OrgProjectConnection`. Ticket sync and PR↔ticket matching are updated to scope by connection, since Jira project keys and internal issue ids are only unique per-site, not across an org's connected sites. + +**Tech Stack:** Flask 3 + SQLAlchemy 2, Postgres (dbmate migrations), Next.js 15 pages router, MUI 5. + +## Global Constraints + +- **No changes to existing tables/constraints.** `Integration`, `OrgProject`, `TeamProjects`, `Ticket`, `TicketState` are untouched — only new tables (`JiraConnection`, `OrgProjectConnection`) and new files. This is deliberate: reverting this feature is "drop 2 new tables + delete new files", never a migration-undo on tables other providers (GitHub/GitLab/Jenkins) also depend on. +- **No implicit sync between `Integration(name='jira')` and `JiraConnection`.** They are independent. If legacy code ever needs "the org's default Jira", that's a deliberate, separate, later change — not automatic mirroring (mirroring two write paths for the same fact is a source of silent drift, not safety). +- Only one `JiraConnection` per org may have `is_default = true` — enforced by a partial unique index, not just app code. +- `(org_id, site_url, email)` is the uniqueness key for a connection, not `(org_id, site_url)` — two different accounts on the same Jira site is a legitimate case (see docs discussion), not a duplicate. +- Ticket idempotency keys and the PR↔ticket matching key-map must be scoped per-connection, not just per-org — Jira issue ids and project keys are site-local, not globally unique, so two connections in the same org can collide (see "Known risks" below). +- Deleting a connection must be a checked, app-level operation (block with a clear error if any `OrgProjectConnection` still references it) — the DB FK (`NO ACTION`, no cascade) will hard-error anyway, but the app should never let a user hit a raw constraint violation. +- `# CLUSTOX:` comments explain WHY, never what. +- The API token is a secret; the email/site_url are not. Never let the token appear in logs, error messages, or test fixtures. +- Backend tests must pass (`cd backend/analytics_server && ./venv/bin/python -m pytest`); frontend must typecheck. + +## Known risks this design must not reintroduce (found by code audit, not hypothetical) + +1. **Ticket idempotency collision**: `_ticket_idempotency_key` in `etl_jira_handler.py` currently builds `f"jira:{org_id}:{issue.id}"` — no connection scoping. Two connections' sites can assign the same small, site-local `issue.id`. Must become `f"jira:{org_id}:{connection_id}:{issue.id}"` once sync is connection-aware (Task 4). +2. **PR↔ticket matching collision**: `get_org_tickets_key_map` in `ticket_matching.py` builds one flat `{TICKET_KEY: ticket_id}` dict per org, no connection scoping. Two connections both having a project called "PROJ" would silently collide ("PROJ-123" maps to whichever ticket the SQL happened to return last), mismatching a PR to the wrong ticket with no error. Must scope the map through `OrgProjectConnection` to only the connection(s) relevant to the PR's repo/team (Task 5). +3. **`get_jira_etl_handler`** currently does `integrations[0]` — hardcoded first element, not a loop. This is the load-bearing change for Task 4, not a minor patch. +4. **No existing rate-limit backoff or concurrency primitive** in the Jira ETL path — N connections will sync N× sequentially. Reuse the existing per-org Redis lock pattern (`api/sync.py`), extended to `{org_id}:{connection_id}:data_sync`, for isolation — do not attempt to introduce new concurrency machinery in this feature. + +--- + +## Task 1: Schema — `JiraConnection` + `OrgProjectConnection` ✅ done + +**Files:** +- `database-docker/db/migrations/20260909120000_clustox_jira_multi_account.sql` +- `backend/analytics_server/mhq/store/models/integrations/jira_connection.py` (+ `__init__.py` export) +- `backend/analytics_server/mhq/store/models/projects/repository.py` (`OrgProjectConnection` model, + `__init__.py` export) + +Migration creates both tables, the `(org_id, site_url, email)` unique index, the partial `is_default` unique index, and both `migrate:down` drops. No existing table touched. + +- [x] Run migration locally (`dbmate up` against the dev DB / docker) and confirm both `migrate:up` and `migrate:down` apply cleanly. Verified `up` against the dev container's DB — both tables and both indexes exist exactly as designed. `down` was not exercised in this session (destructive DB op, held back for explicit approval); the migration's `migrate:down` block is a straightforward two-table `DROP TABLE`, symmetric with `up`. +- [x] Confirm `flake8`/`black` clean on the two new model files. + +## Task 2: Repository/service layer for `JiraConnection` ✅ done + +**Files:** +- Created: `backend/analytics_server/mhq/store/repos/jira_connection.py` +- Created: `backend/analytics_server/tests/store/repos/test_jira_connection.py` + +**Interface:** +```python +def create_jira_connection(org_id, site_url, email, access_token, provider_meta, generated_by) -> JiraConnection: ... +def list_jira_connections(org_id) -> List[JiraConnection]: ... +def get_jira_connection(org_id, connection_id) -> Optional[JiraConnection]: ... +def delete_jira_connection(org_id, connection_id) -> None: ... # raises if OrgProjectConnection rows reference it +def set_default_jira_connection(org_id, connection_id) -> None: ... # unsets any prior default in the same transaction +``` +Reuses `CryptoService`/`get_crypto_service()` exactly as `Integration.access_token_enc_chunks` does — same global keypair, no per-row key management needed (confirmed safe by audit). `delete_jira_connection` and `set_default_jira_connection` both raise `JiraConnectionNotFoundError` when `connection_id` doesn't resolve for the given `org_id` (org-scoped lookup, not just an id match); `delete_jira_connection` additionally raises `JiraConnectionInUseError` when an `OrgProjectConnection` still references it, checked explicitly before any delete so the API layer (Task 3) never has to translate a raw FK `IntegrityError`. `set_default_jira_connection` unsets every other default for the org *before* setting the new one (two statements, one transaction/commit) — the reverse order would momentarily violate `jira_connection_one_default_per_org`. + +- [x] Unit tests: create/list/get/delete, delete-blocked-by-reference, default-switch atomicity (no window where 0 or 2 defaults exist). 9 tests, all passing; full backend suite (469 tests) still green. + +**Bug found live, after Task 6a shipped — second real user-facing failure:** a real user's own connection (site URL pasted straight from the browser's address bar, `https://acme.atlassian.net/`) was stored verbatim by `create_jira_connection` and silently broke every search/sync against it: `jira_project_search.ts`/`etl_jira_handler.py` both build `https://{site_url}/...`, so a stored value that already carries its own scheme becomes `https://https://acme.atlassian.net//...` — a URL that can't resolve, surfacing as an opaque connection timeout (8s, then a 502) with no indication of the real cause. `docs/JIRA_MULTI_ACCOUNT_PLAN.md`'s migration comment always said this column should hold "a normalized host... no scheme/trailing slash," and Task 3's write-up explicitly flagged that nothing enforced it yet — this is that gap, closed. Fixed with a new `_normalize_site_url` (strip scheme, trailing slash/path, lowercase host — mirrors web-server's existing `normalizeJiraSiteUrl`) called from `create_jira_connection` itself, the one place every caller funnels through, rather than trusted to each caller individually. The web-server's connection-add form (`JiraConnectionsManager.tsx`) now also calls `normalizeJiraSiteUrl` before submitting, for the same reason `ConfigureJiraModalBody` (the legacy flow) already did — belt and suspenders, not a substitute for the backend fix. 8 new tests (`TestNormalizeSiteUrl`'s parametrized cases, plus two `create_jira_connection` tests covering the exact reported input). Full backend suite: 556 passed (was 548, +8). + +**Pre-existing data note:** the 3 `JiraConnection` rows already created against this dev DB before the fix landed still hold their un-normalized `site_url` values — a raw SQL repair was attempted and blocked by the environment's own destructive-write guardrail. Simplest fix for existing rows: delete and re-add them through the (now-fixed) UI, which is what was recommended to the user rather than pushing through the guardrail. + +## Task 3: API endpoints ✅ done + +**Decision (asked of the user, since it wasn't derivable from the plan text alone):** the new Next.js routes proxy to new Flask routes backed by Task 2's `JiraConnectionRepoService`, rather than writing `JiraConnection`/`OrgProjectConnection` directly from knex the way `orgs/[org_id]/integration.ts` does for the legacy singleton row. This is the same shape as the Jenkins mapping routes (`pages/api/clustox/jenkins/mappings.ts` → `mhq/api/integrations.py`). Chosen over direct knex writes because it reuses Task 2's delete-blocked check and default-switch atomicity in one place instead of re-implementing them in TypeScript, and because direct writes would have left create/delete/set_default unused outside tests. + +**Files:** +- Backend (new, not in the original file list above — required by the proxy decision): four routes appended to `mhq/api/integrations.py` (`GET`/`POST /orgs/<org_id>/integrations/jira-connections`, `DELETE`/`PATCH /orgs/<org_id>/integrations/jira-connections/<connection_id>`), plus `tests/api/test_jira_connections_routes.py` (10 tests). +- Created: `web-server/pages/api/resources/orgs/[org_id]/jira-connections/index.ts` (GET list, POST create) +- Created: `web-server/pages/api/resources/orgs/[org_id]/jira-connections/[connection_id].ts` (DELETE, PATCH set-default) +- Created: matching `__tests__` for both (12 tests). +- `src/types/request.ts`: added a typed, optional `session` field to `ApiRequest` (previously only reachable via an untyped `(req as any).session`) — needed so the POST route can stamp `generated_by` with the signed-in user's id. +- `src/constants/feature.ts`: added `show_jira_multi_account` (default `false`). + +Left `orgs/[org_id]/integration.ts`'s existing Jira handling completely untouched — these are new, additive routes only. Visibility is gated by `show_jira_multi_account`: both route files 404 (not 403 — while off, the routes should read as not existing) when the flag isn't set on the request. + +**Explicitly not done in this pass** (not asked for by the plan's own checklist, called out so it isn't assumed covered): no live Jira-credential validation on create (the legacy flow's `pages/api/integrations/jira/validate.ts` has no analogue here yet — POST just persists whatever token it's given), and no `site_url` normalization (scheme/trailing-slash stripping) at either the Flask or Next layer — the migration comment describes the column as normalized, but nothing enforces it yet. + +- [x] Auth/authz matches existing org-scoped route conventions (`Endpoint`'s built-in `assertWorkspaceAccess` on `org_id`, same as every other org-scoped BFF route; Flask side uses `query_validator.org_validator(org_id)` like the Jenkins/GitHub/GitLab routes beside it). +- [x] Integration test: create two connections, list returns both, delete one blocked while referenced, set-default swaps atomically. Covered at both layers — the Flask route tests via `FakeJiraConnectionRepoService` (in-memory, reproducing the uniqueness/reference/atomicity rules), and the Next.js tests confirm the proxying, error-forwarding, and feature-flag gate. +- [x] Full backend suite (479 tests) and full web-server typecheck stay green; full web-server jest suite run in progress at hand-off. + +## Task 4: Sync — make it connection-aware ✅ done + +**Files touched (larger than the plan's own list — see call-outs below):** +- `backend/analytics_server/mhq/service/project/sync/etl_jira_handler.py` — as planned. +- `backend/analytics_server/mhq/service/project/sync/etl_provider_handler.py` — new abstract method `get_org_projects_to_sync(org_id)`. +- `backend/analytics_server/mhq/service/project/sync/etl_handler.py` — `sync_org_projects` now asks the handler which projects to sync instead of querying `get_active_org_projects_for_provider` itself; `sync_project_issues` loops a *list* of handlers per provider (one per connection) with per-handler try/except. +- `backend/analytics_server/mhq/service/project/sync/etl_project_factory.py` — `__call__` returns `List[ProjectProviderETLHandler]`, not one handler. +- `backend/analytics_server/mhq/service/project/integration.py` — see gap call-out below. +- `backend/analytics_server/mhq/store/repos/projects.py` — new `get_active_org_projects_for_connection`. +- `backend/analytics_server/mhq/store/repos/jira_connection.py` — new `decrypt_access_token`. +- Tests: `test_etl_jira_handler.py` (+8), `test_projects_connection_scoping.py` (new, 2 tests, the join specifically). + +**Design decision, strict either/or (not additive):** `get_jira_etl_handlers(org_id)` returns `[legacy_handler]` when the org has zero `JiraConnection` rows, or one handler per connection when it has any — never both. Concretely: the moment an org creates its first `JiraConnection`, any pre-existing projects still only reachable through the legacy `Integration` row **stop syncing** until Task 6's project picker moves them onto a connection (writes their `OrgProjectConnection` row). This reads as a real behavioral cliff but is what the plan's own text says ("falling back to legacy... when an org has none") and matches its Global Constraint that the two flows are deliberately independent, never auto-mirrored. Flagging it here in case that cliff is worse in practice than it reads on paper — it would be an easy follow-up to make legacy also cover "projects with no `OrgProjectConnection` row" if wanted, but that's an explicit product call, not a default I should have picked myself. + +**Gap found beyond the plan's stated scope:** `ProjectIntegrationService.get_org_providers` (`integration.py`) only checked the legacy `Integration` table. An org with *only* `JiraConnection` rows and no legacy row would get an empty provider list and `sync_project_issues` would return immediately — a JiraConnection-only org would silently never sync at all. Fixed by also reporting `"jira"` when `JiraConnectionRepoService.list_jira_connections(org_id)` is non-empty. + +**Write side of `OrgProjectConnection` is NOT in this task**, despite the plan's Task 4 text mentioning it ("write an OrgProjectConnection row for every OrgProject synced from a JiraConnection"): nothing in the sync path (`etl_jira_handler.py`/`etl_handler.py`) ever creates `OrgProject` rows — that only happens in `ProjectService.update_team_projects` (`repository_service.py`), which is squarely Task 6 (project selection) territory, not sync. Sync only ever *reads* the `OrgProjectConnection` join (`get_active_org_projects_for_connection`) to know which projects are already claimed by a connection. Task 6, when it adds "pick a connection before picking its projects," is where the write actually belongs. + +**Known risk #4 (rate-limit/concurrency) closed by design, no new primitive needed:** connections sync strictly sequentially inside `sync_project_issues`'s for-loop, and that whole call already runs inside the existing per-org `"{org_id}:data_sync"` Redis lock (`mhq/api/sync.py`). N connections make N sequential passes, same shape as N providers already did — nothing here introduces concurrency, so the plan's suggested lock-key extension wasn't needed. + +**Ticket idempotency key (Known risk #1) — backward compatibility verified explicitly:** the legacy (connection_id is `None`) branch keeps the exact original `f"jira:{org_id}:{issue.id}"` format; only connection-scoped handlers use `f"jira:{org_id}:{connection_id}:{issue.id}"`. Covered by a dedicated test asserting the legacy format is byte-identical to before, plus a two-connections-same-issue-id non-collision test. + +**The new join was verified against real Postgres, not just mocks** — a throwaway smoke script (not committed) inserted two connections × three projects (one inactive, one on a different connection) and confirmed `get_active_org_projects_for_connection` returns exactly the one active project actually owned by the connection asked about, then cleaned up after itself. + +- [x] Test: an org with 0 / 1 / 2+ connections syncs correctly; two connections with colliding `issue.id` do not collide post-fix. +- [x] Full backend suite: 489 passed (was 479 before this task), flake8/black clean on every touched/new file. + +## Task 5: PR↔ticket matching — scope by connection ✅ done + +**Files:** `backend/analytics_server/mhq/store/repos/ticket_matching.py`, `backend/analytics_server/mhq/service/ticket_matching/service.py`, plus their test files (`test_ticket_matching.py` new, `test_service.py` extended). + +**Design refinement on "scope by connection" — asked of and confirmed by the user, then implemented more narrowly than the literal instruction to avoid a self-inflicted regression:** the plan's wording ("restrict... to the connection(s) relevant to the PR's repo/team, instead of one flat org-wide map") reads as *always* scoping every match to the PR's repo/team. Implemented that way, it would have been a real regression: `get_org_tickets_key_map`'s own existing docstring is explicit that matching stays org-wide *on purpose* — "a ticket that was already synced should stay matchable even if its project is later deselected" — and the vast majority of orgs (zero or one Jira connection) never have a colliding key at all. Always scoping by repo/team would have broken that guarantee for every ordinary org to fix a problem that only exists for the rare colliding case. + +What's actually built: matching stays org-wide by default. `get_org_tickets_key_map` now excludes any key shared by 2+ tickets (previously it silently kept "whichever the query happened to return last" — the exact bug the plan's "Known risks" #2 describes) and a new `get_colliding_ticket_key_candidates` exposes those held-back keys with their candidates. Only when a PR's extracted key hits a genuine collision does `TicketMatchingService._resolve_colliding_key` reach for a new `get_relevant_org_project_ids_for_repo(repo_id)` (a `TeamRepos ⋈ TeamProjects` join — nothing in this codebase resolved repo→team→project before this) to find which candidate's project the PR's own tracking team(s) actually selected. Exactly one relevant candidate → matched; zero or more than one → skipped, not guessed (fail closed, consistent with this codebase's existing "never render a plausible wrong answer" pattern for Bitbucket/Jenkins). The per-repo lookup is cached within one `match_org_prs_to_tickets` call so N colliding PRs from the same repo cost one query, not N. + +**Note:** `OrgProjectConnection` isn't referenced directly here — `get_relevant_org_project_ids_for_repo` only needs `TeamRepos`/`TeamProjects`; the connection-vs-legacy distinction is already baked into which projects exist as collision *candidates* (only two same-keyed projects across different connections, or a connection and the legacy flow, can collide in the first place). + +- [x] Regression test: two connections both with a project "PROJ" — a PR referencing "PROJ-123" matches the ticket from the correct connection only (`test_resolves_to_the_ticket_from_the_prs_own_relevant_connection`), plus the no-relevant-match and still-ambiguous-after-scoping fail-closed cases, plus a same-PR test proving a plain non-colliding key and a colliding key both resolve together. +- [x] Full backend suite: 504 passed (was 489 before this task, +15 new), flake8/black clean. + +## Task 6: Frontend — connections management ✅ done; project-picker wiring not started + +**Part 1 — connections list/add/delete/set-default: done.** + +**Files:** +- New: `web-server/src/hooks/useJiraConnections.ts` — list/create/delete/setDefault against the Task 3 API routes. +- New: `web-server/src/components/JiraConnectionsManager.tsx` — the modal body: add-connection form, a table with a set-default star and a delete icon per row (with a confirmation dialog, since delete can be blocked by the backend), server error messages surfaced verbatim (`readApiError`, mirrors `ClustoxJenkinsMapping`'s `mutationError`). +- New: `web-server/src/components/__tests__/JiraConnectionsManager.test.tsx` — 9 tests (list/empty state, validation, create, duplicate-account 409, set-default, delete + confirm, delete-blocked 409, no redundant `feature_flags` param — see the bug below). +- `web-server/src/content/Dashboards/JiraIntegrationCard.tsx` — **the connections manager lives here, not on a separate card.** It was first built as a standalone `JiraConnectionsCard` sitting beside the legacy card, per the plan's literal file list, but that read as two unrelated "Jira" tiles on the integrations page and the user asked for one. `JiraIntegrationCard` now branches on `show_jira_multi_account`: on (the default), it renders a single "Manage connections" action opening this modal; off, it falls back to the original link/unlink flow unchanged (renamed internally to `LegacyJiraIntegrationCard`). `pages/integrations.tsx` no longer has a second Jira card at all. +- `web-server/src/types/request.ts` — added a typed `session?: AuthSession` field to `ApiRequest` (previously only reachable via an untyped cast); needed so the create-connection call can stamp `generated_by` with the signed-in user's id. +- `web-server/src/constants/feature.ts` — `show_jira_multi_account` defaults to `true` (flipped from `false` once the user confirmed they wanted this live, not console-flag-gated). Still a real flag, just on by default. + +**Bug found and fixed after initial ship — real user-facing failure, not just a gap:** `useJiraConnections` originally stamped its own `feature_flags` query param on every request, reasoning (wrongly) that nothing else did. In fact `web-server/middleware.ts` already appends a `feature_flags` param to **every** request in this app (page and API alike), built from cookie-based overrides merged with the defaults — a mechanism that exists at the Next.js middleware layer, invisible to a `src/`/`pages/`-scoped grep (how this was missed the first time). Two query params of the same name parse as an array server-side; `JSON.parse()` on an array coerces it to a comma-joined string via `Array.prototype.toString()`, corrupting the JSON exactly at the join point. This surfaced live as "Unexpected non-whitespace character after JSON at position 32" — 32 being the exact length of the hand-stamped `{"show_jira_multi_account":true}`. Fixed by deleting the manual stamp entirely; the middleware already covers it. Regression-tested (`does not add its own feature_flags param...`). + +**Part 2 — `TeamJiraProjects.tsx`: pick a connection before picking its projects — ✅ done.** Resolved the two open design questions from the original write-up: + +- **How `jira_project_search.ts` authenticates against a specific connection**: an optional `connection_id` query param. Present → look up that `JiraConnection` row (org-scoped, mirrors `JiraConnectionRepoService.get_jira_connection`'s own scoping) and decrypt its token with the same TS `dec()` helper the legacy path already uses. Absent → the exact original legacy `Integration` lookup, byte-for-byte, so an org that has never touched multi-account Jira sees no behavior change at all. Idempotency keys follow the same legacy-preserving pattern as the ticket/sync fixes: `jira:{org_id}:{project.id}` when no connection, `jira:{org_id}:{connection_id}:{project.id}` when one was given. +- **Where the `OrgProjectConnection` row gets written**: `ProjectService._update_org_projects` (`repository_service.py`), the write side of the Task 4 gap. `RawTeamOrgProject` grew an optional `connection_id`; when present, a row is written (after the `OrgProject` it references is already committed — the FK requires that ordering) via a new `ProjectRepoService.save_org_project_connections`. A project saved *without* a `connection_id` is left alone, never de-associated — safe by construction, since the frontend's own "load existing selections" response doesn't yet echo back `connection_id` per project (a resave of an untouched selection would otherwise silently erase a real association). + +**Files:** +- `backend/analytics_server/mhq/service/project/models/org_project.py` — `RawTeamOrgProject.connection_id: Optional[str]`. +- `backend/analytics_server/mhq/api/request_utils.py` — `coerce_org_project` passes it through. +- `backend/analytics_server/mhq/store/repos/projects.py` — new `save_org_project_connections`. +- `backend/analytics_server/mhq/service/project/repository_service.py` — `_update_org_projects` writes the rows, after the projects they reference are committed. +- `backend/analytics_server/tests/service/project/test_repository_service.py` — 5 new tests (`TestOrgProjectConnectionWrites`): new-project case, existing-project case, no-op when nothing has a connection_id, mixed-batch (only the connected one gets a row), and write-ordering (projects before connections). +- `web-server/pages/api/internal/[org_id]/jira_project_search.ts` — connection-aware search, as above. 3 new tests. +- `web-server/pages/api/resources/team_projects.ts` — `connection_id` added to the PUT schema's per-project shape (yup silently strips unlisted fields, so this was required, not optional-to-skip). +- `web-server/src/constants/db.ts` — added `JiraConnection`/`OrgProjectConnection` to the `Table` enum + their `Columns`, needed for `jira_project_search.ts`'s typed `db('JiraConnection')` call. +- `web-server/src/components/Teams/useTeamJiraProjectsConfig.tsx` — pulls in `useJiraConnections`, a `selectedConnectionId` (`''` = legacy, the default), threads it into search and into each saved project's payload. A search result already carries its own `connection_id` (from the route above), so a project keeps the right one even if the picker selection changes after adding it — no separate stamping step needed. +- `web-server/src/components/Teams/TeamJiraProjects.tsx` — the picker itself: a `<TextField select>` with "Legacy Jira account" plus every connection, rendered only when the org has at least one connection (so an org that never added one sees the exact original UI). +- `web-server/src/components/Teams/__tests__/TeamJiraProjects.test.tsx` — 3 new tests for the picker (hidden with no connections, lists legacy + connections, calls the setter on selection). + +**Not done, noted rather than silently skipped:** `GET /teams/<team_id>/projects` (`adapt_org_projects`) doesn't echo back which connection an already-saved project came from, so the picker can't show "this one's from acme.atlassian.net" for previously-saved selections — only for ones picked in the current session. Low-risk (existing associations are never erased by this gap, per the safe-by-construction note above) but worth closing later if the UI needs to display it. + +- [x] Typecheck clean (`tsc --noEmit`, zero errors) for both parts. +- [x] Feature-flagged: `show_jira_multi_account` (default `true`) gates the card's rendering and the API routes it calls; the picker itself additionally self-gates on `connections.length > 0` regardless of the flag's own gating. +- [x] Verified live end-to-end: list/add/delete/set-default all round-trip successfully against the real dev container (confirmed via server logs after the query-param bug fix above). +- [x] Full backend suite: 548 passed (was 543 before this part, +5 new). Full frontend typecheck clean; every touched/new test file passes in isolation (search: 11, picker: 9, connections manager: 9, card: 8, API routes: 22, team_projects: unaffected existing tests still green) — a combined multi-file jest sweep was not completed in this session (see Task 6a's own environment note; same constrained machine, same outcome). + +**Bug found live after shipping Part 2 — search appeared to hang:** the user reported the project search spinner "just showing loader" when searching a real connection. Root cause, found in the server logs, was two compounding pre-existing issues neither specific to this feature nor introduced by it, both now fixed: +1. This app's global `axios` instance is configured with `axiosRetry(axios, { retries: 2 })` (`src/api-helpers/axios.ts`), and Jira's real `/project/search` endpoint was intermittently slow enough to hit the route's own 8s timeout and return a 502. Axios-retry silently retried that 502 twice more — three full ~8s attempts, ~24s total, before ever resolving to the caller. Fixed by opting this specific search call out of retries (`'axios-retry': { retries: 0 }` per-request override) — a slow/failing keystroke's search should fail fast, since the next keystroke's own debounced request is the real retry mechanism here. +2. Search-as-you-type's client-side `AbortController` only tears down the browser's connection to *this app's own* route; it was never propagated to that route's own outbound call to Jira, so a burst of keystrokes piled up that many uncancelled real Jira requests running concurrently in the background, competing for whatever's on the other end and making the genuinely-still-wanted final request slower. Fixed by listening for `res.on('close', ...)` (client disconnect) in `jira_project_search.ts` and forwarding it into an `AbortController` passed as the outbound `axios.get`'s `signal` — `res`, not `req`: `Endpoint.serve()` passes handlers a shallow-spread plain object for `req` (see `transformNextRequest`), which has lost `IncomingMessage`'s `EventEmitter` prototype methods, while `res` is passed through untouched. + +New regression test (`aborts the outbound Jira call when the client disconnects mid-search`) captures the registered `close` handler, invokes it mid-flight, and asserts the outbound call's abort signal actually fires. + +**Environment note:** a full `yarn jest` sweep of the whole web-server suite could not be completed in this session — two attempts both hung with zero output on a memory-constrained dev machine (16GB, multiple other heavy processes/containers), unrelated to the code itself (a targeted run of the new test file passed cleanly, and `tsc --noEmit` — which would catch a type-level regression from either of the two shared-file edits above — is clean). Worth an actual `yarn jest` run on a less constrained machine before merging, as a final broad regression check. + +## Task 7: Backward-compat / rollback tests + +- [x] Org with 0 `JiraConnection` rows behaves identically to today (legacy `Integration` path only). Covered directly: `test_falls_back_to_the_single_legacy_handler_when_the_org_has_no_connections` and `test_legacy_handler_keeps_the_original_key_shape`/`test_legacy_handler_reads_every_active_project_for_the_org_and_provider` (Tasks 4-5) — the legacy handler and its idempotency-key format are byte-for-byte what existed before this feature. +- [ ] Migration `up` then `down` leaves schema identical to pre-migration state. `up` verified live against the dev DB (Task 1). `down` was never exercised — the auto-approval classifier blocked the rollback command as a destructive DB operation both times it was attempted, and it was not re-requested from the user given how much else was already in flight. The migration's `down` block is two straightforward `DROP TABLE` statements, symmetric with what `up` created; still worth an explicit human-approved run before merging. +- [x] Deleting a referenced connection fails loudly (app-level 4xx, not a raw DB error leaking to the client). Covered at every layer: `test_raises_in_use_when_an_org_project_connection_still_references_it` (repo), `test_deleting_a_referenced_connection_is_409_and_leaves_it_in_place` (Flask route), and the Next.js route test asserting the 409 and its message forward correctly. diff --git a/docs/MULTITENANCY_PROPOSAL.md b/docs/MULTITENANCY_PROPOSAL.md new file mode 100644 index 000000000..c94825a2c --- /dev/null +++ b/docs/MULTITENANCY_PROPOSAL.md @@ -0,0 +1,155 @@ +# Middleware Fork — Authentication & Multitenancy + +**For review and approval** +Prepared 2026-08-06 · Clustox/middleware · branch `feat/auth-rbac` + +--- + +## 1. Where we are + +Authentication and role-based access are **built, tested and pushed**. What works today: + +- Login screen; nothing in the app is reachable without signing in +- Two roles: **SuperAdmin** and **Admin** +- SuperAdmin can create users, assign roles, and scope them +- Admins see only what they're scoped to; requesting anything else returns 403 +- The analytics API is locked down — previously `curl http://<host>:9696/…` returned every + team's metrics with no credentials at all +- Sign out, user management page, session expiry + +156 frontend and 155 backend tests passing. + +**One thing worth knowing:** the open-source version we forked ships with *no authentication of +any kind*. Anyone who could reach the URL saw everything, and the API was open to anyone on the +network. That's now closed. + +--- + +## 2. The gap + +The first meeting specified **multitenancy** — each Admin brings their own GitHub/GitLab +integration and their own projects, with SuperAdmin seeing across all of them. + +What we built scopes access by **team** inside a single shared workspace. What's needed scopes by +**tenant**, where each Admin effectively has their own workspace. + +The authentication layer is unaffected and fully reusable. What changes is *what access is measured +against*. + +| Requirement | Status | +|---|---| +| SuperAdmin + Admin roles | ✅ Done | +| Login, sessions, password security | ✅ Done | +| SuperAdmin manages users | ✅ Done | +| API not bypassable | ✅ Done | +| Admin adds **their own** integration | ❌ Currently one shared integration | +| Admin adds **their own** projects | ❌ Projects are currently shared | +| SuperAdmin sees **all** projects | ⚠️ Sees all teams — right idea, wrong dimension | +| SuperAdmin **invites** admins | ❌ Not built | +| Multitenant | ❌ Single shared workspace | + +--- + +## 3. The good news + +We expected multitenancy to be a rewrite. It isn't. + +The upstream database was **designed for multiple tenants** and then bootstrapped to only ever +create one. Every table already carries an organisation ID. The integration table's primary key is +literally *(provider, organisation)* — meaning **one GitHub account per organisation is already how +it was built to work**. + +Only four places in the entire codebase assume a single tenant. Everything else already passes the +organisation through correctly. + +In practical terms: **"one workspace per Admin" is the design the code was already shaped for**, not +something we bolt on. That materially reduces both the cost and the risk. + +--- + +## 4. Proposed model + +**Each Admin gets their own workspace.** + +| | | +|---|---| +| **Admin** | Owns a workspace. Connects their own GitHub/GitLab account, adds their own projects and teams, sees only their own metrics. | +| **SuperAdmin** | Sits above all workspaces. Sees every project across every Admin, invites new Admins, manages users. Does not own a workspace. | + +Concretely, this gives each Admin their own integration credentials, their own project list, and +their own dashboards — with a SuperAdmin view spanning all of them. It matches the meeting notes +directly. + +**On integrations:** the note said this was undecided. Recommendation is that **Admins connect their +own**, with SuperAdmin able to see and revoke any of them. Reasons: it removes the SuperAdmin as a +bottleneck for onboarding, it's what the schema already supports, and it avoids one person holding +credentials for everyone else's repositories. + +--- + +## 5. Decisions needed + +Each has a recommendation. Approving all six as-is is a valid answer. + +**Q1. Does a SuperAdmin have their own workspace, or sit above all of them?** +*Recommend: sits above, owns none.* Keeps "who can see what" simple and avoids the odd case of a +SuperAdmin's personal projects mixing into an org-wide view. + +**Q2. Can two Admins track the same repository?** +*Recommend: yes, allow it.* The consequence is that the same repo gets synced once per Admin, using +each Admin's own credentials — duplicate API calls against GitHub. Acceptable at our scale, and +blocking it would create awkward "someone else claimed this repo" conflicts. + +**Q3. How do invites work — email, or a link?** +*Recommend: invite links first.* A real email invite needs mail infrastructure we don't currently +have. A SuperAdmin-generated single-use link works immediately and can be sent over Slack. Email can +be added later without rework. + +**Q4. What happens to the existing setup?** +There's already a live GitHub connection with two repos and two teams from testing. +*Recommend: it becomes the first Admin's workspace* rather than being discarded, so nothing is +re-entered. + +**Q5. Do we still want team-level access inside a workspace?** +We built the ability to scope an Admin to specific teams. Under the new model an Admin owns their +whole workspace, so it may be unnecessary. +*Recommend: drop it for now.* It's already built and can be re-enabled later if a workspace grows +large enough to need it. Carrying unused permission levels adds confusion, not safety. + +**Q6. Is the sync cost acceptable?** +Today one scheduled job syncs one workspace. Multitenant means syncing every workspace, each +against its own GitHub account and its own rate limit. +*Recommend: proceed, and add per-workspace sync status so failures are visible.* Right now a failed +sync reports success and simply shows no data — we hit this during testing and it cost real time to +diagnose. + +--- + +## 6. Honest costs + +- **Multitenancy is a data-model change**, not a settings toggle. It touches how every request + resolves which workspace it belongs to. +- **Sync is the main scaling risk.** Each workspace syncs independently; GitHub rate limits are + per-token, so this scales reasonably, but sync duration grows with the number of Admins. +- **This is a permanent fork change.** Upstream will not accept it — gated multi-tenant access is + plausibly what their paid product sells. We carry it ourselves from here, which is why we've kept + our changes marked and auditable so upstream updates can still be merged in. + +--- + +## 7. Suggested sequence + +1. **Merge the authentication work now.** It's complete, tested, and valuable on its own — it closes + a real hole where the API was open to anyone on the network. +2. **Approve the decisions above**, then a detailed design for multitenancy. +3. **Build multitenancy**, then invites. + +Splitting it this way means the security fix ships immediately rather than waiting behind a larger +piece of work. + +--- + +## What I need + +Approval on the model in §4, and answers to the six questions in §5 — or a simple "go with the +recommendations". diff --git a/docs/UPSTREAM_SYNC_LOG.md b/docs/UPSTREAM_SYNC_LOG.md new file mode 100644 index 000000000..c1f9cc523 --- /dev/null +++ b/docs/UPSTREAM_SYNC_LOG.md @@ -0,0 +1,93 @@ +# Upstream Sync Log + +A row per merge from `middlewarehq/middleware` into this fork. Append after every sync. + +Procedure: [FORK_STRATEGY.md §5](./FORK_STRATEGY.md#5-the-upstream-sync-procedure). +Cadence: monthly, plus immediately for any upstream security fix. + +Why this file exists: when something breaks three weeks after a sync, the first question is always +"what came in, and what did we have to resolve by hand?" Reconstructing that from git history is +possible but slow. Writing three lines at merge time is not. + +--- + +| Date | Upstream ref merged | Commits | Files changed | Conflicts | Sentinels before → after | Smoke test | PR | Notes | +|---|---|---|---|---|---|---|---|---| +| _(no syncs yet)_ | | | | | | | | | + +## Current divergence baseline + +Recorded 2026-08-06, after authentication, RBAC and multitenancy. **The first sync compares +against these numbers.** + +| | | +|---|---| +| Fork point | `844eb42` | +| Modified upstream files | **30** | +| New files | **48** | +| `CLUSTOX:` sentinel lines | **66** | + +> Superseded an earlier baseline of 19 / 26 / 33, recorded after the auth work and left stale +> through the multitenancy branch. A stale baseline is worse than none: it would have raised a +> false alarm on the very first sync, and the natural response to a tripwire that cries wolf is +> to stop trusting it. Re-record this after every feature branch, not just before a sync. + +Regenerate any time: + +```bash +# modified upstream files +git diff --name-only 844eb42..HEAD | while read -r f; do + git cat-file -e 844eb42:"$f" 2>/dev/null && echo "$f" +done | wc -l + +# sentinel lines +grep -rn "CLUSTOX" --include='*.py' --include='*.ts' --include='*.tsx' \ + --include='*.sql' --include='*.sh' --include='*.txt' --include='*.js' . \ + | grep -v node_modules | grep -v '^./docs/' | wc -l +``` + +**The 26 files carrying real edits** (the other four are generated or append-only: `schema.sql`, +`yarn.lock`, `package.json`, `env.example`). Regenerate with: + +```bash +git diff --name-only 844eb42..HEAD | while read -r f; do + git cat-file -e 844eb42:"$f" 2>/dev/null && echo "$f" +done | grep -vE "schema.sql|yarn.lock|package.json|env.example" +``` + +**Check these three first on any conflict:** + +| File | Why it matters | +|---|---| +| `web-server/src/api-helpers/global.ts` | The single enforcement point for authentication **and** workspace/team scoping across every BFF route. A bad merge here silently disables access control everywhere. | +| `backend/analytics_server/mhq/api/sync.py` | Rewritten to sync every workspace. Upstream's version syncs one org; taking theirs would silently stop syncing all but one workspace, with no error. | +| `web-server/pages/api/resources/orgs/[org_id]/teams/v2.ts` | Names the team `id`, not `team_id`, so central scoping does not cover it. It carries explicit `assertTeamAccess` calls; losing them reopens a cross-workspace hole. | + +Regression tests guarding these: `endpoint-team-scope.test.ts`, `workspace-guard.test.ts`, +`tests/clustox_auth/test_sync_run.py`, and `e2e/multitenancy.spec.ts` — the last covers the +cross-workspace delete that the central guard cannot see. + +--- + +## Column meanings + +| Column | What to record | +|---|---| +| **Upstream ref merged** | The `upstream/main` SHA merged, so the exact state is recoverable | +| **Commits** | `git log --oneline <prev>..upstream/main \| wc -l` | +| **Files changed** | `git diff --stat` summary line | +| **Conflicts** | Count, and the files — name them, they are the ones to re-check next time | +| **Sentinels before → after** | `CLUSTOX:` count either side of the merge. **A drop means a resolution ate one of our changes.** | +| **Smoke test** | Pass/fail against the checklist in FORK_STRATEGY §5 step 4 | +| **PR** | Link to the reviewed sync PR | +| **Notes** | Behaviour changes, migrations, anything that needed a decision | + +## Worked example of a filled row + +| Date | Upstream ref merged | Commits | Files changed | Conflicts | Sentinels before → after | Smoke test | PR | Notes | +|---|---|---|---|---|---|---|---|---| +| 2026-09-01 | `abc1234` | 23 | 41 files, +892/-310 | 2 — `etl_code_factory.py`, `sync_data.py` | 14 → 14 | ✅ | #12 | Upstream added a Bitbucket provider; our registration line moved but survived. One new migration, applied cleanly. | + +Note what the example shows: conflicts in **yellow-zone** registration points, resolved mechanically, +sentinel count unchanged. That is what a healthy sync looks like. Conflicts in red-zone files with a +falling sentinel count is the pattern to escalate on. diff --git a/env.example b/env.example index 99fabc512..081d4ac9b 100644 --- a/env.example +++ b/env.example @@ -22,3 +22,18 @@ NEXT_PUBLIC_APP_ENVIRONMENT="development" DEFAULT_SYNC_DAYS=31 BUILD_DATE=2024-06-05T10:21:34Z MERGE_COMMIT_SHA=5f9ff895ad1d7805edcb22bfe2fcc6129e33bd8c + +# CLUSTOX: auth +NEXTAUTH_URL=http://localhost:3333 +NEXTAUTH_SECRET=generate-with-openssl-rand-base64-32 +INTERNAL_API_TOKEN=generate-with-openssl-rand-hex-32 +SUPERADMIN_EMAIL=admin@clustox.com +SUPERADMIN_PASSWORD=set-a-strong-password + +# CLUSTOX: invite emails (optional -- SMTP_HOST unset means invites still +# create a link to copy/send manually, they just don't also get emailed) +SMTP_HOST= +SMTP_PORT=587 +SMTP_USER= +SMTP_PASSWORD= +SMTP_FROM= diff --git a/setup_utils/cronjob.txt b/setup_utils/cronjob.txt index 5fc95d918..5366b6e4b 100644 --- a/setup_utils/cronjob.txt +++ b/setup_utils/cronjob.txt @@ -1,4 +1,6 @@ #!/usr/bin/env python3 # Every 30 minutes, run the sync script -*/30 * * * * curl -X POST http://localhost:9697/sync >> /var/log/cron/cron.log 2>&1 +# CLUSTOX: routed through trigger_sync.sh so the internal token is sent. +# The sync server rejects unauthenticated requests since auth was added. +*/30 * * * * /app/setup_utils/trigger_sync.sh >> /var/log/cron/cron.log 2>&1 diff --git a/setup_utils/start.sh b/setup_utils/start.sh index 8c86ef367..a5a891ada 100755 --- a/setup_utils/start.sh +++ b/setup_utils/start.sh @@ -20,18 +20,38 @@ fi echo 'MHQ_STARTING SUPERVISOR' -if [ -f "/app/backend/analytics_server/mhq/config/config.ini" ]; then - echo "config.ini found. Setting environment variables from config.ini..." - while IFS='=' read -r key value; do - if [[ "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ && ! -z "$value" ]]; then - echo "$key"="$value" >> ~/.bashrc - fi - done < "../backend/analytics_server/mhq/config/config.ini" -else - echo "config.ini not found. Running generate_config_ini.sh..." - /app/setup_utils/generate_config_ini.sh -t /app/backend/analytics_server/mhq/config +CONFIG_DIR="/app/backend/analytics_server/mhq/config" +CONFIG_FILE="$CONFIG_DIR/config.ini" + +# Provider tokens are encrypted with this key pair. It must be stored in the +# persistent dev_keys volume: replacing it makes existing Jira tokens impossible +# to decrypt. The backend reads this file directly, so copying it into .bashrc +# neither helps Jira nor is safe for private-key material. +if [ ! -f "$CONFIG_FILE" ]; then + echo "config.ini not found. Generating keys for a new installation..." + /app/setup_utils/generate_config_ini.sh -t "$CONFIG_DIR" +fi + +# Do not silently generate a replacement when a mounted key file is corrupt: +# that would leave the database containing credentials encrypted with old keys. +if ! grep -qE '^[[:space:]]*SECRET_PRIVATE_KEY[[:space:]]*=[[:space:]]*[^[:space:]]+' "$CONFIG_FILE" || \ + ! grep -qE '^[[:space:]]*SECRET_PUBLIC_KEY[[:space:]]*=[[:space:]]*[^[:space:]]+' "$CONFIG_FILE"; then + echo "Invalid $CONFIG_FILE: both RSA key entries are required. Refusing to start." + exit 1 fi -source ~/.bashrc +# The Python backend reads config.ini directly and needs nothing further. +# The Node web-server (enc()/dec() in web-server/src/utils/auth-supplementary.ts) +# reads these as plain environment variables, not from the file -- without +# this, SECRET_PUBLIC_KEY/SECRET_PRIVATE_KEY are simply undefined for it on +# every boot, and encrypting any provider token (Jira, GitHub, GitLab) fails +# outright before ever reaching the provider's API. `export`, not just +# `KEY=value`: a plain assignment is only a local shell variable and never +# reaches supervisord's children. +while IFS='=' read -r key value; do + if [[ "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ && ! -z "$value" ]]; then + export "$key=$value" + fi +done < "$CONFIG_FILE" /usr/bin/supervisord -c "/etc/supervisord.conf" diff --git a/setup_utils/trigger_sync.sh b/setup_utils/trigger_sync.sh new file mode 100644 index 000000000..ed3af6fc6 --- /dev/null +++ b/setup_utils/trigger_sync.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# CLUSTOX: cron does not inherit the container environment, so source it before +# calling the sync server, which now requires the internal token. +set -a +[ -f /app/.env ] && . /app/.env +set +a + +curl -s -X POST \ + -H "X-Internal-Token: ${INTERNAL_API_TOKEN}" \ + "http://localhost:${SYNC_SERVER_PORT:-9697}/sync" diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 000000000..a3da551ec --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,45 @@ +# SonarQube configuration for middleware. +# Consumed by the sonar-scanner CLI in the "SonarQube Analysis" stage of the Jenkinsfile. +# sonar.projectVersion is supplied by the pipeline (-Dsonar.projectVersion=$GIT_COMMIT). + +sonar.projectKey=middleware +sonar.projectName=middleware +sonar.sourceEncoding=UTF-8 + +# Polyglot monorepo: Python analytics backend, Next.js web server, TypeScript CLI. +# Sources are listed explicitly rather than using "." so that media_files/, +# database-docker/ and docs/ are not indexed as application code. +sonar.sources=backend/analytics_server,web-server/src,web-server/pages,cli/source + +# Two of these test roots live inside a source root, so they are removed from main +# in sonar.exclusions below and declared here instead -- otherwise the scan fails +# with "file can't be indexed twice". web-server/e2e sits outside sonar.sources, so +# it needs no exclusion. +sonar.tests=backend/analytics_server/tests,cli/source/__tests__,web-server/e2e + +sonar.python.version=3.10,3.11,3.12 + +sonar.exclusions=\ + backend/analytics_server/tests/**,\ + cli/source/__tests__/**,\ + **/*.test.ts,\ + **/*.test.tsx,\ + **/*.spec.ts,\ + **/*.spec.tsx,\ + **/node_modules/**,\ + **/.next/**,\ + **/dist/**,\ + **/build/**,\ + **/__pycache__/**,\ + **/*.pyc,\ + **/*.min.js,\ + web-server/libdefs/**,\ + **/next-env.d.ts,\ + .scannerwork/** + +# No coverage report is produced by this pipeline, so coverage will read as 0%. +sonar.coverage.exclusions=\ + **/tests/**,\ + **/__tests__/**,\ + web-server/e2e/**,\ + backend/dev_scripts/** diff --git a/web-server/e2e/auth.spec.ts b/web-server/e2e/auth.spec.ts new file mode 100644 index 000000000..90fee5082 --- /dev/null +++ b/web-server/e2e/auth.spec.ts @@ -0,0 +1,95 @@ +/** + * Auth and access-control end-to-end coverage. + * + * Running these: + * + * docker compose up -d + * cd web-server && yarn playwright test e2e/auth.spec.ts + * + * The four request-level tests (401/403/health) need no browser and run + * anywhere. The three page-level tests need Chromium plus its system + * libraries, which the middleware-dev image does not ship -- `playwright + * install --with-deps` fails on this Debian base. Run those from the host or + * from CI, where browser dependencies are available. The behaviours they + * cover were each verified manually against a real browser when written. + */ +import { expect, test } from '@playwright/test'; + +// URLs are absolute on purpose: upstream's playwright.config.ts sets no +// baseURL, and adding one would mean touching another upstream file for no +// functional gain. +const APP = 'http://localhost:3333'; +const API = 'http://localhost:9696'; +const SYNC = 'http://localhost:9697'; + +const SUPERADMIN = { + email: process.env.SUPERADMIN_EMAIL || 'admin@clustox.com', + password: process.env.SUPERADMIN_PASSWORD || '' +}; + +const signIn = async (page: any, email: string, password: string) => { + await page.goto(`${APP}/login`); + await page.fill('input[type="email"]', email); + await page.fill('input[type="password"]', password); + await page.click('button[type="submit"]'); + await page.waitForURL((u: URL) => !u.pathname.startsWith('/login'), { + timeout: 30_000 + }); +}; + +test('unauthenticated visitors are redirected to login', async ({ page }) => { + await page.goto(`${APP}/dora-metrics`); + await expect(page).toHaveURL(/\/login/); +}); + +test('bad credentials are rejected without revealing which field was wrong', async ({ + page +}) => { + await page.goto(`${APP}/login`); + await page.fill('input[type="email"]', 'nobody@clustox.com'); + await page.fill('input[type="password"]', 'wrong-password'); + await page.click('button[type="submit"]'); + await expect(page.getByRole('alert')).toContainText('Invalid email or password'); +}); + +test('unauthenticated API requests return 401, not a redirect', async ({ + request +}) => { + const res = await request.get(`${APP}/api/clustox/users`, { + failOnStatusCode: false, + maxRedirects: 0 + }); + expect(res.status()).toBe(401); +}); + +test('the analytics API refuses direct requests without the internal token', async ({ + request +}) => { + const res = await request.get(`${API}/teams/any-id/lead_time`, { + failOnStatusCode: false + }); + expect(res.status()).toBe(403); +}); + +test('the sync API refuses direct requests without the internal token', async ({ + request +}) => { + const res = await request.post(`${SYNC}/sync`, { failOnStatusCode: false }); + expect(res.status()).toBe(403); +}); + +test('health endpoints stay reachable so probes keep working', async ({ + request +}) => { + expect((await request.get(`${API}/`, { failOnStatusCode: false })).status()).toBe(200); + expect((await request.get(`${SYNC}/`, { failOnStatusCode: false })).status()).toBe(200); +}); + +test('superadmin can sign in and reach user management', async ({ page }) => { + test.skip(!SUPERADMIN.password, 'SUPERADMIN_PASSWORD not set'); + + await signIn(page, SUPERADMIN.email, SUPERADMIN.password); + await page.goto(`${APP}/users`); + await expect(page.getByText('Not authorised')).toHaveCount(0); + await expect(page.getByRole('heading', { name: 'Users' })).toBeVisible(); +}); diff --git a/web-server/e2e/benchmarks.spec.ts b/web-server/e2e/benchmarks.spec.ts new file mode 100644 index 000000000..f4f10d96c --- /dev/null +++ b/web-server/e2e/benchmarks.spec.ts @@ -0,0 +1,326 @@ +/** + * Cross-workspace isolation for the per-team DORA benchmark endpoints. + * + * Two endpoints are under test: + * - the settings routes, reused for benchmarks via setting_type=BENCHMARK_SETTING: + * GET/PUT /api/internal/team/{team_id}/settings + * workspace-scoped by Endpoint.serve() asserting team access whenever the + * payload carries a team_id (here, taken from the URL path). + * - the global baseline: + * GET/PUT /api/clustox/benchmarks/global + * GET is open to any authenticated admin (the five numbers already appear + * on every team's own dashboard as target lines); PUT is SUPERADMIN-only, + * because one superadmin's numbers become every unset team's targets. + * + * All five benchmarked metrics are covered -- the four DORA ones plus + * lines_of_code, which shares these exact routes. + * + * Running: + * docker compose up -d + * cd web-server && SUPERADMIN_PASSWORD='<value>' yarn playwright test e2e/benchmarks.spec.ts + * + * The suite provisions its own admins through the API and cleans them up, so + * it does not depend on whatever happens to be in the database. + */ +import { APIRequestContext, expect, request, test } from '@playwright/test'; + +// CLUSTOX: overridable so the suite can run against an instance other than the +// default dev one -- e.g. a second server started from a branch build while the +// usual port is still serving something else. +const APP = process.env.E2E_BASE_URL || 'http://localhost:3333'; + +const SUPERADMIN = { + email: process.env.SUPERADMIN_EMAIL || 'admin@clustox.com', + password: process.env.SUPERADMIN_PASSWORD || '' +}; + +/** A signed-in API context. next-auth needs the CSRF token from a prior GET. */ +const signIn = async (email: string, password: string) => { + const ctx = await request.newContext({ baseURL: APP }); + + const csrf = await (await ctx.get('/api/auth/csrf')).json(); + const res = await ctx.post('/api/auth/callback/credentials', { + form: { csrfToken: csrf.csrfToken, email, password, json: 'true' }, + failOnStatusCode: false + }); + expect(res.status(), `sign-in failed for ${email}`).toBe(200); + + return ctx; +}; + +const unique = (prefix: string) => + `${prefix}.${Date.now()}${Math.floor(Math.random() * 1000)}@clustox.com`; + +/** Provision an admin with their own fresh workspace. */ +const createAdmin = async (su: APIRequestContext, name: string) => { + const email = unique(name.toLowerCase().replace(/\s+/g, '.')); + const password = 'E2eBenchmarksPass123'; + + const res = await su.post('/api/clustox/users', { + data: { name, email, password, role: 'ADMIN', team_ids: [] } + }); + expect(res.status()).toBe(200); + const { user_id, org_id } = await res.json(); + + return { userId: user_id, orgId: org_id as string, email, password }; +}; + +/** + * Remove accounts the suite created, and their workspaces with them. + * + * Without this every run left behind an admin and a workspace, which + * accumulated until the superadmin's cross-workspace view was unreadable. + */ +const deleteUsers = async (su: APIRequestContext, userIds: string[]) => { + for (const id of userIds) { + await su.fetch(`/api/clustox/users/${id}`, { + method: 'DELETE', + failOnStatusCode: false + }); + } +}; + +// CLUSTOX: yup.string().uuid() enforces the RFC-4122 version/variant nibbles, +// so an all-zero placeholder like ...-000000000001 is rejected at schema +// validation -- a test using it would get a 400 while asserting 403, passing +// for the wrong reason. This value is v4/variant-8 shaped but not a real row. +// It must NOT be GLOBAL_BENCHMARK_ENTITY_ID (...-000000000001), which is the +// real global baseline row -- using that as a "nonexistent team" would be +// reasoning about live data. +const FOREIGN_TEAM_ID = '00000000-0000-4000-8000-0000000000ff'; + +// CLUSTOX: all five benchmarked metrics, lines_of_code included. It is a full +// benchmark metric on the same routes as the original four -- same team +// setting, same global fallback -- so an isolation suite that only carried the +// four would leave the newest one unproven on exactly the endpoints where a +// missing key is silently dropped rather than rejected. +// +// The value is in *lines* (average gross lines per merged PR), not seconds or +// a percentage, and it has no upper bound. 200 is a plausible target rather +// than a shape-only placeholder. +const BENCHMARK_PAYLOAD = { + lead_time: 24, + deployment_frequency: 3, + change_failure_rate: 10, + mean_time_to_recovery: 12, + lines_of_code: 200 +}; + +/** + * Every metric cleared, for restoring the shared global row when it had no + * value before. Derived from the payload above so a sixth metric added there + * cannot be left behind set to a test value on the one row every workspace + * inherits from. + */ +const CLEARED_PAYLOAD = Object.fromEntries( + Object.keys(BENCHMARK_PAYLOAD).map((metric) => [metric, null]) +); + +test.describe('benchmarks workspace isolation', () => { + test.skip(!SUPERADMIN.password, 'SUPERADMIN_PASSWORD not set'); + + let su: APIRequestContext; + let alpha: Awaited<ReturnType<typeof createAdmin>>; + let beta: Awaited<ReturnType<typeof createAdmin>>; + let alphaCtx: APIRequestContext; + let betaCtx: APIRequestContext; + let betaTeamId: string; + + test.beforeAll(async () => { + su = await signIn(SUPERADMIN.email, SUPERADMIN.password); + alpha = await createAdmin(su, 'E2e Bench Alpha'); + beta = await createAdmin(su, 'E2e Bench Beta'); + alphaCtx = await signIn(alpha.email, alpha.password); + betaCtx = await signIn(beta.email, beta.password); + + const created = await betaCtx.post( + `/api/resources/orgs/${beta.orgId}/teams/v2`, + { data: { name: 'Bench Beta Team', org_repos: {} } } + ); + expect(created.status()).toBe(200); + betaTeamId = (await created.json()).team.id; + }); + + test.afterAll(async () => { + await deleteUsers(su, [alpha.userId, beta.userId]); + }); + + test('an admin cannot read another workspace team benchmark', async () => { + const res = await alphaCtx.get( + `/api/internal/team/${betaTeamId}/settings`, + { + params: { setting_type: 'BENCHMARK_SETTING' }, + failOnStatusCode: false + } + ); + expect(res.status()).toBe(403); + }); + + test('an admin cannot write another workspace team benchmark', async () => { + const res = await alphaCtx.fetch( + `/api/internal/team/${betaTeamId}/settings`, + { + method: 'PUT', + data: { + setting_type: 'BENCHMARK_SETTING', + setting_data: BENCHMARK_PAYLOAD + }, + failOnStatusCode: false + } + ); + expect(res.status()).toBe(403); + }); + + test('an admin cannot read a nonexistent/foreign team benchmark either', async () => { + // CLUSTOX: same 403, via a placeholder id rather than a real foreign + // team -- covers the case where the id does not resolve to any team in + // the caller's own workspace, not just a team that visibly belongs to + // someone else. + const res = await alphaCtx.get( + `/api/internal/team/${FOREIGN_TEAM_ID}/settings`, + { + params: { setting_type: 'BENCHMARK_SETTING' }, + failOnStatusCode: false + } + ); + expect(res.status()).toBe(403); + }); + + test('a non-superadmin PUT to the global baseline is forbidden', async () => { + const res = await alphaCtx.fetch('/api/clustox/benchmarks/global', { + method: 'PUT', + data: BENCHMARK_PAYLOAD, + failOnStatusCode: false + }); + expect(res.status()).toBe(403); + }); + + test('a non-superadmin GET of the global baseline is allowed', async () => { + // CLUSTOX: deliberately not a 403. The five global numbers already + // appear on every admin's own dashboard as resolved target lines, so + // reading them is open to any authenticated admin; only writing is + // superadmin-only. Asserting 403 here would assert the opposite of the + // intended design. + const res = await alphaCtx.get('/api/clustox/benchmarks/global', { + failOnStatusCode: false + }); + expect(res.status()).toBe(200); + }); + + test('unauthenticated requests to either endpoint are rejected', async ({ + request: req + }) => { + const teamRes = await req.get( + `${APP}/api/internal/team/${betaTeamId}/settings`, + { + params: { setting_type: 'BENCHMARK_SETTING' }, + failOnStatusCode: false, + maxRedirects: 0 + } + ); + expect(teamRes.status()).toBe(401); + + const globalRes = await req.get(`${APP}/api/clustox/benchmarks/global`, { + failOnStatusCode: false, + maxRedirects: 0 + }); + expect(globalRes.status()).toBe(401); + }); + + test('a superadmin can read and write the global baseline', async () => { + // CLUSTOX: this endpoint has no per-run row to delete afterwards -- it's + // the one real, shared global baseline (GLOBAL_BENCHMARK_ENTITY_ID), not + // an ephemeral fixture. Capture whatever is there before writing so the + // test can put it back, instead of leaving the shared row permanently + // changed to test values for every other admin's dashboard. + const before = await su.get('/api/clustox/benchmarks/global', { + failOnStatusCode: false + }); + expect(before.status()).toBe(200); + const original = (await before.json()).setting; + + try { + const putRes = await su.fetch('/api/clustox/benchmarks/global', { + method: 'PUT', + data: BENCHMARK_PAYLOAD, + failOnStatusCode: false + }); + expect(putRes.status()).toBe(200); + + const getRes = await su.get('/api/clustox/benchmarks/global', { + failOnStatusCode: false + }); + expect(getRes.status()).toBe(200); + const body = await getRes.json(); + // CLUSTOX: was `lead_time` alone. Asserting the whole payload is what + // makes a metric that the route quietly drops -- rather than rejects -- + // fail here, which is the failure mode a fifth metric added to an + // existing schema is most likely to hit. + expect(body.setting).toMatchObject(BENCHMARK_PAYLOAD); + } finally { + await su.fetch('/api/clustox/benchmarks/global', { + method: 'PUT', + data: original ?? CLEARED_PAYLOAD, + failOnStatusCode: false + }); + } + }); + + test('a team lines_of_code target round-trips in its own workspace', async () => { + // CLUSTOX: the isolation cases above only ever assert 403, so nothing in + // this file proved the settings route stores what it was given. A route + // that rejected every write would pass all of them. Beta writes to beta's + // own team, which is the allowed direction of the same call alpha is + // refused. + const put = await betaCtx.fetch( + `/api/internal/team/${betaTeamId}/settings`, + { + method: 'PUT', + data: { + setting_type: 'BENCHMARK_SETTING', + setting_data: BENCHMARK_PAYLOAD + }, + failOnStatusCode: false + } + ); + expect(put.status()).toBe(200); + + const read = await betaCtx.get( + `/api/internal/team/${betaTeamId}/settings`, + { + params: { setting_type: 'BENCHMARK_SETTING' }, + failOnStatusCode: false + } + ); + expect(read.status()).toBe(200); + + const setting = (await read.json()).setting; + expect(setting).toMatchObject(BENCHMARK_PAYLOAD); + // Named explicitly as well as via the payload above, so the assertion + // that matters cannot be weakened by editing one shared constant. + expect(setting.lines_of_code).toBe(200); + + // CLUSTOX: no cleanup. The team belongs to beta's workspace, which + // afterAll deletes along with beta -- unlike the global row, which is + // shared and is restored in place. + }); + + test('a lines_of_code target does not leak across workspaces', async () => { + // The isolation guarantee, restated for the fifth metric specifically: + // beta's 200 lines/PR target must not be readable by alpha. + const res = await alphaCtx.get( + `/api/internal/team/${betaTeamId}/settings`, + { + params: { setting_type: 'BENCHMARK_SETTING' }, + failOnStatusCode: false + } + ); + // CLUSTOX: status only. A body assertion here reads like extra safety and + // is not -- the request is refused at the team-access check before any + // team's settings are loaded, so `not.toContain('lines_of_code')` would + // hold identically whether or not beta had ever set that target. An + // assertion that passes regardless of the thing it names is worse than no + // assertion, because it looks like coverage. + expect(res.status()).toBe(403); + }); +}); diff --git a/web-server/e2e/invites.spec.ts b/web-server/e2e/invites.spec.ts new file mode 100644 index 000000000..2802d1164 --- /dev/null +++ b/web-server/e2e/invites.spec.ts @@ -0,0 +1,199 @@ +/** + * Invite links. + * + * The link is a bearer credential handed to someone with no account, so the + * properties worth testing are the ones that make it safe to send over Slack: + * single use, expiring, revocable, and useless to anyone reading the database. + * + * Running: + * docker compose up -d + * cd web-server && yarn playwright test e2e/invites.spec.ts + */ +import { APIRequestContext, expect, request, test } from '@playwright/test'; + +const APP = 'http://localhost:3333'; + +const SUPERADMIN = { + email: process.env.SUPERADMIN_EMAIL || 'admin@clustox.com', + password: process.env.SUPERADMIN_PASSWORD || '' +}; + +const signIn = async (email: string, password: string) => { + const ctx = await request.newContext({ baseURL: APP }); + const csrf = await (await ctx.get('/api/auth/csrf')).json(); + const res = await ctx.post('/api/auth/callback/credentials', { + form: { csrfToken: csrf.csrfToken, email, password, json: 'true' }, + failOnStatusCode: false + }); + expect(res.status(), `sign-in failed for ${email}`).toBe(200); + return ctx; +}; + +const createdEmails: string[] = []; + +const unique = (p: string) => { + const email = `${p}.${Date.now()}${Math.floor(Math.random() * 1000)}@clustox.com`; + createdEmails.push(email); + return email; +}; + +const tokenFrom = (url: string) => url.split('token=')[1]; + +const invite = async (su: APIRequestContext, email: string, role = 'ADMIN') => { + const res = await su.post('/api/clustox/invites', { + data: { name: 'Invited Person', email, role } + }); + expect(res.status()).toBe(200); + return tokenFrom((await res.json()).invite_url); +}; + +test.describe('invite links', () => { + test.skip(!SUPERADMIN.password, 'SUPERADMIN_PASSWORD not set'); + + let su: APIRequestContext; + let anon: APIRequestContext; + // Accounts redeemed during the run, cleaned up afterwards so workspaces do + // not accumulate across runs. + const created = createdEmails; + + test.beforeAll(async () => { + su = await signIn(SUPERADMIN.email, SUPERADMIN.password); + // No cookies: an invitee has no account and therefore no session. + anon = await request.newContext({ baseURL: APP }); + }); + + test.afterAll(async () => { + const users = await (await su.get('/api/clustox/users')).json(); + for (const u of users) { + if (u.role === 'ADMIN' && created.includes(u.email)) { + await su.fetch(`/api/clustox/users/${u.userId}`, { + method: 'DELETE', + failOnStatusCode: false + }); + } + } + }); + + test('only a superadmin can issue an invite', async () => { + const email = unique('perm.check'); + const token = await invite(su, email); + + // Redeem it so the created admin can be used to test the restriction. + await anon.post('/api/clustox/accept-invite', { + data: { token, password: 'InvitedPersonPass123' } + }); + const adminCtx = await signIn(email, 'InvitedPersonPass123'); + + const res = await adminCtx.post('/api/clustox/invites', { + data: { name: 'Nope', email: unique('nope'), role: 'ADMIN' }, + failOnStatusCode: false + }); + expect(res.status()).toBe(403); + }); + + test('an invitee can preview and redeem without a session', async () => { + const email = unique('redeem'); + const token = await invite(su, email); + + const preview = await anon.get( + `/api/clustox/accept-invite?token=${token}`, + { failOnStatusCode: false } + ); + expect(preview.status()).toBe(200); + expect((await preview.json()).email).toBe(email); + + const accept = await anon.post('/api/clustox/accept-invite', { + data: { token, password: 'InvitedPersonPass123' }, + failOnStatusCode: false + }); + expect(accept.status()).toBe(200); + + // and the password they chose actually works + const ctx = await signIn(email, 'InvitedPersonPass123'); + const me = await (await ctx.get('/api/clustox/me')).json(); + expect(me.email).toBe(email); + expect(me.role).toBe('ADMIN'); + expect(me.org_id).toBeTruthy(); + }); + + test('a link cannot be redeemed twice', async () => { + const token = await invite(su, unique('single.use')); + + const first = await anon.post('/api/clustox/accept-invite', { + data: { token, password: 'InvitedPersonPass123' }, + failOnStatusCode: false + }); + expect(first.status()).toBe(200); + + const second = await anon.post('/api/clustox/accept-invite', { + data: { token, password: 'DifferentPersonPass123' }, + failOnStatusCode: false + }); + expect(second.status()).toBe(404); + }); + + test('a revoked link stops working', async () => { + const email = unique('revoked'); + const token = await invite(su, email); + + const pending = await (await su.get('/api/clustox/invites')).json(); + const row = pending.find((i: any) => i.email === email); + expect(row).toBeTruthy(); + + const del = await su.fetch(`/api/clustox/invites/${row.id}`, { + method: 'DELETE' + }); + expect(del.status()).toBe(200); + + const res = await anon.post('/api/clustox/accept-invite', { + data: { token, password: 'InvitedPersonPass123' }, + failOnStatusCode: false + }); + expect(res.status()).toBe(404); + }); + + test('an unknown token is rejected exactly like a spent one', async () => { + // Identical responses, so guessing reveals nothing about which tokens + // exist. + const spent = await invite(su, unique('spent')); + await anon.post('/api/clustox/accept-invite', { + data: { token: spent, password: 'InvitedPersonPass123' } + }); + + const spentRes = await anon.get( + `/api/clustox/accept-invite?token=${spent}`, + { failOnStatusCode: false } + ); + const unknownRes = await anon.get( + '/api/clustox/accept-invite?token=deadbeefdeadbeef', + { failOnStatusCode: false } + ); + + expect(spentRes.status()).toBe(unknownRes.status()); + expect(spentRes.status()).toBe(404); + }); + + test('inviting an email that already has an account is refused', async () => { + const email = unique('dupe'); + const token = await invite(su, email); + await anon.post('/api/clustox/accept-invite', { + data: { token, password: 'InvitedPersonPass123' } + }); + + const res = await su.post('/api/clustox/invites', { + data: { name: 'Dupe', email, role: 'ADMIN' }, + failOnStatusCode: false + }); + expect(res.status()).toBe(409); + }); + + test('the accept page is reachable signed out', async () => { + const token = await invite(su, unique('page')); + const res = await anon.get(`/accept-invite?token=${token}`, { + failOnStatusCode: false, + maxRedirects: 0 + }); + // Not a redirect to /login: the invitee has no account to sign in with. + expect(res.status()).toBe(200); + }); +}); diff --git a/web-server/e2e/jenkins.spec.ts b/web-server/e2e/jenkins.spec.ts new file mode 100644 index 000000000..cb53240ea --- /dev/null +++ b/web-server/e2e/jenkins.spec.ts @@ -0,0 +1,120 @@ +/** + * Jenkins endpoints must obey the same workspace boundary as everything else. + * + * Running: + * docker compose up -d + * cd web-server && yarn playwright test e2e/jenkins.spec.ts + */ +import { APIRequestContext, expect, request, test } from '@playwright/test'; + +const APP = 'http://localhost:3333'; + +const SUPERADMIN = { + email: process.env.SUPERADMIN_EMAIL || 'admin@clustox.com', + password: process.env.SUPERADMIN_PASSWORD || '' +}; + +const signIn = async (email: string, password: string) => { + const ctx = await request.newContext({ baseURL: APP }); + const csrf = await (await ctx.get('/api/auth/csrf')).json(); + const res = await ctx.post('/api/auth/callback/credentials', { + form: { csrfToken: csrf.csrfToken, email, password, json: 'true' }, + failOnStatusCode: false + }); + expect(res.status(), `sign-in failed for ${email}`).toBe(200); + return ctx; +}; + +const unique = (p: string) => + `${p}.${Date.now()}${Math.floor(Math.random() * 1000)}@clustox.com`; + +const createAdmin = async (su: APIRequestContext, name: string) => { + const email = unique(name.toLowerCase().replace(/\s+/g, '.')); + const password = 'E2eJenkinsPass123'; + const res = await su.post('/api/clustox/users', { + data: { name, email, password, role: 'ADMIN', team_ids: [] } + }); + expect(res.status()).toBe(200); + const { user_id, org_id } = await res.json(); + return { userId: user_id, orgId: org_id as string, email, password }; +}; + +test.describe('jenkins workspace isolation', () => { + test.skip(!SUPERADMIN.password, 'SUPERADMIN_PASSWORD not set'); + + let su: APIRequestContext; + let alpha: Awaited<ReturnType<typeof createAdmin>>; + let beta: Awaited<ReturnType<typeof createAdmin>>; + let alphaCtx: APIRequestContext; + + test.beforeAll(async () => { + su = await signIn(SUPERADMIN.email, SUPERADMIN.password); + alpha = await createAdmin(su, 'Jenkins Alpha'); + beta = await createAdmin(su, 'Jenkins Beta'); + alphaCtx = await signIn(alpha.email, alpha.password); + }); + + test.afterAll(async () => { + for (const id of [alpha.userId, beta.userId]) { + await su.fetch(`/api/clustox/users/${id}`, { + method: 'DELETE', + failOnStatusCode: false + }); + } + }); + + test('an admin cannot list another workspace jenkins jobs', async () => { + const res = await alphaCtx.get( + `/api/clustox/jenkins/jobs?org_id=${beta.orgId}`, + { failOnStatusCode: false } + ); + expect(res.status()).toBe(403); + }); + + test('an admin cannot map a job into another workspace', async () => { + // CLUSTOX: the id below must be a schema-valid (v4-shaped) UUID. Payload + // validation runs before assertWorkspaceAccess (see global.ts), so a + // malformed placeholder would 400 before the workspace check ever ran, + // making this test pass for the wrong reason. + const res = await alphaCtx.post('/api/clustox/jenkins/mappings', { + data: { + org_id: beta.orgId, + org_repo_id: '00000000-0000-4000-8000-000000000001', + job_full_name: 'deploy-api' + }, + failOnStatusCode: false + }); + expect(res.status()).toBe(403); + }); + + test('an admin cannot delete a mapping in another workspace', async () => { + // CLUSTOX: same reasoning as above -- repo_workflow_id must pass schema + // validation so the request actually reaches assertWorkspaceAccess. + const res = await alphaCtx.fetch('/api/clustox/jenkins/mappings', { + method: 'DELETE', + data: { + org_id: beta.orgId, + repo_workflow_id: '00000000-0000-4000-8000-000000000001' + }, + failOnStatusCode: false + }); + expect(res.status()).toBe(403); + }); + + test('unauthenticated requests are rejected', async ({ request: req }) => { + const res = await req.get( + `${APP}/api/clustox/jenkins/jobs?org_id=${alpha.orgId}`, + { failOnStatusCode: false, maxRedirects: 0 } + ); + expect(res.status()).toBe(401); + }); + + test('a malformed org_id is a 400, not a 403', async () => { + // Schema validation runs before the access check, so bad input is not + // reported as a permission problem. + const res = await alphaCtx.get('/api/clustox/jenkins/jobs?org_id=nonsense', { + failOnStatusCode: false + }); + expect(res.status()).toBe(400); + }); +}); diff --git a/web-server/e2e/jira-incident-source.spec.ts b/web-server/e2e/jira-incident-source.spec.ts new file mode 100644 index 000000000..b09f876c5 --- /dev/null +++ b/web-server/e2e/jira-incident-source.spec.ts @@ -0,0 +1,173 @@ +/** + * Jira issues as an incident source for Change Failure Rate / MTTR + * (MID-8 -- see docs/JIRA_INTEGRATION_PROPOSAL.md). + * + * Same shape as jira-integration.spec.ts: skips without SUPERADMIN_PASSWORD, + * runs against a real host/CI browser (see e2e/auth.spec.ts for why this + * can't run inside the middleware-dev container), and is deliberately + * non-destructive to whatever this org's actual setting already is -- + * every test restores it in an `afterEach` rather than leaving the org + * opted in/out as a side effect of the test run. + * + * Running: + * docker compose up -d + * cd web-server && yarn playwright test e2e/jira-incident-source.spec.ts + */ +import { expect, test } from '@playwright/test'; + +const APP = 'http://localhost:3333'; +const INCIDENT_SOURCES_SETTING = 'INCIDENT_SOURCES_SETTING'; +const JIRA_INCIDENT_ISSUE_TYPES_SETTING = 'JIRA_INCIDENT_ISSUE_TYPES_SETTING'; +const JIRA_ISSUE_SOURCE = 'JIRA_ISSUE'; + +const SUPERADMIN = { + email: process.env.SUPERADMIN_EMAIL || 'admin@clustox.com', + password: process.env.SUPERADMIN_PASSWORD || '' +}; + +const signIn = async (page: import('@playwright/test').Page) => { + await page.goto(`${APP}/login`); + await page.fill('input[type="email"]', SUPERADMIN.email); + await page.fill('input[type="password"]', SUPERADMIN.password); + await Promise.all([ + page.waitForNavigation({ waitUntil: 'load', timeout: 30_000 }), + page.click('button[type="submit"]') + ]); +}; + +const gotoDoraMetrics = async (page: import('@playwright/test').Page) => { + // Same "Getting app data" first-paint wait as jira-integration.spec.ts's + // gotoIntegrations -- the post-login landing page has its own loading + // state before any other page's content is safe to interact with. + await expect(page.getByText('Getting app data')).toHaveCount(0, { + timeout: 90_000 + }); + await page.goto(`${APP}/dora-metrics`); + await page.getByRole('button', { name: /settings/i }).waitFor({ + timeout: 30_000 + }); +}; + +const getOrgId = async (page: import('@playwright/test').Page) => { + const session = await (await page.request.get(`${APP}/api/auth/session`)).json(); + return session?.org?.id as string | undefined; +}; + +const getSetting = async ( + page: import('@playwright/test').Page, + orgId: string, + settingType: string +) => + ( + await page.request.get( + `${APP}/api/internal/${orgId}/settings?setting_type=${settingType}` + ) + ).json(); + +test.describe('Jira incident source setting', () => { + test.skip(!SUPERADMIN.password, 'SUPERADMIN_PASSWORD not set'); + + let orgId: string | undefined; + let originalSources: string[] = []; + let originalIssueTypes: string[] = []; + + test.beforeEach(async ({ page }) => { + test.setTimeout(180_000); + await signIn(page); + orgId = await getOrgId(page); + expect(orgId, 'signed-in session should resolve an org').toBeTruthy(); + + const sources = await getSetting(page, orgId!, INCIDENT_SOURCES_SETTING); + originalSources = sources.incident_sources || []; + const types = await getSetting( + page, + orgId!, + JIRA_INCIDENT_ISSUE_TYPES_SETTING + ); + originalIssueTypes = types.issue_types || []; + }); + + test.afterEach(async ({ page }) => { + if (!orgId) return; + // Restore this org's real setting -- this test's whole point is + // exercising the toggle, not leaving the environment's actual + // configuration changed behind it. + await page.request.put(`${APP}/api/internal/${orgId}/settings`, { + data: { + setting_type: INCIDENT_SOURCES_SETTING, + setting_data: { incident_sources: originalSources } + } + }); + await page.request.put(`${APP}/api/internal/${orgId}/settings`, { + data: { + setting_type: JIRA_INCIDENT_ISSUE_TYPES_SETTING, + setting_data: { issue_types: originalIssueTypes } + } + }); + }); + + test('enabling the switch and adding an issue type persists both settings', async ({ + page + }) => { + await gotoDoraMetrics(page); + + await page.getByRole('button', { name: /settings/i }).click(); + await page.getByRole('menuitem', { name: 'Configure Jira Incident Source' }).click(); + + const dialog = page.getByRole('dialog'); + const switchInput = dialog.getByRole('checkbox'); + if (!(await switchInput.isChecked())) { + await switchInput.click(); + } + + const issueTypeInput = dialog.getByPlaceholder(/e\.g\. bug, incident/i); + await issueTypeInput.click(); + await issueTypeInput.fill('Bug'); + await page.keyboard.press('Enter'); + + await dialog.getByRole('button', { name: 'Save' }).click(); + await expect(page.getByText('Updated Successfully')).toBeVisible({ + timeout: 10_000 + }); + + const sources = await getSetting(page, orgId!, INCIDENT_SOURCES_SETTING); + expect(sources.incident_sources).toContain(JIRA_ISSUE_SOURCE); + + const types = await getSetting( + page, + orgId!, + JIRA_INCIDENT_ISSUE_TYPES_SETTING + ); + expect(types.issue_types).toContain('Bug'); + }); + + test('disabling the switch removes JIRA_ISSUE but keeps every other configured source', async ({ + page + }) => { + // Seed a known starting state: enabled, plus a source unrelated to + // this ticket, so the test can assert the unrelated one survives. + await page.request.put(`${APP}/api/internal/${orgId}/settings`, { + data: { + setting_type: INCIDENT_SOURCES_SETTING, + setting_data: { incident_sources: ['GIT_REPO', JIRA_ISSUE_SOURCE] } + } + }); + + await gotoDoraMetrics(page); + await page.getByRole('button', { name: /settings/i }).click(); + await page.getByRole('menuitem', { name: 'Configure Jira Incident Source' }).click(); + + const dialog = page.getByRole('dialog'); + const switchInput = dialog.getByRole('checkbox'); + await expect(switchInput).toBeChecked(); + await switchInput.click(); + await dialog.getByRole('button', { name: 'Save' }).click(); + await expect(page.getByText('Updated Successfully')).toBeVisible({ + timeout: 10_000 + }); + + const sources = await getSetting(page, orgId!, INCIDENT_SOURCES_SETTING); + expect(sources.incident_sources).not.toContain(JIRA_ISSUE_SOURCE); + expect(sources.incident_sources).toContain('GIT_REPO'); + }); +}); diff --git a/web-server/e2e/jira-integration.spec.ts b/web-server/e2e/jira-integration.spec.ts new file mode 100644 index 000000000..6fea179ae --- /dev/null +++ b/web-server/e2e/jira-integration.spec.ts @@ -0,0 +1,129 @@ +/** + * Jira linking (Phase 1 -- see docs/JIRA_INTEGRATION_PROPOSAL.md). + * + * Deliberately does not exercise a full successful link: that needs a real + * Jira Cloud site + API token, and this repo has no precedent for that kind + * of dependency in E2E either -- there's no "link GitHub successfully" or + * "link GitLab successfully" E2E test here, for the exact same reason (a + * real PAT would be needed). ConfigureJiraModalBody's validation/success/ + * error logic is covered in isolation instead, by + * src/content/Dashboards/__tests__/ConfigureJiraModalBody.test.tsx (Jest, + * mocked network). + * + * This dev environment's org has a real, live-linked Jira integration + * (checked directly against the DB while writing this) -- these tests are + * deliberately non-destructive: nothing here unlinks it, since redoing that + * link requires real credentials this test run doesn't have and can't + * restore. + * + * Running (see e2e/auth.spec.ts for why this needs a real browser, and + * therefore the host or CI rather than the middleware-dev container): + * docker compose up -d + * cd web-server && yarn playwright test e2e/jira-integration.spec.ts + */ +import { expect, test } from '@playwright/test'; + +const APP = 'http://localhost:3333'; + +const SUPERADMIN = { + email: process.env.SUPERADMIN_EMAIL || 'admin@clustox.com', + password: process.env.SUPERADMIN_PASSWORD || '' +}; + +const signIn = async (page: import('@playwright/test').Page) => { + await page.goto(`${APP}/login`); + await page.fill('input[type="email"]', SUPERADMIN.email); + await page.fill('input[type="password"]', SUPERADMIN.password); + await Promise.all([ + page.waitForNavigation({ waitUntil: 'load', timeout: 30_000 }), + page.click('button[type="submit"]') + ]); +}; + +const gotoIntegrations = async (page: import('@playwright/test').Page) => { + // The sidebar is closed by default (SidebarContext's sidebarToggle starts + // false) and the post-login landing page shows its own "Getting app + // data" loading state first -- see integration-link-status.spec.ts for + // both. + await expect(page.getByText('Getting app data')).toHaveCount(0, { + timeout: 90_000 + }); + await page.getByRole('navigation').click(); + await page + .getByRole('link', { name: 'Manage Integrations', exact: true }) + .click(); + await page.waitForURL(/\/integrations$/, { timeout: 15_000 }); +}; + +test('the Jira card reflects the real link status on first paint after login, without a refresh', async ({ + page +}) => { + test.skip(!SUPERADMIN.password, 'SUPERADMIN_PASSWORD not set'); + test.setTimeout(180_000); + + await signIn(page); + // gotoIntegrations waits out the post-login "Getting app data" loading + // state first -- firing page.request calls before the page has settled + // was observed to starve the (single-process, dev mode) Next server of + // the request budget the page's own hydration needs, which can leave + // /api/auth/session itself answering with no org yet. + await gotoIntegrations(page); + + const session = await (await page.request.get(`${APP}/api/auth/session`)).json(); + const orgId = session?.org?.id; + expect(orgId, 'signed-in session should resolve an org').toBeTruthy(); + + const map = await ( + await page.request.get( + `${APP}/api/integrations/integrations-map?org_id=${orgId}` + ) + ).json(); + const trulyLinked = Boolean(map.jira); + + const jiraCard = page.getByTestId('jira-integration-card'); + if (trulyLinked) { + await expect(jiraCard.getByText('Unlink', { exact: true })).toBeVisible({ + timeout: 5_000 + }); + } else { + await expect(jiraCard.getByText('Link', { exact: true })).toBeVisible({ + timeout: 5_000 + }); + } +}); + +test('declining the unlink confirmation leaves a linked Jira integration untouched', async ({ + page +}) => { + test.skip(!SUPERADMIN.password, 'SUPERADMIN_PASSWORD not set'); + test.setTimeout(180_000); + + await signIn(page); + await gotoIntegrations(page); + + const session = await (await page.request.get(`${APP}/api/auth/session`)).json(); + const orgId = session?.org?.id; + const before = await ( + await page.request.get( + `${APP}/api/integrations/integrations-map?org_id=${orgId}` + ) + ).json(); + test.skip(!before.jira, 'this environment has no Jira integration linked to protect'); + + // Scoped to the Jira card specifically -- GitHub is also linked in this + // environment, so an unscoped getByText('Unlink') would be ambiguous. + const jiraCard = page.getByTestId('jira-integration-card'); + page.once('dialog', (dialog) => dialog.dismiss()); + await jiraCard.getByText('Unlink', { exact: true }).click(); + + // Give the (declined) confirm a moment to resolve, then confirm nothing + // changed -- no DELETE was ever sent. + await page.waitForTimeout(500); + const after = await ( + await page.request.get( + `${APP}/api/integrations/integrations-map?org_id=${orgId}` + ) + ).json(); + expect(after.jira).toBeTruthy(); + await expect(jiraCard.getByText('Unlink', { exact: true })).toBeVisible(); +}); diff --git a/web-server/e2e/multitenancy.spec.ts b/web-server/e2e/multitenancy.spec.ts new file mode 100644 index 000000000..ae826cd28 --- /dev/null +++ b/web-server/e2e/multitenancy.spec.ts @@ -0,0 +1,217 @@ +/** + * Cross-workspace isolation. + * + * This is the property the product promises: an admin owns one workspace and + * cannot reach another's data. Everything here is request-level, so it runs + * without a browser -- unlike the page tests in auth.spec.ts, which need + * Chromium system libraries the dev image does not ship. + * + * Running: + * docker compose up -d + * cd web-server && yarn playwright test e2e/multitenancy.spec.ts + * + * The suite provisions its own admins through the API and cleans them up, so + * it does not depend on whatever happens to be in the database. + */ +import { APIRequestContext, expect, request, test } from '@playwright/test'; + +const APP = 'http://localhost:3333'; +const API = 'http://localhost:9696'; +const SYNC = 'http://localhost:9697'; + +const SUPERADMIN = { + email: process.env.SUPERADMIN_EMAIL || 'admin@clustox.com', + password: process.env.SUPERADMIN_PASSWORD || '' +}; + +/** A signed-in API context. next-auth needs the CSRF token from a prior GET. */ +const signIn = async (email: string, password: string) => { + const ctx = await request.newContext({ baseURL: APP }); + + const csrf = await (await ctx.get('/api/auth/csrf')).json(); + const res = await ctx.post('/api/auth/callback/credentials', { + form: { csrfToken: csrf.csrfToken, email, password, json: 'true' }, + failOnStatusCode: false + }); + expect(res.status(), `sign-in failed for ${email}`).toBe(200); + + return ctx; +}; + +const unique = (prefix: string) => + `${prefix}.${Date.now()}${Math.floor(Math.random() * 1000)}@clustox.com`; + +/** Provision an admin with their own fresh workspace. */ +const createAdmin = async (su: APIRequestContext, name: string) => { + const email = unique(name.toLowerCase().replace(/\s+/g, '.')); + const password = 'E2eWorkspacePass123'; + + const res = await su.post('/api/clustox/users', { + data: { name, email, password, role: 'ADMIN', team_ids: [] } + }); + expect(res.status()).toBe(200); + const { user_id, org_id } = await res.json(); + + return { userId: user_id, orgId: org_id as string, email, password }; +}; + +/** + * Remove accounts the suite created, and their workspaces with them. + * + * Without this every run left behind an admin and a workspace, which + * accumulated until the superadmin's cross-workspace view was unreadable. + */ +const deleteUsers = async (su: APIRequestContext, userIds: string[]) => { + for (const id of userIds) { + await su.fetch(`/api/clustox/users/${id}`, { + method: 'DELETE', + failOnStatusCode: false + }); + } +}; + +test.describe('cross-workspace isolation', () => { + test.skip(!SUPERADMIN.password, 'SUPERADMIN_PASSWORD not set'); + + let su: APIRequestContext; + let alpha: Awaited<ReturnType<typeof createAdmin>>; + let beta: Awaited<ReturnType<typeof createAdmin>>; + let alphaCtx: APIRequestContext; + + test.beforeAll(async () => { + su = await signIn(SUPERADMIN.email, SUPERADMIN.password); + alpha = await createAdmin(su, 'E2e Alpha'); + beta = await createAdmin(su, 'E2e Beta'); + alphaCtx = await signIn(alpha.email, alpha.password); + }); + + test.afterAll(async () => { + await deleteUsers(su, [alpha.userId, beta.userId]); + }); + + test('each admin is given a distinct workspace', async () => { + expect(alpha.orgId).toBeTruthy(); + expect(beta.orgId).toBeTruthy(); + expect(alpha.orgId).not.toBe(beta.orgId); + }); + + test('an admin sees only their own workspace', async () => { + const res = await alphaCtx.get('/api/clustox/workspace-status'); + expect(res.status()).toBe(200); + + const workspaces = await res.json(); + expect(workspaces).toHaveLength(1); + expect(workspaces[0].id).toBe(alpha.orgId); + }); + + test('an admin cannot read another workspace integration', async () => { + const res = await alphaCtx.get( + `/api/resources/orgs/${beta.orgId}/integration`, + { failOnStatusCode: false } + ); + expect(res.status()).toBe(403); + }); + + test('an admin cannot write an integration into another workspace', async () => { + const res = await alphaCtx.post( + `/api/resources/orgs/${beta.orgId}/integration`, + { + data: { + provider: 'github', + the_good_stuff: 'not-a-real-token', + meta_data: {} + }, + failOnStatusCode: false + } + ); + expect(res.status()).toBe(403); + }); + + test('an admin cannot create a team in another workspace', async () => { + const res = await alphaCtx.post( + `/api/resources/orgs/${beta.orgId}/teams/v2`, + { data: { name: 'Intruder', org_repos: {} }, failOnStatusCode: false } + ); + expect(res.status()).toBe(403); + }); + + test('an admin can create a team in their own workspace', async () => { + const res = await alphaCtx.post( + `/api/resources/orgs/${alpha.orgId}/teams/v2`, + { data: { name: 'Alpha Squad', org_repos: {} }, failOnStatusCode: false } + ); + expect(res.status()).toBe(200); + }); + + test('an admin cannot delete another workspace team by supplying their own org_id', async () => { + // The subtle case: the path carries a workspace they DO own, while the + // body names a team they do not. Central scoping keys on org_id, so this + // is only caught by an explicit team check. + const betaCtx = await signIn(beta.email, beta.password); + const created = await betaCtx.post( + `/api/resources/orgs/${beta.orgId}/teams/v2`, + { data: { name: 'Beta Only', org_repos: {} } } + ); + expect(created.status()).toBe(200); + const betaTeamId = (await created.json()).team.id; + + const res = await alphaCtx.fetch( + `/api/resources/orgs/${alpha.orgId}/teams/v2`, + { method: 'DELETE', data: { id: betaTeamId }, failOnStatusCode: false } + ); + expect(res.status()).toBe(403); + + // and the team survived + const stillThere = await betaCtx.get( + `/api/resources/orgs/${beta.orgId}/teams/v2` + ); + const names = (await stillThere.json()).teams.map((t: any) => t.name); + expect(names).toContain('Beta Only'); + }); + + test('an admin cannot manage users', async () => { + const res = await alphaCtx.get('/api/clustox/users', { + failOnStatusCode: false + }); + expect(res.status()).toBe(403); + }); + + test('an admin cannot force a sync across all workspaces', async () => { + const res = await alphaCtx.post('/api/clustox/sync-now', { + failOnStatusCode: false + }); + expect(res.status()).toBe(403); + }); + + test('a superadmin sees every workspace', async () => { + const res = await su.get('/api/clustox/workspace-status'); + const ids = (await res.json()).map((w: any) => w.id); + expect(ids).toEqual(expect.arrayContaining([alpha.orgId, beta.orgId])); + }); +}); + +test.describe('backend reachability', () => { + test('the analytics API refuses requests without the internal token', async ({ + request: req + }) => { + const res = await req.get(`${API}/teams/any-id/lead_time`, { + failOnStatusCode: false + }); + expect(res.status()).toBe(403); + }); + + test('the sync API refuses requests without the internal token', async ({ + request: req + }) => { + const res = await req.post(`${SYNC}/sync`, { failOnStatusCode: false }); + expect(res.status()).toBe(403); + }); + + test('unauthenticated BFF requests are rejected', async ({ request: req }) => { + const res = await req.get(`${APP}/api/clustox/users`, { + failOnStatusCode: false, + maxRedirects: 0 + }); + expect(res.status()).toBe(401); + }); +}); diff --git a/web-server/instrumentation.ts b/web-server/instrumentation.ts new file mode 100644 index 000000000..a2a5793d3 --- /dev/null +++ b/web-server/instrumentation.ts @@ -0,0 +1,8 @@ +// CLUSTOX: seed the first superadmin on server start. The NEXT_RUNTIME check +// keeps this off the Edge runtime, where knex cannot run. +export async function register() { + if (process.env.NEXT_RUNTIME === 'nodejs') { + const { bootstrapSuperadmin } = await import('@/auth/bootstrap'); + await bootstrapSuperadmin(); + } +} diff --git a/web-server/jest.config.js b/web-server/jest.config.js index d8a8cca04..83c83fd40 100644 --- a/web-server/jest.config.js +++ b/web-server/jest.config.js @@ -2,6 +2,13 @@ module.exports = { preset: 'ts-jest/presets/js-with-babel', // Use the TypeScript preset with Babel testEnvironment: 'jsdom', // Use jsdom as the test environment (for browser-like behavior) setupFiles: ['<rootDir>/jest.setup.js'], + setupFilesAfterEnv: ['<rootDir>/jest.setup.after-env.js'], + // tsconfig.json's baseUrl:"." is what makes a bare `from 'src/components/X'` + // import (no @/ alias, no relative path) resolve in the real app -- Jest + // doesn't read tsconfig's baseUrl on its own, so without this it can't + // find any module imported that way. First hit via + // pages/integrations.tsx importing 'src/components/Authenticated'. + modulePaths: ['<rootDir>'], moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], testMatch: [ '**/__tests__/**/*.test.(ts|tsx|js|jsx)', @@ -9,14 +16,63 @@ module.exports = { '**/*.test.tsx' ], transform: { - '^.+\\.(ts|tsx)$': 'ts-jest' // Transform TypeScript files using ts-jest + // tsconfig.json sets jsx:"preserve" so Next's own SWC build handles + // JSX -- ts-jest respects that setting too, which means it never + // lowers JSX to React.createElement()/jsx() calls, leaving raw JSX in + // its output for Node to choke on. Override just for the test + // transform, without touching the real tsconfig. + // + // CLUSTOX: `tsconfig.json` sets `jsx: "preserve"` -- correct for Next, + // which does its own JSX transform, but it makes ts-jest emit raw `<div>` + // into CommonJS and every test that touches a component dies on + // `SyntaxError: Unexpected token '<'`. `testMatch` has always listed + // `*.test.tsx`; until now nothing could actually run under it. Overriding + // only `jsx` here leaves .ts suites (all the existing ones) untouched -- + // the option has no effect on a file with no JSX in it. + '^.+\\.(ts|tsx)$': ['ts-jest', { tsconfig: { jsx: 'react-jsx' } }] }, testPathIgnorePatterns: ['/node_modules/', 'auth.spec.ts'], moduleNameMapper: { + // Checked in order -- first match wins. This must come before the + // '^@/(.*)$' alias rule below: @/mocks/icons/gitlab.svg matches both, + // and the alias rule matching first would just resolve it to another + // raw, still-unparseable .svg file, never reaching this one. Next's + // build treats .svg imports as React components via an SVGR-style + // loader; Jest has no such transform and chokes trying to parse raw + // SVG/XML as JS. First hit via JiraIntegrationCard -> + // githubIntegration.tsx -> gitlab.svg / jira-icon.svg. + '\\.svg$': '<rootDir>/src/mocks/__mocks__/svg.js', '^@/public/(.*)$': '<rootDir>/public/$1', '^@/api/(.*)$': '<rootDir>/pages/api/$1', '^@/(.*)$': '<rootDir>/src/$1', - '^uuid$': require.resolve('uuid') + // CLUSTOX: a handful of modules import as `src/contexts/...` rather than + // `@/contexts/...`. Next resolves that through tsconfig's `baseUrl: "."`; + // jest's `moduleDirectories: ['node_modules', 'src']` does not, because it + // strips no prefix -- it looks for `src/src/contexts`. + '^src/(.*)$': '<rootDir>/src/$1', + // Same reason as the faker mapping below: jsdom's testEnvironment makes + // Jest's resolver prefer this package's ESM build (import-only syntax), + // which then fails to parse since node_modules isn't transformed. + // Forcing the CJS entry (resolved by plain Node here, in the config + // file itself, so it correctly follows the "require" export condition) + // sidesteps that. First hit via FlexBox -> Shared.tsx -> date.ts -> + // mock.ts -> @faker-js/faker, none of which any test imported before. + '^uuid$': require.resolve('uuid'), + // CLUSTOX: faker's `exports` map resolves to `dist/esm` under the jsdom + // environment (its `node` condition is the only one pointing at CJS), and + // that ESM entry kills any suite that reaches it -- `Cannot use import + // statement outside a module`. It is reached constantly and invisibly: + // `@/utils/date` imports `@/utils/mock`, which imports faker, and + // `FlexBox` imports `date`. So *every* component in this codebase was + // untestable for this one reason. Two util modules on this branch + // (`benchmarks.ts`, `benchmarkBand.ts`) were deliberately written + // import-free to dodge it. Pointing at the CJS build it already ships + // fixes the cause instead. + // `require.resolve` from this CJS config selects faker's `require` + // condition, which is the CJS build -- the whole point of the entry -- + // without hardcoding a dist path that a faker major would move and that + // would fail at config load with a bare "Cannot find module". + '^@faker-js/faker$': require.resolve('@faker-js/faker') }, moduleDirectories: ['node_modules', 'src'] }; diff --git a/web-server/jest.setup.after-env.js b/web-server/jest.setup.after-env.js new file mode 100644 index 000000000..252eb2470 --- /dev/null +++ b/web-server/jest.setup.after-env.js @@ -0,0 +1,4 @@ +// Runs after the test framework (expect, jest globals) is installed -- +// unlike jest.setup.js's setupFiles hook, jest-dom's matchers need `expect` +// to already exist to extend it. +require('@testing-library/jest-dom'); diff --git a/web-server/libdefs/ambient.d.ts b/web-server/libdefs/ambient.d.ts index b32daf57e..08f2fb53b 100644 --- a/web-server/libdefs/ambient.d.ts +++ b/web-server/libdefs/ambient.d.ts @@ -44,7 +44,12 @@ declare type IdentityMap = Record< declare type IntegrationsMap = Partial< Record< - 'github' | 'gitlab' | 'bitbucket', + // CLUSTOX: 'jenkins' added so the workspace's linked state is readable + // via useAuth().integrations.jenkins. It is a deployment provider, not a + // code provider -- see CODE_PROVIDER_INTEGRATIONS_MAP. 'jira' is a + // project-tracker integration, not a code provider either -- see + // docs/JIRA_INTEGRATION_PROPOSAL.md. + 'github' | 'gitlab' | 'bitbucket' | 'jenkins' | 'jira', { integrated: Boolean; linked_at: DateString | null; diff --git a/web-server/libdefs/jest-dom.d.ts b/web-server/libdefs/jest-dom.d.ts new file mode 100644 index 000000000..24d506c10 --- /dev/null +++ b/web-server/libdefs/jest-dom.d.ts @@ -0,0 +1,6 @@ +// Side-effect-only import so TS picks up @testing-library/jest-dom's global +// `expect(...).toBeInTheDocument()`-style matcher types everywhere in the +// program. jest.setup.after-env.js requires the same package at runtime; +// that require alone doesn't reach ts-jest's type-checking since it's a +// plain .js file, not part of the TS module graph. +import '@testing-library/jest-dom'; diff --git a/web-server/middleware.ts b/web-server/middleware.ts index 09825d66a..1003bf76b 100644 --- a/web-server/middleware.ts +++ b/web-server/middleware.ts @@ -1,13 +1,48 @@ +import { getToken } from 'next-auth/jwt'; import { NextRequest, NextResponse } from 'next/server'; import { getFeaturesFromReq } from '@/api-helpers/features'; import { defaultFlags } from '@/constants/feature'; +// CLUSTOX: paths reachable without a session. Health checks stay open so +// container and UI status probes keep working. +const PUBLIC_PATHS = [ + '/login', + // CLUSTOX: an invitee has no account yet, so this must be reachable + // signed out -- both the page and the endpoint behind it. + '/accept-invite', + '/api/clustox/accept-invite', + '/api/auth', + '/api/status', + '/api/db_status', + '/api/internal_status', + '/api/hello' +]; + export async function middleware(request: NextRequest) { const flagOverrides = getFeaturesFromReq(request as any); const flags = { ...defaultFlags, ...flagOverrides }; const url = request.nextUrl.clone(); + const isPublic = PUBLIC_PATHS.some((p) => url.pathname.startsWith(p)); + + // CLUSTOX: redirect unauthenticated *page* requests to the sign-in screen. + // API routes are deliberately excluded: they are guarded independently in + // Endpoint.serve(), which returns 401. Redirecting an XHR to an HTML login + // page would surface as a confusing parse error rather than an auth failure. + const isApi = url.pathname.startsWith('/api/'); + if (!isPublic && !isApi) { + const token = await getToken({ + req: request, + secret: process.env.NEXTAUTH_SECRET + }); + if (!token) { + const login = request.nextUrl.clone(); + login.pathname = '/login'; + login.search = ''; + return NextResponse.redirect(login); + } + } // Forward as-is if it's a next-auth URL, except /session if ( @@ -24,9 +59,7 @@ export async function middleware(request: NextRequest) { export const config = { matcher: [ - '/api/auth/:path*', - '/api/integrations/:path*', - '/api/internal/:path*', - '/api/resources/:path*' + // CLUSTOX: widened from the API-only matcher so pages are guarded too. + '/((?!_next/static|_next/image|favicon.ico|assets|.*\\.png$|.*\\.svg$).*)' ] }; diff --git a/web-server/next.config.js b/web-server/next.config.js index 0433ca36e..a6ceb6cca 100644 --- a/web-server/next.config.js +++ b/web-server/next.config.js @@ -49,6 +49,11 @@ const genericPlugins = [ ]; const staticOverrides = { + // CLUSTOX: knex resolves dialects dynamically, which webpack cannot bundle. + // Without this the instrumentation hook (superadmin bootstrap) fails to + // compile. API routes were unaffected, so this only surfaced once + // instrumentation.ts started importing the db helper. + serverExternalPackages: ['knex', 'pg'], staticPageGenerationTimeout: 2000, images: { disableStaticImages: true diff --git a/web-server/package.json b/web-server/package.json index 580e2e282..771cf1552 100644 --- a/web-server/package.json +++ b/web-server/package.json @@ -32,6 +32,7 @@ "axios": "1.7.7", "axios-hooks": "^5.0.2", "axios-retry": "^4.5.0", + "bcryptjs": "^3.0.3", "chart.js": "^3.9.1", "chartjs-plugin-annotation": "^2.0.1", "chartjs-plugin-crosshair": "^1.2.0", @@ -50,6 +51,7 @@ "next": "^15.0.1", "next-auth": "^4.23.0", "next-images": "1.8.4", + "nodemailer": "^9.0.5", "notistack": "^3.0.1", "nprogress": "0.2.0", "pg": "^8.9.0", @@ -102,8 +104,13 @@ "@next/bundle-analyzer": "^12.3.0", "@octokit/types": "^6.34.0", "@playwright/test": "^1.28.0", + "@testing-library/jest-dom": "^6.4.2", + "@testing-library/react": "^14.2.1", + "@testing-library/user-event": "^14.5.2", + "@types/bcryptjs": "^3.0.0", "@types/js-cookie": "^3.0.2", "@types/node": "^16.17.0", + "@types/nodemailer": "^8.0.1", "@types/nprogress": "0.2.0", "@types/pluralize": "^0.0.29", "@types/ramda": "^0.28.7", diff --git a/web-server/pages/_app.tsx b/web-server/pages/_app.tsx index c4fc35587..a6dbced87 100644 --- a/web-server/pages/_app.tsx +++ b/web-server/pages/_app.tsx @@ -2,7 +2,7 @@ import { CacheProvider, EmotionCache } from '@emotion/react'; import CssBaseline from '@mui/material/CssBaseline'; import { LocalizationProvider } from '@mui/x-date-pickers'; import { AdapterDateFns } from '@mui/x-date-pickers/AdapterDateFns'; -import Router from 'next/router'; +import Router, { useRouter } from 'next/router'; import { SnackbarProvider } from 'notistack'; import nProgress from 'nprogress'; import { ReactElement, ReactNode } from 'react'; @@ -54,6 +54,10 @@ interface MyAppProps extends AppProps { function MyApp(props: MyAppProps) { const { Component, emotionCache = clientSideEmotionCache, pageProps } = props; const getLayout = Component.getLayout ?? ((page) => page); + // CLUSTOX: see the AuthConsumer gate below. + const pathname = useRouter().pathname; + const isLoginRoute = + pathname === '/login' || pathname === '/accept-invite'; useScrollTop(); const overrides = useFlagOverrides(); useResizeEventTracking(); @@ -89,7 +93,12 @@ function MyApp(props: MyAppProps) { <ImageUpdateBanner /> <AuthConsumer> {(auth) => - !auth.isInitialized ? ( + // CLUSTOX: the login page must render without + // auth context. AuthProvider initialises by + // fetching session data, which now requires a + // session -- so gating /login on it would wait + // forever for a login that can never happen. + !auth.isInitialized && !isLoginRoute ? ( <Loader /> ) : ( getLayout(<Component {...pageProps} />) diff --git a/web-server/pages/accept-invite.tsx b/web-server/pages/accept-invite.tsx new file mode 100644 index 000000000..da60ac406 --- /dev/null +++ b/web-server/pages/accept-invite.tsx @@ -0,0 +1,228 @@ +import { + AdminPanelSettingsTwoTone, + WorkspacesTwoTone +} from '@mui/icons-material'; +import { + Alert, + Box, + Button, + Card, + CircularProgress, + Divider, + TextField, + Typography +} from '@mui/material'; +import Head from 'next/head'; +import { useRouter } from 'next/router'; +import { signIn } from 'next-auth/react'; +import { FormEvent, ReactElement, useCallback, useEffect, useState } from 'react'; + +import { FlexBox } from '@/components/FlexBox'; + +const MIN_PASSWORD = 12; + +type Preview = { + email: string; + name: string; + role: 'SUPERADMIN' | 'ADMIN'; + orgName: string | null; +}; + +export default function AcceptInvite() { + const router = useRouter(); + const token = (router.query.token as string) || ''; + + const [preview, setPreview] = useState<Preview | null>(null); + const [loading, setLoading] = useState(true); + const [password, setPassword] = useState(''); + const [confirm, setConfirm] = useState(''); + const [error, setError] = useState(''); + const [busy, setBusy] = useState(false); + + const load = useCallback(async () => { + if (!router.isReady) return; + if (!token) { + setLoading(false); + return; + } + + const res = await fetch( + `/api/clustox/accept-invite?token=${encodeURIComponent(token)}` + ); + if (res.ok) setPreview(await res.json()); + setLoading(false); + }, [router.isReady, token]); + + useEffect(() => { + load(); + }, [load]); + + const tooShort = password.length > 0 && password.length < MIN_PASSWORD; + const mismatch = confirm.length > 0 && confirm !== password; + + const onSubmit = async (e: FormEvent) => { + e.preventDefault(); + setBusy(true); + setError(''); + + const res = await fetch('/api/clustox/accept-invite', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ token, password }) + }); + + if (!res.ok) { + setBusy(false); + setError( + res.status === 409 + ? 'An account with this email already exists. Try signing in instead.' + : 'This invitation is no longer valid. Ask for a new link.' + ); + return; + } + + // Sign them straight in: making someone re-enter the password they just + // chose is a pointless extra step. + const signInRes = await signIn('credentials', { + email: preview?.email, + password, + redirect: false + }); + + setBusy(false); + + if (signInRes?.error) { + router.replace('/login'); + return; + } + // CLUSTOX FIX: same AuthProvider one-shot-session issue as login.tsx -- + // router.replace() is a client-side nav that leaves AuthProvider on its + // pre-sign-in (unauthenticated, no org) snapshot. Full navigation forces + // a remount so it fetches session fresh, now with the auth cookie set. + window.location.assign('/'); + }; + + if (loading) + return ( + <FlexBox minHeight="100vh" alignCenter justifyCenter> + <CircularProgress /> + </FlexBox> + ); + + if (!preview) + return ( + <FlexBox minHeight="100vh" alignCenter justifyCenter> + <Card sx={{ p: 4, maxWidth: 460 }}> + <Typography variant="h4" mb={2}> + This invitation is not valid + </Typography> + <Typography color="text.secondary" mb={3}> + The link may have been used already, revoked, or expired. + Invitations last {7} days. Ask whoever invited you for a new one. + </Typography> + <Button variant="outlined" onClick={() => router.replace('/login')}> + Go to sign in + </Button> + </Card> + </FlexBox> + ); + + return ( + <> + <Head> + <title>Accept invitation | MiddlewareHQ + + + + Welcome, {preview.name} + + Choose a password to finish setting up your account. + + + + + ) : ( + + ) + } + > + {preview.role === 'ADMIN' + ? preview.orgName + ? `You will manage the “${preview.orgName}” workspace.` + : 'You will get your own workspace, where you connect your GitHub or GitLab account and add your projects.' + : 'You will be a superadmin, able to see every workspace and manage users.'} + + + + + +
+ + setPassword(e.target.value)} + helperText={ + tooShort + ? `${MIN_PASSWORD - password.length} more characters needed` + : `At least ${MIN_PASSWORD} characters` + } + // A "create account" password field is exactly the kind + // browsers offer to generate/autofill a password into. + // Keeping the label permanently shrunk removes any race + // between that and React learning the value. + InputLabelProps={{ shrink: true }} + /> + setConfirm(e.target.value)} + helperText={mismatch ? 'Passwords do not match' : ' '} + InputLabelProps={{ shrink: true }} + /> + + {error && ( + + {error} + + )} + + + +
+
+ + ); +} + +AcceptInvite.getLayout = (page: ReactElement) => page; diff --git a/web-server/pages/api/auth/[...nextauth].ts b/web-server/pages/api/auth/[...nextauth].ts new file mode 100644 index 000000000..783f09317 --- /dev/null +++ b/web-server/pages/api/auth/[...nextauth].ts @@ -0,0 +1,55 @@ +import NextAuth, { NextAuthOptions } from 'next-auth'; +import CredentialsProvider from 'next-auth/providers/credentials'; + +import { verifyPassword } from '@/auth/password'; +import { getAuthUserByEmail } from '@/auth/queries'; + +export const authOptions: NextAuthOptions = { + session: { strategy: 'jwt', maxAge: 12 * 60 * 60 }, + pages: { signIn: '/login' }, + providers: [ + CredentialsProvider({ + name: 'Credentials', + credentials: { + email: { label: 'Email', type: 'text' }, + password: { label: 'Password', type: 'password' } + }, + async authorize(credentials) { + if (!credentials?.email || !credentials?.password) return null; + + const user = await getAuthUserByEmail(credentials.email); + if (!user) return null; + + const ok = await verifyPassword(credentials.password, user.passwordHash); + if (!ok) return null; + + return { + id: user.userId, + email: user.email, + name: user.name, + role: user.role + }; + } + }) + ], + callbacks: { + async jwt({ token, user }) { + if (user) { + token.userId = user.id; + token.role = user.role; + } + return token; + }, + async session({ session, token }) { + session.user = { + id: token.userId, + email: token.email as string, + name: token.name as string, + role: token.role + }; + return session; + } + } +}; + +export default NextAuth(authOptions); diff --git a/web-server/pages/api/auth/session.ts b/web-server/pages/api/auth/session.ts index 61a99efca..e6d28386f 100644 --- a/web-server/pages/api/auth/session.ts +++ b/web-server/pages/api/auth/session.ts @@ -32,10 +32,21 @@ export const delUserIdCookie = (res: NextApiResponse) => { ]); }; -endpoint.handle.GET(nullSchema, async (_req, res) => { +endpoint.handle.GET(nullSchema, async (req, res) => { + // CLUSTOX: resolve the caller's own workspace. Upstream took whichever + // Organization came back first, which was correct only while exactly one + // existed. A superadmin owns none, so they get no workspace context here -- + // cross-workspace views are served by dedicated endpoints. + const session = (req as any).session; + const orgId: string | null = session?.orgId ?? null; + + if (!orgId) { + return res.send({ org: null }); + } + const [orgDetails, integrationsLinkedAtMap] = await Promise.all([ - getOrgDetails(), - getOrgIntegrations() + getOrgDetails(orgId), + getOrgIntegrations(orgId) ]); const [onboardingState, codeProviderLastSyncedAt] = await Promise.all([ @@ -69,13 +80,15 @@ endpoint.handle.GET(nullSchema, async (_req, res) => { export default endpoint.serve(); -const getOrgDetails = async () => { - return db(Table.Organization).select('*').then(getFirstRow); +// CLUSTOX: scoped to a workspace rather than "the" organization. +const getOrgDetails = async (orgId: string) => { + return db(Table.Organization).select('*').where('id', orgId).then(getFirstRow); }; -export const getOrgIntegrations = async () => { +export const getOrgIntegrations = async (orgId: string) => { return db(Table.Integration) .select('*') + .where('org_id', orgId) .whereNotNull('access_token_enc_chunks') .then(async (rows) => { const integrationsLinkedAtMap = rows.reduce( diff --git a/web-server/pages/api/clustox/accept-invite.ts b/web-server/pages/api/clustox/accept-invite.ts new file mode 100644 index 000000000..03a5f45e7 --- /dev/null +++ b/web-server/pages/api/clustox/accept-invite.ts @@ -0,0 +1,51 @@ +import * as yup from 'yup'; + +import { Endpoint, nullSchema } from '@/api-helpers/global'; +import { acceptInvite, previewInvite } from '@/auth/invites'; +import { Errors, ResponseError } from '@/constants/error'; + +const MIN_PASSWORD = 12; + +const getSchema = yup.object().shape({ + token: yup.string().required() +}); + +const postSchema = yup.object().shape({ + token: yup.string().required(), + password: yup.string().min(MIN_PASSWORD).required() +}); + +/** + * Redeeming an invite. Necessarily unauthenticated -- the whole point is that + * the invitee has no account yet. + * + * The token is the only credential, so: + * - only its hash is stored, and it is returned to the superadmin once + * - unusable tokens are rejected identically whether unknown, spent, revoked + * or expired, so a guesser learns nothing from the response + * - acceptance marks the invite spent, making the link single-use + */ +const endpoint = new Endpoint(nullSchema, { unauthenticated: true }); + +endpoint.handle.GET(getSchema, async (req, res) => { + const preview = await previewInvite(req.payload.token); + if (!preview) throw new ResponseError(Errors.ACCESS_DENIED, 404); + + // Enough to show who the invite is for, and nothing else. + res.send(preview); +}); + +endpoint.handle.POST(postSchema, async (req, res) => { + const result = await acceptInvite(req.payload.token, req.payload.password); + + if (result.ok === false) { + throw new ResponseError( + Errors.ACCESS_DENIED, + result.reason === 'ALREADY_EXISTS' ? 409 : 404 + ); + } + + res.send({ ok: true, email: result.email }); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/clustox/benchmarks/global.ts b/web-server/pages/api/clustox/benchmarks/global.ts new file mode 100644 index 000000000..8c9e17a91 --- /dev/null +++ b/web-server/pages/api/clustox/benchmarks/global.ts @@ -0,0 +1,54 @@ +import * as yup from 'yup'; + +import { internal } from '@/api-helpers/axios'; +import { Endpoint, nullSchema } from '@/api-helpers/global'; +import { assertRole } from '@/auth/guard'; + +// CLUSTOX: writing the global baseline is superadmin-only -- one +// superadmin's numbers become every unset team's targets, so that stays +// locked down. The Flask layer trusts the internal token and has no notion +// of who is calling, so that role check has to live here. +const putSchema = yup.object().shape({ + lead_time: yup.number().min(0).nullable().optional(), + deployment_frequency: yup.number().min(0).nullable().optional(), + change_failure_rate: yup.number().min(0).max(100).nullable().optional(), + mean_time_to_recovery: yup.number().min(0).nullable().optional(), + // CLUSTOX: no `.max()`. Percent is the only bounded metric; an average PR + // of 5,000 lines is a bad number, not an invalid one, and the baseline has + // to be able to record where teams actually are. + lines_of_code: yup.number().min(0).nullable().optional() +}); + +const endpoint = new Endpoint(nullSchema); + +// CLUSTOX: reading the global baseline is deliberately NOT superadmin-gated. +// These numbers already reach every admin's own dashboard as resolved +// target lines on the metric cards (see resolve_benchmarks / the +// `benchmarks` field on team DORA metrics) -- they are not a secret, they're +// the other half of "why did my team's card say this target." Gating the +// read behind SUPERADMIN hid the number from the one form whose entire job +// is to explain where an inherited target came from, with no +// confidentiality benefit, since it's already visible elsewhere to the same +// audience. `Endpoint.serve()` still requires an authenticated session +// (`this.authenticated` defaults to true and is not opted out here), so this +// is "any signed-in admin may read" -- not "anyone may read." Do not add +// `assertRole` back here without re-checking that reasoning; PUT below is +// the one that actually needs it. +endpoint.handle.GET(nullSchema, async (_req, res) => { + const result = await internal.get('/settings/global', { + params: { setting_type: 'BENCHMARK_SETTING' } + }); + res.send(result.data); +}); + +endpoint.handle.PUT(putSchema, async (req, res) => { + assertRole((req as any).session, 'SUPERADMIN'); + + const result = await internal.put('/settings/global', { + setting_type: 'BENCHMARK_SETTING', + setting_data: req.payload + }); + res.send(result.data); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/clustox/invites/[invite_id].ts b/web-server/pages/api/clustox/invites/[invite_id].ts new file mode 100644 index 000000000..b180937e2 --- /dev/null +++ b/web-server/pages/api/clustox/invites/[invite_id].ts @@ -0,0 +1,20 @@ +import * as yup from 'yup'; + +import { Endpoint } from '@/api-helpers/global'; +import { assertRole } from '@/auth/guard'; +import { revokeInvite } from '@/auth/invites'; + +const pathSchema = yup.object().shape({ + invite_id: yup.string().uuid().required() +}); + +const endpoint = new Endpoint(pathSchema); + +/** Revoke a pending invite, so a shared link stops working. */ +endpoint.handle.DELETE(pathSchema, async (req, res) => { + assertRole((req as any).session, 'SUPERADMIN'); + await revokeInvite(req.payload.invite_id); + res.send({ ok: true }); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/clustox/invites/index.ts b/web-server/pages/api/clustox/invites/index.ts new file mode 100644 index 000000000..01e2fd2ba --- /dev/null +++ b/web-server/pages/api/clustox/invites/index.ts @@ -0,0 +1,90 @@ +import * as yup from 'yup'; + +import { Endpoint, nullSchema } from '@/api-helpers/global'; +import { assertRole } from '@/auth/guard'; +import { + createInvite, + listPendingInvites, + markInviteEmailed +} from '@/auth/invites'; +import { getAuthUserByEmail } from '@/auth/queries'; +import { ClustoxRole } from '@/auth/types'; +import { Table } from '@/constants/db'; +import { Errors, ResponseError } from '@/constants/error'; +import { db } from '@/utils/db'; +import { sendInviteEmail } from '@/utils/mailer'; + +// CLUSTOX: hardcoded on purpose. This was built from NEXTAUTH_URL, which the +// server's .env still sets to http://localhost:3333 -- so every invite email +// carried a link only the server itself could open. Nothing else caught it, +// because every other reader of that variable is same-origin. +// +// Set NEXTAUTH_URL to the public domain on the server and this can go back to +// reading it. Note that local invites now also point at production; the token +// is still valid, so swap the host by hand when testing on localhost. +const APP_URL = 'https://middleware.theclustox.com'; + +const postSchema = yup.object().shape({ + name: yup.string().required(), + email: yup.string().email().required(), + role: yup.string().oneOf(['SUPERADMIN', 'ADMIN']).required(), + org_id: yup.string().uuid().nullable().optional() +}); + +const endpoint = new Endpoint(nullSchema); + +endpoint.handle.GET(nullSchema, async (req, res) => { + assertRole((req as any).session, 'SUPERADMIN'); + res.send(await listPendingInvites()); +}); + +endpoint.handle.POST(postSchema, async (req, res) => { + const session = assertRole((req as any).session, 'SUPERADMIN'); + + const existing = await getAuthUserByEmail(req.payload.email); + if (existing) throw new ResponseError(Errors.INSUFFICIENT_PARAMS, 409); + + const role = req.payload.role as ClustoxRole; + const orgId = req.payload.org_id ?? null; + + const { id, token, expiresAt } = await createInvite({ + name: req.payload.name, + email: req.payload.email, + role, + orgId, + createdBy: session.userId + }); + + const inviteUrl = `${APP_URL}/accept-invite?token=${token}`; + + // Adopting an existing workspace is the only case where a name is known + // up front -- a brand-new one is created (and named) on acceptance, so + // there's nothing to show yet. + const org = + role === 'ADMIN' && orgId + ? await db(Table.Organization).where('id', orgId).first() + : null; + + // Best-effort and never fatal: the invite (and its link) already exists + // regardless of whether the email goes out, so a superadmin can always + // fall back to copying it manually. See src/utils/mailer.ts. + const { sent } = await sendInviteEmail({ + to: req.payload.email, + name: req.payload.name, + role, + orgName: org?.name ?? null, + inviteUrl, + expiresAt + }); + if (sent) await markInviteEmailed(id); + + // The raw token is returned exactly once, here. Only its hash is stored, so + // it cannot be recovered later -- a lost link has to be reissued. + res.send({ + invite_url: inviteUrl, + expires_at: expiresAt.toISOString(), + emailed: sent + }); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/clustox/jenkins/jobs.ts b/web-server/pages/api/clustox/jenkins/jobs.ts new file mode 100644 index 000000000..55d8535a9 --- /dev/null +++ b/web-server/pages/api/clustox/jenkins/jobs.ts @@ -0,0 +1,26 @@ +import * as yup from 'yup'; + +import { internal } from '@/api-helpers/axios'; +import { Endpoint } from '@/api-helpers/global'; +import { forwardInternalError } from '@/api-helpers/internal-error'; + +// CLUSTOX: proxies the workspace's Jenkins job list. org_id is validated by +// Endpoint.serve(), which asserts the caller may act on that workspace. +const getSchema = yup.object().shape({ + org_id: yup.string().uuid().required() +}); + +const endpoint = new Endpoint(getSchema); + +endpoint.handle.GET(getSchema, async (req, res) => { + // CLUSTOX: the analytics server distinguishes a base URL it refuses to fetch + // (400, with the address and the reason) from an unreachable Jenkins (502). + // Collapsing both into a generic failure here is what made the setup form + // tell an admin who typed an internal hostname to check his API token. + const jobs = await internal + .get(`/orgs/${req.payload.org_id}/integrations/jenkins/jobs`) + .catch(forwardInternalError); + res.send({ jobs: jobs.data }); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/clustox/jenkins/mappings.ts b/web-server/pages/api/clustox/jenkins/mappings.ts new file mode 100644 index 000000000..d0fb094ac --- /dev/null +++ b/web-server/pages/api/clustox/jenkins/mappings.ts @@ -0,0 +1,63 @@ +// CLUSTOX: BFF route for the Jenkins job-to-repo mapping. GET lists what is +// mapped, POST creates a mapping, DELETE removes it; all three proxy straight +// through to the analytics server, which owns the one-active-deployment-source +// -per-repo invariant. +import * as yup from 'yup'; + +import { internal } from '@/api-helpers/axios'; +import { Endpoint } from '@/api-helpers/global'; +import { forwardInternalError } from '@/api-helpers/internal-error'; + +// Only what every method here shares. A method-specific schema in the +// constructor is validated on *all* methods, which made the others a permanent +// 400 -- fixed once already, so the per-method shapes stay below. +const pathSchema = yup.object().shape({ + org_id: yup.string().uuid().required() +}); + +const postSchema = yup.object().shape({ + org_id: yup.string().uuid().required(), + org_repo_id: yup.string().uuid().required(), + job_full_name: yup.string().required() +}); + +const deleteSchema = yup.object().shape({ + org_id: yup.string().uuid().required(), + repo_workflow_id: yup.string().uuid().required() +}); + +const endpoint = new Endpoint(pathSchema); + +// CLUSTOX: what the mapping table renders as already-mapped. Each row carries +// the repo_workflow_id, which is the only thing that can address a mapping for +// removal -- without this call a mapping can be created but never undone. +endpoint.handle.GET(pathSchema, async (req, res) => { + const result = await internal + .get(`/orgs/${req.payload.org_id}/integrations/jenkins/mappings`) + .catch(forwardInternalError); + res.send({ mappings: result.data }); +}); + +endpoint.handle.POST(postSchema, async (req, res) => { + // One deployment source per repo: creating this mapping deactivates the + // repo's GitHub Actions workflows and switches every team tracking the repo + // to workflow-based deployments, so Jenkins runs are what get counted. + const result = await internal + .post(`/orgs/${req.payload.org_id}/integrations/jenkins/mappings`, { + org_repo_id: req.payload.org_repo_id, + job_full_name: req.payload.job_full_name + }) + .catch(forwardInternalError); + res.send(result.data); +}); + +endpoint.handle.DELETE(deleteSchema, async (req, res) => { + const result = await internal + .delete(`/orgs/${req.payload.org_id}/integrations/jenkins/mappings`, { + data: { repo_workflow_id: req.payload.repo_workflow_id } + }) + .catch(forwardInternalError); + res.send(result.data); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/clustox/me.ts b/web-server/pages/api/clustox/me.ts new file mode 100644 index 000000000..c40a6da84 --- /dev/null +++ b/web-server/pages/api/clustox/me.ts @@ -0,0 +1,30 @@ +import { Endpoint, nullSchema } from '@/api-helpers/global'; +import { assertAuthenticated } from '@/auth/guard'; + +/** + * The signed-in user's identity and current role. + * + * The UI needs the real Clustox role to decide what to show, and upstream's + * AuthContext hardcodes `role` to UserRole.MOM. Rather than rewire that + * context, components read from here. + * + * Presentation only -- every route enforces its own authorization server-side. + */ +const endpoint = new Endpoint(nullSchema); + +endpoint.handle.GET(nullSchema, async (req, res) => { + const session = assertAuthenticated((req as any).session); + + res.send({ + user_id: session.userId, + email: session.email, + name: session.name, + role: session.role, + // The workspace to act on. Resolved server-side per request, so unlike the + // redux copy it can never be a stale value rehydrated from a previous + // session's persisted state. + org_id: session.orgId + }); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/clustox/overview.ts b/web-server/pages/api/clustox/overview.ts new file mode 100644 index 000000000..09e0fa656 --- /dev/null +++ b/web-server/pages/api/clustox/overview.ts @@ -0,0 +1,106 @@ +import * as yup from 'yup'; + +import { handleRequest } from '@/api-helpers/axios'; +import { Endpoint } from '@/api-helpers/global'; +import { assertRole } from '@/auth/guard'; +import { listWorkspaceSummaries } from '@/auth/queries'; +import { Table } from '@/constants/db'; +import { db } from '@/utils/db'; + +type LeadTime = { lead_time: number; pr_count: number }; +type DeployFreq = { + total_deployments: number; + avg_daily_deployment_frequency: number; +}; + +const schema = yup.object().shape({ + from_time: yup.string().required(), + to_time: yup.string().required() +}); + +/** + * DORA metrics rolled up across every workspace. + * + * The dashboards are per team, and a team belongs to a workspace, so a + * superadmin comparing workspaces would otherwise have to switch into each one + * and read them off individually. + * + * Metrics are fetched per team and combined per workspace. Lead time is + * weighted by pull request count rather than averaged flat -- a team with 200 + * PRs and one with 3 do not contribute equally, and a flat mean would let a + * tiny team swing a workspace's figure. This matches how upstream aggregates + * across a team's repositories. + */ +const endpoint = new Endpoint(schema); + +endpoint.handle.GET(schema, async (req, res) => { + assertRole((req as any).session, 'SUPERADMIN'); + + const { from_time, to_time } = req.payload; + const workspaces = await listWorkspaceSummaries(); + + const teams = await db(Table.Team) + .where('is_deleted', false) + .select('id', 'org_id', 'name'); + + const rows = await Promise.all( + workspaces.map(async (w) => { + const wsTeams = teams.filter((t: any) => t.org_id === w.id); + + const perTeam = await Promise.all( + wsTeams.map(async (t: any) => { + try { + const [lead, deploy] = await Promise.all([ + handleRequest(`/teams/${t.id}/lead_time`, { + params: { from_time, to_time } + }), + handleRequest(`/teams/${t.id}/deployment_frequency`, { + params: { from_time, to_time } + }) + ]); + return { lead, deploy }; + } catch { + // One team failing must not blank the whole workspace, for the + // same reason a failing workspace does not stop a sync. + return null; + } + }) + ); + + const ok = perTeam.filter(Boolean) as { lead: LeadTime; deploy: DeployFreq }[]; + + const prCount = ok.reduce((n, r) => n + (r.lead.pr_count ?? 0), 0); + const deployments = ok.reduce( + (n, r) => n + (r.deploy.total_deployments ?? 0), + 0 + ); + + // Weighted by PR count. Undefined rather than 0 when there is nothing to + // average -- "no data" and "zero" are different claims, and conflating + // them is what makes a dashboard untrustworthy. + const leadTime = + prCount > 0 + ? ok.reduce( + (sum, r) => sum + (r.lead.lead_time ?? 0) * (r.lead.pr_count ?? 0), + 0 + ) / prCount + : null; + + return { + id: w.id, + name: w.name, + ownerEmail: w.ownerEmail, + hasIntegration: w.hasIntegration, + teamCount: wsTeams.length, + repoCount: w.repoCount, + prCount, + deployments, + leadTimeSeconds: leadTime + }; + }) + ); + + res.send(rows); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/clustox/sync-now.ts b/web-server/pages/api/clustox/sync-now.ts new file mode 100644 index 000000000..48a0cdfc6 --- /dev/null +++ b/web-server/pages/api/clustox/sync-now.ts @@ -0,0 +1,21 @@ +import { handleSyncServerRequest } from '@/api-helpers/axios'; +import { Endpoint, nullSchema } from '@/api-helpers/global'; +import { assertRole } from '@/auth/guard'; + +/** + * Trigger a sync across every workspace, rather than waiting up to 30 minutes + * for cron. + * + * Superadmin only: this runs every workspace's sync against every workspace's + * provider token, so it is not an action scoped to one admin's workspace. + */ +const endpoint = new Endpoint(nullSchema); + +endpoint.handle.POST(nullSchema, async (req, res) => { + assertRole((req as any).session, 'SUPERADMIN'); + + const body = await handleSyncServerRequest('/sync', { method: 'post' }); + res.send(body); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/clustox/users/[user_id].ts b/web-server/pages/api/clustox/users/[user_id].ts new file mode 100644 index 000000000..07045127b --- /dev/null +++ b/web-server/pages/api/clustox/users/[user_id].ts @@ -0,0 +1,79 @@ +import * as yup from 'yup'; + +import { Endpoint } from '@/api-helpers/global'; +import { assertRole } from '@/auth/guard'; +import { + countSuperadmins, + deleteUserAndEmptyWorkspace, + setUserTeams, + setUserWorkspace, + updateUserRole +} from '@/auth/queries'; +import { ClustoxRole } from '@/auth/types'; +import { Errors, ResponseError } from '@/constants/error'; + +const pathSchema = yup.object().shape({ + user_id: yup.string().uuid().required() +}); + +const patchSchema = yup.object().shape({ + user_id: yup.string().uuid().required(), + role: yup.string().oneOf(['SUPERADMIN', 'ADMIN']).optional(), + team_ids: yup.array().of(yup.string().uuid()).optional(), + // Move an admin into a workspace. Used to adopt a workspace that has no + // owner, rather than recreating its integration and repositories. + org_id: yup.string().uuid().nullable().optional() +}); + +const endpoint = new Endpoint(pathSchema); + +endpoint.handle.PATCH(patchSchema, async (req, res) => { + const session = assertRole((req as any).session, 'SUPERADMIN'); + + // Demoting the last superadmin would leave nobody able to administer the + // instance, which is unrecoverable through the UI. + if (req.payload.role === 'ADMIN') { + const remaining = await countSuperadmins(); + if (remaining <= 1 && session.userId === req.payload.user_id) { + throw new ResponseError(Errors.ACCESS_DENIED, 409); + } + } + + if (req.payload.role) { + // yup's .oneOf() widens to string; the schema has already validated it. + const role = req.payload.role as ClustoxRole; + await updateUserRole(req.payload.user_id, role); + + // A superadmin owns no workspace. Promoting without clearing it would + // leave a user who both sits above every workspace and owns one, which + // no part of the model expects. + if (role === 'SUPERADMIN') { + await setUserWorkspace(req.payload.user_id, null); + } + } + + if (req.payload.org_id !== undefined && req.payload.role !== 'SUPERADMIN') { + await setUserWorkspace(req.payload.user_id, req.payload.org_id ?? null); + } + + if (req.payload.team_ids) { + await setUserTeams(req.payload.user_id, req.payload.team_ids); + } + + res.send({ ok: true }); +}); + +endpoint.handle.DELETE(pathSchema, async (req, res) => { + const session = assertRole((req as any).session, 'SUPERADMIN'); + + // Deleting yourself would immediately end your own session, and if you were + // the last superadmin nobody could administer the instance afterwards. + if (session.userId === req.payload.user_id) { + throw new ResponseError(Errors.ACCESS_DENIED, 409); + } + + const result = await deleteUserAndEmptyWorkspace(req.payload.user_id); + res.send({ ok: true, workspace_removed: result.workspaceRemoved }); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/clustox/users/index.ts b/web-server/pages/api/clustox/users/index.ts new file mode 100644 index 000000000..a4e2b4454 --- /dev/null +++ b/web-server/pages/api/clustox/users/index.ts @@ -0,0 +1,51 @@ +import * as yup from 'yup'; + +import { Endpoint, nullSchema } from '@/api-helpers/global'; +import { assertRole } from '@/auth/guard'; +import { createUser, getAuthUserByEmail, listUsers } from '@/auth/queries'; +import { ClustoxRole } from '@/auth/types'; +import { Errors, ResponseError } from '@/constants/error'; + +const postSchema = yup.object().shape({ + name: yup.string().required(), + email: yup.string().email().required(), + // Enforced here so the UI cannot create a weak account. + password: yup.string().min(12).required(), + role: yup.string().oneOf(['SUPERADMIN', 'ADMIN']).required(), + team_ids: yup.array().of(yup.string().uuid()).required(), + // Optional: place the admin in an existing workspace instead of + // provisioning a new one. Used to adopt the pre-multitenancy workspace. + org_id: yup.string().uuid().nullable().optional() +}); + +const endpoint = new Endpoint(nullSchema); + +endpoint.handle.GET(nullSchema, async (req, res) => { + assertRole((req as any).session, 'SUPERADMIN'); + res.send(await listUsers()); +}); + +endpoint.handle.POST(postSchema, async (req, res) => { + assertRole((req as any).session, 'SUPERADMIN'); + + const existing = await getAuthUserByEmail(req.payload.email); + if (existing) { + throw new ResponseError(Errors.INSUFFICIENT_PARAMS, 409); + } + + // An ADMIN gets a freshly provisioned workspace unless one is named; a + // SUPERADMIN gets none. createUser decides based on the role. + const { userId, orgId } = await createUser({ + name: req.payload.name, + email: req.payload.email, + password: req.payload.password, + // yup's .oneOf() widens to string; the schema has already validated it. + role: req.payload.role as ClustoxRole, + teamIds: req.payload.team_ids, + orgId: req.payload.org_id ?? null + }); + + res.send({ user_id: userId, org_id: orgId }); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/clustox/workspace-status.ts b/web-server/pages/api/clustox/workspace-status.ts new file mode 100644 index 000000000..10a1e6bb1 --- /dev/null +++ b/web-server/pages/api/clustox/workspace-status.ts @@ -0,0 +1,65 @@ +import { handleSyncServerRequest } from '@/api-helpers/axios'; +import { Endpoint, nullSchema } from '@/api-helpers/global'; +import { assertAuthenticated, canAccessAllWorkspaces } from '@/auth/guard'; +import { listWorkspaceSummaries } from '@/auth/queries'; + +type SyncRun = { + org_id: string; + started_at: string | null; + finished_at: string | null; + status: 'RUNNING' | 'SUCCESS' | 'FAILED' | 'SKIPPED'; + detail: string | null; +}; + +/** + * Workspaces with their latest sync outcome. + * + * Answers "did last night's sync actually work?" -- which previously required + * reading a log file before it rotated away. + * + * A superadmin sees every workspace; an admin sees only their own, so the same + * page serves both without leaking the existence of other workspaces. + */ +const endpoint = new Endpoint(nullSchema); + +endpoint.handle.GET(nullSchema, async (req, res) => { + const session = assertAuthenticated((req as any).session); + + const summaries = await listWorkspaceSummaries(); + + const visible = canAccessAllWorkspaces(session) + ? summaries + : summaries.filter((w) => w.id === session.orgId); + + // The sync server owns run history. If it is unreachable, report the + // workspaces without status rather than failing the whole page -- an + // unknown sync state is still more useful than an error screen. + let runs: SyncRun[] = []; + try { + const body = await handleSyncServerRequest<{ results: SyncRun[] }>( + '/sync/status' + ); + runs = body?.results ?? []; + } catch { + runs = []; + } + + res.send( + visible.map((w) => { + const run = runs.find((r) => r.org_id === w.id) ?? null; + return { + ...w, + lastSync: run + ? { + status: run.status, + finished_at: run.finished_at, + started_at: run.started_at, + detail: run.detail + } + : null + }; + }) + ); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/clustox/workspaces.ts b/web-server/pages/api/clustox/workspaces.ts new file mode 100644 index 000000000..fca3c5227 --- /dev/null +++ b/web-server/pages/api/clustox/workspaces.ts @@ -0,0 +1,47 @@ +import * as yup from 'yup'; + +import { Endpoint, nullSchema } from '@/api-helpers/global'; +import { assertRole } from '@/auth/guard'; +import { listUnownedWorkspaces, listWorkspaces } from '@/auth/queries'; + +/** + * Every workspace, for the SuperAdmin workspace switcher. + * + * A SuperAdmin owns no workspace, so without a way to choose one the whole + * org-scoped UI has nothing to render. Selecting a workspace here sets the + * viewing context; it does not change what they are permitted to reach, which + * is everything. + */ +const endpoint = new Endpoint(nullSchema); + +endpoint.handle.GET(nullSchema, async (req, res) => { + assertRole((req as any).session, 'SUPERADMIN'); + + const [all, unowned] = await Promise.all([ + listWorkspaces(), + listUnownedWorkspaces() + ]); + const unownedIds = new Set(unowned.map((w) => w.id)); + + // `owned` lets the user form offer adoption of a workspace that has no + // admin -- notably the pre-multitenancy one holding the real data. + res.send(all.map((w) => ({ ...w, owned: !unownedIds.has(w.id) }))); +}); + +const putSchema = yup.object().shape({ + org_id: yup.string().uuid().required() +}); + +endpoint.handle.PUT(putSchema, async (req, res) => { + assertRole((req as any).session, 'SUPERADMIN'); + + // Viewing context only. Guards never consult this -- a SuperAdmin passes + // every workspace check regardless of what is selected. + res.setHeader( + 'set-cookie', + `clustox-workspace=${req.payload.org_id};Path=/;HttpOnly;SameSite=Lax` + ); + res.send({ org_id: req.payload.org_id }); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/integrations/index.ts b/web-server/pages/api/integrations/index.ts index 235167945..bc3c58459 100644 --- a/web-server/pages/api/integrations/index.ts +++ b/web-server/pages/api/integrations/index.ts @@ -25,6 +25,9 @@ endpoint.handle.GET(getSchema, async (req, res) => { }); endpoint.handle.DELETE(deleteSchema, async (req, res) => { + // CLUSTOX: admins connect their own integrations, so unlinking one is theirs + // to do as well. Endpoint.serve() confines them to their own workspace via + // the org_id in the payload. const data = await db(Table.Integration) .delete() .where({ diff --git a/web-server/pages/api/integrations/integrations-map.ts b/web-server/pages/api/integrations/integrations-map.ts index a216f75d6..119c86e02 100644 --- a/web-server/pages/api/integrations/integrations-map.ts +++ b/web-server/pages/api/integrations/integrations-map.ts @@ -14,8 +14,10 @@ const getSchema = yup.object().shape({ endpoint.handle.GET(getSchema, async (req, res) => { const { org_id } = req.payload; + // CLUSTOX: integrations are per workspace. Endpoint.serve() has already + // checked the caller may access this org_id. const [integrationsLinkedAtMap, codeProviderLastSyncedAt] = await Promise.all( - [getOrgIntegrations(), getLastSyncedAtForCodeProvider(org_id)] + [getOrgIntegrations(org_id), getLastSyncedAtForCodeProvider(org_id)] ); const integrations = {} as IntegrationsMap; Object.entries(integrationsLinkedAtMap).forEach( diff --git a/web-server/pages/api/integrations/jira/__tests__/validate.test.ts b/web-server/pages/api/integrations/jira/__tests__/validate.test.ts new file mode 100644 index 000000000..4db157817 --- /dev/null +++ b/web-server/pages/api/integrations/jira/__tests__/validate.test.ts @@ -0,0 +1,242 @@ +jest.mock('@/auth/session', () => ({ getAuthSession: jest.fn() })); +jest.mock('axios'); + +import axios from 'axios'; + +import { getAuthSession } from '@/auth/session'; + +import validateHandler from '../validate'; + +const mockRes = () => { + const res: any = {}; + res.status = jest.fn(() => res); + res.send = jest.fn(() => res); + return res; +}; + +const mockReq = (body: Record) => + ({ method: 'POST', query: {}, body, headers: {} }) as any; + +const asAuthed = () => + (getAuthSession as jest.Mock).mockResolvedValue({ + userId: 'u1', + email: 'admin@clustox.com', + name: 'Admin', + role: 'SUPERADMIN' + }); + +describe('POST /api/integrations/jira/validate', () => { + beforeEach(() => { + jest.clearAllMocks(); + asAuthed(); + }); + + it('requires an authenticated session, same as every other Endpoint route', async () => { + (getAuthSession as jest.Mock).mockResolvedValue(null); + const res = mockRes(); + + await validateHandler( + mockReq({ + site_url: 'x.atlassian.net', + email: 'a@b.com', + api_token: 't' + }), + res + ); + + expect(res.status).toHaveBeenCalledWith(401); + expect(axios.get).not.toHaveBeenCalled(); + }); + + it('reports valid, with the display name, when Jira answers with a real /myself payload', async () => { + (axios.get as jest.Mock).mockResolvedValue({ + data: { accountId: 'acc-1', displayName: 'Jordan Diaz' } + }); + const res = mockRes(); + + await validateHandler( + mockReq({ + site_url: 'https://mycompany.atlassian.net/', + email: 'jordan@mycompany.com', + api_token: 'tok' + }), + res + ); + + expect(res.send).toHaveBeenCalledWith({ + valid: true, + display_name: 'Jordan Diaz' + }); + // Normalized: protocol and trailing slash stripped before the request. + expect(axios.get).toHaveBeenCalledWith( + 'https://mycompany.atlassian.net/rest/api/3/myself', + expect.objectContaining({ + auth: { username: 'jordan@mycompany.com', password: 'tok' } + }) + ); + }); + + // CLUSTOX: regression test for a real bug found in code review. A regex + // that only stripped a leading protocol and trailing slash left any + // path/query intact -- pasting a full address copied from the browser + // while looking at a board would have built a request against the + // wrong path (a plausible thing for a user to paste, since /rest/api/3 + // is not something they'd normally see or think to strip themselves). + it('normalizes a full URL with a path down to just the hostname before requesting', async () => { + (axios.get as jest.Mock).mockResolvedValue({ + data: { accountId: 'acc-1' } + }); + const res = mockRes(); + + await validateHandler( + mockReq({ + site_url: + 'https://mycompany.atlassian.net/jira/software/projects/ABC/boards/1', + email: 'a@b.com', + api_token: 'tok' + }), + res + ); + + expect(axios.get).toHaveBeenCalledWith( + 'https://mycompany.atlassian.net/rest/api/3/myself', + expect.anything() + ); + }); + + it('falls back to the submitted email as display name when Jira omits displayName', async () => { + (axios.get as jest.Mock).mockResolvedValue({ + data: { accountId: 'acc-1' } + }); + const res = mockRes(); + + await validateHandler( + mockReq({ + site_url: 'mycompany.atlassian.net', + email: 'jordan@mycompany.com', + api_token: 'tok' + }), + res + ); + + expect(res.send).toHaveBeenCalledWith({ + valid: true, + display_name: 'jordan@mycompany.com' + }); + }); + + // CLUSTOX: regression test for a real bug found while building this -- + // id.atlassian.com (a genuine Atlassian domain, just not a Jira Cloud + // *site*) answers this exact path with HTTP 202 and an HTML body, which + // a naive "did the request throw" check accepts as valid. Every real + // Jira /myself response is JSON with an accountId; this must be + // rejected despite the "successful" HTTP status. + it('rejects a 2xx response that is not actually a Jira /myself payload', async () => { + (axios.get as jest.Mock).mockResolvedValue({ + data: 'not jira' + }); + const res = mockRes(); + + await validateHandler( + mockReq({ + site_url: 'id.atlassian.com', + email: 'a@b.com', + api_token: 'tok' + }), + res + ); + + expect(res.send).toHaveBeenCalledWith({ valid: false, reason: 'unknown' }); + }); + + it.each([401, 403])( + 'reports "unauthorized" when Jira responds %d', + async (status) => { + (axios.get as jest.Mock).mockRejectedValue({ response: { status } }); + const res = mockRes(); + + await validateHandler( + mockReq({ + site_url: 'mycompany.atlassian.net', + email: 'a@b.com', + api_token: 'wrong' + }), + res + ); + + expect(res.send).toHaveBeenCalledWith({ + valid: false, + reason: 'unauthorized' + }); + } + ); + + // CLUSTOX: regression test for a real bug found while building this -- + // a nonexistent *.atlassian.net subdomain resolves fine at the DNS/edge + // level and answers with a JSON 404, it does not fail to connect. A + // check that only looked for "no response at all" would misclassify a + // typo'd site as reason:"unknown" instead of the more useful + // reason:"unreachable" ("check the URL"). + it('reports "unreachable" for a 404 (a typo\'d or nonexistent site)', async () => { + (axios.get as jest.Mock).mockRejectedValue({ response: { status: 404 } }); + const res = mockRes(); + + await validateHandler( + mockReq({ + site_url: 'this-does-not-exist.atlassian.net', + email: 'a@b.com', + api_token: 'tok' + }), + res + ); + + expect(res.send).toHaveBeenCalledWith({ + valid: false, + reason: 'unreachable' + }); + }); + + it('reports "unreachable" when there is no response at all (DNS/network failure)', async () => { + (axios.get as jest.Mock).mockRejectedValue({ code: 'ENOTFOUND' }); + const res = mockRes(); + + await validateHandler( + mockReq({ + site_url: 'unreachable.example', + email: 'a@b.com', + api_token: 'tok' + }), + res + ); + + expect(res.send).toHaveBeenCalledWith({ + valid: false, + reason: 'unreachable' + }); + }); + + it('reports "unknown" for any other error status', async () => { + (axios.get as jest.Mock).mockRejectedValue({ response: { status: 500 } }); + const res = mockRes(); + + await validateHandler( + mockReq({ + site_url: 'mycompany.atlassian.net', + email: 'a@b.com', + api_token: 'tok' + }), + res + ); + + expect(res.send).toHaveBeenCalledWith({ valid: false, reason: 'unknown' }); + }); + + it('rejects a request missing required fields before ever calling Jira', async () => { + const res = mockRes(); + + await validateHandler(mockReq({ site_url: 'x.atlassian.net' }), res); + + expect(axios.get).not.toHaveBeenCalled(); + expect(res.status).toHaveBeenCalledWith(400); + }); +}); diff --git a/web-server/pages/api/integrations/jira/validate.ts b/web-server/pages/api/integrations/jira/validate.ts new file mode 100644 index 000000000..e634cd062 --- /dev/null +++ b/web-server/pages/api/integrations/jira/validate.ts @@ -0,0 +1,93 @@ +import axios from 'axios'; +import * as yup from 'yup'; + +import { Endpoint, nullSchema } from '@/api-helpers/global'; + +const postSchema = yup.object().shape({ + site_url: yup.string().required(), + email: yup.string().email().required(), + api_token: yup.string().required() +}); + +// Reduces any of "mycompany.atlassian.net", "https://mycompany.atlassian.net/", +// or a full URL copied from the browser while looking at a board +// ("https://mycompany.atlassian.net/jira/software/projects/ABC/boards/1") +// to just the hostname. A regex that only stripped a leading protocol and +// trailing slash left any path/query intact, which broke the very next +// request below (404s even with a valid site + credentials) for anyone +// who pasted a full address rather than typing the bare domain. Mirrors +// normalizeJiraSiteUrl in src/utils/auth.ts -- kept as two small copies +// rather than a shared import across the client/server boundary, same as +// that file's other provider helpers. +const normalizeSiteUrl = (input: string) => { + const trimmed = input.trim(); + try { + const withProtocol = /^https?:\/\//i.test(trimmed) + ? trimmed + : `https://${trimmed}`; + return new URL(withProtocol).hostname; + } catch { + return trimmed; + } +}; + +const endpoint = new Endpoint(nullSchema); + +// CLUSTOX: Jira's Cloud REST API does not send permissive CORS headers for +// arbitrary third-party origins doing Basic-auth token requests (unlike +// GitHub/GitLab's APIs, which checkGitHubValidity/checkGitLabValidity call +// directly from the browser). A direct browser fetch to +// https://{site}/rest/api/3/myself fails with a CORS error before the +// request even reaches Jira, regardless of whether the credentials are +// valid. Proxying through our own backend (server-to-server, no CORS) +// sidesteps that -- this route exists only for that reason, and does not +// persist anything; linking still goes through the existing generic +// /api/resources/orgs/[org_id]/integration route unchanged. +endpoint.handle.POST(postSchema, async (req, res) => { + const { site_url, email, api_token } = req.payload; + const domain = normalizeSiteUrl(site_url); + + try { + const response = await axios.get( + `https://${domain}/rest/api/3/myself`, + { + auth: { username: email, password: api_token }, + timeout: 8000 + } + ); + + // CLUSTOX: a 2xx status alone doesn't mean this was actually Jira's + // /myself endpoint -- e.g. id.atlassian.com (a real Atlassian domain, + // just not a Jira Cloud *site*) answers this same path with HTTP 202 + // and an HTML body, which axios treats as success. Every genuine + // Jira /myself response is JSON with an accountId; require that + // instead of trusting the status code alone, or a wrong-but-plausible + // domain reports as "linked" without ever having verified anything. + if (!response.data || typeof response.data !== 'object' || !response.data.accountId) { + return res.send({ valid: false, reason: 'unknown' }); + } + + return res.send({ + valid: true, + display_name: response.data.displayName ?? email + }); + } catch (error: any) { + const status = error?.response?.status; + return res.send({ + valid: false, + // Distinguishes "reached Jira, credentials were wrong" from "that + // site doesn't exist" from "couldn't reach it at all" -- Atlassian's + // edge answers a nonexistent *.atlassian.net subdomain with a JSON + // 404 (not a DNS failure), so a typo'd site shows up as a 404 here, + // not as the network-level "unreachable" case. + reason: + status === 401 || status === 403 + ? 'unauthorized' + : status === 404 || !error?.response + ? 'unreachable' + : 'unknown' + }); + } +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/internal/[org_id]/__tests__/jira_project_search.test.ts b/web-server/pages/api/internal/[org_id]/__tests__/jira_project_search.test.ts new file mode 100644 index 000000000..83018b679 --- /dev/null +++ b/web-server/pages/api/internal/[org_id]/__tests__/jira_project_search.test.ts @@ -0,0 +1,287 @@ +jest.mock('@/auth/session', () => ({ getAuthSession: jest.fn() })); +jest.mock('axios'); +jest.mock('@/utils/db', () => ({ db: jest.fn() })); +jest.mock('@/utils/auth-supplementary', () => ({ dec: jest.fn() })); + +import axios from 'axios'; + +import { getAuthSession } from '@/auth/session'; +import { dec } from '@/utils/auth-supplementary'; +import { db } from '@/utils/db'; + +import searchHandler from '../jira_project_search'; + +const ORG_ID = '11111111-1111-4111-8111-111111111111'; + +const mockRes = () => { + const res: any = {}; + res.status = jest.fn(() => res); + res.send = jest.fn(() => res); + // CLUSTOX: the real res is an untouched http.ServerResponse (EventEmitter), + // and the handler listens for 'close' to abort the outbound Jira call if + // the client disconnects mid-search -- see jira_project_search.ts's own + // comment. This plain mock needs at least a no-op .on to match that shape. + res.on = jest.fn(); + return res; +}; + +const mockReq = (query: Record = {}) => + ({ method: 'GET', query: { org_id: ORG_ID, ...query }, body: {}, headers: {} }) as any; + +const asAuthed = () => + (getAuthSession as jest.Mock).mockResolvedValue({ + userId: 'u1', + email: 'admin@clustox.com', + name: 'Admin', + role: 'SUPERADMIN' + }); + +// CLUSTOX: this route reads an Integration row via a real knex chain +// (db('Integration').select(...).where(...).first()), not a one-shot +// call -- stub the whole chain rather than just `db` itself. Mirrors the +// same auth/no-DB test setup already used by +// pages/api/integrations/jira/__tests__/validate.test.ts for the rest. +const mockDbChain = (row: any) => { + const chain: any = {}; + chain.select = jest.fn(() => chain); + chain.where = jest.fn(() => chain); + chain.first = jest.fn().mockResolvedValue(row); + (db as unknown as jest.Mock).mockReturnValue(chain); + return chain; +}; + +describe('GET /api/internal/[org_id]/jira_project_search', () => { + beforeEach(() => { + jest.clearAllMocks(); + asAuthed(); + }); + + it('requires an authenticated session', async () => { + (getAuthSession as jest.Mock).mockResolvedValue(null); + mockDbChain(undefined); + const res = mockRes(); + + await searchHandler(mockReq(), res); + + expect(res.status).toHaveBeenCalledWith(401); + expect(axios.get).not.toHaveBeenCalled(); + }); + + it('reports not-linked, and never calls Jira, when the org has no Jira integration row', async () => { + mockDbChain(undefined); + const res = mockRes(); + + await searchHandler(mockReq(), res); + + expect(res.status).toHaveBeenCalledWith(404); + expect(axios.get).not.toHaveBeenCalled(); + }); + + it('searches Jira and maps results, scoping idempotency_key by org_id', async () => { + mockDbChain({ + provider_meta: { site_url: 'mycompany.atlassian.net', email: 'a@b.com' }, + access_token_enc_chunks: ['enc1'] + }); + (dec as jest.Mock).mockReturnValue('decrypted-token'); + (axios.get as jest.Mock).mockResolvedValue({ + data: { + values: [ + { id: '10001', key: 'PAY', name: 'Payments' }, + { id: '10002', key: 'ENG', name: 'Engineering' } + ] + } + }); + const res = mockRes(); + + await searchHandler(mockReq({ search_text: 'pay' }), res); + + expect(axios.get).toHaveBeenCalledWith( + 'https://mycompany.atlassian.net/rest/api/3/project/search', + expect.objectContaining({ + auth: { username: 'a@b.com', password: 'decrypted-token' }, + params: { maxResults: 50, query: 'pay' } + }) + ); + expect(res.status).toHaveBeenCalledWith(200); + expect(res.send).toHaveBeenCalledWith([ + { + id: '10001', + key: 'PAY', + name: 'Payments', + idempotency_key: `jira:${ORG_ID}:10001`, + provider: 'jira' + }, + { + id: '10002', + key: 'ENG', + name: 'Engineering', + idempotency_key: `jira:${ORG_ID}:10002`, + provider: 'jira' + } + ]); + }); + + it('omits the query param when no search_text was given', async () => { + mockDbChain({ + provider_meta: { site_url: 'mycompany.atlassian.net', email: 'a@b.com' }, + access_token_enc_chunks: ['enc1'] + }); + (dec as jest.Mock).mockReturnValue('decrypted-token'); + (axios.get as jest.Mock).mockResolvedValue({ data: { values: [] } }); + const res = mockRes(); + + await searchHandler(mockReq(), res); + + expect(axios.get).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ params: { maxResults: 50 } }) + ); + }); + + it.each([401, 403])( + 'reports 401 when Jira rejects the stored credentials (%d)', + async (status) => { + mockDbChain({ + provider_meta: { site_url: 'mycompany.atlassian.net', email: 'a@b.com' }, + access_token_enc_chunks: ['enc1'] + }); + (dec as jest.Mock).mockReturnValue('decrypted-token'); + (axios.get as jest.Mock).mockRejectedValue({ response: { status } }); + const res = mockRes(); + + await searchHandler(mockReq(), res); + + expect(res.status).toHaveBeenCalledWith(401); + } + ); + + it('aborts the outbound Jira call when the client disconnects mid-search', async () => { + // Regression test for the pile-up bug: search-as-you-type fires a new + // request on every keystroke, and without this, a superseded request's + // outbound Jira call keeps running to completion (or its own 8s + // timeout) regardless of the client having already moved on. + mockDbChain({ + provider_meta: { site_url: 'mycompany.atlassian.net', email: 'a@b.com' }, + access_token_enc_chunks: ['enc1'] + }); + (dec as jest.Mock).mockReturnValue('decrypted-token'); + let resolveAxios: (value: unknown) => void; + (axios.get as jest.Mock).mockReturnValue( + new Promise((resolve) => { + resolveAxios = resolve; + }) + ); + const res = mockRes(); + + const handlerPromise = searchHandler(mockReq(), res); + // Let the handler run up to (and register) res.on('close', ...) before + // simulating the disconnect -- several awaited steps (schema + // validation, the DB lookup) sit before that line, so a single + // microtask tick isn't enough; a macrotask (setTimeout) flushes all + // of them. jsdom has no setImmediate. + await new Promise((resolve) => setTimeout(resolve, 0)); + + const [, onClose] = (res.on as jest.Mock).mock.calls[0]; + const [, axiosConfig] = (axios.get as jest.Mock).mock.calls[0]; + expect(axiosConfig.signal.aborted).toBe(false); + + onClose(); + + expect(axiosConfig.signal.aborted).toBe(true); + + resolveAxios!({ data: { values: [] } }); + await handlerPromise; + }); + + it('reports 502 when Jira is unreachable', async () => { + mockDbChain({ + provider_meta: { site_url: 'mycompany.atlassian.net', email: 'a@b.com' }, + access_token_enc_chunks: ['enc1'] + }); + (dec as jest.Mock).mockReturnValue('decrypted-token'); + (axios.get as jest.Mock).mockRejectedValue(new Error('network down')); + const res = mockRes(); + + await searchHandler(mockReq(), res); + + expect(res.status).toHaveBeenCalledWith(502); + }); +}); + +// CLUSTOX: Jira multi-account support. See +// docs/JIRA_MULTI_ACCOUNT_PLAN.md Task 6 part 2. +describe('GET /api/internal/[org_id]/jira_project_search with connection_id', () => { + const CONNECTION_ID = '22222222-2222-4222-8222-222222222222'; + + beforeEach(() => { + jest.clearAllMocks(); + asAuthed(); + }); + + it('searches the given connection’s Jira site, not the legacy Integration row', async () => { + const chain = mockDbChain({ + site_url: 'other.atlassian.net', + email: 'b@other.com', + access_token_enc_chunks: ['enc2'] + }); + (dec as jest.Mock).mockReturnValue('other-decrypted-token'); + (axios.get as jest.Mock).mockResolvedValue({ + data: { values: [{ id: '5001', key: 'TEST', name: 'Test Project' }] } + }); + const res = mockRes(); + + await searchHandler( + mockReq({ connection_id: CONNECTION_ID, search_text: 'test' }), + res + ); + + expect(db).toHaveBeenCalledWith('JiraConnection'); + expect(chain.where).toHaveBeenCalledWith({ + org_id: ORG_ID, + id: CONNECTION_ID + }); + expect(axios.get).toHaveBeenCalledWith( + 'https://other.atlassian.net/rest/api/3/project/search', + expect.objectContaining({ + auth: { username: 'b@other.com', password: 'other-decrypted-token' } + }) + ); + expect(res.status).toHaveBeenCalledWith(200); + expect(res.send).toHaveBeenCalledWith([ + { + id: '5001', + key: 'TEST', + name: 'Test Project', + // Scoped by connection_id too, not just org_id -- a second + // connection with a colliding site-local project id must not + // collide with this one. + idempotency_key: `jira:${ORG_ID}:${CONNECTION_ID}:5001`, + provider: 'jira', + connection_id: CONNECTION_ID + } + ]); + }); + + it('reports 404 without ever calling Jira when the connection does not resolve', async () => { + mockDbChain(undefined); + const res = mockRes(); + + await searchHandler(mockReq({ connection_id: CONNECTION_ID }), res); + + expect(res.status).toHaveBeenCalledWith(404); + expect(axios.get).not.toHaveBeenCalled(); + }); + + it('never falls back to the legacy Integration row for a connection that fails to resolve', async () => { + // A missing/foreign connection_id must not silently search whatever + // the legacy Integration row happens to point at -- that would search + // (or worse, appear to succeed against) the wrong Jira site entirely. + mockDbChain(undefined); + const res = mockRes(); + + await searchHandler(mockReq({ connection_id: CONNECTION_ID }), res); + + expect(db).toHaveBeenCalledWith('JiraConnection'); + expect(db).not.toHaveBeenCalledWith('Integration'); + }); +}); diff --git a/web-server/pages/api/internal/[org_id]/git_org_repos.ts b/web-server/pages/api/internal/[org_id]/git_org_repos.ts index 6c3d2fa6a..e96b2588c 100644 --- a/web-server/pages/api/internal/[org_id]/git_org_repos.ts +++ b/web-server/pages/api/internal/[org_id]/git_org_repos.ts @@ -1,45 +1,61 @@ -import * as yup from 'yup' +import * as yup from 'yup'; + import { + getBitbucketCredentials, getGithubToken, getGitlabToken, getGitHubGraphQLUrl, - replaceURL, -} from '@/api/internal/[org_id]/utils' -import { Endpoint } from '@/api-helpers/global' -import { Integration } from '@/constants/integrations' -import { BaseRepo } from '@/types/resources' + replaceURL +} from '@/api/internal/[org_id]/utils'; +import { Endpoint } from '@/api-helpers/global'; +import { Integration } from '@/constants/integrations'; +import { BaseRepo } from '@/types/resources'; +import { adaptBitbucketRepo, isBitbucketApiUrl } from '@/utils/bitbucketRepos'; const pathSchema = yup.object({ - org_id: yup.string().required(), -}) + org_id: yup.string().required() +}); const getSchema = yup.object({ provider: yup .string() - .oneOf([Integration.GITHUB, Integration.GITLAB]) + .oneOf([Integration.GITHUB, Integration.GITLAB, Integration.BITBUCKET]) .required(), org: yup.string().required(), first: yup.number().integer().min(1).max(100).optional(), after: yup.string().optional(), - select_all: yup.boolean().optional(), -}) + select_all: yup.boolean().optional() +}); -const endpoint = new Endpoint(pathSchema) +const endpoint = new Endpoint(pathSchema); -type PageInfo = { hasNextPage: boolean; endCursor: string | null } -type RepoItem = Record +type PageInfo = { hasNextPage: boolean; endCursor: string | null }; +type RepoItem = Record; export async function fetchRepos(params: { - provider: string - token: string - org: string - first?: number - after?: string | null + provider: string; + token: string; + org: string; + first?: number; + after?: string | null; }): Promise<{ - totalCount: number | null - pageInfo: PageInfo | null - repos: RepoItem[] + totalCount: number | null; + pageInfo: PageInfo | null; + repos: RepoItem[]; }> { - const { provider, token, org, first = 50, after = null } = params + const { provider, token, org, first = 50, after = null } = params; + + if (provider === Integration.BITBUCKET) { + // CLUSTOX: `token` carries "email:token" for Bitbucket -- Basic auth + // needs both, and the shared signature has one credential slot. + const [email, ...tokenParts] = token.split(':'); + return fetchBitbucketRepos({ + org, + email, + token: tokenParts.join(':'), + first, + after + }); + } if (provider === Integration.GITHUB) { const query = ` @@ -73,29 +89,29 @@ export async function fetchRepos(params: { } } } - ` + `; - const graphqlUrl = await getGitHubGraphQLUrl() + const graphqlUrl = await getGitHubGraphQLUrl(); const res = await fetch(graphqlUrl, { method: 'POST', headers: { Authorization: `bearer ${token}`, - 'Content-Type': 'application/json', + 'Content-Type': 'application/json' }, body: JSON.stringify({ query, - variables: { org, first, after }, - }), - }) + variables: { org, first, after } + }) + }); if (!res.ok) { - throw new Error(`GitHub API returned ${res.status}`) + throw new Error(`GitHub API returned ${res.status}`); } - const { data, errors } = await res.json() - if (errors) throw new Error(JSON.stringify(errors)) + const { data, errors } = await res.json(); + if (errors) throw new Error(JSON.stringify(errors)); - const r = data.organization?.repositories + const r = data.organization?.repositories; const repos = r?.nodes.map( (repo: any) => @@ -108,14 +124,14 @@ export async function fetchRepos(params: { web_url: repo.url, language: repo.primaryLanguage?.name, branch: repo.defaultBranchRef?.name, - provider: Integration.GITHUB, + provider: Integration.GITHUB }) as BaseRepo - ) || [] + ) || []; return { totalCount: r?.totalCount || 0, pageInfo: r?.pageInfo || null, - repos, - } + repos + }; } const query = ` @@ -151,33 +167,33 @@ export async function fetchRepos(params: { } } } - ` + `; - const gqlUrl = await replaceURL('https://gitlab.com/api/graphql') + const gqlUrl = await replaceURL('https://gitlab.com/api/graphql'); const res = await fetch(gqlUrl, { method: 'POST', headers: { Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json', + 'Content-Type': 'application/json' }, body: JSON.stringify({ query, variables: { group: org, first, - after, - }, - }), - }) + after + } + }) + }); if (!res.ok) { - throw new Error(`GitLab API returned ${res.status}`) + throw new Error(`GitLab API returned ${res.status}`); } - const { data, errors } = await res.json() - if (errors) throw new Error(JSON.stringify(errors)) + const { data, errors } = await res.json(); + if (errors) throw new Error(JSON.stringify(errors)); - const projects = data.group?.projects || { nodes: [], pageInfo: null } + const projects = data.group?.projects || { nodes: [], pageInfo: null }; const repos = projects.nodes.map( (repo: any) => ({ @@ -188,55 +204,112 @@ export async function fetchRepos(params: { web_url: repo.webUrl, branch: repo.repository?.rootRef || null, parent: repo.fullPath.split('/').slice(0, -1).join('/'), - provider: Integration.GITLAB, + provider: Integration.GITLAB }) as BaseRepo - ) + ); return { totalCount: null, pageInfo: projects.pageInfo, - repos, + repos + }; +} + +async function fetchBitbucketRepos(params: { + org: string; + email: string; + token: string; + first: number; + after: string | null; +}): Promise<{ + totalCount: number | null; + pageInfo: PageInfo | null; + repos: RepoItem[]; +}> { + const { org, email, token, first, after } = params; + // CLUSTOX: Bitbucket paginates by opaque `next` URLs, not cursors. The + // `after` value IS the next-page URL, threaded through the cursor field the + // shared contract already has -- the UI never inspects it. It is also + // client-supplied, and this fetch carries the org's Basic auth header, so + // an unvalidated cursor is an SSRF that exfiltrates the token to any host + // the caller names. Only Bitbucket's own API may be fetched. + if (after && !isBitbucketApiUrl(after)) { + throw new Error('Invalid Bitbucket pagination cursor'); } + const url = + after || + `https://api.bitbucket.org/2.0/repositories/${encodeURIComponent( + org + )}?pagelen=${first}`; + + const res = await fetch(url, { + headers: { + Authorization: `Basic ${Buffer.from(`${email}:${token}`).toString( + 'base64' + )}` + } + }); + if (!res.ok) { + // Status only -- the body can echo request details and this error + // surfaces in the browser. + throw new Error(`Bitbucket API returned ${res.status}`); + } + + const body = await res.json(); + return { + totalCount: body.size ?? null, + pageInfo: { + hasNextPage: Boolean(body.next), + endCursor: body.next ?? null + }, + repos: (body.values || []).map((repo: RepoItem) => + adaptBitbucketRepo(repo, org) + ) + }; } export async function selectAllRepos(params: { - provider: string - token: string - org: string + provider: string; + token: string; + org: string; }): Promise { - const all: RepoItem[] = [] - let after: string | null = null - let pageInfo: PageInfo | null = null + const all: RepoItem[] = []; + let after: string | null = null; + let pageInfo: PageInfo | null = null; do { const { repos, pageInfo: pi } = await fetchRepos({ ...params, first: 100, - after, - }) - all.push(...repos) - pageInfo = pi - after = pageInfo?.endCursor ?? null - } while (pageInfo?.hasNextPage) - return all + after + }); + all.push(...repos); + pageInfo = pi; + after = pageInfo?.endCursor ?? null; + } while (pageInfo?.hasNextPage); + return all; } endpoint.handle.GET(getSchema, async (req, res) => { - const { org_id, provider, org, first, after, select_all } = req.payload + const { org_id, provider, org, first, after, select_all } = req.payload; const token = provider === Integration.GITHUB ? await getGithubToken(org_id) - : await getGitlabToken(org_id) + : provider === Integration.BITBUCKET + ? await getBitbucketCredentials(org_id).then( + (c) => `${c.email}:${c.token}` + ) + : await getGitlabToken(org_id); if (select_all) { const repos = await selectAllRepos({ provider, token, - org, - }) + org + }); return res.status(200).send({ totalCount: repos.length, pageInfo: null, - repos, - }) + repos + }); } const { totalCount, pageInfo, repos } = await fetchRepos({ @@ -244,10 +317,10 @@ endpoint.handle.GET(getSchema, async (req, res) => { token, org, first, - after, - }) + after + }); - return res.status(200).send({ totalCount, pageInfo, repos }) -}) + return res.status(200).send({ totalCount, pageInfo, repos }); +}); -export default endpoint.serve() +export default endpoint.serve(); diff --git a/web-server/pages/api/internal/[org_id]/jira_project_search.ts b/web-server/pages/api/internal/[org_id]/jira_project_search.ts new file mode 100644 index 000000000..ab5ec2baf --- /dev/null +++ b/web-server/pages/api/internal/[org_id]/jira_project_search.ts @@ -0,0 +1,168 @@ +import axios from 'axios'; +import * as yup from 'yup'; + +import { Endpoint } from '@/api-helpers/global'; +import { Integration } from '@/constants/integrations'; +import { dec } from '@/utils/auth-supplementary'; +import { db } from '@/utils/db'; + +const pathSchema = yup.object().shape({ + org_id: yup.string().uuid().required() +}); + +const getSchema = yup.object().shape({ + search_text: yup.string().optional().nullable(), + // CLUSTOX: Jira multi-account support -- see + // docs/JIRA_MULTI_ACCOUNT_PLAN.md Task 6 part 2. Omitted entirely means + // "search the legacy single-account Integration row", unchanged from + // before this existed. + connection_id: yup.string().uuid().optional().nullable() +}); + +export type JiraProjectSearchResult = { + id: string; + key: string; + name: string; + idempotency_key: string; + provider: Integration.JIRA; + // CLUSTOX: threaded back through team_projects.ts's PUT so + // ProjectService can record which JiraConnection this project came + // from. Absent for a legacy-flow result, same reasoning as above. + connection_id?: string; +}; + +const endpoint = new Endpoint(pathSchema); + +// CLUSTOX: Jira integration, Phase 2 (project selection) -- live project +// search for the team-creation picker. Unlike the GitHub/GitLab repo +// search this mirrors (searchGithubRepos/gitlabSearch in +// pages/api/internal/[org_id]/utils.ts), there's no pre-synced OrgProject +// catalog to search yet in this phase -- Jira's own /project/search +// endpoint already supports server-side name/key filtering, so querying +// it directly here is simpler than standing up a sync job just to make +// this box searchable. See docs/JIRA_INTEGRATION_PROPOSAL.md. +endpoint.handle.GET(getSchema, async (req, res) => { + const { org_id, search_text, connection_id } = req.payload; + + const credentials = connection_id + ? await getJiraConnectionCredentials(org_id, connection_id) + : await getLegacyJiraCredentials(org_id); + + if (!credentials) { + return res.status(404).send({ + error: connection_id + ? 'That Jira connection was not found for this workspace' + : 'Jira is not linked for this workspace' + }); + } + const { siteUrl, email, apiToken } = credentials; + + // CLUSTOX: search-as-you-type on the client aborts a superseded request + // (see useTeamJiraProjectsConfig's controllerRef), but that only tears + // down the browser's connection to *this* route -- without this, the + // outbound call to Jira below keeps running to completion (or its own + // 8s timeout) regardless, so a burst of keystrokes piles up that many + // uncancelled real Jira requests concurrently. `res` (unlike `req`, + // which Endpoint.serve() passes to handlers as a shallow-spread plain + // object -- see transformNextRequest -- and so has lost its + // EventEmitter prototype methods) is the real, untouched + // http.ServerResponse, whose 'close' event fires exactly when the + // client disconnects, so forwarding it into an AbortController here + // lets the Jira call stop too instead of running to completion for + // nothing. + const outboundAbort = new AbortController(); + res.on('close', () => outboundAbort.abort()); + + try { + const response = await axios.get( + `https://${siteUrl}/rest/api/3/project/search`, + { + auth: { username: email, password: apiToken }, + params: { + maxResults: 50, + ...(search_text ? { query: search_text } : {}) + }, + signal: outboundAbort.signal, + timeout: 8000 + } + ); + + const projects: JiraProjectSearchResult[] = ( + response.data?.values || [] + ).map((project: { id: string; key: string; name: string }) => ({ + id: String(project.id), + key: project.key, + name: project.name, + // Scoped by org_id, not the bare Jira id -- two different orgs' + // independent Jira sites can otherwise land on the same small, + // site-local project id. Further scoped by connection_id when one + // was given: two connections in the same org are two independent + // Jira sites too, and a bare org_id scope would collide their + // site-local project ids exactly the way ticket idempotency keys + // could (see docs/JIRA_MULTI_ACCOUNT_PLAN.md, "Known risks" #1). The + // legacy (no connection_id) branch keeps the original key shape -- + // changing it would orphan every project already selected through + // the single-account flow. + idempotency_key: connection_id + ? `jira:${org_id}:${connection_id}:${project.id}` + : `jira:${org_id}:${project.id}`, + provider: Integration.JIRA, + ...(connection_id ? { connection_id } : {}) + })); + + return res.status(200).send(projects); + } catch (error: any) { + // Surfaced as a plain failure rather than a 500 -- an expired/revoked + // Jira token, or Jira itself being briefly unreachable, isn't our own + // server breaking. + const status = error?.response?.status; + return res.status(status === 401 || status === 403 ? 401 : 502).send({ + error: 'Could not reach Jira to search projects.' + }); + } +}); + +export default endpoint.serve(); + +type JiraCredentials = { siteUrl: string; email: string; apiToken: string }; + +const getLegacyJiraCredentials = async ( + org_id: string +): Promise => { + const integration = await db('Integration') + .select('provider_meta', 'access_token_enc_chunks') + .where({ org_id, name: Integration.JIRA }) + .first(); + + const siteUrl = integration?.provider_meta?.site_url; + const email = integration?.provider_meta?.email; + const apiToken = integration?.access_token_enc_chunks + ? dec(integration.access_token_enc_chunks) + : null; + + return siteUrl && email && apiToken ? { siteUrl, email, apiToken } : null; +}; + +const getJiraConnectionCredentials = async ( + org_id: string, + connection_id: string +): Promise => { + // CLUSTOX: org_id in the where-clause, not just the connection's own id + // -- a connection_id alone must never resolve across orgs, same + // reasoning as JiraConnectionRepoService.get_jira_connection on the + // Python side. + const connection = await db('JiraConnection') + .select('site_url', 'email', 'access_token_enc_chunks') + .where({ org_id, id: connection_id }) + .first(); + + if (!connection?.site_url || !connection?.email) return null; + + const apiToken = connection.access_token_enc_chunks + ? dec(connection.access_token_enc_chunks) + : null; + + return apiToken + ? { siteUrl: connection.site_url, email: connection.email, apiToken } + : null; +}; diff --git a/web-server/pages/api/internal/[org_id]/utils.ts b/web-server/pages/api/internal/[org_id]/utils.ts index e80975b54..8632f9eaa 100644 --- a/web-server/pages/api/internal/[org_id]/utils.ts +++ b/web-server/pages/api/internal/[org_id]/utils.ts @@ -315,26 +315,27 @@ export const getGitHubCustomDomain = async (): Promise => { return head(provider_meta || [])?.custom_domain || null; } catch (error) { - console.error('Error occured while getting custom domain from database:', error); + console.error( + 'Error occured while getting custom domain from database:', + error + ); return null; } }; -const normalizeSlashes = (url: string) => - url.replace(/(? url.replace(/(? { const customDomain = await getGitHubCustomDomain(); - const base = customDomain - ? `${customDomain}/api/v3` - : DEFAULT_GH_URL; + const base = customDomain ? `${customDomain}/api/v3` : DEFAULT_GH_URL; return normalizeSlashes(`${base}/${path}`); }; - export const getGitHubGraphQLUrl = async (): Promise => { const customDomain = await getGitHubCustomDomain(); - return customDomain ? `${customDomain}/api/graphql` : `${DEFAULT_GH_URL}/graphql`; + return customDomain + ? `${customDomain}/api/graphql` + : `${DEFAULT_GH_URL}/graphql`; }; export const getGithubToken = async (org_id: ID) => { @@ -360,3 +361,22 @@ export const getGitlabToken = async (org_id: ID) => { .then(getFirstRow) .then((r) => dec(r.access_token_enc_chunks)); }; + +// CLUSTOX: Bitbucket authenticates with Basic auth (email, Atlassian API +// token), so both halves come back together. The email is not a secret and +// lives in provider_meta -- the same home GitLab uses for custom_domain -- +// while the token is encrypted like every other provider's. +export const getBitbucketCredentials = async (org_id: ID) => { + return await db('Integration') + .select() + .where({ + org_id, + name: Integration.BITBUCKET + }) + .returning('*') + .then(getFirstRow) + .then((r) => ({ + email: r.provider_meta?.email as string, + token: dec(r.access_token_enc_chunks) + })); +}; diff --git a/web-server/pages/api/internal/bitbucket/token-check.ts b/web-server/pages/api/internal/bitbucket/token-check.ts new file mode 100644 index 000000000..f0ed055e3 --- /dev/null +++ b/web-server/pages/api/internal/bitbucket/token-check.ts @@ -0,0 +1,61 @@ +import * as yup from 'yup'; + +import { Endpoint, nullSchema } from '@/api-helpers/global'; + +const postSchema = yup.object().shape({ + email: yup.string().email().required(), + token: yup.string().required() +}); + +const endpoint = new Endpoint(nullSchema); + +// CLUSTOX: GitHub's modal validates its PAT straight from the browser; +// Bitbucket cannot -- api.bitbucket.org sends no CORS headers for Basic auth +// from foreign origins, so the preflight dies before any request. The check +// runs here instead. GitLab's modal already validates server-side, so this is +// the established pattern, not a new one. +// +// The token passes through this handler and is never logged, never persisted +// here, and never echoed in a response -- the reply carries a boolean and the +// account nickname only. Linking (and encryption) stays with the existing +// /api/resources/orgs//integration flow. +endpoint.handle.POST(postSchema, async (req, res) => { + const { email, token } = req.payload; + const headers = { + Authorization: `Basic ${Buffer.from(`${email}:${token}`).toString( + 'base64' + )}` + }; + + // CLUSTOX: the try/catch is secret hygiene, not politeness. An uncaught + // fetch error propagates to the global handler, which logs the whole + // request -- token included -- into the server log. A network failure here + // must degrade to a clean "invalid" answer instead. + let response: Response; + try { + response = await fetch('https://api.bitbucket.org/2.0/user', { headers }); + } catch (e) { + return res.send({ valid: false, reason: 'network_error' }); + } + + if (response.ok) { + const user = await response.json(); + return res.send({ valid: true, nickname: user.nickname ?? null }); + } + + // CLUSTOX: a scoped token missing the account read scope gets 403 from + // /2.0/user with otherwise-valid credentials; a wrong pair gets 401. The + // sync's own validity check uses /2.0/user, so a 403 token must still be + // rejected -- but with a message naming the actual problem. (An earlier + // version probed /2.0/workspaces to tell these apart; that endpoint is + // dead -- 410 CHANGE-2770 -- so the probe made this reason unreachable.) + if (response.status === 403) { + return res.send({ valid: false, reason: 'missing_account_scope' }); + } + + // 401 and everything else: wrong email, wrong token, revoked or expired -- + // the API cannot tell these apart, so neither can our copy. + return res.send({ valid: false, reason: 'invalid_credentials' }); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/internal/team/[team_id]/__tests__/sprints.test.ts b/web-server/pages/api/internal/team/[team_id]/__tests__/sprints.test.ts new file mode 100644 index 000000000..f17bb4e9f --- /dev/null +++ b/web-server/pages/api/internal/team/[team_id]/__tests__/sprints.test.ts @@ -0,0 +1,64 @@ +jest.mock('@/auth/session', () => ({ getAuthSession: jest.fn() })); +jest.mock('@/api-helpers/axios', () => ({ handleRequest: jest.fn() })); + +import { handleRequest } from '@/api-helpers/axios'; +import { getAuthSession } from '@/auth/session'; + +import sprintsHandler from '../sprints'; + +const TEAM_ID = '33333333-3333-4333-8333-333333333333'; + +const mockRes = () => { + const res: any = {}; + res.status = jest.fn(() => res); + res.send = jest.fn(() => res); + return res; +}; + +const mockReq = () => + ({ method: 'GET', query: { team_id: TEAM_ID }, body: {}, headers: {} }) as any; + +const asAuthed = () => + (getAuthSession as jest.Mock).mockResolvedValue({ + userId: 'u1', + email: 'admin@clustox.com', + name: 'Admin', + role: 'SUPERADMIN' + }); + +// CLUSTOX: Jira integration -- the Sprint rollup chart. See +// docs/JIRA_INTEGRATION_PROPOSAL.md §6D. +describe('GET /api/internal/team/[team_id]/sprints', () => { + beforeEach(() => { + jest.clearAllMocks(); + asAuthed(); + }); + + it('proxies to the backend with no date-range params', async () => { + (handleRequest as jest.Mock).mockResolvedValue([]); + const res = mockRes(); + + await sprintsHandler(mockReq(), res); + + expect(handleRequest).toHaveBeenCalledWith(`/teams/${TEAM_ID}/sprints`); + }); + + it('sends back whatever the backend returned', async () => { + const payload = [ + { + name: 'PZDA Sprint 1', + state: 'closed', + start_date: '2026-07-20T07:55:00.130Z', + end_date: '2026-08-03T05:00:00.000Z', + planned_count: 355, + completed_count: 272 + } + ]; + (handleRequest as jest.Mock).mockResolvedValue(payload); + const res = mockRes(); + + await sprintsHandler(mockReq(), res); + + expect(res.send).toHaveBeenCalledWith(payload); + }); +}); diff --git a/web-server/pages/api/internal/team/[team_id]/__tests__/ticket_insights.test.ts b/web-server/pages/api/internal/team/[team_id]/__tests__/ticket_insights.test.ts new file mode 100644 index 000000000..4404c26af --- /dev/null +++ b/web-server/pages/api/internal/team/[team_id]/__tests__/ticket_insights.test.ts @@ -0,0 +1,95 @@ +jest.mock('@/auth/session', () => ({ getAuthSession: jest.fn() })); +jest.mock('@/api-helpers/axios', () => ({ handleRequest: jest.fn() })); + +import { handleRequest } from '@/api-helpers/axios'; +import { getAuthSession } from '@/auth/session'; + +import ticketInsightsHandler from '../ticket_insights'; + +const TEAM_ID = '33333333-3333-4333-8333-333333333333'; + +const mockRes = () => { + const res: any = {}; + res.status = jest.fn(() => res); + res.send = jest.fn(() => res); + return res; +}; + +const mockReq = (query: Record) => + ({ method: 'GET', query: { team_id: TEAM_ID, ...query }, body: {}, headers: {} }) as any; + +const asAuthed = () => + (getAuthSession as jest.Mock).mockResolvedValue({ + userId: 'u1', + email: 'admin@clustox.com', + name: 'Admin', + role: 'SUPERADMIN' + }); + +// CLUSTOX: Jira integration, Phase 4 (§6C/§6E). See +// docs/JIRA_INTEGRATION_PROPOSAL.md. +describe('GET /api/internal/team/[team_id]/ticket_insights', () => { + beforeEach(() => { + jest.clearAllMocks(); + asAuthed(); + }); + + it('proxies to the backend with isoDateString-formatted timestamps, not toISOString', async () => { + (handleRequest as jest.Mock).mockResolvedValue({ + cycle_time_by_project: [], + prs_without_ticket_count: 0 + }); + const res = mockRes(); + + await ticketInsightsHandler( + mockReq({ from_date: '2026-05-01', to_date: '2026-08-01' }), + res + ); + + expect(handleRequest).toHaveBeenCalledWith( + `/teams/${TEAM_ID}/ticket_insights`, + { + params: { + // isoDateString uses a real +hh:mm/-hh:mm offset, never a bare + // "Z" -- Python's datetime.fromisoformat() on the backend + // (pre-3.11) rejects the latter. + from_time: expect.not.stringContaining('Z'), + to_time: expect.not.stringContaining('Z') + } + } + ); + }); + + it('sends back whatever the backend returned', async () => { + const payload = { + cycle_time_by_project: [ + { + project_key: 'PZDA', + project_name: 'Project Zero Deposit Africa', + ticket_count: 5, + avg_total_seconds: 3600, + avg_seconds_by_category: { 'In Progress': 3600 } + } + ], + prs_without_ticket_count: 2 + }; + (handleRequest as jest.Mock).mockResolvedValue(payload); + const res = mockRes(); + + await ticketInsightsHandler( + mockReq({ from_date: '2026-05-01', to_date: '2026-08-01' }), + res + ); + + expect(res.send).toHaveBeenCalledWith(payload); + }); + + it('rejects a request missing the required date range', async () => { + const res = mockRes(); + + await ticketInsightsHandler(mockReq({}), res); + + expect(handleRequest).not.toHaveBeenCalled(); + expect(res.status).toHaveBeenCalledWith(400); + }); +}); diff --git a/web-server/pages/api/internal/team/[team_id]/__tests__/unlinked_prs.test.ts b/web-server/pages/api/internal/team/[team_id]/__tests__/unlinked_prs.test.ts new file mode 100644 index 000000000..87c088b47 --- /dev/null +++ b/web-server/pages/api/internal/team/[team_id]/__tests__/unlinked_prs.test.ts @@ -0,0 +1,87 @@ +jest.mock('@/auth/session', () => ({ getAuthSession: jest.fn() })); +jest.mock('@/api-helpers/axios', () => ({ handleRequest: jest.fn() })); + +import { handleRequest } from '@/api-helpers/axios'; +import { getAuthSession } from '@/auth/session'; + +import unlinkedPrsHandler from '../unlinked_prs'; + +const TEAM_ID = '33333333-3333-4333-8333-333333333333'; + +const mockRes = () => { + const res: any = {}; + res.status = jest.fn(() => res); + res.send = jest.fn(() => res); + return res; +}; + +const mockReq = (query: Record) => + ({ method: 'GET', query: { team_id: TEAM_ID, ...query }, body: {}, headers: {} }) as any; + +const asAuthed = () => + (getAuthSession as jest.Mock).mockResolvedValue({ + userId: 'u1', + email: 'admin@clustox.com', + name: 'Admin', + role: 'SUPERADMIN' + }); + +// CLUSTOX: Jira integration, Phase 4 (§6E) -- the Data Hygiene card's +// drill-down. See docs/JIRA_INTEGRATION_PROPOSAL.md. +describe('GET /api/internal/team/[team_id]/unlinked_prs', () => { + beforeEach(() => { + jest.clearAllMocks(); + asAuthed(); + }); + + it('proxies to the backend with isoDateString-formatted timestamps, not toISOString', async () => { + (handleRequest as jest.Mock).mockResolvedValue([]); + const res = mockRes(); + + await unlinkedPrsHandler( + mockReq({ from_date: '2026-05-01', to_date: '2026-08-01' }), + res + ); + + expect(handleRequest).toHaveBeenCalledWith( + `/teams/${TEAM_ID}/unlinked_prs`, + { + params: { + from_time: expect.not.stringContaining('Z'), + to_time: expect.not.stringContaining('Z') + } + } + ); + }); + + it('sends back whatever the backend returned', async () => { + const payload = [ + { + id: 'pr-1', + title: 'feat(payments): add refund flow', + url: 'https://github.com/org/repo/pull/42', + head_branch: 'feat/refund-flow', + author: 'jordan', + merged_at: '2026-07-15T10:00:00+00:00' + } + ]; + (handleRequest as jest.Mock).mockResolvedValue(payload); + const res = mockRes(); + + await unlinkedPrsHandler( + mockReq({ from_date: '2026-05-01', to_date: '2026-08-01' }), + res + ); + + expect(res.send).toHaveBeenCalledWith(payload); + }); + + it('rejects a request missing the required date range', async () => { + const res = mockRes(); + + await unlinkedPrsHandler(mockReq({}), res); + + expect(handleRequest).not.toHaveBeenCalled(); + expect(res.status).toHaveBeenCalledWith(400); + }); +}); diff --git a/web-server/pages/api/internal/team/[team_id]/contributors.ts b/web-server/pages/api/internal/team/[team_id]/contributors.ts new file mode 100644 index 000000000..7d8985643 --- /dev/null +++ b/web-server/pages/api/internal/team/[team_id]/contributors.ts @@ -0,0 +1,42 @@ +import { endOfDay, startOfDay } from 'date-fns'; +import * as yup from 'yup'; + +import { internal } from '@/api-helpers/axios'; +import { Endpoint } from '@/api-helpers/global'; +import { isoDateString } from '@/utils/date'; + +// CLUSTOX: backs the contributor filter on the DORA metrics page. team_id is +// validated by Endpoint.serve(), which asserts the caller may read that team. +const pathSchema = yup.object().shape({ + team_id: yup.string().uuid().required() +}); + +const getSchema = yup.object().shape({ + team_id: yup.string().uuid().required(), + from_date: yup.date().required(), + to_date: yup.date().required() +}); + +const endpoint = new Endpoint(pathSchema); + +endpoint.handle.GET(getSchema, async (req, res) => { + const { team_id, from_date: rawFromDate, to_date: rawToDate } = req.payload; + + // The same day-boundary normalisation the DORA route applies to the very + // same picker values (see dora_metrics.ts). Both routes are handed + // `dates.start` / `dates.end`, which carry the wall-clock time of day, so + // forwarding them raw made this window strictly narrower than the metrics + // window: a PR merged later today than the current clock time counted + // towards Lead Time but was missing from the dropdown that is supposed to + // list exactly the people with data in view. + const from_date = isoDateString(startOfDay(new Date(rawFromDate))); + const to_date = isoDateString(endOfDay(new Date(rawToDate))); + + const contributors = await internal.get(`/teams/${team_id}/contributors`, { + params: { from_time: from_date, to_time: to_date } + }); + + res.send(contributors.data); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/internal/team/[team_id]/dora_metrics.ts b/web-server/pages/api/internal/team/[team_id]/dora_metrics.ts index 9ec5eb28b..c164063c0 100644 --- a/web-server/pages/api/internal/team/[team_id]/dora_metrics.ts +++ b/web-server/pages/api/internal/team/[team_id]/dora_metrics.ts @@ -14,8 +14,14 @@ import { fetchLeadTimeStats, fetchChangeFailureRateStats, fetchMeanTimeToRestoreStats, - fetchDeploymentFrequencyStats + fetchDeploymentFrequencyStats, + // CLUSTOX: per-team DORA benchmarks. + fetchTeamBenchmarks, + // CLUSTOX: lines of code. + fetchLocStats } from '@/utils/cockpitMetricUtils'; +// CLUSTOX: contributor filter. +import { stripContributorFilters } from '@/utils/contributorFilters'; import { isoDateString, getAggregateAndTrendsIntervalTime } from '@/utils/date'; import { getBranchesAndRepoFilter, @@ -33,7 +39,10 @@ const getSchema = yup.object().shape({ branches: yup.string().optional().nullable(), from_date: yup.date().required(), to_date: yup.date().required(), - branch_mode: yup.string().oneOf(Object.values(ActiveBranchMode)).required() + branch_mode: yup.string().oneOf(Object.values(ActiveBranchMode)).required(), + // CLUSTOX: contributor filter -- git usernames, optional so an unfiltered + // dashboard sends exactly what it always did. + authors: yup.array().of(yup.string()).optional() }); const endpoint = new Endpoint(pathSchema); @@ -49,32 +58,63 @@ endpoint.handle.GET(getSchema, async (req, res) => { from_date: rawFromDate, to_date: rawToDate, branches, - branch_mode + branch_mode, + // CLUSTOX: contributor filter. + authors } = req.payload; const from_date = isoDateString(startOfDay(new Date(rawFromDate))); const to_date = isoDateString(endOfDay(new Date(rawToDate))); - const [branchAndRepoFilters, unsyncedRepos] = await Promise.all([ + const [branchAndRepoFilters, unsyncedRepos, benchmarks] = await Promise.all([ getBranchesAndRepoFilter({ orgId: org_id, teamId, branchMode: branch_mode as ActiveBranchMode, branches }), - getUnsyncedRepos(teamId) + getUnsyncedRepos(teamId), + // CLUSTOX: per-team DORA benchmarks. Deliberately fetched here and sent + // below, not on any other route: `metrics_summary` -- the only slice the + // four cards read `benchmarks` from -- is written solely by whatever this + // handler returns. + // + // The `catch` is load-bearing. Inside a Promise.all, a rejection here + // would take down the entire DORA response and blank all four cards over + // an optional decoration. That is not hypothetical: an unrelated 400 on a + // sibling call in this same Promise.all rendered "something went wrong" + // across the whole dashboard during the contributor-filter work. Targets + // are already optional, and every card treats `undefined` as "no target", + // so degrading to an unbenchmarked dashboard is the correct failure. + fetchTeamBenchmarks(teamId).catch(() => undefined) ]); const [prFilters, workflowFilters] = await Promise.all([ - updatePrFilterParams(teamId, {}, branchAndRepoFilters).then( + // CLUSTOX: contributor filter -- `authors` narrows Lead Time by PR author, + // `eventActors` narrows Deployment Frequency by the actor who triggered the + // run. Both are no-ops when the selection is empty. + updatePrFilterParams(teamId, {}, { ...branchAndRepoFilters, authors }).then( ({ pr_filter }) => ({ pr_filter }) ), getWorkFlowFiltersAsPayloadForSingleTeam({ orgId: org_id, - teamId: teamId + teamId: teamId, + eventActors: authors }) + // END CLUSTOX ]); + // CLUSTOX: contributor filter -- Change Failure Rate and MTTR stay team-wide + // (no defensible per-contributor definition until Jira incident ownership + // lands), so they get a copy of the filters with the contributor keys + // removed. With nothing selected these are identical to the originals, which + // is what keeps unfiltered dashboards byte-for-byte unchanged. + const { + prFilter: prFilterWithoutAuthors, + workflowFilter: workflowFilterWithoutEventActors + } = stripContributorFilters(prFilters, workflowFilters); + // END CLUSTOX + const { currTrendsTimeObject, prevTrendsTimeObject, @@ -90,7 +130,8 @@ endpoint.handle.GET(getSchema, async (req, res) => { changeFailureRateResponse, deploymentFrequencyResponse, leadtimePrs, - teamRepos + teamRepos, + locResponse ] = await Promise.all([ fetchLeadTimeStats({ teamId, @@ -118,7 +159,7 @@ endpoint.handle.GET(getSchema, async (req, res) => { }, currTrendsTimeObject, prevTrendsTimeObject, - prFilter: prFilters + prFilter: prFilterWithoutAuthors }), fetchChangeFailureRateStats({ teamId, @@ -132,8 +173,8 @@ endpoint.handle.GET(getSchema, async (req, res) => { }, currTrendsTimeObject, prevTrendsTimeObject, - prFilter: prFilters, - workflowFilter: workflowFilters + prFilter: prFilterWithoutAuthors, + workflowFilter: workflowFilterWithoutEventActors }), fetchDeploymentFrequencyStats({ teamId, @@ -150,10 +191,47 @@ endpoint.handle.GET(getSchema, async (req, res) => { workflowFilter: workflowFilters, prFilter: prFilters }), - getTeamLeadTimePRs(teamId, from_date, to_date, prFilters).then( + // CLUSTOX: deliberately the *unfiltered* pr filter. This list lands in + // redux as `summary_prs`, which has a second consumer: the Change Failure + // Rate "See details" overlay (content/DoraMetrics/Incidents.tsx) uses it + // as the denominator in `percent(revertedPrCount, prs.length)` against an + // unfiltered revert count. Narrowing it to one contributor there reports + // e.g. 12 reverts over 8 PRs = 150% on a card labelled team-wide. + // TeamInsightsBody reads the same slice. One slice cannot be both + // per-contributor and team-wide, so it stays team-wide; the lead time + // *metric* above is still filtered. + getTeamLeadTimePRs(teamId, from_date, to_date, prFilterWithoutAuthors).then( (r) => r.data ), - getTeamRepos(teamId) + getTeamRepos(teamId), + // CLUSTOX: lines of code. Sits here rather than in the earlier Promise.all + // because it needs `prFilters` -- the same filtered object lead time gets, + // so the contributor filter and branch mode narrow LOC identically instead + // of leaving one card on the dashboard reporting team-wide numbers. + // + // The `catch` carries the same soft-failure contract as `fetchTeamBenchmarks` + // above, and for the same reason: inside a Promise.all an unhandled + // rejection here would take down the whole DORA response and blank all five + // cards over one optional metric. That is not hypothetical on this project + // -- a payload key nested one level too deep made the analytics server 400 + // and the entire dashboard went blank on a single click. `loc_stats` and + // `loc_trends` are optional on the response type and every consumer must + // treat their absence as "not measured", so degrading to a dashboard + // without the LOC card is the correct failure. + fetchLocStats({ + teamId, + currStatsTimeObject: { + from_time: isoDateString(currentCycleStartDay), + to_time: isoDateString(currentCycleEndDay) + }, + prevStatsTimeObject: { + from_time: isoDateString(prevCycleStartDay), + to_time: isoDateString(prevCycleEndDay) + }, + currTrendsTimeObject, + prevTrendsTimeObject, + prFilter: prFilters + }).catch(() => undefined) ]); return res.send({ @@ -173,7 +251,18 @@ endpoint.handle.GET(getSchema, async (req, res) => { deploymentFrequencyResponse.deployment_frequency_trends, lead_time_prs: leadtimePrs, assigned_repos: teamRepos, - unsynced_repos: unsyncedRepos + unsynced_repos: unsyncedRepos, + // CLUSTOX: read by all four DORA cards as + // `metrics_summary.benchmarks..target` / `.source`. + benchmarks, + // CLUSTOX: lines of code. The dora_metrics payload is written wholesale + // into the `metrics_summary` slice, so these land as + // `metrics_summary.loc_stats` / `.loc_trends`, which is where the LOC card + // reads them. Spread flat like every other metric rather than nested under + // one key -- a card reading `loc_stats.current.avg_pr_size` must not have + // to know which fetcher produced it. + loc_stats: locResponse?.loc_stats, + loc_trends: locResponse?.loc_trends } as TeamDoraMetricsApiResponseType); }); diff --git a/web-server/pages/api/internal/team/[team_id]/sprints.ts b/web-server/pages/api/internal/team/[team_id]/sprints.ts new file mode 100644 index 000000000..581210494 --- /dev/null +++ b/web-server/pages/api/internal/team/[team_id]/sprints.ts @@ -0,0 +1,34 @@ +import * as yup from 'yup'; + +import { handleRequest } from '@/api-helpers/axios'; +import { Endpoint } from '@/api-helpers/global'; + +// CLUSTOX: Jira integration -- the Sprint rollup chart (docs/ +// JIRA_INTEGRATION_PROPOSAL.md §6D). No date-range params -- a sprint's +// own start/end dates are its natural window, unlike every other Jira +// widget on this page, which scopes to the DORA Metrics page's selected +// period. +export type Sprint = { + name: string; + state: string; + start_date: string | null; + end_date: string | null; + planned_count: number; + completed_count: number; +}; + +const pathSchema = yup.object().shape({ + team_id: yup.string().uuid().required() +}); + +const endpoint = new Endpoint(pathSchema); + +endpoint.handle.GET(yup.object(), async (req, res) => { + const { team_id } = req.payload; + + const sprints = await handleRequest(`/teams/${team_id}/sprints`); + + res.send(sprints); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/internal/team/[team_id]/ticket_insights.ts b/web-server/pages/api/internal/team/[team_id]/ticket_insights.ts new file mode 100644 index 000000000..8b3ae0bc7 --- /dev/null +++ b/web-server/pages/api/internal/team/[team_id]/ticket_insights.ts @@ -0,0 +1,54 @@ +import * as yup from 'yup'; + +import { handleRequest } from '@/api-helpers/axios'; +import { Endpoint } from '@/api-helpers/global'; +import { isoDateString } from '@/utils/date'; + +// CLUSTOX: Jira integration, Phase 4 (§6C/§6E) -- the DORA Metrics +// page's Jira widget. Deliberately its own standalone proxy, not folded +// into dora_metrics.ts (which already aggregates the 4 existing DORA +// cards' own calls) -- keeps this additive to that endpoint rather than +// risking a regression in it. See docs/JIRA_INTEGRATION_PROPOSAL.md. +export type TicketInsights = { + cycle_time_by_project: { + project_key: string; + project_name: string; + ticket_count: number; + avg_total_seconds: number; + avg_seconds_by_category: Record; + }[]; + prs_without_ticket_count: number; +}; + +const pathSchema = yup.object().shape({ + team_id: yup.string().uuid().required() +}); + +const getSchema = yup.object().shape({ + from_date: yup.date().required(), + to_date: yup.date().required() +}); + +const endpoint = new Endpoint(pathSchema); + +endpoint.handle.GET(getSchema, async (req, res) => { + const { team_id, from_date, to_date } = req.payload; + + const insights = await handleRequest( + `/teams/${team_id}/ticket_insights`, + { + params: { + // isoDateString, not Date#toISOString -- the latter's trailing + // "Z" isn't accepted by Python's datetime.fromisoformat() on the + // backend (pre-3.11), the same reason every other DORA proxy + // already uses this helper instead. + from_time: isoDateString(new Date(from_date)), + to_time: isoDateString(new Date(to_date)) + } + } + ); + + res.send(insights); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/internal/team/[team_id]/ticket_lead_time.ts b/web-server/pages/api/internal/team/[team_id]/ticket_lead_time.ts new file mode 100644 index 000000000..d9672fca7 --- /dev/null +++ b/web-server/pages/api/internal/team/[team_id]/ticket_lead_time.ts @@ -0,0 +1,47 @@ +import * as yup from 'yup'; + +import { handleRequest } from '@/api-helpers/axios'; +import { Endpoint } from '@/api-helpers/global'; +import { isoDateString } from '@/utils/date'; + +// CLUSTOX: Jira integration -- the extended Lead Time breakdown's +// "ticket created -> first commit" leading phase (docs/ +// JIRA_INTEGRATION_PROPOSAL.md §6A). Its own standalone proxy, not +// folded into the existing /teams/[team_id]/lead_time-backed flow -- +// that one feeds the untouched, org-wide "Lead Time for Changes" DORA +// card and must keep returning exactly what it always has. +export type TicketLeadTimeMetrics = { + matched_pr_count: number; + avg_ticket_to_first_commit_seconds: number; + avg_commit_only_lead_time_seconds: number; + avg_extended_lead_time_seconds: number; +}; + +const pathSchema = yup.object().shape({ + team_id: yup.string().uuid().required() +}); + +const getSchema = yup.object().shape({ + from_date: yup.date().required(), + to_date: yup.date().required() +}); + +const endpoint = new Endpoint(pathSchema); + +endpoint.handle.GET(getSchema, async (req, res) => { + const { team_id, from_date, to_date } = req.payload; + + const metrics = await handleRequest( + `/teams/${team_id}/ticket_lead_time`, + { + params: { + from_time: isoDateString(new Date(from_date)), + to_time: isoDateString(new Date(to_date)) + } + } + ); + + res.send(metrics); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/internal/team/[team_id]/unlinked_prs.ts b/web-server/pages/api/internal/team/[team_id]/unlinked_prs.ts new file mode 100644 index 000000000..3435e4cf2 --- /dev/null +++ b/web-server/pages/api/internal/team/[team_id]/unlinked_prs.ts @@ -0,0 +1,51 @@ +import * as yup from 'yup'; + +import { handleRequest } from '@/api-helpers/axios'; +import { Endpoint } from '@/api-helpers/global'; +import { isoDateString } from '@/utils/date'; + +// CLUSTOX: Jira integration, Phase 4 (§6E) -- the Data Hygiene card's +// drill-down. Its own standalone proxy, not folded into ticket_insights.ts +// -- that one loads on every DORA Metrics page view, this one only when +// someone actually opens the unmatched-PRs list. See +// docs/JIRA_INTEGRATION_PROPOSAL.md. +export type UnlinkedPr = { + id: string; + title: string; + url: string; + head_branch: string; + author: string; + merged_at: string; +}; + +const pathSchema = yup.object().shape({ + team_id: yup.string().uuid().required() +}); + +const getSchema = yup.object().shape({ + from_date: yup.date().required(), + to_date: yup.date().required() +}); + +const endpoint = new Endpoint(pathSchema); + +endpoint.handle.GET(getSchema, async (req, res) => { + const { team_id, from_date, to_date } = req.payload; + + const prs = await handleRequest( + `/teams/${team_id}/unlinked_prs`, + { + params: { + // isoDateString, not Date#toISOString -- see ticket_insights.ts + // for why (Python's datetime.fromisoformat() pre-3.11 rejects a + // bare "Z" offset). + from_time: isoDateString(new Date(from_date)), + to_time: isoDateString(new Date(to_date)) + } + } + ); + + res.send(prs); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/resources/__tests__/team_projects.test.ts b/web-server/pages/api/resources/__tests__/team_projects.test.ts new file mode 100644 index 000000000..65db71aab --- /dev/null +++ b/web-server/pages/api/resources/__tests__/team_projects.test.ts @@ -0,0 +1,120 @@ +jest.mock('@/auth/session', () => ({ getAuthSession: jest.fn() })); +jest.mock('@/api-helpers/axios', () => ({ handleRequest: jest.fn() })); + +import { handleRequest } from '@/api-helpers/axios'; +import { getAuthSession } from '@/auth/session'; + +import projectsHandler from '../team_projects'; + +const TEAM_ID = '22222222-2222-4222-8222-222222222222'; + +const mockRes = () => { + const res: any = {}; + res.status = jest.fn(() => res); + res.send = jest.fn(() => res); + return res; +}; + +const mockReq = (method: string, query: Record, body: Record = {}) => + ({ method, query, body, headers: {} }) as any; + +const asAuthed = () => + (getAuthSession as jest.Mock).mockResolvedValue({ + userId: 'u1', + email: 'admin@clustox.com', + name: 'Admin', + role: 'SUPERADMIN' + }); + +// CLUSTOX: this route is a thin proxy to the Python backend's +// /teams//projects -- mirrors team_repos.ts, which this file's +// GET/PUT handlers are modeled on. See +// docs/JIRA_INTEGRATION_PROPOSAL.md. +describe('/api/resources/team_projects', () => { + beforeEach(() => { + jest.clearAllMocks(); + asAuthed(); + }); + + describe('GET', () => { + it('fetches the current team projects and stamps team_id onto every row', async () => { + (handleRequest as jest.Mock).mockResolvedValue([ + { org_project_id: 'proj-1', key: 'PAY', name: 'Payments' } + ]); + const res = mockRes(); + + await projectsHandler(mockReq('GET', { team_id: TEAM_ID }), res); + + expect(handleRequest).toHaveBeenCalledWith(`/teams/${TEAM_ID}/projects`); + expect(res.send).toHaveBeenCalledWith([ + { org_project_id: 'proj-1', key: 'PAY', name: 'Payments', team_id: TEAM_ID } + ]); + }); + }); + + describe('PUT', () => { + it('forwards the full replacement project set to the backend, with team_id on each item', async () => { + (handleRequest as jest.Mock).mockResolvedValue([ + { + org_project_id: 'proj-1', + key: 'PAY', + name: 'Payments', + provider: 'jira', + idempotency_key: 'jira:org-1:10001' + } + ]); + const res = mockRes(); + + await projectsHandler( + mockReq('PUT', { team_id: TEAM_ID }, { + projects: [ + { + key: 'PAY', + name: 'Payments', + provider: 'jira', + idempotency_key: 'jira:org-1:10001' + } + ] + }), + res + ); + + expect(handleRequest).toHaveBeenCalledWith( + `/teams/${TEAM_ID}/projects`, + { + method: 'PUT', + data: { + projects: [ + { + key: 'PAY', + name: 'Payments', + provider: 'jira', + idempotency_key: 'jira:org-1:10001', + team_id: TEAM_ID + } + ] + } + } + ); + expect(res.send).toHaveBeenCalledWith([ + { + org_project_id: 'proj-1', + key: 'PAY', + name: 'Payments', + provider: 'jira', + idempotency_key: 'jira:org-1:10001', + team_id: TEAM_ID + } + ]); + }); + + it('rejects an empty body missing the required projects array', async () => { + const res = mockRes(); + + await projectsHandler(mockReq('PUT', { team_id: TEAM_ID }, {}), res); + + expect(handleRequest).not.toHaveBeenCalled(); + expect(res.status).toHaveBeenCalledWith(400); + }); + }); +}); diff --git a/web-server/pages/api/resources/__tests__/team_repo_project_mappings.test.ts b/web-server/pages/api/resources/__tests__/team_repo_project_mappings.test.ts new file mode 100644 index 000000000..81982fc4b --- /dev/null +++ b/web-server/pages/api/resources/__tests__/team_repo_project_mappings.test.ts @@ -0,0 +1,117 @@ +jest.mock('@/auth/session', () => ({ getAuthSession: jest.fn() })); +jest.mock('@/api-helpers/axios', () => ({ handleRequest: jest.fn() })); + +import { handleRequest } from '@/api-helpers/axios'; +import { getAuthSession } from '@/auth/session'; + +import mappingsHandler from '../team_repo_project_mappings'; + +const TEAM_ID = '22222222-2222-4222-8222-222222222222'; +const REPO_ID = '33333333-3333-4333-8333-333333333333'; +const PROJECT_ID = '44444444-4444-4444-8444-444444444444'; + +const mockRes = () => { + const res: any = {}; + res.status = jest.fn(() => res); + res.send = jest.fn(() => res); + return res; +}; + +const mockReq = ( + method: string, + query: Record, + body: Record = {} +) => ({ method, query, body, headers: {} }) as any; + +const asAuthed = () => + (getAuthSession as jest.Mock).mockResolvedValue({ + userId: 'u1', + email: 'admin@clustox.com', + name: 'Admin', + role: 'SUPERADMIN' + }); + +// CLUSTOX: explicit, informational repo<->Jira-project pairing -- a thin +// proxy to the Python backend's /teams//repo_project_mappings, +// same "GET the current set / PUT the full replacement set" shape as +// team_projects.ts. See docs/JIRA_MULTI_ACCOUNT_PLAN.md's follow-up on +// Jira<->repo relationships. +describe('/api/resources/team_repo_project_mappings', () => { + beforeEach(() => { + jest.clearAllMocks(); + asAuthed(); + }); + + describe('GET', () => { + it('fetches the current mappings for the team', async () => { + (handleRequest as jest.Mock).mockResolvedValue([ + { org_repo_id: REPO_ID, org_project_id: PROJECT_ID } + ]); + const res = mockRes(); + + await mappingsHandler(mockReq('GET', { team_id: TEAM_ID }), res); + + expect(handleRequest).toHaveBeenCalledWith( + `/teams/${TEAM_ID}/repo_project_mappings` + ); + expect(res.send).toHaveBeenCalledWith([ + { org_repo_id: REPO_ID, org_project_id: PROJECT_ID } + ]); + }); + }); + + describe('PUT', () => { + it('forwards the full replacement mapping set to the backend', async () => { + (handleRequest as jest.Mock).mockResolvedValue([ + { org_repo_id: REPO_ID, org_project_id: PROJECT_ID } + ]); + const res = mockRes(); + + await mappingsHandler( + mockReq( + 'PUT', + { team_id: TEAM_ID }, + { mappings: [{ org_repo_id: REPO_ID, org_project_id: PROJECT_ID }] } + ), + res + ); + + expect(handleRequest).toHaveBeenCalledWith( + `/teams/${TEAM_ID}/repo_project_mappings`, + { + method: 'PUT', + data: { + mappings: [{ org_repo_id: REPO_ID, org_project_id: PROJECT_ID }] + } + } + ); + expect(res.send).toHaveBeenCalledWith([ + { org_repo_id: REPO_ID, org_project_id: PROJECT_ID } + ]); + }); + + it('accepts an empty mappings array (clearing every pairing)', async () => { + (handleRequest as jest.Mock).mockResolvedValue([]); + const res = mockRes(); + + await mappingsHandler( + mockReq('PUT', { team_id: TEAM_ID }, { mappings: [] }), + res + ); + + expect(handleRequest).toHaveBeenCalledWith( + `/teams/${TEAM_ID}/repo_project_mappings`, + { method: 'PUT', data: { mappings: [] } } + ); + }); + + it('rejects a body missing the required mappings array', async () => { + const res = mockRes(); + + await mappingsHandler(mockReq('PUT', { team_id: TEAM_ID }, {}), res); + + expect(handleRequest).not.toHaveBeenCalled(); + expect(res.status).toHaveBeenCalledWith(400); + }); + }); +}); diff --git a/web-server/pages/api/resources/orgs/[org_id]/jira-connections/[connection_id].ts b/web-server/pages/api/resources/orgs/[org_id]/jira-connections/[connection_id].ts new file mode 100644 index 000000000..2ccdfbecf --- /dev/null +++ b/web-server/pages/api/resources/orgs/[org_id]/jira-connections/[connection_id].ts @@ -0,0 +1,51 @@ +// CLUSTOX: BFF routes for a single JiraConnection -- see +// docs/JIRA_MULTI_ACCOUNT_PLAN.md Task 3. DELETE removes it (blocked +// server-side while an OrgProjectConnection still references it), PATCH +// marks it the org's default. Both proxy through to the analytics server, +// which owns those invariants -- see index.ts's header for why this isn't a +// direct knex write. +import * as yup from 'yup'; + +import { internal } from '@/api-helpers/axios'; +import { Endpoint } from '@/api-helpers/global'; +import { forwardInternalError } from '@/api-helpers/internal-error'; +import { ResponseError } from '@/constants/error'; +import { ApiRequest } from '@/types/request'; + +const pathSchema = yup.object().shape({ + org_id: yup.string().uuid().required(), + connection_id: yup.string().uuid().required() +}); + +// See index.ts's assertFeatureEnabled for why this is a 404, not a 403. +const assertFeatureEnabled = (req: ApiRequest) => { + if (!req.meta?.features?.show_jira_multi_account) { + throw new ResponseError('Not found', 404); + } +}; + +const endpoint = new Endpoint(pathSchema); + +endpoint.handle.DELETE(pathSchema, async (req, res) => { + assertFeatureEnabled(req); + + const { org_id, connection_id } = req.payload; + + const result = await internal + .delete(`/orgs/${org_id}/integrations/jira-connections/${connection_id}`) + .catch(forwardInternalError); + res.send(result.data); +}); + +endpoint.handle.PATCH(pathSchema, async (req, res) => { + assertFeatureEnabled(req); + + const { org_id, connection_id } = req.payload; + + const result = await internal + .patch(`/orgs/${org_id}/integrations/jira-connections/${connection_id}`) + .catch(forwardInternalError); + res.send(result.data); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/resources/orgs/[org_id]/jira-connections/__tests__/[connection_id].test.ts b/web-server/pages/api/resources/orgs/[org_id]/jira-connections/__tests__/[connection_id].test.ts new file mode 100644 index 000000000..9e56e5c38 --- /dev/null +++ b/web-server/pages/api/resources/orgs/[org_id]/jira-connections/__tests__/[connection_id].test.ts @@ -0,0 +1,127 @@ +jest.mock('@/auth/session', () => ({ getAuthSession: jest.fn() })); +jest.mock('@/api-helpers/axios', () => ({ + internal: { delete: jest.fn(), patch: jest.fn() } +})); + +import { internal } from '@/api-helpers/axios'; +import { getAuthSession } from '@/auth/session'; + +import handler from '../[connection_id]'; + +const ORG_ID = '11111111-1111-4111-8111-111111111111'; +const CONNECTION_ID = '33333333-3333-4333-8333-333333333333'; + +const mockRes = () => { + const res: any = {}; + res.status = jest.fn(() => res); + res.send = jest.fn(() => res); + return res; +}; + +// CLUSTOX: feature flags reach the handler via req.query.feature_flags (see +// getFlagsFromRequest), not a `meta` field set directly on the request -- +// transformNextRequest recomputes `meta` from that query param and would +// overwrite anything set here. +const mockReq = ( + method: string, + features: Record = { show_jira_multi_account: true } +) => + ({ + method, + query: { + org_id: ORG_ID, + connection_id: CONNECTION_ID, + feature_flags: JSON.stringify(features) + }, + body: {}, + headers: {} + }) as any; + +const asAuthed = () => + (getAuthSession as jest.Mock).mockResolvedValue({ + userId: 'u1', + email: 'admin@clustox.com', + name: 'Admin', + role: 'SUPERADMIN' + }); + +describe('/api/resources/orgs/[org_id]/jira-connections/[connection_id]', () => { + beforeEach(() => { + jest.clearAllMocks(); + asAuthed(); + }); + + describe('DELETE', () => { + it('proxies to the analytics server', async () => { + (internal.delete as jest.Mock).mockResolvedValue({ data: { ok: true } }); + const res = mockRes(); + + await handler(mockReq('DELETE'), res); + + expect(internal.delete).toHaveBeenCalledWith( + `/orgs/${ORG_ID}/integrations/jira-connections/${CONNECTION_ID}` + ); + expect(res.send).toHaveBeenCalledWith({ ok: true }); + }); + + it('forwards a 409 when the connection is still referenced', async () => { + (internal.delete as jest.Mock).mockRejectedValue({ + response: { + status: 409, + data: { error: 'JiraConnection still has projects synced from it' } + } + }); + const res = mockRes(); + + await handler(mockReq('DELETE'), res); + + expect(res.status).toHaveBeenCalledWith(409); + expect(res.send).toHaveBeenCalledWith({ + message: 'JiraConnection still has projects synced from it' + }); + }); + + it('forwards a 404 for an unknown connection', async () => { + (internal.delete as jest.Mock).mockRejectedValue({ + response: { status: 404, data: { error: 'No JiraConnection' } } + }); + const res = mockRes(); + + await handler(mockReq('DELETE'), res); + + expect(res.status).toHaveBeenCalledWith(404); + }); + + it('is a 404 while the feature flag is off', async () => { + const res = mockRes(); + + await handler(mockReq('DELETE', {}), res); + + expect(res.status).toHaveBeenCalledWith(404); + expect(internal.delete).not.toHaveBeenCalled(); + }); + }); + + describe('PATCH', () => { + it('proxies the set-default request to the analytics server', async () => { + (internal.patch as jest.Mock).mockResolvedValue({ data: { ok: true } }); + const res = mockRes(); + + await handler(mockReq('PATCH'), res); + + expect(internal.patch).toHaveBeenCalledWith( + `/orgs/${ORG_ID}/integrations/jira-connections/${CONNECTION_ID}` + ); + expect(res.send).toHaveBeenCalledWith({ ok: true }); + }); + + it('is a 404 while the feature flag is off', async () => { + const res = mockRes(); + + await handler(mockReq('PATCH', {}), res); + + expect(res.status).toHaveBeenCalledWith(404); + expect(internal.patch).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/web-server/pages/api/resources/orgs/[org_id]/jira-connections/__tests__/index.test.ts b/web-server/pages/api/resources/orgs/[org_id]/jira-connections/__tests__/index.test.ts new file mode 100644 index 000000000..8339ba6c3 --- /dev/null +++ b/web-server/pages/api/resources/orgs/[org_id]/jira-connections/__tests__/index.test.ts @@ -0,0 +1,151 @@ +jest.mock('@/auth/session', () => ({ getAuthSession: jest.fn() })); +jest.mock('@/api-helpers/axios', () => ({ + internal: { get: jest.fn(), post: jest.fn() } +})); + +import { internal } from '@/api-helpers/axios'; +import { getAuthSession } from '@/auth/session'; + +import handler from '../index'; + +const ORG_ID = '11111111-1111-4111-8111-111111111111'; + +const mockRes = () => { + const res: any = {}; + res.status = jest.fn(() => res); + res.send = jest.fn(() => res); + return res; +}; + +// CLUSTOX: feature flags reach the handler via req.query.feature_flags (see +// getFlagsFromRequest), not a `meta` field set directly on the request -- +// transformNextRequest recomputes `meta` from that query param and would +// overwrite anything set here. +const mockReq = ( + method: string, + body: Record = {}, + features: Record | null = { show_jira_multi_account: true } +) => + ({ + method, + query: { + org_id: ORG_ID, + ...(features ? { feature_flags: JSON.stringify(features) } : {}) + }, + body, + headers: {} + }) as any; + +const asAuthed = () => + (getAuthSession as jest.Mock).mockResolvedValue({ + userId: 'u1', + email: 'admin@clustox.com', + name: 'Admin', + role: 'SUPERADMIN' + }); + +describe('/api/resources/orgs/[org_id]/jira-connections', () => { + beforeEach(() => { + jest.clearAllMocks(); + asAuthed(); + }); + + describe('GET', () => { + it('lists the connections the analytics server returns', async () => { + (internal.get as jest.Mock).mockResolvedValue({ + data: [{ id: 'c1', site_url: 'acme.atlassian.net', is_default: true }] + }); + const res = mockRes(); + + await handler(mockReq('GET'), res); + + expect(internal.get).toHaveBeenCalledWith( + `/orgs/${ORG_ID}/integrations/jira-connections` + ); + expect(res.send).toHaveBeenCalledWith([ + { id: 'c1', site_url: 'acme.atlassian.net', is_default: true } + ]); + }); + + it('is a 404 while the feature flag is off', async () => { + const res = mockRes(); + + await handler(mockReq('GET', {}, {}), res); + + expect(res.status).toHaveBeenCalledWith(404); + expect(internal.get).not.toHaveBeenCalled(); + }); + }); + + describe('POST', () => { + const body = { + site_url: 'acme.atlassian.net', + email: 'a@acme.com', + access_token: 'raw-token' + }; + + it('creates a connection and stamps the signed-in user as generated_by', async () => { + (internal.post as jest.Mock).mockResolvedValue({ + data: { id: 'c1', site_url: 'acme.atlassian.net', is_default: false } + }); + const res = mockRes(); + + await handler(mockReq('POST', body), res); + + expect(internal.post).toHaveBeenCalledWith( + `/orgs/${ORG_ID}/integrations/jira-connections`, + { + site_url: 'acme.atlassian.net', + email: 'a@acme.com', + access_token: 'raw-token', + provider_meta: undefined, + generated_by: 'u1' + } + ); + expect(res.status).toHaveBeenCalledWith(201); + expect(res.send).toHaveBeenCalledWith({ + id: 'c1', + site_url: 'acme.atlassian.net', + is_default: false + }); + }); + + it('forwards the analytics server 409 for a duplicate account, with its message', async () => { + (internal.post as jest.Mock).mockRejectedValue({ + response: { + status: 409, + data: { error: 'a@acme.com is already connected to acme.atlassian.net' } + } + }); + const res = mockRes(); + + await handler(mockReq('POST', body), res); + + expect(res.status).toHaveBeenCalledWith(409); + expect(res.send).toHaveBeenCalledWith({ + message: 'a@acme.com is already connected to acme.atlassian.net' + }); + }); + + it('rejects a body missing the required access_token', async () => { + const res = mockRes(); + + await handler( + mockReq('POST', { site_url: 'acme.atlassian.net', email: 'a@acme.com' }), + res + ); + + expect(internal.post).not.toHaveBeenCalled(); + expect(res.status).toHaveBeenCalledWith(400); + }); + + it('is a 404 while the feature flag is off', async () => { + const res = mockRes(); + + await handler(mockReq('POST', body, {}), res); + + expect(res.status).toHaveBeenCalledWith(404); + expect(internal.post).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/web-server/pages/api/resources/orgs/[org_id]/jira-connections/index.ts b/web-server/pages/api/resources/orgs/[org_id]/jira-connections/index.ts new file mode 100644 index 000000000..98e454dfc --- /dev/null +++ b/web-server/pages/api/resources/orgs/[org_id]/jira-connections/index.ts @@ -0,0 +1,68 @@ +// CLUSTOX: BFF routes for Jira multi-account support -- see +// docs/JIRA_MULTI_ACCOUNT_PLAN.md Task 3. GET lists an org's JiraConnection +// rows, POST creates one; both proxy straight through to the analytics +// server (mirrors pages/api/clustox/jenkins/mappings.ts), which owns +// encrypt-on-write and the (org_id, site_url, email) uniqueness check. +// Deliberately additive: orgs/[org_id]/integration.ts's legacy single-Jira +// row is untouched by this file. +import * as yup from 'yup'; + +import { internal } from '@/api-helpers/axios'; +import { Endpoint } from '@/api-helpers/global'; +import { forwardInternalError } from '@/api-helpers/internal-error'; +import { ResponseError } from '@/constants/error'; +import { ApiRequest } from '@/types/request'; + +const pathSchema = yup.object().shape({ + org_id: yup.string().uuid().required() +}); + +const postSchema = yup.object().shape({ + org_id: yup.string().uuid().required(), + site_url: yup.string().required(), + email: yup.string().email().required(), + access_token: yup.string().required(), + provider_meta: yup.object().optional() +}); + +// CLUSTOX: still landing -- see show_jira_multi_account's own comment in +// constants/feature.ts. A 404 rather than a 403: while the feature is off, +// these routes should read as not existing at all, not as a permission an +// admin might think to ask for. +const assertFeatureEnabled = (req: ApiRequest) => { + if (!req.meta?.features?.show_jira_multi_account) { + throw new ResponseError('Not found', 404); + } +}; + +const endpoint = new Endpoint(pathSchema); + +endpoint.handle.GET(pathSchema, async (req, res) => { + assertFeatureEnabled(req); + + const result = await internal + .get(`/orgs/${req.payload.org_id}/integrations/jira-connections`) + .catch(forwardInternalError); + res.send(result.data); +}); + +endpoint.handle.POST(postSchema, async (req, res) => { + assertFeatureEnabled(req); + + const { org_id, site_url, email, access_token, provider_meta } = req.payload; + + const result = await internal + .post(`/orgs/${org_id}/integrations/jira-connections`, { + site_url, + email, + access_token, + provider_meta, + // The analytics server records who linked a connection; only this BFF + // layer knows who is actually signed in. + generated_by: req.session?.userId ?? null + }) + .catch(forwardInternalError); + res.status(201).send(result.data); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/resources/orgs/[org_id]/teams/index.ts b/web-server/pages/api/resources/orgs/[org_id]/teams/index.ts index f5a9c3f64..f5a70a084 100644 --- a/web-server/pages/api/resources/orgs/[org_id]/teams/index.ts +++ b/web-server/pages/api/resources/orgs/[org_id]/teams/index.ts @@ -8,6 +8,8 @@ import { import { getTeamRepos } from '@/api/resources/team_repos'; import { Endpoint } from '@/api-helpers/global'; import { getTeamMembersFilterSettingForOrg } from '@/api-helpers/team'; +// CLUSTOX: per-team scoping for the team picker. +import { getAccessibleTeamIds } from '@/auth/guard'; import { getTeamV2Mock } from '@/mocks/teams'; import { FetchTeamsResponse } from '@/types/resources'; import { db } from '@/utils/db'; @@ -28,7 +30,18 @@ endpoint.handle.GET(getSchema, async (req, res) => { } const { include_teams, org_id, user_id } = req.payload; - res.send(await getOrgTeams(org_id, user_id, include_teams)); + const result = await getOrgTeams(org_id, user_id, include_teams); + + // CLUSTOX: an admin must only see their assigned teams in the picker. + // Filtered here rather than in getOrgTeams because that helper is also + // imported by other routes which do their own scoping. + const accessible = await getAccessibleTeamIds((req as any).session); + const allowed = new Set(accessible); + + res.send({ + ...result, + teams: result.teams.filter((t: { id: ID }) => allowed.has(t.id)) + }); }); export const getOrgTeams = async ( diff --git a/web-server/pages/api/resources/orgs/[org_id]/teams/v2.ts b/web-server/pages/api/resources/orgs/[org_id]/teams/v2.ts index bf0db917e..c2b6d061f 100644 --- a/web-server/pages/api/resources/orgs/[org_id]/teams/v2.ts +++ b/web-server/pages/api/resources/orgs/[org_id]/teams/v2.ts @@ -13,6 +13,9 @@ import { } from '@/api/resources/orgs/[org_id]/onboarding'; import { handleRequest } from '@/api-helpers/axios'; import { Endpoint } from '@/api-helpers/global'; +// CLUSTOX: workspace scoping for the team picker, plus explicit team +// checks on routes that name the team `id` rather than `team_id`. +import { assertTeamAccess, getAccessibleTeamIds } from '@/auth/guard'; import { Row, Table } from '@/constants/db'; import { CIProvider, @@ -29,6 +32,17 @@ import { import { db, dbRaw, getFirstRow } from '@/utils/db'; import groupBy from '@/utils/objectArray'; +// CLUSTOX: every code provider a team's repos can actually come from -- +// see this constant's two call sites below for the bug this fixes +// (Bitbucket repos silently excluded from both the team list's repo +// count and the edit form's repo picker). Mirrors +// ClustoxJenkinsMapping.tsx's own codeProviders list. +const GIT_CODE_PROVIDERS = [ + Integration.GITHUB, + Integration.GITLAB, + Integration.BITBUCKET +]; + const repoSchema = yup.object().shape({ idempotency_key: yup.string().required(), deployment_type: yup.string().required(), @@ -89,18 +103,32 @@ endpoint.handle.GET(getSchema, async (req, res) => { const teams = await getQuery; const teamReposMap = await getTeamReposMap( org_id, - providers?.length - ? (providers as Integration[]) - : [Integration.GITHUB, Integration.GITLAB] + // CLUSTOX: a real, live user-reported bug -- this default previously + // omitted Bitbucket, so a team whose only repo was a Bitbucket one + // showed "0 Repositories" on the team list and (this map is also + // what the edit form's initial repo selection is built from) an + // empty repo picker when editing it, even though the repo was + // genuinely saved and DORA metrics for it were computing correctly + // (that path already scoped its own query independently). Every + // code provider a team can actually have repos from, not just the + // two this happened to be written against first. + providers?.length ? (providers as Integration[]) : GIT_CODE_PROVIDERS ); + // CLUSTOX: an admin must only see their assigned teams in the picker. + const accessible = await getAccessibleTeamIds((req as any).session); + const allowed = new Set(accessible); + res.send({ - teams: teams, + teams: teams.filter((t: { id: ID }) => allowed.has(t.id)), teamReposMap }); }); endpoint.handle.POST(postSchema, async (req, res) => { + // CLUSTOX: an admin owns their workspace, so creating teams inside it is + // theirs to do. Endpoint.serve() has already confirmed the caller may act on + // this org_id, which is what confines them to their own workspace. if (req.meta?.features?.use_mock_data) { return res.send(getTeamV2Mock); } @@ -155,6 +183,11 @@ endpoint.handle.PATCH(patchSchema, async (req, res) => { } const { org_id, id, name, org_repos } = req.payload; + + // CLUSTOX: this route names the team `id`, not `team_id`, so the central + // scoping in Endpoint.serve() does not see it. Without this an admin could + // pass their own org_id in the path and a team id from another workspace. + await assertTeamAccess((req as any).session, id); const orgReposList: ReqRepoWithProvider[] = []; forEachObjIndexed((repos, org) => { repos.forEach((repo) => { @@ -195,6 +228,11 @@ endpoint.handle.DELETE(deleteSchema, async (req, res) => { return res.send(getTeamV2Mock); } + // CLUSTOX: as with PATCH above, the team arrives as `id` so the central + // scoping does not cover it. Deleting another workspace's team would + // otherwise be reachable by guessing a uuid. + await assertTeamAccess((req as any).session, req.payload.id); + const data = await dbRaw.transaction(async (trx) => { const deletedTeamRow = await trx('Team') .update({ @@ -301,7 +339,12 @@ const updateReposWorkflows = async ( .whereIn('name', reposForWorkflows) .where('org_id', org_id) .andWhere('is_active', true) - .and.whereIn('provider', [Integration.GITHUB, Integration.GITLAB]); + // CLUSTOX: was missing Bitbucket here -- silently dead code, since + // the CIRCLE_CI branch just below explicitly handles + // Integration.BITBUCKET but this filter meant dbReposForWorkflows + // (and therefore groupedRepos) could never actually contain one. + // Same bug as GIT_CODE_PROVIDERS' other call site above. + .and.whereIn('provider', GIT_CODE_PROVIDERS); const groupedRepos = groupBy(dbReposForWorkflows, 'name'); diff --git a/web-server/pages/api/resources/share.ts b/web-server/pages/api/resources/share.ts index 12f53e514..c97ea7f8f 100644 --- a/web-server/pages/api/resources/share.ts +++ b/web-server/pages/api/resources/share.ts @@ -5,6 +5,13 @@ import { Endpoint, nullSchema } from '@/api-helpers/global'; import { Columns, Table } from '@/constants/db'; import { db } from '@/utils/db'; +// CLUSTOX: hardcoded for the same reason as the invite link -- this was built +// from NEXTAUTH_URL, which the server's .env still sets to localhost, so a +// share link generated server-side pointed at a host the recipient cannot +// reach. Only used when the client does not send its own base_url. +// See pages/api/clustox/invites/index.ts. +const APP_URL = 'https://middleware.theclustox.com'; + const getSchema = yup.object().shape({ share_link_id: yup.string().required() }); @@ -14,7 +21,10 @@ const postSchema = yup.object().shape({ base_url: yup.string().optional() }); -const endpoint = new Endpoint(nullSchema, { unauthenticated: true }); +// CLUSTOX: was unauthenticated, which let anyone POST unbounded rows into +// URLShortenerData and read stored link state. Everyone who should open a +// share link has an account, so requiring one costs nothing. +const endpoint = new Endpoint(nullSchema); endpoint.handle.GET(getSchema, async (req, res) => { const { share_link_id } = req.payload; @@ -52,7 +62,7 @@ endpoint.handle.POST(postSchema, async (req, res) => { response.url = shareLink; res.send(response); } - shareLink = `${process.env.NEXTAUTH_URL}?share=${id}`; + shareLink = `${APP_URL}?share=${id}`; response.url = shareLink; res.send(response); } catch (e) { diff --git a/web-server/pages/api/resources/team_projects.ts b/web-server/pages/api/resources/team_projects.ts new file mode 100644 index 000000000..f654272e8 --- /dev/null +++ b/web-server/pages/api/resources/team_projects.ts @@ -0,0 +1,69 @@ +import * as yup from 'yup'; + +import { handleRequest } from '@/api-helpers/axios'; +import { Endpoint, nullSchema } from '@/api-helpers/global'; +import { Row } from '@/constants/db'; + +// CLUSTOX: Jira integration, Phase 2 (project selection) -- a team's +// selected Jira project(s). Deliberately its own standalone resource +// (mirrors team_repos.ts's shape, not wired into teams/v2.ts's combined +// create/update-team request) rather than folded into the existing +// team-repos save flow: project selection has none of repos' deployment +// type/workflow concerns, and keeping it separate means this can't +// regress the already-working repo-selection flow. See +// docs/JIRA_INTEGRATION_PROPOSAL.md. +export type TeamProject = Row<'TeamProjects'> & Row<'OrgProject'>; + +const getSchema = yup.object().shape({ + team_id: yup.string().uuid().required() +}); + +const putSchema = yup.object().shape({ + team_id: yup.string().uuid().required(), + projects: yup + .array() + .of( + yup.object().shape({ + key: yup.string().required(), + name: yup.string().required(), + provider: yup.string().required(), + idempotency_key: yup.string().required(), + // CLUSTOX: which JiraConnection this project was picked under -- + // see docs/JIRA_MULTI_ACCOUNT_PLAN.md Task 6 part 2. Optional: a + // project from the legacy single-account flow (or a non-Jira + // provider) carries none, and yup would otherwise silently strip + // this field before it reaches the forwarding call below. + connection_id: yup.string().uuid().optional().nullable() + }) + ) + .required() +}); + +const endpoint = new Endpoint(nullSchema); + +endpoint.handle.GET(getSchema, async (req, res) => { + res.send(await getTeamProjects(req.payload.team_id)); +}); + +export const getTeamProjects = (team_id: ID) => + handleRequest(`/teams/${team_id}/projects`).then((projects) => + projects.map((p) => ({ ...p, team_id })) + ); + +endpoint.handle.PUT(putSchema, async (req, res) => { + const { team_id, projects } = req.payload; + + const updated = await handleRequest( + `/teams/${team_id}/projects`, + { + method: 'PUT', + data: { + projects: projects.map((project) => ({ ...project, team_id })) + } + } + ); + + res.send(updated.map((p) => ({ ...p, team_id }))); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/api/resources/team_repo_project_mappings.ts b/web-server/pages/api/resources/team_repo_project_mappings.ts new file mode 100644 index 000000000..029fe91f6 --- /dev/null +++ b/web-server/pages/api/resources/team_repo_project_mappings.ts @@ -0,0 +1,59 @@ +import * as yup from 'yup'; + +import { handleRequest } from '@/api-helpers/axios'; +import { Endpoint, nullSchema } from '@/api-helpers/global'; + +// CLUSTOX: explicit, informational repo<->Jira-project pairing within a +// team -- see docs/JIRA_MULTI_ACCOUNT_PLAN.md's follow-up on Jira<->repo +// relationships. Mirrors team_projects.ts's shape exactly ("GET the +// current set / PUT the full replacement set"); deliberately its own +// standalone resource for the same reason that one is separate from the +// repo-selection save flow. Never read by PR<->ticket matching -- see +// TeamRepoProjectMapping's own docstring on the backend. +export type TeamRepoProjectMapping = { + org_repo_id: string; + org_project_id: string; +}; + +const getSchema = yup.object().shape({ + team_id: yup.string().uuid().required() +}); + +const putSchema = yup.object().shape({ + team_id: yup.string().uuid().required(), + mappings: yup + .array() + .of( + yup.object().shape({ + org_repo_id: yup.string().uuid().required(), + org_project_id: yup.string().uuid().required() + }) + ) + .required() +}); + +const endpoint = new Endpoint(nullSchema); + +endpoint.handle.GET(getSchema, async (req, res) => { + res.send( + await handleRequest( + `/teams/${req.payload.team_id}/repo_project_mappings` + ) + ); +}); + +endpoint.handle.PUT(putSchema, async (req, res) => { + const { team_id, mappings } = req.payload; + + const updated = await handleRequest( + `/teams/${team_id}/repo_project_mappings`, + { + method: 'PUT', + data: { mappings } + } + ); + + res.send(updated); +}); + +export default endpoint.serve(); diff --git a/web-server/pages/dora-metrics/index.tsx b/web-server/pages/dora-metrics/index.tsx index 8dcb801b2..88684a3b4 100644 --- a/web-server/pages/dora-metrics/index.tsx +++ b/web-server/pages/dora-metrics/index.tsx @@ -1,6 +1,8 @@ import ExtendedSidebarLayout from 'src/layouts/ExtendedSidebarLayout'; import { Authenticated } from '@/components/Authenticated'; +// CLUSTOX: contributor filter. +import { ContributorFilter } from '@/components/ContributorFilter'; import { FlexBox } from '@/components/FlexBox'; import Loader from '@/components/Loader'; import { FetchState } from '@/constants/ui-states'; @@ -17,6 +19,12 @@ function Page() { ); const { integrationList } = useAuth(); + // CLUSTOX: contributor filter -- rendered alongside the team/date selectors + // through PageWrapper's existing `additionalFilters` seam, so no upstream + // layout component is edited. + const additionalFilters = []; + // END CLUSTOX + return ( {integrationList.length > 0 ? : } diff --git a/web-server/pages/integrations.tsx b/web-server/pages/integrations.tsx index 2ec60be91..9382d3df0 100644 --- a/web-server/pages/integrations.tsx +++ b/web-server/pages/integrations.tsx @@ -1,19 +1,27 @@ -import { AddRounded, InfoRounded } from '@mui/icons-material'; +import { AddRounded, BuildRounded, InfoRounded } from '@mui/icons-material'; import { LoadingButton } from '@mui/lab'; -import { Button, Card, Divider } from '@mui/material'; +import { Button, Card, Divider, useTheme } from '@mui/material'; import { differenceInMilliseconds } from 'date-fns'; import { millisecondsInMinute } from 'date-fns/constants'; import { useEffect, useMemo } from 'react'; import { Authenticated } from 'src/components/Authenticated'; import { handleApi } from '@/api-helpers/axios-api-instance'; +// CLUSTOX: the two halves of the Jenkins integration card below -- the +// credentials form and the job-to-repo mapping table. +import { ClustoxJenkinsMapping } from '@/components/ClustoxJenkinsMapping'; +import { ClustoxJenkinsSetup } from '@/components/ClustoxJenkinsSetup'; +// END CLUSTOX import { FlexBox } from '@/components/FlexBox'; import { Line } from '@/components/Text'; import { ROUTES } from '@/constants/routes'; import { FetchState } from '@/constants/ui-states'; +import { BitbucketIntegrationCard } from '@/content/Dashboards/BitbucketIntegrationCard'; import { GithubIntegrationCard } from '@/content/Dashboards/GithubIntegrationCard'; import { GitlabIntegrationCard } from '@/content/Dashboards/GitlabIntegrationCard'; +import { JiraIntegrationCard } from '@/content/Dashboards/JiraIntegrationCard'; import { PageWrapper } from '@/content/PullRequests/PageWrapper'; +import { useModal } from '@/contexts/ModalContext'; import { useAuth } from '@/hooks/useAuth'; import { useBoolState, useEasyState } from '@/hooks/useEasyState'; import ExtendedSidebarLayout from '@/layouts/ExtendedSidebarLayout'; @@ -58,8 +66,13 @@ Integrations.getLayout = (page: PageLayout) => ( export default Integrations; const Content = () => { - const { orgId, integrations, integrationList } = useAuth(); - const hasCodeProviderLinked = integrationList.length > 0; + const { + orgId, + integrations, + integrationList, + codeProviderIntegrationList, + hasCodeProviderLinked + } = useAuth(); const teamCount = useSelector((s) => s.team.teams?.length); const dispatch = useDispatch(); const loadedTeams = useBoolState(false); @@ -69,7 +82,7 @@ const Content = () => { hasCodeProviderLinked && !teamCount && loadedTeams.value; const lastSyncMap = useMemo(() => { - return integrationList + return codeProviderIntegrationList .map((item) => { const linkedAt = integrations[item as 'github' | 'gitlab' | 'bitbucket'].linked_at; @@ -83,7 +96,7 @@ const Content = () => { return diff; }) .filter(Boolean); - }, [integrationList, integrations]); + }, [codeProviderIntegrationList, integrations]); const showForceSyncBtn = useMemo(() => { if (!hasCodeProviderLinked) return false; @@ -160,9 +173,14 @@ const Content = () => { - + + + + {/* CLUSTOX: Jenkins as a third deployment-detection provider. */} + + {/* END CLUSTOX */} {showCreationCTA && ( @@ -236,3 +254,71 @@ const Content = () => { ); }; + +/** + * CLUSTOX: Jenkins is a deployment provider set up through a credentials + * form rather than OAuth, so unlike GithubIntegrationCard/GitlabIntegrationCard + * it opens a modal instead of a redirect, and switches between the setup + * form and the job-mapping table based on link state. + */ +const JenkinsIntegrationCard = () => { + const theme = useTheme(); + const { integrations } = useAuth(); + const isJenkinsLinked = Boolean(integrations.jenkins?.integrated); + const { addModal, closeAllModals } = useModal(); + + const openSetup = () => + addModal({ + title: 'Connect Jenkins', + body: , + showCloseIcon: true + }); + + const openMapping = () => + addModal({ + title: 'Map Jenkins jobs to repositories', + body: , + showCloseIcon: true + }); + + return ( + + {isJenkinsLinked && ( + + + Linked + + + )} + + + + Jenkins + + + + Map deployments from Jenkins jobs + + + + + + ); +}; + diff --git a/web-server/pages/login.tsx b/web-server/pages/login.tsx new file mode 100644 index 000000000..53295e5af --- /dev/null +++ b/web-server/pages/login.tsx @@ -0,0 +1,149 @@ +import { Box, Button, Card, TextField, Typography } from '@mui/material'; +import Head from 'next/head'; +import { signIn } from 'next-auth/react'; +import { FormEvent, ReactElement, useState } from 'react'; + +import { useAutofillSync } from '@/hooks/useAutofillSync'; + +export default function Login() { + const [email, setEmail] = useState(''); + const [password, setPassword] = useState(''); + const [error, setError] = useState(''); + const [busy, setBusy] = useState(false); + const emailRef = useAutofillSync(email, setEmail); + const passwordRef = useAutofillSync(password, setPassword); + + const onSubmit = async (e: FormEvent) => { + e.preventDefault(); + setBusy(true); + setError(''); + + let res: Awaited>; + try { + res = await signIn('credentials', { email, password, redirect: false }); + } catch { + setBusy(false); + // CLUSTOX FIX: signIn() throwing (a network/server failure) used to be + // indistinguishable from a bad password -- both fell through to the + // same generic message. Kept the wording deliberately vague about + // *why*, since we still don't want to reveal which case happened, but + // this path no longer requires the request to have actually reached + // the server and returned an { error }. + setError('Something went wrong. Please try again.'); + return; + } + + setBusy(false); + + if (res?.error) { + // Deliberately does not distinguish unknown email from wrong password. + setError('Invalid email or password'); + return; + } + // CLUSTOX FIX: a router.replace() here is a client-side navigation -- + // AuthProvider fetches /api/auth/session exactly once, on mount, and + // nothing re-triggers that fetch afterwards. Since AuthProvider is + // mounted at the app root (above /login itself), it had already fetched + // and cached the pre-login (unauthenticated, no org) session before this + // sign-in happened, and a client-side route change doesn't remount it -- + // every org-scoped read (integrations included) kept serving that stale + // snapshot until a manual hard refresh forced a remount. A full + // navigation here remounts AuthProvider so it fetches session fresh, + // this time with the auth cookie actually present. + window.location.assign('/'); + }; + + return ( + <> + + Sign in | MiddlewareHQ + + + + + `0 0 12px ${t.palette.primary.main}` + }} + /> + + Middleware + + + + + Sign in to your workspace + + + Use the account your Super Admin set up for you. + + +
+ setEmail(e.target.value)} + required + inputRef={emailRef} + // Deterministic fix for the autofill-overlap: keep the label + // permanently shrunk instead of trying to detect the exact + // moment the browser autofills, which is inherently racy + // against React's own render timing. + InputLabelProps={{ shrink: true }} + /> + setPassword(e.target.value)} + required + inputRef={passwordRef} + InputLabelProps={{ shrink: true }} + /> + {error && ( + + {error} + + )} + + + + + SELF-HOSTED · YOUR DATA STAYS ON YOUR INFRASTRUCTURE + +
+
+ + ); +} + +Login.getLayout = (page: ReactElement) => page; diff --git a/web-server/pages/settings.tsx b/web-server/pages/settings.tsx index 71f649bf8..30d040f94 100644 --- a/web-server/pages/settings.tsx +++ b/web-server/pages/settings.tsx @@ -13,7 +13,11 @@ import { PageLayout } from '@/types/resources'; function Settings() { useRedirectWithSession(); - const { integrationList } = useAuth(); + // CLUSTOX: hasCodeProviderLinked, not integrationList.length -- Content + // renders SyncDaysSetting, a code-repo sync-cadence control that has + // nothing to configure for a Jira-only org. See + // docs/JIRA_INTEGRATION_PROPOSAL.md. + const { hasCodeProviderLinked } = useAuth(); const isLoading = useSelector( (s) => s.org.requests?.defaultSyncDays === FetchState.REQUEST ); @@ -31,7 +35,7 @@ function Settings() { showEvenIfNoTeamSelected={true} isLoading={isLoading} > - {integrationList.length > 0 && } + {hasCodeProviderLinked && } ); diff --git a/web-server/pages/teams/index.tsx b/web-server/pages/teams/index.tsx index 4fb1d020d..83c85d7c0 100644 --- a/web-server/pages/teams/index.tsx +++ b/web-server/pages/teams/index.tsx @@ -16,7 +16,10 @@ import { depFn } from '@/utils/fn'; function Page() { useRedirectWithSession(); const dispatch = useDispatch(); - const { orgId, integrationList } = useAuth(); + // CLUSTOX: hasCodeProviderLinked, not integrationList.length -- a + // Jira-only org has no repos to build a team around. See + // docs/JIRA_INTEGRATION_PROPOSAL.md. + const { orgId, hasCodeProviderLinked } = useAuth(); const teamsList = useSelector((state) => state.team.teams); const loading = useBoolState(!Boolean(teamsList.length)); @@ -46,7 +49,7 @@ function Page() { showEvenIfNoTeamSelected hideAllSelectors > - {integrationList.length && !loading.value ? ( + {hasCodeProviderLinked && !loading.value ? ( {teamsList.length ? : } diff --git a/web-server/pages/users.tsx b/web-server/pages/users.tsx new file mode 100644 index 000000000..59fd48f6f --- /dev/null +++ b/web-server/pages/users.tsx @@ -0,0 +1,750 @@ +import { + AdminPanelSettingsTwoTone, + LockTwoTone, + MoreVertTwoTone, + PersonAddAlt1TwoTone, + WorkspacesTwoTone +} from '@mui/icons-material'; +import { + Alert, + Button, + Chip, + CircularProgress, + Dialog, + DialogActions, + DialogContent, + DialogTitle, + Divider, + IconButton, + ListItemIcon, + ListItemText, + Menu, + MenuItem, + Table, + TableBody, + TableCell, + TableContainer, + TableHead, + TableRow, + TextField, + Tooltip, + Typography, + useTheme +} from '@mui/material'; +import Head from 'next/head'; +import { useSnackbar } from 'notistack'; +import { FormEvent, useCallback, useEffect, useMemo, useState } from 'react'; +import ExtendedSidebarLayout from 'src/layouts/ExtendedSidebarLayout'; + +import { FlexBox } from '@/components/FlexBox'; +import { RoleChip } from '@/components/RoleChip'; +import { Line } from '@/components/Text'; +import { UserAvatar } from '@/components/UserAvatar'; +import { PageWrapper } from '@/content/PullRequests/PageWrapper'; +import { useAutofillSync } from '@/hooks/useAutofillSync'; +import { PageLayout } from '@/types/resources'; + +type ClustoxRole = 'SUPERADMIN' | 'ADMIN'; + +type UserRow = { + userId: string; + email: string; + name: string; + role: ClustoxRole; + orgId: string | null; + orgName: string | null; +}; + +type InviteRow = { + id: string; + email: string; + name: string; + role: ClustoxRole; + orgName: string | null; + expired: boolean; + emailed: boolean; +}; + +// CLUSTOX: users and pending invites rendered as one roster instead of a +// table plus a separate strip of chips above it -- an invite is just a +// person who hasn't finished signing up yet, not a different kind of thing. +type RosterRow = + | { kind: 'user'; key: string; name: string; email: string; role: ClustoxRole; orgName: string | null; user: UserRow } + | { kind: 'invite'; key: string; name: string; email: string; role: ClustoxRole; orgName: string | null; invite: InviteRow }; + +const NEW_WORKSPACE = '__new__'; + +const emptyForm = { + name: '', + email: '', + role: 'ADMIN' as ClustoxRole, + // NEW_WORKSPACE provisions a fresh one; otherwise adopt an existing + // workspace that has no owner. + org_id: NEW_WORKSPACE +}; + +/** Identifiers read as data, not prose. */ +const Mono = ({ children }: { children: React.ReactNode }) => ( + + {children} + +); + +const StatusChip = ({ status }: { status: 'active' | 'invited' | 'expired' }) => ( + +); + +const ROLE_OPTIONS: { + value: ClustoxRole; + label: string; + description: string; + icon: typeof WorkspacesTwoTone; +}[] = [ + { + value: 'ADMIN', + label: 'Admin', + description: + 'Gets their own workspace, connects their own GitHub or GitLab, and sees only their own projects.', + icon: WorkspacesTwoTone + }, + { + value: 'SUPERADMIN', + label: 'Superadmin', + description: + 'Sees every workspace, manages users and roles, and owns no workspace of their own.', + icon: AdminPanelSettingsTwoTone + } +]; + +/** Role picker as description cards, not a plain toggle-button pair. */ +const RolePicker = ({ + value, + onChange +}: { + value: ClustoxRole; + onChange: (role: ClustoxRole) => void; +}) => { + const theme = useTheme(); + + return ( + + {ROLE_OPTIONS.map((opt) => { + const Icon = opt.icon; + const selected = value === opt.value; + return ( + onChange(opt.value)} + sx={{ + cursor: 'pointer', + border: '1.5px solid', + borderColor: selected + ? theme.colors.primary.main + : theme.colors.alpha.trueWhite[30], + background: selected + ? theme.colors.alpha.black[10] + : 'transparent', + transition: 'border-color 0.15s' + }} + > + + + + {opt.label} + + + + {opt.description} + + + ); + })} + + ); +}; + +function UsersPage() { + const theme = useTheme(); + const { enqueueSnackbar } = useSnackbar(); + + const [users, setUsers] = useState([]); + const [workspaces, setWorkspaces] = useState< + { id: string; name: string; owned: boolean }[] + >([]); + const [loading, setLoading] = useState(true); + const [forbidden, setForbidden] = useState(false); + + const [form, setForm] = useState(emptyForm); + const [busy, setBusy] = useState(false); + const [formError, setFormError] = useState(''); + + const [inviteOpen, setInviteOpen] = useState(false); + const [inviteLink, setInviteLink] = useState(''); + const [inviteEmailed, setInviteEmailed] = useState(false); + const [invites, setInvites] = useState([]); + + const [menuAnchor, setMenuAnchor] = useState(null); + const [menuRow, setMenuRow] = useState(null); + + const nameRef = useAutofillSync(form.name, (name) => + setForm((f) => ({ ...f, name })) + ); + const emailRef = useAutofillSync(form.email, (email) => + setForm((f) => ({ ...f, email })) + ); + + const loadUsers = useCallback(async () => { + const res = await fetch('/api/clustox/users'); + if (res.status === 403) { + setForbidden(true); + setLoading(false); + return; + } + setUsers(await res.json()); + setLoading(false); + }, []); + + const loadWorkspaces = useCallback(async () => { + const res = await fetch('/api/clustox/workspaces'); + if (res.ok) setWorkspaces(await res.json()); + }, []); + + const loadInvites = useCallback(async () => { + const res = await fetch('/api/clustox/invites'); + if (res.ok) setInvites(await res.json()); + }, []); + + useEffect(() => { + loadUsers(); + loadWorkspaces(); + loadInvites(); + }, [loadUsers, loadWorkspaces, loadInvites]); + + const roster: RosterRow[] = useMemo( + () => [ + ...users.map((u): RosterRow => ({ + kind: 'user', + key: u.userId, + name: u.name, + email: u.email, + role: u.role, + orgName: u.orgName, + user: u + })), + ...invites.map((i): RosterRow => ({ + kind: 'invite', + key: i.id, + name: i.name, + email: i.email, + role: i.role, + orgName: i.orgName, + invite: i + })) + ], + [users, invites] + ); + + const counts = useMemo( + () => ({ + total: users.length, + admins: users.filter((u) => u.role === 'ADMIN').length, + workspaces: new Set(users.map((u) => u.orgId).filter(Boolean)).size, + pending: invites.length + }), + [users, invites] + ); + + const createInvite = async (e: FormEvent) => { + e.preventDefault(); + setBusy(true); + setFormError(''); + + const res = await fetch('/api/clustox/invites', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + name: form.name, + email: form.email, + role: form.role, + org_id: form.org_id === NEW_WORKSPACE ? null : form.org_id + }) + }); + + setBusy(false); + + if (!res.ok) { + setFormError( + res.status === 409 + ? 'A user with that email already exists.' + : 'Could not create the invitation.' + ); + return; + } + + const { invite_url, emailed } = await res.json(); + // Shown once and only once: only the hash is stored, so a lost link + // cannot be recovered and has to be reissued. + setInviteLink(invite_url); + setInviteEmailed(emailed); + await loadInvites(); + }; + + const revokeInvite = async (id: string) => { + setMenuAnchor(null); + await fetch(`/api/clustox/invites/${id}`, { method: 'DELETE' }); + await loadInvites(); + enqueueSnackbar('Invitation revoked', { variant: 'success' }); + }; + + const resendInvite = async (invite: InviteRow) => { + setMenuAnchor(null); + // No dedicated resend endpoint -- revoke the old one and issue a fresh + // link with the same details, which is exactly what "resend" means + // given only a hash of the link is ever stored server-side. + await fetch(`/api/clustox/invites/${invite.id}`, { method: 'DELETE' }); + const res = await fetch('/api/clustox/invites', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + name: invite.name, + email: invite.email, + role: invite.role + }) + }); + await loadInvites(); + if (!res.ok) { + enqueueSnackbar('Could not resend the invitation.', { variant: 'error' }); + return; + } + const { emailed } = await res.json(); + enqueueSnackbar( + emailed ? `Invite resent to ${invite.email}` : `New invite link created for ${invite.email}, but it couldn't be emailed`, + { variant: emailed ? 'success' : 'warning' } + ); + }; + + const changeRole = async (user: UserRow, role: ClustoxRole) => { + setMenuAnchor(null); + const res = await fetch(`/api/clustox/users/${user.userId}`, { + method: 'PATCH', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ role }) + }); + + if (!res.ok) { + enqueueSnackbar( + res.status === 409 + ? 'You cannot demote the last superadmin.' + : 'Could not change role.', + { variant: 'error' } + ); + return; + } + + await loadUsers(); + enqueueSnackbar(`${user.name} is now ${role.toLowerCase()}`, { + variant: 'success' + }); + }; + + if (loading) + return ( + + + + ); + + if (forbidden) + return ( + + + + + + You don't have access to this page + + + Only Superadmins can manage users and roles. Ask your Superadmin if + someone needs to be added or removed. + + + ); + + return ( + <> + + Users | MiddlewareHQ + + + + {/* Summary + primary action */} + + + + {counts.total} {counts.total === 1 ? 'user' : 'users'} + + + {counts.admins} admin{counts.admins === 1 ? '' : 's'} across{' '} + {counts.workspaces} workspace{counts.workspaces === 1 ? '' : 's'} + {counts.pending > 0 && + ` · ${counts.pending} pending invite${counts.pending === 1 ? '' : 's'}`} + + + + + + + {/* Roster */} + + + + + User + Role + Status + Workspace + + + + + {roster.map((row) => ( + + + + + + {row.name} + + {row.email} + + + + + + + + + + + + + + + {row.role === 'SUPERADMIN' ? ( + + + + + All workspaces + + + + ) : ( + + + {row.orgName ?? '—'} + + )} + + + + { + setMenuRow(row); + setMenuAnchor(e.currentTarget); + }} + > + + + + + ))} + + {roster.length === 0 && ( + + + + No users yet. + + Invite an admin to create their first workspace. + + + + + )} + +
+
+
+ + {/* Row actions */} + setMenuAnchor(null)} + > + {menuRow?.kind === 'user' && + (menuRow.user.role === 'ADMIN' ? ( + changeRole(menuRow.user, 'SUPERADMIN')}> + + + + + + ) : ( + changeRole(menuRow.user, 'ADMIN')}> + + + + + + ))} + + {menuRow?.kind === 'invite' && [ + resendInvite(menuRow.invite)}> + + + + + , + revokeInvite(menuRow.invite.id)} + sx={{ color: 'error.main' }} + > + + + + + + ]} + + + {/* Invite */} + !busy && setInviteOpen(false)} + maxWidth="sm" + fullWidth + > + {inviteLink ? ( + <> + Invitation ready + + + + {inviteEmailed ? ( + + Emailed. The link is also below in case it needs to be + forwarded — only its fingerprint is stored, so it cannot be + shown again if lost. + + ) : ( + + Couldn't email this one (SMTP isn't set up, or the + send failed) — copy the link below and send it yourself. + Only its fingerprint is stored, so it cannot be shown again + if lost. + + )} + e.target.select()} + /> + + Single use, expires in 7 days. The recipient chooses their own + password. + + + + + + + + + + ) : ( +
+ Invite a teammate + + + + setForm({ ...form, email: e.target.value })} + inputRef={emailRef} + InputLabelProps={{ shrink: true }} + /> + setForm({ ...form, name: e.target.value })} + inputRef={nameRef} + InputLabelProps={{ shrink: true }} + /> + + + + Role + + setForm({ ...form, role })} + /> + + + {form.role === 'ADMIN' && ( + setForm({ ...form, org_id: e.target.value })} + helperText={ + form.org_id === NEW_WORKSPACE + ? 'A new workspace is created and named after them' + : 'They take ownership of this existing workspace, keeping its integration and repositories' + } + > + + Create a new workspace + + {workspaces + .filter((w) => !w.owned) + .map((w) => ( + + Adopt “{w.name}” (currently unowned) + + ))} + + )} + + {formError && ( + + {formError} + + )} + + + + + + + + + )} +
+ + ); +} + +UsersPage.getLayout = (page: PageLayout) => ( + + + + Users +
+ } + pageTitle="Users" + hideAllSelectors + showEvenIfNoTeamSelected + showDate={false} + > + {page} + + +); + +export default UsersPage; diff --git a/web-server/pages/workspaces.tsx b/web-server/pages/workspaces.tsx new file mode 100644 index 000000000..142332a84 --- /dev/null +++ b/web-server/pages/workspaces.tsx @@ -0,0 +1,335 @@ +import { + CheckCircleTwoTone, + ErrorTwoTone, + HelpOutlineTwoTone, + LinkOffTwoTone, + SyncTwoTone, + WorkspacesTwoTone +} from '@mui/icons-material'; +import { + Alert, + Button, + Chip, + CircularProgress, + Tab, + Table, + TableBody, + TableCell, + TableContainer, + TableHead, + TableRow, + Tabs, + Tooltip, + Typography, + useTheme +} from '@mui/material'; +import { formatDistanceToNow } from 'date-fns'; +import Head from 'next/head'; +import { useSnackbar } from 'notistack'; +import { useCallback, useEffect, useState } from 'react'; +import ExtendedSidebarLayout from 'src/layouts/ExtendedSidebarLayout'; + +import { BenchmarkSettingsForm } from '@/components/BenchmarkSettingsForm'; +import { ClustoxWorkspaceMetrics } from '@/components/ClustoxWorkspaceMetrics'; +import { FlexBox } from '@/components/FlexBox'; +import { Line } from '@/components/Text'; +import { PageWrapper } from '@/content/PullRequests/PageWrapper'; +import { useClustoxUser } from '@/hooks/useClustoxUser'; +import { PageLayout } from '@/types/resources'; + +type SyncStatus = 'RUNNING' | 'SUCCESS' | 'FAILED' | 'SKIPPED'; + +type Workspace = { + id: string; + name: string; + ownerEmail: string | null; + hasIntegration: boolean; + repoCount: number; + teamCount: number; + lastSync: { + status: SyncStatus; + finished_at: string | null; + started_at: string | null; + detail: string | null; + } | null; +}; + +const Mono = ({ children }: { children: React.ReactNode }) => ( + + {children} + +); + +const relative = (iso: string | null) => { + if (!iso) return 'never'; + try { + return `${formatDistanceToNow(new Date(iso))} ago`; + } catch { + return 'unknown'; + } +}; + +function WorkspacesPage() { + const theme = useTheme(); + const { enqueueSnackbar } = useSnackbar(); + const { isSuperadmin, loading: userLoading } = useClustoxUser(); + + const [workspaces, setWorkspaces] = useState([]); + const [loading, setLoading] = useState(true); + const [syncing, setSyncing] = useState(false); + const [tab, setTab] = useState<'health' | 'metrics' | 'benchmarks'>('health'); + + const load = useCallback(async () => { + const res = await fetch('/api/clustox/workspace-status'); + if (res.ok) setWorkspaces(await res.json()); + setLoading(false); + }, []); + + useEffect(() => { + load(); + }, [load]); + + const syncNow = async () => { + setSyncing(true); + try { + const res = await fetch('/api/clustox/sync-now', { method: 'POST' }); + if (!res.ok) throw new Error('sync failed'); + const body = await res.json(); + enqueueSnackbar(body.message ?? 'Sync finished', { + variant: 'success', + autoHideDuration: 6000 + }); + await load(); + } catch { + enqueueSnackbar('Could not reach the sync server', { variant: 'error' }); + } finally { + setSyncing(false); + } + }; + + const statusChip = (w: Workspace) => { + if (!w.hasIntegration) + return ( + + } + label="Not connected" + /> + + ); + + if (!w.lastSync) + return ( + + } + label="Never synced" + /> + + ); + + if (w.lastSync.status === 'FAILED') + return ( + + } + label="Failed" + /> + + ); + + if (w.lastSync.status === 'RUNNING') + return } label="Running" />; + + return ( + } + label="Synced" + /> + ); + }; + + if (loading || userLoading) + return ( + + + + ); + + const failing = workspaces.filter((w) => w.lastSync?.status === 'FAILED'); + const unconnected = workspaces.filter((w) => !w.hasIntegration); + + return ( + <> + + Workspaces | MiddlewareHQ + + + + + + + {workspaces.length}{' '} + {workspaces.length === 1 ? 'workspace' : 'workspaces'} + + + {failing.length > 0 + ? `${failing.length} failing to sync` + : 'All connected workspaces syncing'} + + + + {isSuperadmin && ( + + )} + + + {failing.length > 0 && ( + + {failing.length === 1 + ? `${failing[0].name} failed its last sync.` + : `${failing.length} workspaces failed their last sync.`}{' '} + Their metrics are stale until it succeeds. + + )} + + {unconnected.length > 0 && failing.length === 0 && ( + + {unconnected.length}{' '} + {unconnected.length === 1 ? 'workspace has' : 'workspaces have'} no + code provider connected yet, so{' '} + {unconnected.length === 1 ? 'it has' : 'they have'} nothing to sync. + + )} + + {isSuperadmin && ( + setTab(v)} + sx={{ borderBottom: 1, borderColor: 'divider' }} + > + + + + + )} + + {isSuperadmin && tab === 'metrics' && } + + {isSuperadmin && tab === 'benchmarks' && ( + + )} + + {(!isSuperadmin || tab === 'health') && ( + + + + + Workspace + Last sync + When + + Repos + + + Teams + + + + + {workspaces.map((w) => ( + + + + + + {w.name} + + {w.ownerEmail ?? 'no owner'} + + + + + {statusChip(w)} + + + {w.hasIntegration + ? relative(w.lastSync?.finished_at ?? null) + : '—'} + + + + {w.repoCount} + + + {w.teamCount} + + + ))} + +
+
+ )} + + {(!isSuperadmin || tab === 'health') && ( + + Syncs run automatically every 30 minutes. A failed workspace does + not affect the others. + + )} +
+ + ); +} + +WorkspacesPage.getLayout = (page: PageLayout) => ( + + + + Workspaces + + } + pageTitle="Workspaces" + hideAllSelectors + showEvenIfNoTeamSelected + showDate={false} + > + {page} + + +); + +export default WorkspacesPage; diff --git a/web-server/src/api-helpers/__tests__/dora-metrics-additive.test.ts b/web-server/src/api-helpers/__tests__/dora-metrics-additive.test.ts new file mode 100644 index 000000000..68cb4eadf --- /dev/null +++ b/web-server/src/api-helpers/__tests__/dora-metrics-additive.test.ts @@ -0,0 +1,299 @@ +/** + * The other half of the regression guarantee: the DORA response itself. + * + * `noBenchmarksRegression.test.tsx` proves the five cards render with no band + * and no caption once the payload reaches them. This proves the payload gets + * there unchanged -- that adding benchmarks and lines of code to the + * dora_metrics route did not disturb the four `*_stats` / `*_trends` keys the + * original cards have always read. + * + * CLUSTOX: the two additions are wrapped in `.catch(() => undefined)` in the + * route precisely so an optional decoration cannot blank the dashboard. That + * contract had no test. Deleting either `.catch` looks harmless in review and + * costs nothing until the day one of those calls 400s -- which has already + * happened once on this project, when a payload key nested one level too deep + * made the analytics server reject a sibling call inside the same Promise.all + * and the whole dashboard went to "something went wrong". + */ +jest.mock('@/auth/session', () => ({ getAuthSession: jest.fn() })); +jest.mock('@/auth/queries', () => ({ + getTeamOrgId: jest.fn(), + getTeamIdsForOrg: jest.fn(), + getAllTeamIds: jest.fn() +})); +jest.mock('@/utils/cockpitMetricUtils', () => ({ + fetchLeadTimeStats: jest.fn(), + fetchChangeFailureRateStats: jest.fn(), + fetchMeanTimeToRestoreStats: jest.fn(), + fetchDeploymentFrequencyStats: jest.fn(), + fetchTeamBenchmarks: jest.fn(), + fetchLocStats: jest.fn() +})); +jest.mock('@/api/resources/team_repos', () => ({ + getTeamRepos: jest.fn() +})); +jest.mock('@/api/resources/teams/[team_id]/unsynced_repos', () => ({ + getUnsyncedRepos: jest.fn() +})); +jest.mock('@/api/internal/team/[team_id]/insights', () => ({ + getTeamLeadTimePRs: jest.fn() +})); +jest.mock('@/api-helpers/team', () => ({ + updatePrFilterParams: jest.fn() +})); +jest.mock('@/utils/filterUtils', () => ({ + getBranchesAndRepoFilter: jest.fn(), + getWorkFlowFiltersAsPayloadForSingleTeam: jest.fn() +})); + +// CLUSTOX: the route under test. `jest.mock` calls are hoisted above every +// import, so this reading first is only an import-order rule, not a mocking +// hazard. +import handler from '@/api/internal/team/[team_id]/dora_metrics'; +import { getTeamRepos } from '@/api/resources/team_repos'; +import { getUnsyncedRepos } from '@/api/resources/teams/[team_id]/unsynced_repos'; +import { updatePrFilterParams } from '@/api-helpers/team'; +import { getTeamOrgId } from '@/auth/queries'; +import { getAuthSession } from '@/auth/session'; +import { + fetchChangeFailureRateStats, + fetchDeploymentFrequencyStats, + fetchLeadTimeStats, + fetchLocStats, + fetchMeanTimeToRestoreStats, + fetchTeamBenchmarks +} from '@/utils/cockpitMetricUtils'; +import { + getBranchesAndRepoFilter, + getWorkFlowFiltersAsPayloadForSingleTeam +} from '@/utils/filterUtils'; + +const TEAM_ID = '00000000-0000-4000-8000-0000000000ff'; +const ORG_ID = '00000000-0000-4000-8000-0000000000ee'; + +/** + * The four original metrics, exactly as their fetchers return them. Every + * assertion below compares the response against these by identity, so a route + * that reshaped, renamed or re-nested any of them fails. + */ +const LEAD_TIME = { + lead_time_stats: { + current: { lead_time: 86400, pr_count: 12 }, + previous: { lead_time: 93600, pr_count: 10 } + }, + lead_time_trends: { current: { a: 1 }, previous: { b: 2 } } +}; +const DEPLOYMENT_FREQUENCY = { + deployment_frequency_stats: { + current: { avg_weekly_deployment_frequency: 4, total_deployments: 16 }, + previous: { avg_weekly_deployment_frequency: 3, total_deployments: 12 } + }, + deployment_frequency_trends: { current: { c: 3 }, previous: { d: 4 } } +}; +const CHANGE_FAILURE_RATE = { + change_failure_rate_stats: { + current: { change_failure_rate: 12 }, + previous: { change_failure_rate: 15 } + }, + change_failure_rate_trends: { current: { e: 5 }, previous: { f: 6 } } +}; +const MEAN_TIME_TO_RESTORE = { + mean_time_to_restore_stats: { + current: { mean_time_to_recovery: 4200, incident_count: 3 }, + previous: { mean_time_to_recovery: 5400, incident_count: 4 } + }, + mean_time_to_restore_trends: { current: { g: 7 }, previous: { h: 8 } } +}; +const LEAD_TIME_PRS = [{ id: 'pr-1' }]; +const TEAM_REPOS = [{ id: 'repo-1' }]; +const UNSYNCED_REPOS = ['repo-2']; + +const mockRes = () => { + const res: any = {}; + res.status = jest.fn(() => res); + res.send = jest.fn(() => res); + return res; +}; + +const request = () => + ({ + method: 'GET', + query: { + team_id: TEAM_ID, + org_id: ORG_ID, + from_date: '2026-07-01T00:00:00+00:00', + to_date: '2026-08-01T00:00:00+00:00', + branch_mode: 'PROD' + }, + body: {}, + headers: {} + }) as any; + +const call = async () => { + const res = mockRes(); + await handler(request(), res); + return res; +}; + +/** Everything the route needs that is not the subject of a given test. */ +const stubTheHappyPath = () => { + (getAuthSession as jest.Mock).mockResolvedValue({ + userId: 'ad1', + email: 'lead@clustox.com', + role: 'ADMIN', + orgId: ORG_ID + }); + (getTeamOrgId as jest.Mock).mockResolvedValue(ORG_ID); + + (getBranchesAndRepoFilter as jest.Mock).mockResolvedValue({}); + (getWorkFlowFiltersAsPayloadForSingleTeam as jest.Mock).mockResolvedValue({}); + (updatePrFilterParams as jest.Mock).mockResolvedValue({ pr_filter: null }); + (getUnsyncedRepos as jest.Mock).mockResolvedValue(UNSYNCED_REPOS); + (getTeamRepos as jest.Mock).mockResolvedValue(TEAM_REPOS); + + (fetchLeadTimeStats as jest.Mock).mockResolvedValue(LEAD_TIME); + (fetchDeploymentFrequencyStats as jest.Mock).mockResolvedValue( + DEPLOYMENT_FREQUENCY + ); + (fetchChangeFailureRateStats as jest.Mock).mockResolvedValue( + CHANGE_FAILURE_RATE + ); + (fetchMeanTimeToRestoreStats as jest.Mock).mockResolvedValue( + MEAN_TIME_TO_RESTORE + ); +}; + +const expectTheFourOriginalMetricsIntact = (body: any) => { + expect(body.lead_time_stats).toBe(LEAD_TIME.lead_time_stats); + expect(body.lead_time_trends).toBe(LEAD_TIME.lead_time_trends); + expect(body.deployment_frequency_stats).toBe( + DEPLOYMENT_FREQUENCY.deployment_frequency_stats + ); + expect(body.deployment_frequency_trends).toBe( + DEPLOYMENT_FREQUENCY.deployment_frequency_trends + ); + expect(body.change_failure_rate_stats).toBe( + CHANGE_FAILURE_RATE.change_failure_rate_stats + ); + expect(body.change_failure_rate_trends).toBe( + CHANGE_FAILURE_RATE.change_failure_rate_trends + ); + expect(body.mean_time_to_restore_stats).toBe( + MEAN_TIME_TO_RESTORE.mean_time_to_restore_stats + ); + expect(body.mean_time_to_restore_trends).toBe( + MEAN_TIME_TO_RESTORE.mean_time_to_restore_trends + ); + expect(body.assigned_repos).toBe(TEAM_REPOS); + expect(body.unsynced_repos).toBe(UNSYNCED_REPOS); +}; + +describe('dora_metrics is additive', () => { + beforeEach(() => { + jest.clearAllMocks(); + stubTheHappyPath(); + ( + require('@/api/internal/team/[team_id]/insights') + .getTeamLeadTimePRs as jest.Mock + ).mockResolvedValue({ + data: LEAD_TIME_PRS + }); + }); + + it('sends the four original metrics unchanged when nothing is benchmarked', async () => { + // The zero-config state every installation is in until an admin sets a + // target: the benchmarks route resolves every metric to a null target. + (fetchTeamBenchmarks as jest.Mock).mockResolvedValue({ + lead_time: { target: null, source: null }, + deployment_frequency: { target: null, source: null }, + change_failure_rate: { target: null, source: null }, + mean_time_to_recovery: { target: null, source: null }, + lines_of_code: { target: null, source: null } + }); + (fetchLocStats as jest.Mock).mockResolvedValue({ + loc_stats: { current: {}, previous: {} }, + loc_trends: { current: {}, previous: {} } + }); + + const res = await call(); + const [body] = res.send.mock.calls[0]; + + expectTheFourOriginalMetricsIntact(body); + // Present but empty -- every card reads `benchmarks?.?.target` + // and must see null, not a missing key it could mistake for a fetch error. + expect(body.benchmarks.lines_of_code).toEqual({ + target: null, + source: null + }); + }); + + it('still sends the four original metrics when the benchmarks call fails', async () => { + // CLUSTOX: this is what the `.catch(() => undefined)` buys. Degrading to + // an unbenchmarked dashboard is correct; taking the dashboard down over an + // optional decoration is not. + (fetchTeamBenchmarks as jest.Mock).mockRejectedValue(new Error('500')); + (fetchLocStats as jest.Mock).mockResolvedValue({ + loc_stats: {}, + loc_trends: {} + }); + + const res = await call(); + const [body] = res.send.mock.calls[0]; + + // The success path never touches res.status at all; the global error + // handler is the only thing that sets one. + expect(res.status).not.toHaveBeenCalled(); + expectTheFourOriginalMetricsIntact(body); + expect(body.benchmarks).toBeUndefined(); + }); + + it('still sends the four original metrics when the LOC call fails', async () => { + (fetchTeamBenchmarks as jest.Mock).mockResolvedValue(undefined); + (fetchLocStats as jest.Mock).mockRejectedValue(new Error('400')); + + const res = await call(); + const [body] = res.send.mock.calls[0]; + + // The success path never touches res.status at all; the global error + // handler is the only thing that sets one. + expect(res.status).not.toHaveBeenCalled(); + expectTheFourOriginalMetricsIntact(body); + // `undefined`, not a zeroed-out object. The LOC card distinguishes "not + // measured" from a measured zero, and only absence can mean the former. + expect(body.loc_stats).toBeUndefined(); + expect(body.loc_trends).toBeUndefined(); + }); + + it('still sends the four original metrics when both new calls fail', async () => { + (fetchTeamBenchmarks as jest.Mock).mockRejectedValue(new Error('500')); + (fetchLocStats as jest.Mock).mockRejectedValue(new Error('400')); + + const res = await call(); + const [body] = res.send.mock.calls[0]; + + // The success path never touches res.status at all; the global error + // handler is the only thing that sets one. + expect(res.status).not.toHaveBeenCalled(); + expectTheFourOriginalMetricsIntact(body); + }); + + it('control: an unguarded fetcher failing DOES take the response down', async () => { + // CLUSTOX: without this the three cases above prove nothing -- they would + // pass just as well if a rejection inside this route were invisible to + // this harness. Lead time carries no `.catch` (it is not optional; there + // is no dashboard without it), so its failure is exactly the outcome the + // two new calls are deliberately shielded from. Same setup, same + // assertions, opposite result. + (fetchTeamBenchmarks as jest.Mock).mockResolvedValue(undefined); + (fetchLocStats as jest.Mock).mockResolvedValue(undefined); + (fetchLeadTimeStats as jest.Mock).mockRejectedValue(new Error('upstream')); + + const res = await call(); + + // An error status, whichever one parseError lands on -- the point is that + // the response stopped being the dashboard, not which code says so. + expect(res.status).toHaveBeenCalled(); + expect(res.status.mock.calls[0][0]).toBeGreaterThanOrEqual(400); + expect(res.send.mock.calls[0][0]).not.toHaveProperty('lead_time_stats'); + }); +}); diff --git a/web-server/src/api-helpers/__tests__/endpoint-auth.test.ts b/web-server/src/api-helpers/__tests__/endpoint-auth.test.ts new file mode 100644 index 000000000..c92844ba4 --- /dev/null +++ b/web-server/src/api-helpers/__tests__/endpoint-auth.test.ts @@ -0,0 +1,80 @@ +jest.mock('@/auth/session', () => ({ getAuthSession: jest.fn() })); + +import { Endpoint, nullSchema } from '@/api-helpers/global'; +import { getAuthSession } from '@/auth/session'; + +const mockRes = () => { + const res: any = {}; + res.status = jest.fn(() => res); + res.send = jest.fn(() => res); + return res; +}; + +const mockReq = () => + ({ method: 'GET', query: {}, body: {}, headers: {} }) as any; + +describe('Endpoint auth enforcement', () => { + beforeEach(() => jest.clearAllMocks()); + + it('rejects with 401 when no session and the endpoint is authenticated', async () => { + (getAuthSession as jest.Mock).mockResolvedValue(null); + + const endpoint = new Endpoint(nullSchema); + endpoint.handle.GET(nullSchema, async (_req, res) => res.send({ ok: true })); + + const res = mockRes(); + await endpoint.serve()(mockReq(), res); + + expect(res.status).toHaveBeenCalledWith(401); + }); + + it('allows the request through when a session exists', async () => { + (getAuthSession as jest.Mock).mockResolvedValue({ + userId: 'u1', + email: 'a@clustox.com', + name: 'A', + role: 'ADMIN' + }); + + const endpoint = new Endpoint(nullSchema); + endpoint.handle.GET(nullSchema, async (_req, res) => res.send({ ok: true })); + + const res = mockRes(); + await endpoint.serve()(mockReq(), res); + + expect(res.send).toHaveBeenCalledWith({ ok: true }); + }); + + it('skips the check for endpoints marked unauthenticated', async () => { + (getAuthSession as jest.Mock).mockResolvedValue(null); + + const endpoint = new Endpoint(nullSchema, { unauthenticated: true }); + endpoint.handle.GET(nullSchema, async (_req, res) => res.send({ ok: true })); + + const res = mockRes(); + await endpoint.serve()(mockReq(), res); + + expect(res.send).toHaveBeenCalledWith({ ok: true }); + expect(getAuthSession).not.toHaveBeenCalled(); + }); + + it('exposes the session on the request object', async () => { + const session = { + userId: 'u1', + email: 'a@clustox.com', + name: 'A', + role: 'ADMIN' as const + }; + (getAuthSession as jest.Mock).mockResolvedValue(session); + + let seen: any = null; + const endpoint = new Endpoint(nullSchema); + endpoint.handle.GET(nullSchema, async (req, res) => { + seen = (req as any).session; + res.send({ ok: true }); + }); + + await endpoint.serve()(mockReq(), mockRes()); + expect(seen).toEqual(session); + }); +}); diff --git a/web-server/src/api-helpers/__tests__/endpoint-team-scope.test.ts b/web-server/src/api-helpers/__tests__/endpoint-team-scope.test.ts new file mode 100644 index 000000000..183dd17f1 --- /dev/null +++ b/web-server/src/api-helpers/__tests__/endpoint-team-scope.test.ts @@ -0,0 +1,105 @@ +jest.mock('@/auth/session', () => ({ getAuthSession: jest.fn() })); +jest.mock('@/auth/queries', () => ({ + getTeamOrgId: jest.fn(), + getTeamIdsForOrg: jest.fn(), + getAllTeamIds: jest.fn() +})); + +import { Endpoint, nullSchema } from '@/api-helpers/global'; +import { getTeamOrgId } from '@/auth/queries'; +import { getAuthSession } from '@/auth/session'; + +const mockRes = () => { + const res: any = {}; + res.status = jest.fn(() => res); + res.send = jest.fn(() => res); + return res; +}; + +const reqWithTeam = (team_id?: string) => + ({ + method: 'GET', + query: team_id ? { team_id } : {}, + body: {}, + headers: {} + }) as any; + +const asAdmin = () => + (getAuthSession as jest.Mock).mockResolvedValue({ + userId: 'ad1', + email: 'lead@clustox.com', + name: 'Lead', + role: 'ADMIN', + orgId: 'workspace-1' + }); + +const asSuperadmin = () => + (getAuthSession as jest.Mock).mockResolvedValue({ + userId: 'su1', + email: 'boss@clustox.com', + name: 'Boss', + role: 'SUPERADMIN', + orgId: null + }); + +const buildEndpoint = () => { + const endpoint = new Endpoint(nullSchema); + endpoint.handle.GET(nullSchema, async (_req, res) => res.send({ ok: true })); + return endpoint; +}; + +describe('Endpoint team scoping', () => { + beforeEach(() => jest.clearAllMocks()); + + it('403s when an admin requests a team outside their workspace', async () => { + asAdmin(); + (getTeamOrgId as jest.Mock).mockResolvedValue('workspace-2'); + + const res = mockRes(); + await buildEndpoint().serve()(reqWithTeam('team-cgpt'), res); + + expect(res.status).toHaveBeenCalledWith(403); + }); + + it('allows an admin any team inside their own workspace', async () => { + asAdmin(); + (getTeamOrgId as jest.Mock).mockResolvedValue('workspace-1'); + + const res = mockRes(); + await buildEndpoint().serve()(reqWithTeam('team-zda'), res); + + expect(res.send).toHaveBeenCalledWith({ ok: true }); + }); + + it('allows a superadmin any team', async () => { + asSuperadmin(); + + const res = mockRes(); + await buildEndpoint().serve()(reqWithTeam('team-anything'), res); + + expect(res.send).toHaveBeenCalledWith({ ok: true }); + expect(getTeamOrgId).not.toHaveBeenCalled(); + }); + + it('does not attempt a team check when the payload has no team_id', async () => { + asAdmin(); + + const res = mockRes(); + await buildEndpoint().serve()(reqWithTeam(undefined), res); + + expect(res.send).toHaveBeenCalledWith({ ok: true }); + expect(getTeamOrgId).not.toHaveBeenCalled(); + }); + + it('skips the team check entirely on unauthenticated endpoints', async () => { + (getAuthSession as jest.Mock).mockResolvedValue(null); + + const endpoint = new Endpoint(nullSchema, { unauthenticated: true }); + endpoint.handle.GET(nullSchema, async (_req, res) => res.send({ ok: true })); + + const res = mockRes(); + await endpoint.serve()(reqWithTeam('team-zda'), res); + + expect(res.send).toHaveBeenCalledWith({ ok: true }); + }); +}); diff --git a/web-server/src/api-helpers/axios.ts b/web-server/src/api-helpers/axios.ts index d5b031fe7..c516b8db1 100644 --- a/web-server/src/api-helpers/axios.ts +++ b/web-server/src/api-helpers/axios.ts @@ -31,6 +31,18 @@ export const internalSyncServer = axios.create({ baseURL: process.env.INTERNAL_SYNC_API_BASE_URL }); +// CLUSTOX: the Flask servers reject any request without this shared secret. +// Stamped here because every outbound call funnels through these two +// instances, so there is no route that can forget it. +const internalTokenInterceptor = (req: any) => { + req.headers = req.headers || {}; + req.headers['X-Internal-Token'] = process.env.INTERNAL_API_TOKEN; + return req; +}; + +internal.interceptors.request.use(internalTokenInterceptor); +internalSyncServer.interceptors.request.use(internalTokenInterceptor); + axiosRetry(internal, { retries: 2, retryCondition(error) { diff --git a/web-server/src/api-helpers/global.ts b/web-server/src/api-helpers/global.ts index 7cc16985d..ce94bbbdc 100644 --- a/web-server/src/api-helpers/global.ts +++ b/web-server/src/api-helpers/global.ts @@ -1,6 +1,13 @@ import { NextApiRequest } from 'next/types'; import { AnySchema, InferType, object } from 'yup'; +// CLUSTOX: enforce the `authenticated` flag that upstream declared but never read. +import { + assertAuthenticated, + assertTeamAccess, + assertWorkspaceAccess +} from '@/auth/guard'; +import { getAuthSession } from '@/auth/session'; import { Errors, ResponseError } from '@/constants/error'; import { ApiRequest, ApiResponse, HttpMethods } from '@/types/request'; @@ -78,6 +85,16 @@ export class Endpoint { return async (nextReq: NextApiRequest, res: ApiResponse) => { try { const req = transformNextRequest(nextReq); + + // CLUSTOX: single enforcement point for all BFF routes. Routes opt out + // with `new Endpoint(schema, { unauthenticated: true })`. + let session = null; + if (this.authenticated) { + session = await getAuthSession(nextReq); + assertAuthenticated(session); + (req as any).session = session; + } + if (this.pathSchema) { await this.pathSchema.validate(req.payload); } @@ -91,6 +108,23 @@ export class Endpoint { await schema.validate(req.payload); } + // CLUSTOX: routes addressing a specific workspace or team are scoped + // centrally here rather than in each of the ~18 team routes and ~20 org + // routes. Doing it at the choke point means a newly added route is + // protected by default instead of relying on the author remembering a + // guard call. + // + // Runs *after* schema validation so malformed input is a 400 with a + // useful message. Checking first turned a client sending + // "/orgs/undefined/integration" into an opaque 403. + if (this.authenticated) { + const orgId = (req.payload as any)?.org_id; + if (orgId) await assertWorkspaceAccess(session, String(orgId)); + + const teamId = (req.payload as any)?.team_id; + if (teamId) await assertTeamAccess(session, String(teamId)); + } + await handler(req, res); } catch (err) { const error = parseError(err); diff --git a/web-server/src/api-helpers/internal-error.ts b/web-server/src/api-helpers/internal-error.ts new file mode 100644 index 000000000..18f203972 --- /dev/null +++ b/web-server/src/api-helpers/internal-error.ts @@ -0,0 +1,28 @@ +import { AxiosError } from 'axios'; + +import { ResponseError } from '@/constants/error'; + +/** + * CLUSTOX: re-throws an analytics-server failure with the server's own status + * and message. + * + * The BFF's generic error path (parseError) reads `message` off whatever was + * thrown, and for an axios failure that string is always "Request failed with + * status code 409" -- the server's explanation never reaches the browser, so + * every failure arrives looking identical. Jenkins depends on the difference: + * a 400 naming a base URL this server refuses to fetch and a 409 naming the + * workflow already holding a job id are both things only an admin can fix, and + * both read as "try again" without the text. + */ +export const forwardInternalError = (err: unknown): never => { + const response = (err as AxiosError<{ error?: string }>)?.response; + const message = response?.data?.error; + + // A timeout, a refused connection or an HTML error page carries nothing + // worth showing, so it keeps the generic 400/500 handling rather than + // surfacing a fragment of someone else's stack trace to an admin. + if (response?.status && typeof message === 'string' && message) + throw new ResponseError(message, response.status); + + throw err; +}; diff --git a/web-server/src/api-helpers/team.ts b/web-server/src/api-helpers/team.ts index 23065c999..aa2061b55 100644 --- a/web-server/src/api-helpers/team.ts +++ b/web-server/src/api-helpers/team.ts @@ -42,11 +42,17 @@ export const getTeamMembersFilterSettingForOrg = (orgId: ID) => export const updatePrFilterParams = async ( _teamId: ID, params: T, - filters?: Partial<{ branches: string; repo_filters: RepoFilterConfig }> + filters?: Partial<{ + branches: string; + repo_filters: RepoFilterConfig; + // CLUSTOX: contributor filter -- git usernames, as synced. + authors: string[]; + }> ) => { const updatedParams = { base_branches: filters?.branches?.split(','), - repo_filters: filters?.repo_filters + repo_filters: filters?.repo_filters, + authors: filters?.authors?.length ? filters.authors : undefined }; const reducedParams = reject(isNil, updatedParams); const pr_filter = equals({}, reducedParams) ? null : reducedParams; diff --git a/web-server/src/auth/__tests__/bootstrap.test.ts b/web-server/src/auth/__tests__/bootstrap.test.ts new file mode 100644 index 000000000..ee2d82b6d --- /dev/null +++ b/web-server/src/auth/__tests__/bootstrap.test.ts @@ -0,0 +1,57 @@ +jest.mock('@/auth/queries', () => ({ + countSuperadmins: jest.fn(), + createUser: jest.fn() +})); +jest.mock('@/utils/db', () => ({ db: jest.fn() })); + +import { bootstrapSuperadmin } from '@/auth/bootstrap'; +import { countSuperadmins, createUser } from '@/auth/queries'; + +describe('bootstrapSuperadmin', () => { + const env = process.env; + + beforeEach(() => { + jest.clearAllMocks(); + process.env = { ...env }; + }); + + afterAll(() => { + process.env = env; + }); + + it('does nothing when a superadmin already exists', async () => { + (countSuperadmins as jest.Mock).mockResolvedValue(1); + process.env.SUPERADMIN_EMAIL = 'a@clustox.com'; + process.env.SUPERADMIN_PASSWORD = 'pw'; + + await bootstrapSuperadmin(); + expect(createUser).not.toHaveBeenCalled(); + }); + + it('does not create a user when env vars are missing', async () => { + (countSuperadmins as jest.Mock).mockResolvedValue(0); + delete process.env.SUPERADMIN_EMAIL; + delete process.env.SUPERADMIN_PASSWORD; + + await bootstrapSuperadmin(); + expect(createUser).not.toHaveBeenCalled(); + }); + + it('creates the superadmin when none exists and env vars are set', async () => { + (countSuperadmins as jest.Mock).mockResolvedValue(0); + (createUser as jest.Mock).mockResolvedValue('new-id'); + process.env.SUPERADMIN_EMAIL = 'boss@clustox.com'; + process.env.SUPERADMIN_PASSWORD = 'strong-password'; + + await bootstrapSuperadmin(); + + expect(createUser).toHaveBeenCalledWith( + expect.objectContaining({ + email: 'boss@clustox.com', + password: 'strong-password', + role: 'SUPERADMIN', + teamIds: [] + }) + ); + }); +}); diff --git a/web-server/src/auth/__tests__/guard.test.ts b/web-server/src/auth/__tests__/guard.test.ts new file mode 100644 index 000000000..f1754efc8 --- /dev/null +++ b/web-server/src/auth/__tests__/guard.test.ts @@ -0,0 +1,69 @@ +jest.mock('@/auth/queries', () => ({ + getTeamIdsForOrg: jest.fn(), + getTeamOrgId: jest.fn(), + getAllTeamIds: jest.fn() +})); + +import { + assertAuthenticated, + assertRole, + canAccessAllTeams +} from '@/auth/guard'; +import { AuthSession } from '@/auth/types'; + +const superadmin: AuthSession = { + userId: 'su1', + email: 'boss@clustox.com', + name: 'Boss', + role: 'SUPERADMIN', + orgId: null +}; +const admin: AuthSession = { + userId: 'ad1', + email: 'lead@clustox.com', + name: 'Lead', + role: 'ADMIN', + orgId: 'workspace-1' +}; + +describe('assertAuthenticated', () => { + it('returns the session when present', () => { + expect(assertAuthenticated(admin)).toBe(admin); + }); + + it('throws 401 when the session is null', () => { + expect(() => assertAuthenticated(null)).toThrow( + expect.objectContaining({ status: 401 }) + ); + }); +}); + +describe('assertRole', () => { + it('passes when the role matches', () => { + expect(assertRole(superadmin, 'SUPERADMIN')).toBe(superadmin); + }); + + it('throws 403 when an admin attempts a superadmin action', () => { + expect(() => assertRole(admin, 'SUPERADMIN')).toThrow( + expect.objectContaining({ status: 403 }) + ); + }); + + it('throws 401 when unauthenticated, not 403', () => { + expect(() => assertRole(null, 'SUPERADMIN')).toThrow( + expect.objectContaining({ status: 401 }) + ); + }); +}); + +describe('canAccessAllTeams', () => { + it('is true for superadmin', () => { + expect(canAccessAllTeams(superadmin)).toBe(true); + }); + it('is false for admin', () => { + expect(canAccessAllTeams(admin)).toBe(false); + }); +}); + +// assertTeamAccess and getAccessibleTeamIds are workspace-derived since +// multitenancy; they are covered in workspace-guard.test.ts. diff --git a/web-server/src/auth/__tests__/password.test.ts b/web-server/src/auth/__tests__/password.test.ts new file mode 100644 index 000000000..65dd09d44 --- /dev/null +++ b/web-server/src/auth/__tests__/password.test.ts @@ -0,0 +1,31 @@ +import { hashPassword, verifyPassword } from '@/auth/password'; + +describe('password', () => { + it('produces a hash that is not the plaintext', async () => { + const hash = await hashPassword('correct-horse-battery'); + expect(hash).not.toBe('correct-horse-battery'); + expect(hash.length).toBeGreaterThan(50); + }); + + it('verifies a correct password', async () => { + const hash = await hashPassword('correct-horse-battery'); + await expect(verifyPassword('correct-horse-battery', hash)).resolves.toBe( + true + ); + }); + + it('rejects an incorrect password', async () => { + const hash = await hashPassword('correct-horse-battery'); + await expect(verifyPassword('wrong-password', hash)).resolves.toBe(false); + }); + + it('produces different hashes for the same input (salted)', async () => { + const a = await hashPassword('same-input'); + const b = await hashPassword('same-input'); + expect(a).not.toBe(b); + }); + + it('returns false rather than throwing on a malformed hash', async () => { + await expect(verifyPassword('anything', 'not-a-hash')).resolves.toBe(false); + }); +}); diff --git a/web-server/src/auth/__tests__/queries.test.ts b/web-server/src/auth/__tests__/queries.test.ts new file mode 100644 index 000000000..6610ee347 --- /dev/null +++ b/web-server/src/auth/__tests__/queries.test.ts @@ -0,0 +1,80 @@ +const mockDb = jest.fn(); +jest.mock('@/utils/db', () => ({ + db: (...args: any[]) => mockDb(...args) +})); + +import { getAuthUserByEmail, getTeamIdsForUser } from '@/auth/queries'; + +const chain = (result: any) => { + const c: any = {}; + c.select = jest.fn(() => c); + c.join = jest.fn(() => c); + c.where = jest.fn(() => c); + c.andWhere = jest.fn(() => c); + c.first = jest.fn(() => Promise.resolve(result)); + c.then = (fn: any) => Promise.resolve(result).then(fn); + return c; +}; + +describe('getAuthUserByEmail', () => { + beforeEach(() => mockDb.mockReset()); + + it('maps a row to AuthUserRow, including the workspace', async () => { + mockDb.mockReturnValue( + chain({ + user_id: 'u1', + primary_email: 'a@clustox.com', + name: 'Ayesha', + role: 'ADMIN', + org_id: 'workspace-1', + password_hash: 'hashed' + }) + ); + + await expect(getAuthUserByEmail('a@clustox.com')).resolves.toEqual({ + userId: 'u1', + email: 'a@clustox.com', + name: 'Ayesha', + role: 'ADMIN', + orgId: 'workspace-1', + passwordHash: 'hashed' + }); + }); + + it('maps a superadmin with no workspace to a null orgId', async () => { + mockDb.mockReturnValue( + chain({ + user_id: 'su1', + primary_email: 'boss@clustox.com', + name: 'Boss', + role: 'SUPERADMIN', + org_id: null, + password_hash: 'hashed' + }) + ); + + await expect(getAuthUserByEmail('boss@clustox.com')).resolves.toMatchObject({ + role: 'SUPERADMIN', + orgId: null + }); + }); + + it('returns null when no row matches', async () => { + mockDb.mockReturnValue(chain(undefined)); + await expect(getAuthUserByEmail('nobody@clustox.com')).resolves.toBeNull(); + }); +}); + +describe('getTeamIdsForUser', () => { + beforeEach(() => mockDb.mockReset()); + + it('returns a flat array of team ids', async () => { + mockDb.mockReturnValue(chain([{ team_id: 't1' }, { team_id: 't2' }])); + await expect(getTeamIdsForUser('u1')).resolves.toEqual(['t1', 't2']); + }); + + it('returns an empty array when the user has no teams', async () => { + mockDb.mockReturnValue(chain([])); + await expect(getTeamIdsForUser('u1')).resolves.toEqual([]); + }); +}); diff --git a/web-server/src/auth/__tests__/session.test.ts b/web-server/src/auth/__tests__/session.test.ts new file mode 100644 index 000000000..053e0f80b --- /dev/null +++ b/web-server/src/auth/__tests__/session.test.ts @@ -0,0 +1,146 @@ +jest.mock('next-auth/jwt', () => ({ getToken: jest.fn() })); +jest.mock('@/auth/queries', () => ({ + getAuthUserById: jest.fn(), + listWorkspaces: jest.fn(), + workspaceExists: jest.fn() +})); + +import { getToken } from 'next-auth/jwt'; + +import { + getAuthUserById, + listWorkspaces, + workspaceExists +} from '@/auth/queries'; +import { getAuthSession } from '@/auth/session'; + +const req = { cookies: {} } as any; + +describe('getAuthSession', () => { + beforeEach(() => jest.clearAllMocks()); + + it('returns null when there is no token', async () => { + (getToken as jest.Mock).mockResolvedValue(null); + await expect(getAuthSession(req)).resolves.toBeNull(); + expect(getAuthUserById).not.toHaveBeenCalled(); + }); + + it('returns null when the token is valid but the user no longer exists', async () => { + (getToken as jest.Mock).mockResolvedValue({ userId: 'deleted-user' }); + (getAuthUserById as jest.Mock).mockResolvedValue(null); + + await expect(getAuthSession(req)).resolves.toBeNull(); + }); + + it('resolves the session from the database, not the token', async () => { + (getToken as jest.Mock).mockResolvedValue({ + userId: 'u1', + // Stale claims from sign-in time. These must be ignored. + role: 'SUPERADMIN', + email: 'old@clustox.com', + name: 'Old Name' + }); + (getAuthUserById as jest.Mock).mockResolvedValue({ + userId: 'u1', + email: 'current@clustox.com', + name: 'Current Name', + role: 'ADMIN', + orgId: 'workspace-1' + }); + + await expect(getAuthSession(req)).resolves.toEqual({ + userId: 'u1', + email: 'current@clustox.com', + name: 'Current Name', + role: 'ADMIN', + orgId: 'workspace-1' + }); + }); + + it('carries the admin workspace onto the session', async () => { + (getToken as jest.Mock).mockResolvedValue({ userId: 'u1' }); + (getAuthUserById as jest.Mock).mockResolvedValue({ + userId: 'u1', + email: 'a@clustox.com', + name: 'A', + role: 'ADMIN', + orgId: 'workspace-1' + }); + + const session = await getAuthSession(req); + expect(session?.orgId).toBe('workspace-1'); + }); + + it('drops a superadmin into the oldest workspace when none is selected', async () => { + (getToken as jest.Mock).mockResolvedValue({ userId: 'su1' }); + (getAuthUserById as jest.Mock).mockResolvedValue({ + userId: 'su1', + email: 'boss@clustox.com', + name: 'Boss', + role: 'SUPERADMIN', + orgId: null + }); + (listWorkspaces as jest.Mock).mockResolvedValue([ + { id: 'workspace-1', name: 'First' }, + { id: 'workspace-2', name: 'Second' } + ]); + + const session = await getAuthSession(req); + expect(session?.role).toBe('SUPERADMIN'); + // Viewing context, not ownership -- guards let them into any workspace. + expect(session?.orgId).toBe('workspace-1'); + }); + + it('honours a superadmin selected workspace', async () => { + (getToken as jest.Mock).mockResolvedValue({ userId: 'su1' }); + (getAuthUserById as jest.Mock).mockResolvedValue({ + userId: 'su1', + email: 'boss@clustox.com', + name: 'Boss', + role: 'SUPERADMIN', + orgId: null + }); + (workspaceExists as jest.Mock).mockResolvedValue(true); + + const session = await getAuthSession({ + cookies: { 'clustox-workspace': 'workspace-2' } + } as any); + expect(session?.orgId).toBe('workspace-2'); + }); + + it('ignores a stale selected workspace that no longer exists', async () => { + (getToken as jest.Mock).mockResolvedValue({ userId: 'su1' }); + (getAuthUserById as jest.Mock).mockResolvedValue({ + userId: 'su1', + email: 'boss@clustox.com', + name: 'Boss', + role: 'SUPERADMIN', + orgId: null + }); + (workspaceExists as jest.Mock).mockResolvedValue(false); + (listWorkspaces as jest.Mock).mockResolvedValue([ + { id: 'workspace-1', name: 'First' } + ]); + + const session = await getAuthSession({ + cookies: { 'clustox-workspace': 'deleted-workspace' } + } as any); + expect(session?.orgId).toBe('workspace-1'); + }); + + it('reflects a demotion immediately rather than at token expiry', async () => { + (getToken as jest.Mock).mockResolvedValue({ + userId: 'u1', + role: 'SUPERADMIN' + }); + (getAuthUserById as jest.Mock).mockResolvedValue({ + userId: 'u1', + email: 'a@clustox.com', + name: 'A', + role: 'ADMIN' + }); + + const session = await getAuthSession(req); + expect(session?.role).toBe('ADMIN'); + }); +}); diff --git a/web-server/src/auth/__tests__/workspace-guard.test.ts b/web-server/src/auth/__tests__/workspace-guard.test.ts new file mode 100644 index 000000000..b30b3786a --- /dev/null +++ b/web-server/src/auth/__tests__/workspace-guard.test.ts @@ -0,0 +1,116 @@ +jest.mock('@/auth/queries', () => ({ + getTeamIdsForUser: jest.fn(), + getAllTeamIds: jest.fn(), + getTeamOrgId: jest.fn(), + getTeamIdsForOrg: jest.fn() +})); + +import { + assertTeamAccess, + assertWorkspaceAccess, + getAccessibleTeamIds +} from '@/auth/guard'; +import { getTeamIdsForOrg, getTeamOrgId } from '@/auth/queries'; +import { AuthSession } from '@/auth/types'; + +const superadmin: AuthSession = { + userId: 'su1', + email: 'boss@clustox.com', + name: 'Boss', + role: 'SUPERADMIN', + orgId: null +}; + +const admin: AuthSession = { + userId: 'ad1', + email: 'lead@clustox.com', + name: 'Lead', + role: 'ADMIN', + orgId: 'workspace-1' +}; + +const adminWithoutWorkspace: AuthSession = { + ...admin, + userId: 'ad2', + orgId: null +}; + +describe('assertWorkspaceAccess', () => { + it('lets a superadmin into any workspace', async () => { + await expect( + assertWorkspaceAccess(superadmin, 'workspace-9') + ).resolves.toBeUndefined(); + }); + + it('lets an admin into their own workspace', async () => { + await expect( + assertWorkspaceAccess(admin, 'workspace-1') + ).resolves.toBeUndefined(); + }); + + it('403s an admin reaching into another workspace', async () => { + await expect( + assertWorkspaceAccess(admin, 'workspace-2') + ).rejects.toMatchObject({ status: 403 }); + }); + + it('403s an admin who has no workspace at all', async () => { + await expect( + assertWorkspaceAccess(adminWithoutWorkspace, 'workspace-1') + ).rejects.toMatchObject({ status: 403 }); + }); + + it('401s when unauthenticated', async () => { + await expect( + assertWorkspaceAccess(null, 'workspace-1') + ).rejects.toMatchObject({ status: 401 }); + }); +}); + +describe('assertTeamAccess is now workspace-derived', () => { + beforeEach(() => jest.clearAllMocks()); + + it('lets an admin reach any team inside their own workspace', async () => { + // No explicit ClustoxUserTeamAccess row: ownership of the workspace is + // what grants access now. + (getTeamOrgId as jest.Mock).mockResolvedValue('workspace-1'); + await expect(assertTeamAccess(admin, 'team-a')).resolves.toBeUndefined(); + }); + + it('403s an admin reaching a team in another workspace', async () => { + (getTeamOrgId as jest.Mock).mockResolvedValue('workspace-2'); + await expect(assertTeamAccess(admin, 'team-b')).rejects.toMatchObject({ + status: 403 + }); + }); + + it('403s on a team that does not exist', async () => { + (getTeamOrgId as jest.Mock).mockResolvedValue(null); + await expect(assertTeamAccess(admin, 'team-ghost')).rejects.toMatchObject({ + status: 403 + }); + }); + + it('lets a superadmin reach any team without a lookup', async () => { + await expect( + assertTeamAccess(superadmin, 'team-anything') + ).resolves.toBeUndefined(); + expect(getTeamOrgId).not.toHaveBeenCalled(); + }); +}); + +describe('getAccessibleTeamIds', () => { + beforeEach(() => jest.clearAllMocks()); + + it('gives an admin every team in their workspace', async () => { + (getTeamIdsForOrg as jest.Mock).mockResolvedValue(['t1', 't2']); + await expect(getAccessibleTeamIds(admin)).resolves.toEqual(['t1', 't2']); + expect(getTeamIdsForOrg).toHaveBeenCalledWith('workspace-1'); + }); + + it('gives an admin with no workspace nothing', async () => { + await expect(getAccessibleTeamIds(adminWithoutWorkspace)).resolves.toEqual( + [] + ); + }); +}); diff --git a/web-server/src/auth/bootstrap.ts b/web-server/src/auth/bootstrap.ts new file mode 100644 index 000000000..c7d3f0ef0 --- /dev/null +++ b/web-server/src/auth/bootstrap.ts @@ -0,0 +1,31 @@ +import { countSuperadmins, createUser } from './queries'; + +export const bootstrapSuperadmin = async (): Promise => { + const existing = await countSuperadmins(); + if (existing > 0) return; + + const email = process.env.SUPERADMIN_EMAIL; + const password = process.env.SUPERADMIN_PASSWORD; + + if (!email || !password) { + // Fail loudly but do not invent a default account. A known-value fallback + // is worse than having no superadmin at all. + console.error( + '[CLUSTOX] No superadmin exists and SUPERADMIN_EMAIL / SUPERADMIN_PASSWORD ' + + 'are not set. Nobody can administer this instance until they are.' + ); + return; + } + + // No workspace: a superadmin sits above every workspace rather than owning + // one, so createUser leaves org_id null for this role. + await createUser({ + name: 'Superadmin', + email, + password, + role: 'SUPERADMIN', + teamIds: [] + }); + + console.info(`[CLUSTOX] Bootstrapped superadmin ${email}`); +}; diff --git a/web-server/src/auth/guard.ts b/web-server/src/auth/guard.ts new file mode 100644 index 000000000..11d7c5c5e --- /dev/null +++ b/web-server/src/auth/guard.ts @@ -0,0 +1,83 @@ +import { Errors, ResponseError } from '@/constants/error'; + +import { getTeamIdsForOrg, getTeamOrgId } from './queries'; +import { AuthSession, ClustoxRole } from './types'; + +export const assertAuthenticated = ( + session: AuthSession | null +): AuthSession => { + if (!session) throw new ResponseError(Errors.SESSION_USER_NOT_FOUND, 401); + return session; +}; + +export const assertRole = ( + session: AuthSession | null, + role: ClustoxRole +): AuthSession => { + const authed = assertAuthenticated(session); + if (authed.role !== role) throw new ResponseError(Errors.ACCESS_DENIED, 403); + return authed; +}; + +/** A superadmin sits above every workspace rather than owning one. */ +export const canAccessAllWorkspaces = (session: AuthSession): boolean => + session.role === 'SUPERADMIN'; + +/** Retained name for readability at call sites that talk about teams. */ +export const canAccessAllTeams = canAccessAllWorkspaces; + +/** + * Gate a request against a specific workspace. + * + * An admin may only touch the workspace they own. An admin with no workspace + * is denied everything -- that state means their account exists but has not + * been provisioned, and silently showing them an empty app reads as data loss + * rather than as a permissions problem. + */ +export const assertWorkspaceAccess = async ( + session: AuthSession | null, + orgId: string +): Promise => { + const authed = assertAuthenticated(session); + if (canAccessAllWorkspaces(authed)) return; + + if (!authed.orgId || authed.orgId !== orgId) { + throw new ResponseError(Errors.ACCESS_DENIED, 403); + } +}; + +/** + * Gate a request against a team. + * + * Access is derived from which workspace the team belongs to, not from a + * per-team grant. An admin owns their whole workspace, so listing individual + * teams would be redundant. ClustoxUserTeamAccess is left in place unused, so + * finer-grained scoping can be reinstated inside a workspace later without a + * migration. + */ +export const assertTeamAccess = async ( + session: AuthSession | null, + teamId: string +): Promise => { + const authed = assertAuthenticated(session); + if (canAccessAllWorkspaces(authed)) return; + + const teamOrgId = await getTeamOrgId(teamId); + // A missing team is denied rather than 404'd, so an admin cannot probe for + // which team ids exist outside their workspace. + if (!teamOrgId || !authed.orgId || teamOrgId !== authed.orgId) { + throw new ResponseError(Errors.ACCESS_DENIED, 403); + } +}; + +/** Every team the session may see. */ +export const getAccessibleTeamIds = async ( + session: AuthSession +): Promise => { + if (canAccessAllWorkspaces(session)) { + const { getAllTeamIds } = await import('./queries'); + return getAllTeamIds(); + } + if (!session.orgId) return []; + return getTeamIdsForOrg(session.orgId); +}; diff --git a/web-server/src/auth/invites.ts b/web-server/src/auth/invites.ts new file mode 100644 index 000000000..8b10249be --- /dev/null +++ b/web-server/src/auth/invites.ts @@ -0,0 +1,203 @@ +import { createHash, randomBytes } from 'node:crypto'; + +import { Table } from '@/constants/db'; +import { db } from '@/utils/db'; + +import { createUser } from './queries'; +import { ClustoxRole } from './types'; + +/** Days a link stays usable. Long enough to survive a weekend. */ +export const INVITE_TTL_DAYS = 7; + +export type PendingInvite = { + id: string; + email: string; + name: string; + role: ClustoxRole; + orgId: string | null; + orgName: string | null; + createdAt: string; + expiresAt: string; + expired: boolean; + // Whether the invite email actually went out -- separate from the link + // existing, since SMTP is optional and best-effort (see + // src/utils/mailer.ts). A superadmin can always fall back to copying the + // link, but the list should be able to say whether that's necessary. + emailed: boolean; +}; + +/** + * The link is a bearer credential, so only its hash is stored -- reading the + * table must not let anyone accept an invitation. + * + * SHA-256 rather than bcrypt is deliberate. The token is 32 random bytes, so + * it is not brute-forceable and does not need a slow hash; and a fast + * deterministic hash makes acceptance a single indexed lookup rather than a + * scan comparing against every pending row. + */ +const hashToken = (token: string) => + createHash('sha256').update(token).digest('hex'); + +export const createInvite = async (input: { + name: string; + email: string; + role: ClustoxRole; + orgId: string | null; + createdBy: string; +}): Promise<{ id: string; token: string; expiresAt: Date }> => { + const token = randomBytes(32).toString('hex'); + const expiresAt = new Date( + Date.now() + INVITE_TTL_DAYS * 24 * 60 * 60 * 1000 + ); + + const [row] = await db(Table.ClustoxInvite) + .insert({ + token_hash: hashToken(token), + email: input.email, + name: input.name, + role: input.role, + org_id: input.role === 'ADMIN' ? input.orgId : null, + created_by: input.createdBy, + expires_at: expiresAt + }) + .returning('id'); + + const id = typeof row === 'string' ? row : row.id; + + return { id, token, expiresAt }; +}; + +/** Best-effort delivery marker -- see the `emailed` field on PendingInvite. */ +export const markInviteEmailed = async (inviteId: string): Promise => { + await db(Table.ClustoxInvite) + .where('id', inviteId) + .update({ emailed_at: new Date() }); +}; + +export const listPendingInvites = async (): Promise => { + const rows = await db(Table.ClustoxInvite) + .leftJoin( + Table.Organization, + `${Table.Organization}.id`, + `${Table.ClustoxInvite}.org_id` + ) + .whereNull(`${Table.ClustoxInvite}.accepted_at`) + .whereNull(`${Table.ClustoxInvite}.revoked_at`) + .select( + `${Table.ClustoxInvite}.id`, + `${Table.ClustoxInvite}.email`, + `${Table.ClustoxInvite}.name`, + `${Table.ClustoxInvite}.role`, + `${Table.ClustoxInvite}.org_id`, + `${Table.ClustoxInvite}.created_at`, + `${Table.ClustoxInvite}.expires_at`, + `${Table.ClustoxInvite}.emailed_at`, + `${Table.Organization}.name as org_name` + ) + .orderBy(`${Table.ClustoxInvite}.created_at`, 'desc'); + + const now = Date.now(); + + return rows.map((r: any) => ({ + id: r.id, + email: r.email, + name: r.name, + role: r.role as ClustoxRole, + orgId: r.org_id ?? null, + orgName: r.org_name ?? null, + createdAt: new Date(r.created_at).toISOString(), + expiresAt: new Date(r.expires_at).toISOString(), + // Surfaced rather than hidden: an expired invite still explains why + // someone's link stopped working. + expired: new Date(r.expires_at).getTime() < now, + emailed: Boolean(r.emailed_at) + })); +}; + +export const revokeInvite = async (inviteId: string): Promise => { + await db(Table.ClustoxInvite) + .where('id', inviteId) + .whereNull('accepted_at') + .update({ revoked_at: new Date() }); +}; + +export type InvitePreview = { + email: string; + name: string; + role: ClustoxRole; + orgName: string | null; +}; + +/** + * Look up a usable invite by its raw token. + * + * Returns null for anything unusable -- unknown, spent, revoked or expired -- + * without distinguishing between them. The endpoint is unauthenticated, so + * telling a caller that a token exists but is expired would confirm a valid + * token to someone guessing. + */ +export const findUsableInvite = async (token: string) => { + const row = await db(Table.ClustoxInvite) + .where('token_hash', hashToken(token)) + .whereNull('accepted_at') + .whereNull('revoked_at') + .first(); + + if (!row) return null; + if (new Date(row.expires_at).getTime() < Date.now()) return null; + + return row; +}; + +export const previewInvite = async ( + token: string +): Promise => { + const row = await findUsableInvite(token); + if (!row) return null; + + const org = row.org_id + ? await db(Table.Organization).where('id', row.org_id).first() + : null; + + return { + email: row.email, + name: row.name, + role: row.role as ClustoxRole, + orgName: org?.name ?? null + }; +}; + +/** + * Redeem an invite, creating the account with a password the invitee chose. + * + * The invite is marked spent in the same step, so a link that leaks after use + * grants nothing. + */ +export const acceptInvite = async ( + token: string, + password: string +): Promise<{ ok: true; email: string } | { ok: false; reason: string }> => { + const row = await findUsableInvite(token); + if (!row) return { ok: false, reason: 'INVALID' }; + + const existing = await db(Table.Users) + .where('primary_email', row.email) + .andWhere('is_deleted', false) + .first(); + if (existing) return { ok: false, reason: 'ALREADY_EXISTS' }; + + const { userId } = await createUser({ + name: row.name, + email: row.email, + password, + role: row.role as ClustoxRole, + teamIds: [], + orgId: row.org_id ?? null + }); + + await db(Table.ClustoxInvite) + .where('id', row.id) + .update({ accepted_at: new Date(), accepted_by: userId }); + + return { ok: true, email: row.email }; +}; diff --git a/web-server/src/auth/password.ts b/web-server/src/auth/password.ts new file mode 100644 index 000000000..5f2f1e848 --- /dev/null +++ b/web-server/src/auth/password.ts @@ -0,0 +1,19 @@ +import { compare, hash } from 'bcryptjs'; + +const COST = 12; + +export const hashPassword = (plain: string): Promise => + hash(plain, COST); + +export const verifyPassword = async ( + plain: string, + passwordHash: string +): Promise => { + try { + return await compare(plain, passwordHash); + } catch { + // A malformed stored hash must read as "wrong password", never as an + // exception that a caller might treat as success. + return false; + } +}; diff --git a/web-server/src/auth/queries.ts b/web-server/src/auth/queries.ts new file mode 100644 index 000000000..549cbfa4a --- /dev/null +++ b/web-server/src/auth/queries.ts @@ -0,0 +1,411 @@ +import { randomUUID } from 'node:crypto'; + +import { Columns, Table } from '@/constants/db'; +import { db } from '@/utils/db'; + +import { hashPassword } from './password'; +import { AuthUserListItem, AuthUserRow, ClustoxRole } from './types'; + +export const getAuthUserByEmail = async ( + email: string +): Promise => { + const row = await db(Table.ClustoxUserAuth) + .join(Table.Users, `${Table.Users}.id`, `${Table.ClustoxUserAuth}.user_id`) + .where(`${Table.Users}.primary_email`, email) + .andWhere(`${Table.Users}.is_deleted`, false) + .select( + `${Table.ClustoxUserAuth}.user_id`, + `${Table.ClustoxUserAuth}.role`, + `${Table.ClustoxUserAuth}.password_hash`, + `${Table.Users}.primary_email`, + `${Table.Users}.name`, + `${Table.Users}.org_id` + ) + .first(); + + if (!row) return null; + + return { + userId: row.user_id, + email: row.primary_email, + name: row.name, + role: row.role as ClustoxRole, + orgId: row.org_id ?? null, + passwordHash: row.password_hash + }; +}; + +/** + * Current auth state for a user id, straight from the database. + * + * Sessions are JWTs, so the token alone proves only that someone signed in at + * some point. It says nothing about whether the account still exists or still + * holds the role it had at sign-in. Every request resolves both here instead. + */ +export const getAuthUserById = async ( + userId: string +): Promise<{ + userId: string; + email: string; + name: string; + role: ClustoxRole; + orgId: string | null; +} | null> => { + const row = await db(Table.ClustoxUserAuth) + .join(Table.Users, `${Table.Users}.id`, `${Table.ClustoxUserAuth}.user_id`) + .where(`${Table.ClustoxUserAuth}.user_id`, userId) + .andWhere(`${Table.Users}.is_deleted`, false) + .select( + `${Table.ClustoxUserAuth}.user_id`, + `${Table.ClustoxUserAuth}.role`, + `${Table.Users}.primary_email`, + `${Table.Users}.name`, + `${Table.Users}.org_id` + ) + .first(); + + if (!row) return null; + + return { + userId: row.user_id, + email: row.primary_email, + name: row.name, + role: row.role as ClustoxRole, + // null for SUPERADMIN, who sits above every workspace. + orgId: row.org_id ?? null + }; +}; + +export const getTeamIdsForUser = async (userId: string): Promise => { + const rows = await db(Table.ClustoxUserTeamAccess) + .select(Columns[Table.ClustoxUserTeamAccess].team_id) + .where(Columns[Table.ClustoxUserTeamAccess].user_id, userId); + return rows.map((r: { team_id: string }) => r.team_id); +}; + +export const getAllTeamIds = async (): Promise => { + const rows = await db(Table.Team).select('id').where('is_deleted', false); + return rows.map((r: { id: string }) => r.id); +}; + +/** Which workspace a team belongs to, or null if the team does not exist. */ +export const getTeamOrgId = async (teamId: string): Promise => { + const row = await db(Table.Team) + .select('org_id') + .where('id', teamId) + .andWhere('is_deleted', false) + .first(); + return row?.org_id ?? null; +}; + +/** Every workspace, oldest first. Used by the SuperAdmin switcher. */ +export const listWorkspaces = async (): Promise< + { id: string; name: string }[] +> => { + const rows = await db(Table.Organization) + .select('id', 'name') + .orderBy('created_at', 'asc'); + return rows.map((r: { id: string; name: string }) => ({ + id: r.id, + name: r.name + })); +}; + +export type WorkspaceSummary = { + id: string; + name: string; + ownerEmail: string | null; + hasIntegration: boolean; + repoCount: number; + teamCount: number; +}; + +/** + * Every workspace with enough context to answer "is this one healthy?". + * + * Deliberately excludes sync status: that lives on the sync server, so the + * route composes the two rather than this query reaching across a service + * boundary. + */ +export const listWorkspaceSummaries = async (): Promise => { + const [orgs, owners, integrations, repos, teams] = await Promise.all([ + db(Table.Organization).select('id', 'name').orderBy('created_at', 'asc'), + db(Table.ClustoxUserAuth) + .join(Table.Users, `${Table.Users}.id`, `${Table.ClustoxUserAuth}.user_id`) + .where(`${Table.ClustoxUserAuth}.role`, 'ADMIN') + .andWhere(`${Table.Users}.is_deleted`, false) + .select(`${Table.Users}.org_id`, `${Table.Users}.primary_email`), + db(Table.Integration) + .whereNotNull('access_token_enc_chunks') + .select('org_id'), + db(Table.OrgRepo).where('is_active', true).select('org_id'), + db(Table.Team).where('is_deleted', false).select('org_id') + ]); + + const countBy = (rows: { org_id: string }[], orgId: string) => + rows.filter((r) => r.org_id === orgId).length; + + return orgs.map((o: { id: string; name: string }) => ({ + id: o.id, + name: o.name, + ownerEmail: + owners.find((u: { org_id: string }) => u.org_id === o.id) + ?.primary_email ?? null, + hasIntegration: integrations.some( + (i: { org_id: string }) => i.org_id === o.id + ), + repoCount: countBy(repos, o.id), + teamCount: countBy(teams, o.id) + })); +}; + +/** Does this workspace exist? Guards the SuperAdmin's selected workspace. */ +export const workspaceExists = async (orgId: string): Promise => { + const row = await db(Table.Organization).select('id').where('id', orgId).first(); + return Boolean(row); +}; + +/** Every live team in a workspace. */ +export const getTeamIdsForOrg = async (orgId: string): Promise => { + const rows = await db(Table.Team) + .select('id') + .where('org_id', orgId) + .andWhere('is_deleted', false); + return rows.map((r: { id: string }) => r.id); +}; + +export const countSuperadmins = async (): Promise => { + const rows = await db(Table.ClustoxUserAuth) + .where(Columns[Table.ClustoxUserAuth].role, 'SUPERADMIN') + .select(Columns[Table.ClustoxUserAuth].user_id); + return rows.length; +}; + +export const listUsers = async (): Promise => { + const rows = await db(Table.ClustoxUserAuth) + .join(Table.Users, `${Table.Users}.id`, `${Table.ClustoxUserAuth}.user_id`) + .leftJoin( + Table.Organization, + `${Table.Organization}.id`, + `${Table.Users}.org_id` + ) + .where(`${Table.Users}.is_deleted`, false) + .select( + `${Table.ClustoxUserAuth}.user_id`, + `${Table.ClustoxUserAuth}.role`, + `${Table.Users}.primary_email`, + `${Table.Users}.name`, + `${Table.Users}.org_id`, + // left join: null for a superadmin, who owns no workspace + `${Table.Organization}.name as org_name` + ); + + const access = await db(Table.ClustoxUserTeamAccess).select( + 'user_id', + 'team_id' + ); + + return rows.map((r: any) => ({ + userId: r.user_id, + email: r.primary_email, + name: r.name, + role: r.role as ClustoxRole, + orgId: r.org_id ?? null, + orgName: r.org_name ?? null, + teamIds: access + .filter((a: any) => a.user_id === r.user_id) + .map((a: any) => a.team_id) + })); +}; + +/** + * Create a workspace and return its id. + * + * Upstream only ever created one Organization, named "default", at boot. The + * schema was always multi-workspace -- every table carries org_id and + * Integration is keyed on (name, org_id) -- so provisioning more is simply + * doing what the schema already allows. + */ +export const createWorkspace = async (name: string): Promise => { + // Organization.id has no database default -- upstream's bootstrap supplies + // one from Python. The exported `db` helper is a table function, not the + // knex instance, so there is no db.raw() here either. Generate it in JS. + const id = randomUUID(); + + await db(Table.Organization).insert({ + id, + name, + // Organization.domain carries a UNIQUE constraint. Deriving it from the + // workspace name meant two admins with the same name could not both exist + // -- "Ali Khan" joining twice would fail on a duplicate key. The id is + // unique by construction and nothing reads this column. + domain: id, + created_at: new Date() + }); + + return id; +}; + +/** + * Create a user. + * + * An ADMIN gets their own workspace, provisioned here. A SUPERADMIN gets none + * (org_id null) because they sit above every workspace rather than owning one. + */ +export const createUser = async (input: { + name: string; + email: string; + password: string; + role: ClustoxRole; + teamIds: string[]; + /** Existing workspace to place an ADMIN in. Omit to provision a new one. */ + orgId?: string | null; +}): Promise<{ userId: string; orgId: string | null }> => { + const passwordHash = await hashPassword(input.password); + + let orgId: string | null = null; + if (input.role === 'ADMIN') { + orgId = input.orgId ?? (await createWorkspace(input.name)); + } + + const [user] = await db(Table.Users) + .insert({ + org_id: orgId, + name: input.name, + primary_email: input.email, + is_deleted: false + }) + .returning('id'); + + const userId = typeof user === 'string' ? user : user.id; + + await db(Table.ClustoxUserAuth).insert({ + user_id: userId, + password_hash: passwordHash, + role: input.role + }); + + if (input.teamIds.length) { + await db(Table.ClustoxUserTeamAccess).insert( + input.teamIds.map((team_id) => ({ user_id: userId, team_id })) + ); + } + + return { userId, orgId }; +}; + +/** + * Workspaces with no admin owning them. + * + * The pre-multitenancy workspace is the motivating case: it holds all the real + * data but predates the idea of ownership, so it belongs to nobody. Adopting it + * is preferable to recreating its integration and repositories from scratch. + */ +export const listUnownedWorkspaces = async (): Promise< + { id: string; name: string }[] +> => { + const [orgs, owned] = await Promise.all([ + db(Table.Organization).select('id', 'name').orderBy('created_at', 'asc'), + db(Table.ClustoxUserAuth) + .join(Table.Users, `${Table.Users}.id`, `${Table.ClustoxUserAuth}.user_id`) + .where(`${Table.ClustoxUserAuth}.role`, 'ADMIN') + .andWhere(`${Table.Users}.is_deleted`, false) + .whereNotNull(`${Table.Users}.org_id`) + .select(`${Table.Users}.org_id`) + ]); + + const ownedIds = new Set(owned.map((r: { org_id: string }) => r.org_id)); + return orgs.filter((o: { id: string }) => !ownedIds.has(o.id)); +}; + +/** + * Remove a user, and their workspace if removing them would strand it empty. + * + * The user is soft-deleted, matching how upstream treats Users and Teams. The + * workspace is only hard-deleted when it holds nothing worth keeping -- no + * integration, no repositories, and no other owner. A workspace with real data + * is left behind unowned instead, where it can be adopted; silently destroying + * synced history because the last admin left would be the wrong default. + * + * Returns whether the workspace was removed, so callers can say what happened. + */ +export const deleteUserAndEmptyWorkspace = async ( + userId: string +): Promise<{ workspaceRemoved: boolean }> => { + const user = await db(Table.Users).where('id', userId).first(); + const orgId: string | null = user?.org_id ?? null; + + // org_id is cleared as well as flagging the row deleted: a removed user + // should not still own a workspace, and the Users.org_id foreign key would + // otherwise block removing the workspace below. + await db(Table.Users) + .where('id', userId) + .update({ is_deleted: true, org_id: null, updated_at: new Date() }); + await db(Table.ClustoxUserAuth).where('user_id', userId).delete(); + await db(Table.ClustoxUserTeamAccess).where('user_id', userId).delete(); + + if (!orgId) return { workspaceRemoved: false }; + + const [otherOwners, repos, integrations] = await Promise.all([ + db(Table.Users) + .where('org_id', orgId) + .andWhere('is_deleted', false) + .whereNot('id', userId) + .select('id'), + db(Table.OrgRepo).where('org_id', orgId).select('id'), + db(Table.Integration).where('org_id', orgId).select('org_id') + ]); + + const worthKeeping = + otherOwners.length > 0 || repos.length > 0 || integrations.length > 0; + if (worthKeeping) return { workspaceRemoved: false }; + + // Nothing references the workspace except its own teams, and the foreign + // keys have no cascade, so those go first. + await db(Table.ClustoxUserTeamAccess) + .whereIn( + 'team_id', + db(Table.Team).where('org_id', orgId).select('id') + ) + .delete(); + await db(Table.Team).where('org_id', orgId).delete(); + await db(Table.ClustoxSyncRun).where('org_id', orgId).delete(); + await db(Table.ClustoxInvite).where('org_id', orgId).update({ org_id: null }); + await db(Table.Organization).where('id', orgId).delete(); + + return { workspaceRemoved: true }; +}; + +/** Move an admin into a different workspace, or detach them from one. */ +export const setUserWorkspace = async ( + userId: string, + orgId: string | null +): Promise => { + await db(Table.Users) + .where('id', userId) + .update({ org_id: orgId, updated_at: new Date() }); +}; + +export const updateUserRole = async ( + userId: string, + role: ClustoxRole +): Promise => { + await db(Table.ClustoxUserAuth) + .where(Columns[Table.ClustoxUserAuth].user_id, userId) + .update({ role, updated_at: new Date() }); +}; + +export const setUserTeams = async ( + userId: string, + teamIds: string[] +): Promise => { + await db(Table.ClustoxUserTeamAccess) + .where(Columns[Table.ClustoxUserTeamAccess].user_id, userId) + .delete(); + + if (teamIds.length) { + await db(Table.ClustoxUserTeamAccess).insert( + teamIds.map((team_id) => ({ user_id: userId, team_id })) + ); + } +}; diff --git a/web-server/src/auth/session.ts b/web-server/src/auth/session.ts new file mode 100644 index 000000000..6fcc91052 --- /dev/null +++ b/web-server/src/auth/session.ts @@ -0,0 +1,62 @@ +import { getToken } from 'next-auth/jwt'; +import { NextApiRequest } from 'next/types'; + +import { getAuthUserById, listWorkspaces, workspaceExists } from './queries'; +import { AuthSession } from './types'; + +const WORKSPACE_COOKIE = 'clustox-workspace'; + +/** + * Which workspace a SuperAdmin is currently looking at. + * + * A SuperAdmin owns no workspace, but the entire UI is workspace-scoped, so + * without one there is nothing to render. Their selection is a *viewing* + * context only -- guards let a SuperAdmin into every workspace regardless of + * what is selected here. + * + * Falls back to the oldest workspace so a fresh SuperAdmin lands somewhere + * useful rather than on an empty app. + */ +const resolveViewingWorkspace = async ( + req: NextApiRequest +): Promise => { + const selected = req.cookies?.[WORKSPACE_COOKIE]; + if (selected && (await workspaceExists(selected))) return selected; + + const workspaces = await listWorkspaces(); + return workspaces[0]?.id ?? null; +}; + +/** + * Resolve the caller's session. + * + * The JWT is only used to identify *who* is claiming to be signed in. Whether + * that account still exists, and what role it currently holds, are read from + * the database on every request. + * + * Without this, a JWT stays valid for its full lifetime regardless of what + * happens to the account behind it: a deleted user keeps a working session, + * and a demoted superadmin keeps superadmin powers until the token expires. + */ +export const getAuthSession = async ( + req: NextApiRequest +): Promise => { + const token = await getToken({ req, secret: process.env.NEXTAUTH_SECRET }); + if (!token?.userId) return null; + + const user = await getAuthUserById(token.userId as string); + if (!user) return null; + + const orgId = + user.role === 'SUPERADMIN' + ? await resolveViewingWorkspace(req) + : user.orgId; + + return { + userId: user.userId, + email: user.email, + name: user.name, + role: user.role, + orgId + }; +}; diff --git a/web-server/src/auth/types.ts b/web-server/src/auth/types.ts new file mode 100644 index 000000000..ceb341398 --- /dev/null +++ b/web-server/src/auth/types.ts @@ -0,0 +1,35 @@ +export type ClustoxRole = 'SUPERADMIN' | 'ADMIN'; + +export interface AuthSession { + userId: string; + email: string; + name: string; + role: ClustoxRole; + /** + * The workspace this user owns. + * + * null for SUPERADMIN, who sits above every workspace rather than owning + * one. Upstream's Users.org_id is already nullable, so this needs no schema + * change -- the column was built for exactly this. + */ + orgId: string | null; +} + +export interface AuthUserRow { + userId: string; + email: string; + name: string; + role: ClustoxRole; + orgId: string | null; + passwordHash: string; +} + +export interface AuthUserListItem { + userId: string; + email: string; + name: string; + role: ClustoxRole; + orgId: string | null; + orgName: string | null; + teamIds: string[]; +} diff --git a/web-server/src/components/AccountMenu/index.tsx b/web-server/src/components/AccountMenu/index.tsx new file mode 100644 index 000000000..8515ea236 --- /dev/null +++ b/web-server/src/components/AccountMenu/index.tsx @@ -0,0 +1,123 @@ +import { + ExpandMoreRounded, + LogoutTwoTone, + ManageAccountsTwoTone +} from '@mui/icons-material'; +import { + Divider, + ListItemIcon, + ListItemText, + Menu, + MenuItem, + Typography, + useTheme +} from '@mui/material'; +import { signOut } from 'next-auth/react'; +import NextLink from 'next/link'; +import { FC, useState } from 'react'; + +import { FlexBox } from '@/components/FlexBox'; +import { RoleChip } from '@/components/RoleChip'; +import { UserAvatar } from '@/components/UserAvatar'; +import { ROUTES } from '@/constants/routes'; +import { useClustoxUser } from '@/hooks/useClustoxUser'; + +/** + * CLUSTOX: the account identity + sign-out surface, moved to the top-right + * of every page (rendered once, inside PageHeader) instead of living as a + * static block at the bottom of the sidebar. "Manage users" is included + * here as a shortcut for Superadmins, mirroring its (also role-gated) + * sidebar entry. + * + * No "Profile settings" entry: there's no per-user profile page in this + * app yet (the existing /settings page is org-level sync config, not + * personal account settings) -- adding a menu item that opens the wrong + * thing would be worse than not having one. + */ +export const AccountMenu: FC = () => { + const { user, loading, isSuperadmin } = useClustoxUser(); + const [anchor, setAnchor] = useState(null); + const theme = useTheme(); + + if (loading || !user) return null; + + return ( + <> + setAnchor(e.currentTarget)} + > + + + + {user.name} + + + + + + + setAnchor(null)} + anchorOrigin={{ vertical: 'bottom', horizontal: 'right' }} + transformOrigin={{ vertical: 'top', horizontal: 'right' }} + PaperProps={{ sx: { width: 250, mt: 1 } }} + > + + + {user.name} + + + {user.email} + + + + + {isSuperadmin && ( + + setAnchor(null)}> + + + + + + + )} + + { + await signOut({ redirect: false }); + window.location.assign('/login'); + }} + sx={{ color: 'error.main' }} + > + + + + + + + + ); +}; diff --git a/web-server/src/components/BenchmarkSettingsForm.tsx b/web-server/src/components/BenchmarkSettingsForm.tsx new file mode 100644 index 000000000..43defbfa5 --- /dev/null +++ b/web-server/src/components/BenchmarkSettingsForm.tsx @@ -0,0 +1,465 @@ +import { LoadingButton } from '@mui/lab'; +import { + Alert, + Button, + CircularProgress, + Divider, + TextField +} from '@mui/material'; +import { useSnackbar } from 'notistack'; +import { FC, useCallback, useEffect, useMemo, useState } from 'react'; + +import { handleApi } from '@/api-helpers/axios-api-instance'; +import { useDoraMetricsFetchArgs } from '@/hooks/useDoraMetricsFetchArgs'; +import { + useSingleTeamConfig, + useStateBranchConfig +} from '@/hooks/useStateTeamConfig'; +import { fetchTeamDoraMetrics } from '@/slices/dora_metrics'; +import { useDispatch, useSelector } from '@/store'; +import { + ActiveBranchMode, + FetchTeamSettingsAPIResponse +} from '@/types/resources'; +import { BenchmarkMetric } from '@/utils/benchmarks'; + +import { FlexBox } from './FlexBox'; +import { Line } from './Text'; + +// CLUSTOX: the API stores seconds for these two, but nobody thinks in +// seconds when setting a target. Hours are entered/displayed; converted at +// the read/write boundary only, so the rest of the component (validation, +// dirty-checking, payload building) never has to know which unit a metric is +// really in. +const SECONDS_PER_HOUR = 3600; +// CLUSTOX: `lines_of_code` is deliberately NOT in here. Its unit is lines, +// not hours -- adding it would multiply every target the admin types by 3600 +// on save and divide it back on load, so a 200-line target would be stored +// as 720000 and compared against an actual measured in lines. +const DURATION_METRICS: ReadonlySet = new Set([ + 'lead_time', + 'mean_time_to_recovery' +]); + +const METRIC_LABEL: Record = { + lead_time: 'Lead time', + deployment_frequency: 'Deployment frequency', + change_failure_rate: 'Change failure rate', + mean_time_to_recovery: 'Mean time to recovery', + // CLUSTOX: "Average PR size" rather than "Lines of code" -- the target is + // per merged PR, and an admin reading the raw metric name would reasonably + // type a weekly total into it. + lines_of_code: 'Average PR size' +}; + +// CLUSTOX: derived from METRIC_LABEL rather than written out again, because a +// `Record` is exhaustiveness-checked by tsc and a bare +// `BenchmarkMetric[]` is not. Spelled as its own array, a sixth metric added +// to the union but forgotten here would compile clean and simply never render +// a field -- silently unconfigurable, with no error anywhere to find. +const METRICS = Object.keys(METRIC_LABEL) as BenchmarkMetric[]; + +const UNIT_LABEL: Record = { + lead_time: 'hours', + deployment_frequency: 'per week', + change_failure_rate: '%', + mean_time_to_recovery: 'hours', + lines_of_code: 'lines' +}; + +type BenchmarkSettingValues = Record; +type FormValues = Record; + +const EMPTY_FORM_VALUES: FormValues = { + lead_time: '', + deployment_frequency: '', + change_failure_rate: '', + mean_time_to_recovery: '', + lines_of_code: '' +}; + +/** seconds/percent/deployments-per-week -> the string shown in the input. */ +const toDisplayValue = (metric: BenchmarkMetric, apiValue: number | null) => { + if (apiValue === null || apiValue === undefined) return ''; + const display = DURATION_METRICS.has(metric) + ? apiValue / SECONDS_PER_HOUR + : apiValue; + return String(display); +}; + +/** + * The string in the input -> the payload value, or `undefined` to omit the + * key entirely. + * + * CLUSTOX: an empty string must become "key absent", never "key: 0" -- the + * backend treats an absent key as "inherit" and an explicit 0 as a real + * target of zero. Collapsing empty to 0 here would silently turn every + * cleared field into "fail this metric at every value above zero" the next + * time the form is saved. + */ +const toApiValue = ( + metric: BenchmarkMetric, + raw: string +): number | undefined => { + if (raw.trim() === '') return undefined; + const n = Number(raw); + if (Number.isNaN(n)) return undefined; + return DURATION_METRICS.has(metric) ? n * SECONDS_PER_HOUR : n; +}; + +const roundForDisplay = (n: number) => Math.round(n * 100) / 100; + +/** `24h (default)` / `3.5/week (default)` / `12% (default)`. */ +const formatInheritedPlaceholder = ( + metric: BenchmarkMetric, + apiValue: number +): string => { + const display = DURATION_METRICS.has(metric) + ? apiValue / SECONDS_PER_HOUR + : apiValue; + const n = roundForDisplay(display); + switch (metric) { + case 'lead_time': + case 'mean_time_to_recovery': + return `${n}h (default)`; + case 'deployment_frequency': + return `${n}/week (default)`; + case 'change_failure_rate': + return `${n}% (default)`; + case 'lines_of_code': + return `${n} lines (default)`; + } +}; + +/** Mirrors `validate_benchmark_payload` on the backend, for instant feedback + * without a round trip. The server is still the source of truth -- see the + * catch block in `handleSave`, which surfaces whatever it rejects even when + * this check passed. */ +const clientErrorFor = ( + metric: BenchmarkMetric, + raw: string +): string | null => { + if (raw.trim() === '') return null; + const n = Number(raw); + if (Number.isNaN(n)) return 'Must be a number'; + if (n < 0) return 'Must not be negative'; + if (metric === 'change_failure_rate' && n > 100) + return 'Must be between 0 and 100'; + return null; +}; + +const GLOBAL_URL = '/clustox/benchmarks/global'; +const teamUrl = (teamId: string) => `/internal/team/${teamId}/settings`; + +/** + * A validation message coming back from `validate_benchmark_payload` + * (`mhq/service/settings/benchmarks.py`) always contains the offending + * metric's key verbatim, e.g. "change_failure_rate must be between 0 and + * 100". Flask's default error page wraps that text in HTML, so tags are + * stripped before searching -- this way the field is found whether the BFF + * forwarded a clean JSON message or the raw werkzeug error page. + */ +const fieldFromServerError = ( + raw: unknown +): { metric: BenchmarkMetric | null; message: string } => { + const text = String(raw ?? '') + .replace(/<[^>]+>/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + const metric = METRICS.find((m) => text.includes(m)) ?? null; + return { metric, message: text || 'Could not save benchmarks.' }; +}; + +export const BenchmarkSettingsForm: FC<{ + scope: 'team' | 'global'; + /** Called after a successful save. Team-scope callers use this to close + * the modal that hosts the form; the global section on the Workspaces + * page renders inline and has none to close. */ + onClose?: () => void; +}> = ({ scope, onClose }) => { + const { enqueueSnackbar } = useSnackbar(); + const { singleTeamId } = useSingleTeamConfig(); + const teamId = scope === 'team' ? singleTeamId : null; + const dispatch = useDispatch(); + const doraMetricsFetchArgs = useDoraMetricsFetchArgs(); + const branches = useStateBranchConfig(); + const activeBranchMode = useSelector((s) => s.app.branchMode); + + const [values, setValues] = useState(null); + const [initialValues, setInitialValues] = useState(null); + const [inherited, setInherited] = useState( + null + ); + const [loading, setLoading] = useState(true); + const [loadFailed, setLoadFailed] = useState(false); + const [saving, setSaving] = useState(false); + const [serverErrors, setServerErrors] = useState< + Partial> + >({}); + const [formError, setFormError] = useState(null); + + useEffect(() => { + if (scope === 'team' && !teamId) { + setLoading(false); + return; + } + + let cancelled = false; + setLoading(true); + setLoadFailed(false); + + const request = + scope === 'global' + ? handleApi<{ setting: BenchmarkSettingValues }>(GLOBAL_URL) + : handleApi>( + teamUrl(teamId), + { params: { setting_type: 'BENCHMARK_SETTING' } } + ); + + request + .then((data) => { + if (cancelled) return; + const next = METRICS.reduce((acc, metric) => { + acc[metric] = toDisplayValue(metric, data.setting?.[metric] ?? null); + return acc; + }, {} as FormValues); + setValues(next); + setInitialValues(next); + }) + .catch(() => { + if (!cancelled) setLoadFailed(true); + }) + .finally(() => { + if (!cancelled) setLoading(false); + }); + + return () => { + cancelled = true; + }; + }, [scope, teamId]); + + // CLUSTOX: the placeholder wants to show what clearing a field reverts to. + // For a team, that's the global baseline. `GET /clustox/benchmarks/global` + // is readable by any authenticated admin (see the CLUSTOX comment on that + // route) specifically so this call succeeds for the workspace admin/EM who + // ordinarily opens this form, not just for the platform superadmin. Still + // wrapped as best-effort -- on any failure (a logged-out session, a + // network error) the placeholder falls back to a generic hint instead of a + // figure, rather than the form breaking over something that isn't the + // user's fault. + useEffect(() => { + if (scope !== 'team') return; + let cancelled = false; + handleApi<{ setting: BenchmarkSettingValues }>(GLOBAL_URL) + .then((data) => { + if (!cancelled) setInherited(data.setting); + }) + .catch(() => { + if (!cancelled) setInherited(null); + }); + return () => { + cancelled = true; + }; + }, [scope]); + + const clientErrors = useMemo(() => { + if (!values) return {}; + return METRICS.reduce( + (acc, metric) => { + const err = clientErrorFor(metric, values[metric]); + if (err) acc[metric] = err; + return acc; + }, + {} as Partial> + ); + }, [values]); + + const isDirty = useMemo(() => { + if (!values || !initialValues) return false; + return METRICS.some((metric) => values[metric] !== initialValues[metric]); + }, [values, initialValues]); + + const handleChange = useCallback((metric: BenchmarkMetric, raw: string) => { + setValues((v) => ({ ...(v ?? EMPTY_FORM_VALUES), [metric]: raw })); + setServerErrors((errs) => { + if (!errs[metric]) return errs; + const next = { ...errs }; + delete next[metric]; + return next; + }); + }, []); + + const handleDiscard = useCallback(() => { + if (!initialValues) return; + setValues(initialValues); + setServerErrors({}); + setFormError(null); + }, [initialValues]); + + const handleSave = useCallback(async () => { + if (!values || (scope === 'team' && !teamId)) return; + + const payload = METRICS.reduce((acc, metric) => { + const apiValue = toApiValue(metric, values[metric]); + if (apiValue !== undefined) acc[metric] = apiValue; + return acc; + }, {} as Partial); + + setSaving(true); + setServerErrors({}); + setFormError(null); + + try { + if (scope === 'global') { + await handleApi(GLOBAL_URL, { method: 'PUT', data: payload }); + } else { + await handleApi(teamUrl(teamId), { + method: 'PUT', + data: { setting_type: 'BENCHMARK_SETTING', setting_data: payload } + }); + } + + setInitialValues(values); + + // CLUSTOX: the targets ride on the dora_metrics response, so without + // this the cards keep drawing the OLD target line until a full page + // reload -- the admin saves, the modal closes, and nothing on screen + // changes. Both sibling modals in the same menu + // (TeamProductionBranchSelector, TeamIncidentPRsFilter) already refetch + // on save; this matches them. Team scope only: the global form lives on + // the Workspaces page, where no DORA state is mounted to refresh. + if (scope === 'team') { + await dispatch( + fetchTeamDoraMetrics({ + ...doraMetricsFetchArgs, + branches: + activeBranchMode === ActiveBranchMode.PROD ? null : branches + }) + ); + } + + enqueueSnackbar('Benchmarks updated', { + variant: 'success', + autoHideDuration: 3000 + }); + // CLUSTOX: only close on success. The existing team-settings modals + // (TeamProductionBranchSelector, TeamIncidentPRsFilter) close + // unconditionally and rely on a toast to report failure -- that can't + // work here, since a rejected value has to stay on screen, next to the + // field it was rejected for, for the admin to fix. + onClose?.(); + } catch (err: any) { + // CLUSTOX: server errors are shown against the field named in the + // message, never as a toast -- see the comment above `handleSave`. + const raw = err?.data?.message ?? err?.data ?? err?.message; + const { metric, message } = fieldFromServerError(raw); + if (metric) setServerErrors({ [metric]: message }); + else setFormError(message); + } finally { + setSaving(false); + } + }, [ + values, + scope, + teamId, + enqueueSnackbar, + onClose, + dispatch, + doraMetricsFetchArgs, + activeBranchMode, + branches + ]); + + if (scope === 'team' && !teamId) { + return ( + Select a team to configure its benchmarks. + ); + } + + if (loading) { + return ( + + + + ); + } + + if (loadFailed || !values) { + return ( + + Could not load benchmark settings. Try again shortly. + + ); + } + + return ( + + {scope === 'global' && ( + <> + + Global DORA Benchmarks + + + The default target every team is measured against unless it sets its + own. + + + + )} + + {formError && {formError}} + + + {METRICS.map((metric) => { + const errorText = clientErrors[metric] || serverErrors[metric]; + const placeholder = + scope === 'team' && inherited?.[metric] != null + ? formatInheritedPlaceholder(metric, inherited[metric] as number) + : 'not set'; + + return ( + handleChange(metric, e.target.value)} + /> + ); + })} + + + + + 0} + sx={{ width: '140px' }} + onClick={handleSave} + > + Save + + + + ); +}; diff --git a/web-server/src/components/ClustoxJenkinsMapping.tsx b/web-server/src/components/ClustoxJenkinsMapping.tsx new file mode 100644 index 000000000..adce27cdb --- /dev/null +++ b/web-server/src/components/ClustoxJenkinsMapping.tsx @@ -0,0 +1,483 @@ +import { LinkOff } from '@mui/icons-material'; +import { LoadingButton } from '@mui/lab'; +import { + Alert, + Button, + CircularProgress, + Dialog, + DialogActions, + DialogContent, + DialogTitle, + IconButton, + MenuItem, + Table, + TableBody, + TableCell, + TableContainer, + TableHead, + TableRow, + TextField, + Tooltip, + useTheme +} from '@mui/material'; +import { useCallback, useEffect, useMemo, useState } from 'react'; + +import { handleApi } from '@/api-helpers/axios-api-instance'; +import { FlexBox } from '@/components/FlexBox'; +import { Line } from '@/components/Text'; +import { Integration } from '@/constants/integrations'; +import { useAuth } from '@/hooks/useAuth'; +// CLUSTOX: server-resolved workspace, consistent with ClustoxJenkinsSetup. +import { useClustoxUser } from '@/hooks/useClustoxUser'; +import { readApiError } from '@/utils/api-error'; + +type JenkinsJob = { name: string; full_name: string; url: string }; +type WorkspaceRepo = { id: string; name: string; provider: Integration }; +type JenkinsMapping = { + repo_workflow_id: string; + org_repo_id: string; + job_full_name: string; + repo_name: string; +}; +type Pending = + | { + action: 'map'; + repoId: string; + repoName: string; + jobFullName: string; + deactivatesWorkflows: boolean; + teamCount: number; + } + | { + action: 'unmap'; + repoId: string; + repoName: string; + jobFullName: string; + repoWorkflowId: string; + teamCount: number; + }; + +const codeProviders = [ + Integration.GITHUB, + Integration.GITLAB, + Integration.BITBUCKET +]; + +type TeamReposResponse = { + teamReposMap: Record; +}; + +// CLUSTOX: the change is per repo, not per team, so it lands on every team +// tracking the repo -- including teams this admin is not currently looking at. +const everyTeamSentence = (teamCount: number) => + `This affects every team tracking this repo${ + teamCount > 0 + ? ` (${teamCount === 1 ? '1 team' : `${teamCount} teams`})` + : '' + }, not only the team you are viewing.`; + +// CLUSTOX: the server's own words whenever it sent any, because the cases it +// reports are ones retrying cannot fix -- a 409 means another workflow already +// holds this job id, and "try again" is actively wrong advice there. A 5xx or a +// dropped connection carries nothing to show, so those keep the fallback. +const mutationError = (err: unknown, fallback: string) => { + const { status, message } = readApiError(err); + return status && status < 500 && message ? message : fallback; +}; + +/** + * CLUSTOX: maps each repo to the Jenkins job whose builds count as its + * deployments. + * + * Selections are read back from the server on every load, so a mapped repo + * still reads as mapped after a reload. It used to show "Select a Jenkins job" + * for a repo that was mapped and ingesting builds, which reads as a lost + * configuration and invites an admin to map it a second time. + */ +export const ClustoxJenkinsMapping = () => { + const theme = useTheme(); + const { orgId: contextOrgId } = useAuth(); + const { orgId: sessionOrgId } = useClustoxUser(); + const orgId = sessionOrgId ?? contextOrgId; + + const [repos, setRepos] = useState([]); + const [jobs, setJobs] = useState([]); + // CLUSTOX: the live mappings, keyed by repo. The backend allows one active + // deployment source per repo, so a repo has at most one entry here. + const [mappings, setMappings] = useState>({}); + // CLUSTOX: repos whose active workflow could be double-counted with a new + // Jenkins mapping. Sourced from teams/v2's repo_workflows, the only + // existing endpoint that already joins OrgRepo with active workflow rows -- + // it does not expose which provider the active workflow belongs to, so + // this is "has an active workflow-based deployment source" rather than + // strictly "has an active GitHub Actions workflow". + const [reposWithActiveWorkflow, setReposWithActiveWorkflow] = useState< + Set + >(new Set()); + // CLUSTOX: how many teams track each repo. Mapping switches the deployment + // source for all of them, and an admin looking at one team's numbers has no + // other way to know he is changing what the other teams see. + const [teamCounts, setTeamCounts] = useState>({}); + const [rowErrors, setRowErrors] = useState>({}); + const [savingRepoId, setSavingRepoId] = useState(null); + const [pending, setPending] = useState(null); + const [loading, setLoading] = useState(true); + const [failed, setFailed] = useState(false); + + const fetchMappings = useCallback(async () => { + const res = await handleApi<{ mappings: JenkinsMapping[] }>( + '/clustox/jenkins/mappings', + { params: { org_id: orgId } } + ); + return Object.fromEntries( + (res.mappings || []).map((mapping) => [mapping.org_repo_id, mapping]) + ) as Record; + }, [orgId]); + + const fetchTeamState = useCallback(async () => { + const res = await handleApi( + `/resources/orgs/${orgId}/teams/v2`, + { params: { providers: codeProviders } } + ); + + const withWorkflow = new Set(); + const counts: Record = {}; + Object.entries(res.teamReposMap || {}).forEach(([teamId, teamRepos]) => { + teamRepos.forEach((repo) => { + if (repo.repo_workflows?.length) withWorkflow.add(repo.id); + // teams/v2 left-joins TeamRepos, so repos belonging to no team arrive + // under a null key. Counting that bucket would tell an admin one team + // tracks a repo that none does. + if (teamId && teamId !== 'null' && teamId !== 'undefined') + counts[repo.id] = (counts[repo.id] || 0) + 1; + }); + }); + return { withWorkflow, counts }; + }, [orgId]); + + const load = useCallback(async () => { + if (!orgId) return; + setLoading(true); + setFailed(false); + try { + const [repoRows, jobsRes, mappingsByRepo, teamState] = await Promise.all([ + handleApi(`/resources/orgs/${orgId}/repos`), + handleApi<{ jobs: JenkinsJob[] }>('/clustox/jenkins/jobs', { + params: { org_id: orgId } + }), + fetchMappings(), + fetchTeamState() + ]); + + setRepos(repoRows || []); + setJobs(jobsRes.jobs || []); + setMappings(mappingsByRepo); + setReposWithActiveWorkflow(teamState.withWorkflow); + setTeamCounts(teamState.counts); + } catch (e) { + console.error('Failed to load Jenkins mapping data', e); + setFailed(true); + } finally { + setLoading(false); + } + }, [fetchMappings, fetchTeamState, orgId]); + + useEffect(() => { + load(); + }, [load]); + + // CLUSTOX: re-read what the server now holds after a mapping changed, rather + // than patching local state to match a guess. Mapping and unmapping both move + // rows this screen only sees indirectly -- displaced workflows, restored + // ones, deployment types -- and the repo_workflow_id needed to undo a fresh + // mapping exists nowhere until it is fetched. The Jenkins job list is left + // alone: it costs a round trip to Jenkins and nothing here changes it. + const refreshMappingState = useCallback(async () => { + const [mappingsByRepo, teamState] = await Promise.all([ + fetchMappings(), + fetchTeamState() + ]); + setMappings(mappingsByRepo); + setReposWithActiveWorkflow(teamState.withWorkflow); + setTeamCounts(teamState.counts); + }, [fetchMappings, fetchTeamState]); + + const submitMapping = useCallback( + async (repoId: string, jobFullName: string) => { + setRowErrors((errs) => ({ ...errs, [repoId]: '' })); + setSavingRepoId(repoId); + try { + await handleApi('/clustox/jenkins/mappings', { + method: 'post', + data: { + org_id: orgId, + org_repo_id: repoId, + job_full_name: jobFullName + } + }); + await refreshMappingState(); + } catch (e) { + console.error('Failed to map Jenkins job', e); + setRowErrors((errs) => ({ + ...errs, + [repoId]: mutationError(e, 'Could not save this mapping. Try again.') + })); + } finally { + setSavingRepoId(null); + } + }, + [orgId, refreshMappingState] + ); + + const submitUnmapping = useCallback( + async (repoId: string, repoWorkflowId: string) => { + setRowErrors((errs) => ({ ...errs, [repoId]: '' })); + setSavingRepoId(repoId); + try { + await handleApi('/clustox/jenkins/mappings', { + method: 'delete', + data: { org_id: orgId, repo_workflow_id: repoWorkflowId } + }); + await refreshMappingState(); + } catch (e) { + console.error('Failed to remove Jenkins mapping', e); + setRowErrors((errs) => ({ + ...errs, + [repoId]: mutationError( + e, + 'Could not remove this mapping. Try again.' + ) + })); + } finally { + setSavingRepoId(null); + } + }, + [orgId, refreshMappingState] + ); + + const askToUnmap = useCallback( + (repo: WorkspaceRepo) => { + const mapping = mappings[repo.id]; + if (!mapping) return; + setPending({ + action: 'unmap', + repoId: repo.id, + repoName: repo.name, + jobFullName: mapping.job_full_name, + repoWorkflowId: mapping.repo_workflow_id, + teamCount: teamCounts[repo.id] || 0 + }); + }, + [mappings, teamCounts] + ); + + const handleSelect = useCallback( + (repo: WorkspaceRepo, jobFullName: string) => { + // The empty option on a mapped row means "stop using Jenkins here", which + // is the same destructive change as the explicit button beside it. + if (!jobFullName) { + if (mappings[repo.id]) askToUnmap(repo); + return; + } + if (jobFullName === mappings[repo.id]?.job_full_name) return; + + // CLUSTOX: confirmed for every repo, not only those with a workflow to + // displace. Mapping now also switches the repo to workflow-based + // deployments for every team tracking it, so even a repo with no active + // workflow changes where its deployment numbers come from. + setPending({ + action: 'map', + repoId: repo.id, + repoName: repo.name, + jobFullName, + deactivatesWorkflows: + reposWithActiveWorkflow.has(repo.id) && !mappings[repo.id], + teamCount: teamCounts[repo.id] || 0 + }); + }, + [askToUnmap, mappings, reposWithActiveWorkflow, teamCounts] + ); + + const confirmPending = useCallback(async () => { + if (!pending) return; + if (pending.action === 'map') + await submitMapping(pending.repoId, pending.jobFullName); + else await submitUnmapping(pending.repoId, pending.repoWorkflowId); + setPending(null); + }, [pending, submitMapping, submitUnmapping]); + + const noJobs = useMemo(() => !loading && !jobs.length, [loading, jobs]); + + return ( + + {failed && ( + + Could not load repos or Jenkins jobs. Try again in a moment. + + )} + + {noJobs && !failed && ( + + Jenkins didn't return any jobs for this instance. + + )} + + {loading ? ( + + + + ) : ( + + + + + Repository + Jenkins job + + + + {repos.map((repo) => ( + + + + {repo.name} + + {repo.provider} + + + + + + + handleSelect(repo, e.target.value as string) + } + SelectProps={{ displayEmpty: true }} + > + + Select a Jenkins job + + {jobs.map((job) => ( + + {job.full_name} + + ))} + + {savingRepoId === repo.id && ( + + )} + {mappings[repo.id] && savingRepoId !== repo.id && ( + <> + + Mapped + + + askToUnmap(repo)} + > + + + + + )} + + {rowErrors[repo.id] && ( + + {rowErrors[repo.id]} + + )} + + + ))} + {!repos.length && ( + + + No repos in this workspace yet. + + + )} + +
+
+ )} + + setPending(null)}> + + {pending?.action === 'unmap' + ? `Remove ${pending?.repoName}'s Jenkins mapping?` + : `Map ${pending?.repoName} to Jenkins?`} + + + + {pending?.action === 'unmap' ? ( + <> + + {pending.repoName} will stop counting runs of{' '} + {pending.jobFullName} as deployments, and + go back to the deployment source it had before it was mapped — + its GitHub Actions workflows if this mapping switched any off, + otherwise merged pull requests. + + {everyTeamSentence(pending.teamCount)} + + Builds already ingested from Jenkins are kept. You can map the + job again at any time. + + + ) : ( + <> + + {pending?.repoName} will start counting runs of{' '} + {pending?.jobFullName} as its deployments, + instead of counting merged pull requests. Deployment frequency + and lead time will be measured from Jenkins from now on. + + {pending?.action === 'map' && pending.deactivatesWorkflows && ( + + This repo's active GitHub Actions deployment workflows + will be switched off so the same deployment isn't + counted twice. + + )} + {everyTeamSentence(pending?.teamCount ?? 0)} + + You can undo this by removing the mapping. + + + )} + + + + + + {pending?.action === 'unmap' ? 'Remove mapping' : 'Map job'} + + + +
+ ); +}; diff --git a/web-server/src/components/ClustoxJenkinsSetup.tsx b/web-server/src/components/ClustoxJenkinsSetup.tsx new file mode 100644 index 000000000..da00cd95d --- /dev/null +++ b/web-server/src/components/ClustoxJenkinsSetup.tsx @@ -0,0 +1,228 @@ +import { LoadingButton } from '@mui/lab'; +import { TextField } from '@mui/material'; +import { useSnackbar } from 'notistack'; +import { FC, useCallback } from 'react'; + +import { handleApi } from '@/api-helpers/axios-api-instance'; +import { FlexBox } from '@/components/FlexBox'; +import { Line } from '@/components/Text'; +import { Integration } from '@/constants/integrations'; +import { useAuth } from '@/hooks/useAuth'; +// CLUSTOX: server-resolved workspace, immune to stale persisted redux state +// (see ConfigureGitlabModalBody for the incident this pattern fixed). +import { useClustoxUser } from '@/hooks/useClustoxUser'; +import { useBoolState, useEasyState } from '@/hooks/useEasyState'; +import { fetchCurrentOrg } from '@/slices/auth'; +import { useDispatch } from '@/store'; +import { readApiError } from '@/utils/api-error'; +import { linkProvider, unlinkProvider } from '@/utils/auth'; +import { checkDomainWithRegex } from '@/utils/domainCheck'; +import { depFn } from '@/utils/fn'; + +const CONNECTION_ERROR = + 'Could not connect to Jenkins with these credentials. Check the base URL, username, and API token.'; + +/** + * CLUSTOX: credentials form for connecting a Jenkins instance. + * + * There is no dry-run endpoint -- the only server-side call that actually + * reaches Jenkins is the job list, which reads credentials out of the + * Integration row. So "validate before saving" is implemented as save, then + * probe, then roll back the row if the probe fails, rather than a true + * pre-save check. That keeps a failed attempt from leaving the workspace + * looking "linked" with credentials nothing can use. + */ +export const ClustoxJenkinsSetup: FC<{ onLinked?: () => void }> = ({ + onLinked +}) => { + const baseUrl = useEasyState(''); + const username = useEasyState(''); + const apiToken = useEasyState(''); + + const baseUrlError = useEasyState(''); + const usernameError = useEasyState(''); + const tokenError = useEasyState(''); + + const { orgId: contextOrgId } = useAuth(); + const { orgId: sessionOrgId } = useClustoxUser(); + const orgId = sessionOrgId ?? contextOrgId; + + const dispatch = useDispatch(); + const { enqueueSnackbar } = useSnackbar(); + const isLoading = useBoolState(false); + + const handleSubmit = useCallback(async () => { + depFn(baseUrlError.set, ''); + depFn(usernameError.set, ''); + depFn(tokenError.set, ''); + + let hasError = false; + if (!baseUrl.value || !checkDomainWithRegex(baseUrl.value)) { + depFn( + baseUrlError.set, + 'Enter a valid URL, including the http(s):// scheme' + ); + hasError = true; + } + if (!username.value) { + depFn(usernameError.set, 'Username is required'); + hasError = true; + } + if (!apiToken.value) { + depFn(tokenError.set, 'API token is required'); + hasError = true; + } + if (!orgId) { + depFn( + baseUrlError.set, + 'No workspace selected. Reload the page and try again.' + ); + hasError = true; + } + if (hasError) return; + + depFn(isLoading.true); + // CLUSTOX: which of the two calls below failed decides whether the server's + // message is about Jenkins at all. A 400 from the credential save is a + // schema complaint about this form; a 400 from the probe is the analytics + // server refusing the address, and that text is the whole point. + let credentialsSaved = false; + try { + await linkProvider(apiToken.value, orgId, Integration.JENKINS, { + base_url: baseUrl.value, + username: username.value + }); + credentialsSaved = true; + + // CLUSTOX: the actual connection probe -- this is the first call that + // hits the real Jenkins instance with the credentials just saved. + await handleApi('/clustox/jenkins/jobs', { + params: { org_id: orgId } + }); + + await dispatch(fetchCurrentOrg()); + enqueueSnackbar('Jenkins connected successfully', { + variant: 'success', + autoHideDuration: 2000 + }); + onLinked?.(); + } catch (e) { + // CLUSTOX: message only. If the failing call is the credential save, the + // axios error object carries config.data with the raw API token in it, + // which would then sit in the browser console. + console.error( + 'Failed to connect Jenkins', + e instanceof Error ? e.message : String(e) + ); + // CLUSTOX: don't leave a workspace "linked" to credentials that don't + // work -- the mapping screen would otherwise open to a broken job list. + // True of a refused address too: nothing here can reach that Jenkins. + await unlinkProvider(orgId, Integration.JENKINS).catch(() => {}); + + // CLUSTOX: a 400 from the probe is the server naming something about this + // URL it will not fetch -- a private, loopback or link-local address, or + // a missing scheme. Reported as CONNECTION_ERROR it sends the admin to + // re-check a username and token that are fine. + const { status, message } = readApiError(e); + depFn( + baseUrlError.set, + credentialsSaved && status === 400 && message + ? message + : CONNECTION_ERROR + ); + } finally { + depFn(isLoading.false); + } + }, [ + apiToken.value, + baseUrl.value, + baseUrlError.set, + dispatch, + enqueueSnackbar, + isLoading.false, + isLoading.true, + onLinked, + orgId, + tokenError.set, + username.value, + usernameError.set + ]); + + return ( + + + Enter your Jenkins connection details below{' '} + + * + + + + + { + baseUrl.set(e.currentTarget.value); + baseUrlError.set(''); + }} + label="Jenkins Base URL" + placeholder="https://jenkins.example.com" + /> + + {baseUrlError.value} + + + + + { + username.set(e.currentTarget.value); + usernameError.set(''); + }} + label="Username" + /> + + {usernameError.value} + + + + + { + apiToken.set(e.currentTarget.value); + tokenError.set(''); + }} + label="API Token" + type="password" + onKeyDown={(e) => { + if (e.key === 'Enter') { + e.preventDefault(); + handleSubmit(); + } + }} + /> + + {tokenError.value} + + + + + + Connect Jenkins + + + + ); +}; diff --git a/web-server/src/components/ClustoxUserFooter.tsx b/web-server/src/components/ClustoxUserFooter.tsx new file mode 100644 index 000000000..00304de03 --- /dev/null +++ b/web-server/src/components/ClustoxUserFooter.tsx @@ -0,0 +1,26 @@ +import { ClustoxWorkspaceSwitcher } from '@/components/ClustoxWorkspaceSwitcher'; +import { FlexBox } from '@/components/FlexBox'; +import { useClustoxUser } from '@/hooks/useClustoxUser'; + +/** + * CLUSTOX: the Superadmin's workspace switcher, at the bottom of the + * sidebar. + * + * Identity (name/role) and sign-out used to live here too, but that's now + * covered by the account menu at the top-right of every page (see + * PageHeader -> AccountMenu) -- kept only the piece that has no other home. + */ +export const ClustoxUserFooter = () => { + const { user, loading, isSuperadmin } = useClustoxUser(); + + // ClustoxWorkspaceSwitcher itself renders null for a non-Superadmin, but + // checking here too avoids Sidebar rendering an empty padded box between + // its two surrounding Dividers for every Admin. + if (loading || !user || !isSuperadmin) return null; + + return ( + + + + ); +}; diff --git a/web-server/src/components/ClustoxWorkspaceMetrics.tsx b/web-server/src/components/ClustoxWorkspaceMetrics.tsx new file mode 100644 index 000000000..a62b4dfa9 --- /dev/null +++ b/web-server/src/components/ClustoxWorkspaceMetrics.tsx @@ -0,0 +1,205 @@ +import { + Alert, + CircularProgress, + MenuItem, + Table, + TableBody, + TableCell, + TableContainer, + TableHead, + TableRow, + TextField, + Typography, + useTheme +} from '@mui/material'; +import { useCallback, useEffect, useState } from 'react'; + +import { FlexBox } from '@/components/FlexBox'; +import { Line } from '@/components/Text'; + +type Row = { + id: string; + name: string; + ownerEmail: string | null; + hasIntegration: boolean; + teamCount: number; + repoCount: number; + prCount: number; + deployments: number; + leadTimeSeconds: number | null; +}; + +const WINDOWS = [ + { days: 7, label: 'Last 7 days' }, + { days: 30, label: 'Last 30 days' }, + { days: 90, label: 'Last 90 days' } +]; + +const Mono = ({ children }: { children: React.ReactNode }) => ( + + {children} + +); + +/** Seconds to the coarsest unit that still reads precisely. */ +const duration = (seconds: number | null) => { + if (seconds === null) return '—'; + const h = seconds / 3600; + if (h < 1) return `${Math.round(seconds / 60)}m`; + if (h < 48) return `${Math.round(h)}h`; + return `${(h / 24).toFixed(1)}d`; +}; + +/** + * CLUSTOX: DORA metrics across every workspace, for a superadmin. + * + * Without this, comparing workspaces means switching into each one and reading + * its dashboard separately. + */ +export const ClustoxWorkspaceMetrics = () => { + const theme = useTheme(); + const [rows, setRows] = useState([]); + const [days, setDays] = useState(30); + const [loading, setLoading] = useState(true); + const [failed, setFailed] = useState(false); + + const load = useCallback(async () => { + setLoading(true); + setFailed(false); + + const to = new Date(); + const from = new Date(to.getTime() - days * 24 * 60 * 60 * 1000); + const params = new URLSearchParams({ + // The analytics API rejects timestamps without timezone info. + from_time: from.toISOString().replace('Z', '+00:00'), + to_time: to.toISOString().replace('Z', '+00:00') + }); + + try { + const res = await fetch(`/api/clustox/overview?${params}`); + if (!res.ok) throw new Error('failed'); + setRows(await res.json()); + } catch { + setFailed(true); + } finally { + setLoading(false); + } + }, [days]); + + useEffect(() => { + load(); + }, [load]); + + const withData = rows.filter((r) => r.prCount > 0); + + return ( + + + + {withData.length} of {rows.length} workspaces have activity in this + window + + setDays(Number(e.target.value))} + > + {WINDOWS.map((w) => ( + + {w.label} + + ))} + + + + {failed && ( + + Could not load metrics. The analytics server may be unreachable. + + )} + + {loading ? ( + + + + ) : ( + + + + + Workspace + + Lead time + + + Deployments + + + PRs + + + Teams + + + + + {rows.map((r) => { + const idle = r.prCount === 0; + return ( + + + + {r.name} + + {r.ownerEmail ?? 'no owner'} + + + + + {idle ? ( + + {r.hasIntegration ? 'no activity' : 'not connected'} + + ) : ( + {duration(r.leadTimeSeconds)} + )} + + + {idle ? '—' : r.deployments} + + + {idle ? '—' : r.prCount} + + + {r.teamCount} + + + ); + })} + +
+
+ )} + + + Lead time is weighted by pull request count, so a workspace with few + merges cannot skew the comparison. Workspaces with no merged pull + requests in the window show no figure rather than a zero. + +
+ ); +}; diff --git a/web-server/src/components/ClustoxWorkspaceSwitcher.tsx b/web-server/src/components/ClustoxWorkspaceSwitcher.tsx new file mode 100644 index 000000000..ff572122f --- /dev/null +++ b/web-server/src/components/ClustoxWorkspaceSwitcher.tsx @@ -0,0 +1,73 @@ +import { FormControl, InputLabel, MenuItem, Select } from '@mui/material'; +import { useCallback, useEffect, useState } from 'react'; + +import { useClustoxUser } from '@/hooks/useClustoxUser'; + +type Workspace = { id: string; name: string }; + +/** + * CLUSTOX: lets a SuperAdmin choose which workspace they are looking at. + * + * A SuperAdmin owns no workspace, but every dashboard in the app is + * workspace-scoped, so without a selection there is nothing to show. This is a + * viewing control, not a permission one -- a SuperAdmin may reach every + * workspace regardless of what is selected. + */ +export const ClustoxWorkspaceSwitcher = () => { + const { isSuperadmin, loading } = useClustoxUser(); + const [workspaces, setWorkspaces] = useState([]); + const [current, setCurrent] = useState(''); + + const load = useCallback(async () => { + const [wsRes, sessionRes] = await Promise.all([ + fetch('/api/clustox/workspaces'), + fetch('/api/auth/session') + ]); + if (!wsRes.ok) return; + + setWorkspaces(await wsRes.json()); + + if (sessionRes.ok) { + const session = await sessionRes.json(); + if (session?.org?.id) setCurrent(session.org.id); + } + }, []); + + useEffect(() => { + if (!loading && isSuperadmin) load(); + }, [loading, isSuperadmin, load]); + + const onChange = async (orgId: string) => { + setCurrent(orgId); + await fetch('/api/clustox/workspaces', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ org_id: orgId }) + }); + // Full reload: the workspace is resolved server-side per request, and + // every cached dashboard slice belongs to the previous workspace. + window.location.reload(); + }; + + if (loading || !isSuperadmin || workspaces.length === 0) return null; + + return ( + + + Workspace + + + + ); +}; diff --git a/web-server/src/components/ConfigureJiraIncidentSourceModalBody.tsx b/web-server/src/components/ConfigureJiraIncidentSourceModalBody.tsx new file mode 100644 index 000000000..70b294af4 --- /dev/null +++ b/web-server/src/components/ConfigureJiraIncidentSourceModalBody.tsx @@ -0,0 +1,191 @@ +import { Autocomplete, Chip, TextField } from '@mui/material'; +import { LoadingButton } from '@mui/lab'; +import { useSnackbar } from 'notistack'; +import { FC, useCallback, useEffect } from 'react'; + +import { FlexBox } from '@/components/FlexBox'; +import { IOSSwitch } from '@/components/Shared'; +import { Line } from '@/components/Text'; +import { useAuth } from '@/hooks/useAuth'; +// CLUSTOX: server-resolved workspace, immune to stale persisted redux state -- +// same reasoning as ConfigureJiraModalBody. +import { useClustoxUser } from '@/hooks/useClustoxUser'; +import { useBoolState, useEasyState } from '@/hooks/useEasyState'; +import { handleApi } from '@/api-helpers/axios-api-instance'; + +const INCIDENT_SOURCES_SETTING = 'INCIDENT_SOURCES_SETTING'; +const JIRA_INCIDENT_ISSUE_TYPES_SETTING = 'JIRA_INCIDENT_ISSUE_TYPES_SETTING'; +const JIRA_ISSUE_SOURCE = 'JIRA_ISSUE'; + +const COMMON_ISSUE_TYPES = ['Bug', 'Incident', 'Task', 'Story']; + +type IncidentSourcesSetting = { incident_sources: string[] }; +type JiraIncidentIssueTypesSetting = { issue_types: string[] }; + +// CLUSTOX: Jira issues as an incident source for Change Failure Rate / +// MTTR -- org-scoped, opt-in (see docs/JIRA_INTEGRATION_PROPOSAL.md). No +// dedicated org-settings page exists yet, so this is surfaced from the +// same DoraMetricsConfigurationSettings menu as the team-scoped Incident +// PR filters, using the generic /orgs/{org_id}/settings route every +// other setting on this page already reads/writes. +export const ConfigureJiraIncidentSourceModalBody: FC<{ + onClose: () => void; +}> = ({ onClose }) => { + const { orgId: contextOrgId } = useAuth(); + const { orgId: sessionOrgId } = useClustoxUser(); + const orgId = sessionOrgId ?? contextOrgId; + const { enqueueSnackbar } = useSnackbar(); + + const isLoading = useBoolState(true); + const isSaving = useBoolState(false); + const enabled = useEasyState(false); + const issueTypes = useEasyState([]); + + useEffect(() => { + if (!orgId) return; + let cancelled = false; + + Promise.all([ + // CLUSTOX: this BFF route (pages/api/internal/[org_id]/settings.ts) + // already unwraps the backend's { setting: T, ... } envelope to + // just T before responding -- see its GET handler. + handleApi(`/internal/${orgId}/settings`, { + params: { setting_type: INCIDENT_SOURCES_SETTING } + }), + handleApi(`/internal/${orgId}/settings`, { + params: { setting_type: JIRA_INCIDENT_ISSUE_TYPES_SETTING } + }) + ]) + .then(([sources, types]) => { + if (cancelled) return; + enabled.set( + (sources.incident_sources || []).includes(JIRA_ISSUE_SOURCE) + ); + issueTypes.set(types.issue_types || []); + }) + .catch(() => { + if (cancelled) return; + enqueueSnackbar('Could not load the current setting', { + variant: 'error', + autoHideDuration: 3000 + }); + }) + .finally(() => { + if (!cancelled) isLoading.false(); + }); + + return () => { + cancelled = true; + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [orgId]); + + const handleSave = useCallback(async () => { + if (!orgId) return; + isSaving.true(); + try { + const currentSources = await handleApi( + `/internal/${orgId}/settings`, + { params: { setting_type: INCIDENT_SOURCES_SETTING } } + ); + + const withoutJira = (currentSources.incident_sources || []).filter( + (source: string) => source !== JIRA_ISSUE_SOURCE + ); + const nextSources = enabled.value + ? [...withoutJira, JIRA_ISSUE_SOURCE] + : withoutJira; + + await handleApi(`/internal/${orgId}/settings`, { + method: 'PUT', + data: { + setting_type: INCIDENT_SOURCES_SETTING, + setting_data: { incident_sources: nextSources } + } + }); + + await handleApi(`/internal/${orgId}/settings`, { + method: 'PUT', + data: { + setting_type: JIRA_INCIDENT_ISSUE_TYPES_SETTING, + setting_data: { issue_types: issueTypes.value } + } + }); + + enqueueSnackbar('Updated Successfully', { + variant: 'success', + autoHideDuration: 3000 + }); + onClose(); + } catch (e: any) { + enqueueSnackbar('Something went wrong', { + variant: 'error', + autoHideDuration: 3000 + }); + } finally { + isSaving.false(); + } + }, [orgId, enabled.value, issueTypes.value, enqueueSnackbar, onClose, isSaving]); + + return ( + + + Treat Jira issues of the selected type(s) as incidents for Change + Failure Rate and MTTR, alongside whatever other incident sources + this org already has configured. + + + + Use Jira issues as an incident source + enabled.set(isEnabled)} + /> + + + + Which issue types count as an incident + issueTypes.set(value)} + renderTags={(value, getTagProps) => + value.map((option, index) => ( + + )) + } + renderInput={(params) => ( + + )} + /> + + + + + Save + + + + ); +}; diff --git a/web-server/src/components/ContributorFilter.tsx b/web-server/src/components/ContributorFilter.tsx new file mode 100644 index 000000000..30e885124 --- /dev/null +++ b/web-server/src/components/ContributorFilter.tsx @@ -0,0 +1,209 @@ +import { + Autocomplete, + CircularProgress, + ListItem, + TextField +} from '@mui/material'; +import { useRouter } from 'next/router'; +import pluralize from 'pluralize'; +import { useCallback, useEffect, useMemo, useState } from 'react'; + +import { handleApi } from '@/api-helpers/axios-api-instance'; +import { FlexBox } from '@/components/FlexBox'; +import { Line } from '@/components/Text'; +import { useSingleTeamConfig } from '@/hooks/useStateTeamConfig'; +import { + CONTRIBUTOR_QUERY_KEY as QUERY_KEY, + contributorSelectionKey, + parseSelectedFromQuery +} from '@/utils/contributorQuery'; + +type Contributor = { + username: string; + pr_count: number; +}; + +/** + * CLUSTOX: single source of truth for the contributor selection. It lives in + * the URL -- unlike the team/date selectors, which are redux-only -- so a + * filtered dashboard survives a refresh and can be shared as a link. Cards + * that need to know "is a contributor selected, and who" read this hook + * directly instead of taking it as a prop, matching how they already source + * team/date themselves (see useCurrentDateRangeLabel and friends). + */ +export const useSelectedContributors = (): string[] => { + const { query } = useRouter(); + // CLUSTOX: keyed off the raw query value (a string), not the whole `query` + // object -- that object gets a new identity on every navigation, including + // unrelated ones like OverlayPageContext pushing its own `overlays` param + // when a "See details" panel opens. Depending on the whole object would + // refetch the DORA summary on every such click. `contributorSelectionKey` + // also flattens the repeated-param array form, which would otherwise be a + // fresh identity on every render and defeat the memo entirely. + const raw = query[QUERY_KEY] as string | string[]; + const key = contributorSelectionKey(raw); + return useMemo(() => parseSelectedFromQuery(key), [key]); +}; + +export const ContributorFilter = () => { + const router = useRouter(); + const { singleTeamId, dates } = useSingleTeamConfig(); + const selected = useSelectedContributors(); + + const [contributors, setContributors] = useState([]); + const [loading, setLoading] = useState(false); + const [failed, setFailed] = useState(false); + // CLUSTOX: true once a fetch for the *current* team/date has resolved + // successfully. Gates the stale-selection cleanup below so it can't run + // against a still-loading or still-failed list. + const [settled, setSettled] = useState(false); + const [notice, setNotice] = useState(null); + + const setSelected = useCallback( + (usernames: string[]) => { + const nextQuery: Record = { + ...router.query + }; + if (usernames.length) nextQuery[QUERY_KEY] = usernames.join(','); + else delete nextQuery[QUERY_KEY]; + router.replace({ query: nextQuery }, undefined, { shallow: true }); + }, + [router] + ); + + useEffect(() => { + if (!singleTeamId) { + setContributors([]); + setSettled(false); + return; + } + + let cancelled = false; + setLoading(true); + setFailed(false); + setSettled(false); + + handleApi(`internal/team/${singleTeamId}/contributors`, { + params: { from_date: dates.start, to_date: dates.end } + }) + .then((data) => { + if (cancelled) return; + setContributors(data || []); + setSettled(true); + }) + .catch(() => { + if (cancelled) return; + setContributors([]); + setFailed(true); + }) + .finally(() => { + if (!cancelled) setLoading(false); + }); + + return () => { + cancelled = true; + }; + }, [singleTeamId, dates.start, dates.end]); + + // Drop selections the fresh list no longer backs, visibly -- a silent drop + // would move the DORA numbers on screen with no explanation for why. + useEffect(() => { + if (!settled || !selected.length) return; + + const valid = new Set(contributors.map((c) => c.username)); + const stale = selected.filter((u) => !valid.has(u)); + if (!stale.length) return; + + setSelected(selected.filter((u) => valid.has(u))); + setNotice( + `${stale.join(', ')} ${ + stale.length > 1 ? "aren't" : "isn't" + } active in this range anymore, so ${ + stale.length > 1 ? "they've" : "it's" + } been removed from the filter.` + ); + }, [settled, contributors, selected, setSelected]); + + useEffect(() => { + if (!notice) return; + const timeout = setTimeout(() => setNotice(null), 6000); + return () => clearTimeout(timeout); + }, [notice]); + + const isEmpty = settled && contributors.length === 0; + // Whatever else is going on, a selection must stay removable. Disabling the + // control on a failed fetch left chips that could not be deleted while the + // dashboard carried on sending `authors`, and the cards carried on saying + // "authored by alice" -- the only way out was hand-editing the URL. With + // nothing selected this is exactly the old behaviour. + const disabled = + !singleTeamId || (!selected.length && (loading || failed || isEmpty)); + + // Rendered from the selection, not from an intersection with the fetched + // list, so chips survive a fetch that failed or has not landed yet. Stale + // selections are still dropped -- but only by the effect above, which runs + // on a *successful* fetch and says so in a notice. + const value = useMemo(() => { + const fetched = new Map(contributors.map((c) => [c.username, c])); + return selected.map( + (username) => fetched.get(username) || { username, pr_count: 0 } + ); + }, [contributors, selected]); + + const helperText = notice + ? notice + : loading + ? 'Loading contributors…' + : failed + ? 'Could not load contributors.' + : isEmpty + ? 'No contributors in this range' + : undefined; + + return ( + + multiple + size="small" + disabled={disabled} + options={contributors} + value={value} + loading={loading} + sx={{ minWidth: 280 }} + getOptionLabel={(option) => option.username} + isOptionEqualToValue={(option, val) => option.username === val.username} + onChange={(_, newValue) => setSelected(newValue.map((c) => c.username))} + renderOption={(props, option) => ( + + + {option.username} + + {option.pr_count} {pluralize('PR', option.pr_count)} + + + + )} + renderInput={(params) => ( + + {loading ? ( + + ) : null} + {params.InputProps.endAdornment} + + ) + }} + /> + )} + /> + ); +}; diff --git a/web-server/src/components/DoraMetricsConfigurationSettings.tsx b/web-server/src/components/DoraMetricsConfigurationSettings.tsx index 068b66a8a..7d95c2497 100644 --- a/web-server/src/components/DoraMetricsConfigurationSettings.tsx +++ b/web-server/src/components/DoraMetricsConfigurationSettings.tsx @@ -2,6 +2,8 @@ import SettingsIcon from '@mui/icons-material/Settings'; import { Button, Menu, MenuItem } from '@mui/material'; import { useCallback, useRef, useEffect } from 'react'; +import { BenchmarkSettingsForm } from '@/components/BenchmarkSettingsForm'; +import { ConfigureJiraIncidentSourceModalBody } from '@/components/ConfigureJiraIncidentSourceModalBody'; import { FlexBox } from '@/components/FlexBox'; import { TeamIncidentPRsFilter } from '@/components/TeamIncidentPRsFilter'; import { TeamProductionBranchSelector } from '@/components/TeamProductionBranchSelector'; @@ -45,6 +47,38 @@ export const DoraMetricsConfigurationSettings = () => { }); }, [addModal, closeModal]); + // CLUSTOX: Jira issues as an incident source (MID-8) -- org-scoped, + // unlike the two team-scoped modals above, but surfaced from the same + // menu since there's no dedicated org-settings page yet. See + // ConfigureJiraIncidentSourceModalBody. + const openJiraIncidentSourceModal = useCallback(async () => { + const modal = addModal({ + title: `Configure Jira Incident Source`, + body: ( + closeModal(modal.key)} + /> + ), + showCloseIcon: true + }); + }, [addModal, closeModal]); + + // CLUSTOX: per-team DORA benchmark targets. Same modal shape as the other + // team settings above -- a menu item opens a dedicated form, which closes + // itself on save. + const openBenchmarkSettingsModal = useCallback(async () => { + const modal = addModal({ + title: `Configure DORA Benchmarks`, + body: ( + closeModal(modal.key)} + /> + ), + showCloseIcon: true + }); + }, [addModal, closeModal]); + const anchorEl = useRef(null); const open = useBoolState(false); @@ -95,6 +129,22 @@ export const DoraMetricsConfigurationSettings = () => { > Configure Filters for Incident PRs + { + open.false(); + openJiraIncidentSourceModal(); + }} + > + Configure Jira Incident Source + + { + open.false(); + openBenchmarkSettingsModal(); + }} + > + Configure DORA Benchmarks + ); diff --git a/web-server/src/components/JiraConnectionsManager.tsx b/web-server/src/components/JiraConnectionsManager.tsx new file mode 100644 index 000000000..b8e761533 --- /dev/null +++ b/web-server/src/components/JiraConnectionsManager.tsx @@ -0,0 +1,310 @@ +import { DeleteOutline, StarRounded, StarOutlineRounded } from '@mui/icons-material'; +import { LoadingButton } from '@mui/lab'; +import { + Alert, + Button, + CircularProgress, + Dialog, + DialogActions, + DialogContent, + DialogTitle, + IconButton, + Table, + TableBody, + TableCell, + TableContainer, + TableHead, + TableRow, + TextField, + Tooltip, + useTheme +} from '@mui/material'; +import { useCallback, useState } from 'react'; + +import { FlexBox } from '@/components/FlexBox'; +import { Line } from '@/components/Text'; +// CLUSTOX: server-resolved workspace, consistent with every other +// integration modal (ConfigureJiraModalBody, ClustoxJenkinsSetup, ...). +import { useClustoxUser } from '@/hooks/useClustoxUser'; +import { JiraConnection, useJiraConnections } from '@/hooks/useJiraConnections'; +import { readApiError } from '@/utils/api-error'; +import { normalizeJiraSiteUrl } from '@/utils/auth'; + +// CLUSTOX: the server's own words whenever it sent any -- a 409 here means +// "already connected" (create) or "still has projects synced from it" +// (delete), and both are things only the admin looking at this screen can +// fix. Mirrors ClustoxJenkinsMapping's mutationError. +const mutationError = (err: unknown, fallback: string) => { + const { status, message } = readApiError(err); + return status && status < 500 && message ? message : fallback; +}; + +const emptyForm = { site_url: '', email: '', access_token: '' }; + +/** + * CLUSTOX: Jira multi-account support -- see + * docs/JIRA_MULTI_ACCOUNT_PLAN.md Task 6. Lists an org's JiraConnection + * rows and lets an admin add, remove, or change the default one. + * Deliberately separate from ConfigureJiraModalBody/JiraIntegrationCard, + * which keep managing the legacy, single-account Integration row + * unchanged -- the two flows are independent by design (see the plan's + * Global Constraints). + */ +export const JiraConnectionsManager = () => { + const theme = useTheme(); + const { orgId } = useClustoxUser(); + const { connections, loading, failed, create, remove, setDefault } = + useJiraConnections(orgId); + + const [form, setForm] = useState(emptyForm); + const [formError, setFormError] = useState(''); + const [creating, setCreating] = useState(false); + const [rowBusyId, setRowBusyId] = useState(null); + const [rowError, setRowError] = useState(''); + const [pendingDelete, setPendingDelete] = useState( + null + ); + + const submitCreate = useCallback(async () => { + if (!form.site_url || !form.email || !form.access_token) { + setFormError('Please fill in all three fields'); + return; + } + setFormError(''); + setCreating(true); + try { + // CLUSTOX: matches ConfigureJiraModalBody's legacy-flow behavior -- + // a URL pasted straight from the browser's address bar + // ("https://acme.atlassian.net/") must become the bare host, or + // the backend builds an unreachable, doubly-prefixed URL from it + // at search/sync time. The backend also normalizes this now (the + // authoritative fix, since a form is never the only caller), but + // showing the actual value that gets saved here avoids a + // could-look-wrong round trip. + await create({ ...form, site_url: normalizeJiraSiteUrl(form.site_url) }); + setForm(emptyForm); + } catch (e) { + setFormError( + mutationError(e, 'Could not add this connection. Try again.') + ); + } finally { + setCreating(false); + } + }, [create, form]); + + const submitSetDefault = useCallback( + async (connection: JiraConnection) => { + setRowError(''); + setRowBusyId(connection.id); + try { + await setDefault(connection.id); + } catch (e) { + setRowError( + mutationError(e, 'Could not set this connection as default.') + ); + } finally { + setRowBusyId(null); + } + }, + [setDefault] + ); + + const confirmDelete = useCallback(async () => { + if (!pendingDelete) return; + setRowError(''); + setRowBusyId(pendingDelete.id); + try { + await remove(pendingDelete.id); + setPendingDelete(null); + } catch (e) { + setRowError( + mutationError( + e, + 'Could not remove this connection. Try again.' + ) + ); + setPendingDelete(null); + } finally { + setRowBusyId(null); + } + }, [pendingDelete, remove]); + + return ( + + {failed && ( + + Could not load Jira connections. Try again in a moment. + + )} + + + Add a connection + + setForm({ ...form, site_url: e.target.value })} + InputLabelProps={{ shrink: true }} + /> + setForm({ ...form, email: e.target.value })} + InputLabelProps={{ shrink: true }} + /> + + setForm({ ...form, access_token: e.target.value }) + } + InputLabelProps={{ shrink: true }} + /> + + Add + + + {formError && ( + + {formError} + + )} + + + {loading ? ( + + + + ) : ( + + + + + Site + Email + Default + Actions + + + + {connections.map((connection) => ( + + {connection.site_url} + {connection.email} + + + + submitSetDefault(connection)} + > + {connection.is_default ? ( + + ) : ( + + )} + + + + + + {rowBusyId === connection.id ? ( + + ) : ( + + setPendingDelete(connection)} + > + + + + )} + + + ))} + {!connections.length && ( + + + No Jira connections yet. + + + )} + +
+
+ )} + {rowError && ( + + {rowError} + + )} + + setPendingDelete(null)} + > + + Remove {pendingDelete?.site_url}? + + + + + This connection's projects and their synced data are kept, + but nothing more will sync from it once removed. + + + If any team's projects still come from this connection, + removal is blocked until they're moved elsewhere. + + + + + + + Remove connection + + + +
+ ); +}; diff --git a/web-server/src/components/PageHeader.tsx b/web-server/src/components/PageHeader.tsx index d1d7a9237..76849280b 100644 --- a/web-server/src/components/PageHeader.tsx +++ b/web-server/src/components/PageHeader.tsx @@ -14,6 +14,7 @@ import { SidebarContext } from 'src/contexts/SidebarContext'; import { TeamSelectorModes } from '@/types/resources'; +import { AccountMenu } from './AccountMenu'; import { AvatarPageTitle } from './AvatarPageTitle'; import { BranchSelector } from './BranchSelector'; import { FlexBox, FlexBoxProps } from './FlexBox'; @@ -102,6 +103,10 @@ export const PageHeader: FC< {Boolean(subRoutes?.length) && } + {/* CLUSTOX: rendered once, here, so the account menu appears at + the top-right of every authenticated page without each page + needing to wire it in individually. */} + {showSelectorSection && ( <> diff --git a/web-server/src/components/RoleChip.tsx b/web-server/src/components/RoleChip.tsx new file mode 100644 index 000000000..19b8923c9 --- /dev/null +++ b/web-server/src/components/RoleChip.tsx @@ -0,0 +1,22 @@ +import { Chip } from '@mui/material'; +import { FC } from 'react'; + +import { ClustoxRole } from '@/auth/types'; + +/** + * CLUSTOX: shared role pill, used anywhere a Superadmin/Admin distinction is + * shown (Manage Users table, the account menu, the sidebar footer). Was + * previously reimplemented ad hoc in each of those places. + */ +export const RoleChip: FC<{ role: ClustoxRole; size?: 'small' | 'medium' }> = ({ + role, + size = 'small' +}) => ( + +); diff --git a/web-server/src/components/TeamIncidentPRsFilter.tsx b/web-server/src/components/TeamIncidentPRsFilter.tsx index fe3202bc0..6899b4082 100644 --- a/web-server/src/components/TeamIncidentPRsFilter.tsx +++ b/web-server/src/components/TeamIncidentPRsFilter.tsx @@ -19,7 +19,8 @@ import { FC, useEffect } from 'react'; import { FormProvider, useForm } from 'react-hook-form'; import * as yup from 'yup'; -import { useAuth } from '@/hooks/useAuth'; +// CLUSTOX: contributor filter -- shared fetch arguments, see the hook's note. +import { useDoraMetricsFetchArgs } from '@/hooks/useDoraMetricsFetchArgs'; import { useEasyState } from '@/hooks/useEasyState'; import { useSingleTeamConfig, @@ -90,8 +91,8 @@ export const TeamIncidentPRsFilter: FC<{ onClose: () => void; }> = ({ onClose }) => { const dispatch = useDispatch(); - const { orgId } = useAuth(); - const { singleTeamId, dates } = useSingleTeamConfig(); + const { singleTeamId } = useSingleTeamConfig(); + const doraMetricsFetchArgs = useDoraMetricsFetchArgs(); const branches = useStateBranchConfig(); const isSaving = useEasyState(false); const { enqueueSnackbar } = useSnackbar(); @@ -138,11 +139,11 @@ export const TeamIncidentPRsFilter: FC<{ await dispatch(updateTeamIncidentPRsFilter(updateConfArgs)).then( async (response) => { if (response.meta.requestStatus === 'fulfilled') { + // CLUSTOX: contributor filter -- saving the incident PR filter + // refetches DORA, and the selection has to survive that. Shared + // hook, so a future argument can't be missed here again. const fetchDoraArgs = { - orgId: orgId, - teamId: singleTeamId, - fromDate: dates.start, - toDate: dates.end, + ...doraMetricsFetchArgs, branches: activeBranchMode === ActiveBranchMode.PROD ? null : branches }; diff --git a/web-server/src/components/TeamProductionBranchSelector.tsx b/web-server/src/components/TeamProductionBranchSelector.tsx index 10f1b0c15..b599cc280 100644 --- a/web-server/src/components/TeamProductionBranchSelector.tsx +++ b/web-server/src/components/TeamProductionBranchSelector.tsx @@ -12,6 +12,8 @@ import { track } from '@/constants/events'; import { ROUTES } from '@/constants/routes'; import { isRoleLessThanEM } from '@/constants/useRoute'; import { useAuth } from '@/hooks/useAuth'; +// CLUSTOX: contributor filter -- shared fetch arguments, see the hook's note. +import { useDoraMetricsFetchArgs } from '@/hooks/useDoraMetricsFetchArgs'; import { useEasyState } from '@/hooks/useEasyState'; import { useSingleTeamConfig, @@ -58,8 +60,9 @@ export const TeamProductionBranchSelector: FC<{ onClose: () => void; }> = ({ onClose }) => { const dispatch = useDispatch(); - const { orgId, integrationSet } = useAuth(); - const { singleTeamId, dates } = useSingleTeamConfig(); + const { integrationSet } = useAuth(); + const { singleTeamId } = useSingleTeamConfig(); + const doraMetricsFetchArgs = useDoraMetricsFetchArgs(); const branches = useStateBranchConfig(); const isCodeIntegrationLinked = integrationSet.has(IntegrationGroup.CODE); const isSaving = useEasyState(false); @@ -149,11 +152,11 @@ export const TeamProductionBranchSelector: FC<{ ); } + // CLUSTOX: contributor filter -- saving prod branches refetches + // DORA, and the selection has to survive that. Shared hook, so a + // future argument can't be missed here again. const fetchDoraArgs = { - orgId: orgId, - teamId: singleTeamId, - fromDate: dates.start, - toDate: dates.end, + ...doraMetricsFetchArgs, branches: activeBranchMode === ActiveBranchMode.PROD ? null : branches }; @@ -182,9 +185,7 @@ export const TeamProductionBranchSelector: FC<{ dispatch, onClose, activeBranchMode, - orgId, - dates.start, - dates.end, + doraMetricsFetchArgs, branches, enqueueSnackbar, updateActiveProdBranch diff --git a/web-server/src/components/TeamSelector/useTeamSelectorSetup.tsx b/web-server/src/components/TeamSelector/useTeamSelectorSetup.tsx index c9178d5c7..d050005fd 100644 --- a/web-server/src/components/TeamSelector/useTeamSelectorSetup.tsx +++ b/web-server/src/components/TeamSelector/useTeamSelectorSetup.tsx @@ -21,7 +21,10 @@ export const useTeamSelectorSetup = ({ mode }: UseTeamSelectorSetupArgs) => { const dispatch = useDispatch(); const { dateRange, singleTeam, setRange, partiallyUnselected } = useStateTeamConfig(); - const { orgId, integrationList } = useAuth(); + // CLUSTOX: hasCodeProviderLinked, not integrationList.length -- a + // Jira-only org has no teams/repos for this fetch to be worth firing. + // See docs/JIRA_INTEGRATION_PROPOSAL.md. + const { orgId, hasCodeProviderLinked } = useAuth(); const [showAllTeams, setShowAllTeams] = useState(true); const activeBranchMode = useSelector((state) => state.app.branchMode); const isAllBranchMode = activeBranchMode === ActiveBranchMode.ALL; @@ -65,8 +68,8 @@ export const useTeamSelectorSetup = ({ mode }: UseTeamSelectorSetupArgs) => { useEffect(() => { if (!orgId) return; - if (integrationList.length) fetchAllTeams(); - }, [fetchAllTeams, integrationList.length, orgId]); + if (hasCodeProviderLinked) fetchAllTeams(); + }, [fetchAllTeams, hasCodeProviderLinked, orgId]); const dateRangeLabel = !partiallyUnselected ? `${format(dateRange[0], 'do MMM')} to ${format(dateRange[1], 'do MMM')}` diff --git a/web-server/src/components/Teams/CreateTeams.tsx b/web-server/src/components/Teams/CreateTeams.tsx index 7f57d8a8c..ebd9a36c7 100644 --- a/web-server/src/components/Teams/CreateTeams.tsx +++ b/web-server/src/components/Teams/CreateTeams.tsx @@ -5,7 +5,10 @@ import { Autocomplete, Button, Card, + Checkbox, CircularProgress, + Divider, + FormControlLabel, MenuItem, Select, Table, @@ -24,7 +27,7 @@ import { Pagination } from '@mui/material'; import { useSnackbar } from 'notistack'; -import { FC, useCallback, useMemo, useState } from 'react'; +import { FC, ReactNode, useCallback, useMemo, useState } from 'react'; import { useTeamCRUD, @@ -35,10 +38,13 @@ import { Integration } from '@/constants/integrations'; import { useModal } from '@/contexts/ModalContext'; import { useBoolState, useEasyState } from '@/hooks/useEasyState'; import GitlabIcon from '@/mocks/icons/gitlab.svg'; +import JiraIcon from '@/mocks/icons/jira-icon.svg'; import { BaseRepo, DeploymentSources } from '@/types/resources'; import { trimWithEllipsis } from '@/utils/stringFormatting'; import { BatchImportModal } from './BatchImportModal'; +import { TeamJiraProjects } from './TeamJiraProjects'; +import { TeamRepoProjectMappings } from './TeamRepoProjectMappings'; import AnimatedInputWrapper from '../AnimatedInputWrapper/AnimatedInputWrapper'; import { FlexBox } from '../FlexBox'; @@ -65,7 +71,7 @@ export const CreateEditTeams: FC = ({ ); }; -const TeamsCRUD: FC = ({ onSave, onDiscard }) => { +const TeamsCRUD: FC = ({ onSave, onDiscard, teamId }) => { const { isPageLoading, editingTeam, isEditing } = useTeamCRUD(); return ( <> @@ -85,9 +91,37 @@ const TeamsCRUD: FC = ({ onSave, onDiscard }) => { ) : ( <> - - - + {/* CLUSTOX: two independent data sources, each with its own + save action -- a repo pick and a Jira project pick save + through entirely separate requests (TeamJiraProjects has + its own hook, its own endpoint, its own button), so each + gets its own labeled section rather than looking like + one shared form with two "Save" buttons in it. See + docs/JIRA_INTEGRATION_PROPOSAL.md. */} + } title="Code repositories"> + + + + + {isEditing && ( + <> + + } + title="Jira integration" + > + + {/* CLUSTOX: explicit, informational repo<->project + pairing -- see docs/JIRA_MULTI_ACCOUNT_PLAN.md's + follow-up on Jira<->repo relationships. Lives in + the same section as the project picker above + (rather than its own DataSourceSection) since it + only ever makes sense once that picker already has + a selection to pair against. */} + + + + )} )} @@ -96,6 +130,30 @@ const TeamsCRUD: FC = ({ onSave, onDiscard }) => { ); }; +// CLUSTOX: a small eyebrow (icon + uppercase label) above each data-source +// section, so "Code repositories" and "Jira integration" read as two +// distinct, independently-saved sources sharing one page -- not one +// shared form. Deliberately lighter-weight than the h6-style headings +// each section already renders internally ("Team Name", "Jira +// Projects"): this labels the section, those label the field. +const DataSourceSection: FC<{ + icon: ReactNode; + title: string; + children: ReactNode; +}> = ({ icon, title, children }) => ( + + + {icon} + + {title} + + + + {children} + + +); + export const Loader: FC<{ label?: string }> = ({ label = 'Loading...' }) => { return ( @@ -168,7 +226,10 @@ const TeamRepos: FC = () => { selectedRepos, raiseTeamRepoError, loadingRepos, - handleReposSearch + handleReposSearch, + isEditing, + isJiraOnlyTeam, + setIsJiraOnlyTeam } = useTeamCRUD(); const { addModal, closeModal } = useModal(); @@ -228,6 +289,22 @@ const TeamRepos: FC = () => { Select repositories for this team using name or link + {/* CLUSTOX: opt-in escape hatch for a Jira-only org (no Git repo at + all) -- create-time only, and off by default, so every existing + repo-based team creation flow is completely unaffected unless an + admin explicitly checks this. See saveDisabled's own comment in + useTeamsConfig.tsx. */} + {!isEditing && ( + setIsJiraOnlyTeam(e.target.checked)} + /> + } + label="This team has no Git repository (Jira-only)" + /> + )} = ({ teamName, selectedRepos, onDiscard, - saveDisabled + saveDisabled, + isEditing, + isJiraOnlyTeam } = useTeamCRUD(); const { enqueueSnackbar } = useSnackbar(); @@ -366,7 +445,12 @@ const ActionTray: FC = ({ autoHideDuration: 2000 }); } - if (!selectedRepos.length) { + // CLUSTOX: unchanged from before -- still required by default. + // The only way past this is the explicit "Jira-only, no Git + // repo" checkbox below, or (while editing) a team that already + // started with zero repos -- see saveDisabled's own comment in + // useTeamsConfig.tsx. + if (!selectedRepos.length && !(isJiraOnlyTeam && !isEditing)) { return enqueueSnackbar('Please select at least one repository', { variant: 'error', autoHideDuration: 2000 @@ -385,7 +469,11 @@ const ActionTray: FC = ({ } }} > - {isSaveLoading ? : 'Save'} + {isSaveLoading ? ( + + ) : ( + 'Save repositories' + )} + ); +}; + +const LinkedIcon = () => { + const isVisible = useBoolState(false); + useEffect(() => { + setTimeout(isVisible.true, 200); + }, [isVisible.true]); + return ( + + + + + + + + + + + ); +}; diff --git a/web-server/src/content/Dashboards/ConfigureBitbucketModalBody.tsx b/web-server/src/content/Dashboards/ConfigureBitbucketModalBody.tsx new file mode 100644 index 000000000..d61dd5de6 --- /dev/null +++ b/web-server/src/content/Dashboards/ConfigureBitbucketModalBody.tsx @@ -0,0 +1,248 @@ +import { LoadingButton } from '@mui/lab'; +import { Link, TextField } from '@mui/material'; +import { useSnackbar } from 'notistack'; +import { FC, useCallback } from 'react'; + +import { FlexBox } from '@/components/FlexBox'; +import { Line } from '@/components/Text'; +import { Integration } from '@/constants/integrations'; +import { useAuth } from '@/hooks/useAuth'; +// CLUSTOX: server-resolved workspace, immune to stale persisted redux state. +import { useClustoxUser } from '@/hooks/useClustoxUser'; +import { useBoolState, useEasyState } from '@/hooks/useEasyState'; +import { fetchCurrentOrg } from '@/slices/auth'; +import { fetchTeams } from '@/slices/team'; +import { useDispatch } from '@/store'; +import { checkBitbucketValidity, linkProvider } from '@/utils/auth'; +import { depFn } from '@/utils/fn'; + +// A plain shape check, not deliverability -- the value is only ever the Basic +// auth username, so the test is "looks like an email", nothing deeper. +const EMAIL_PATTERN = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + +export const ConfigureBitbucketModalBody: FC<{ + onClose: () => void; +}> = ({ onClose }) => { + const email = useEasyState(''); + const token = useEasyState(''); + const { orgId: contextOrgId } = useAuth(); + // CLUSTOX: prefer the server-resolved workspace. The redux copy is persisted + // by redux-persist and can rehydrate empty from a previous session, which + // previously produced a POST to /orgs/undefined/integration. + const { orgId: sessionOrgId } = useClustoxUser(); + const orgId = sessionOrgId ?? contextOrgId; + const { enqueueSnackbar } = useSnackbar(); + const dispatch = useDispatch(); + const isLoading = useBoolState(); + + const emailError = useEasyState(''); + const tokenError = useEasyState(''); + + const handleEmailChange = (value: string) => { + email.set(value); + emailError.set(''); + }; + + const handleTokenChange = (value: string) => { + token.set(value); + tokenError.set(''); + }; + + const handleSubmission = useCallback(async () => { + // CLUSTOX: trim both fields -- a token copied from Atlassian's UI often + // arrives with a trailing newline, which corrupts the Basic auth pair + // into a guaranteed 401 that reads exactly like a wrong token. + const emailValue = email.value.trim(); + const tokenValue = token.value.trim(); + try { + if (!EMAIL_PATTERN.test(emailValue)) { + emailError.set('Enter the Atlassian account email for this token'); + throw Error('Invalid email'); + } + if (!tokenValue) { + tokenError.set('Please enter an API token'); + throw Error('Empty token'); + } + // CLUSTOX: refuse rather than POST to /orgs/undefined/integration. + if (!orgId) { + tokenError.set('No workspace selected. Reload the page and try again.'); + throw Error('No workspace'); + } + } catch (e) { + console.error(e); + return; + } + + depFn(isLoading.true); + await checkBitbucketValidity(emailValue, tokenValue) + .then(({ valid, reason }) => { + if (!valid) { + // CLUSTOX: name the real problem when the server could tell it + // apart. A scoped token without account read access works against + // repositories but fails the sync's own /2.0/user validity check, + // so "invalid credentials" would send the user hunting the wrong + // bug. + throw new Error( + reason === 'missing_account_scope' + ? 'This token works but is missing the Account read scope. Recreate it with account, repository and pullrequest read scopes.' + : 'Invalid email or API token' + ); + } + }) + .then(async () => { + try { + // CLUSTOX: the email rides in provider_meta -- it is the Basic auth + // username, not a secret. Only the token is encrypted. + return await linkProvider(tokenValue, orgId, Integration.BITBUCKET, { + email: emailValue + }); + } catch (e: any) { + throw new Error( + `Failed to link Bitbucket${e?.message ? `: ${e?.message}` : ''}`, + { cause: e } + ); + } + }) + .then(() => { + dispatch(fetchCurrentOrg()); + dispatch(fetchTeams({ org_id: orgId })); + enqueueSnackbar('Bitbucket linked successfully', { + variant: 'success', + autoHideDuration: 2000 + }); + onClose(); + }) + .catch((e) => { + tokenError.set(e.message); + console.error(`Error while linking token: ${e.message}`, e); + }) + .finally(isLoading.false); + }, [ + dispatch, + email.value, + emailError, + enqueueSnackbar, + isLoading.false, + isLoading.true, + onClose, + orgId, + token.value, + tokenError + ]); + + const focusTokenInput = useCallback(() => { + if (!token.value) document.getElementById('bitbucket-api-token')?.focus(); + else handleSubmission(); + }, [token.value, handleSubmission]); + + return ( + + + + Enter your Atlassian account email and API token{' '} + + * + + + + { + if (e.key === 'Enter') { + e.preventDefault(); + e.stopPropagation(); + e.nativeEvent.stopImmediatePropagation(); + focusTokenInput(); + return; + } + }} + error={!!emailError.value} + sx={{ width: '100%' }} + value={email.value} + onChange={(e) => handleEmailChange(e.currentTarget.value)} + label="Atlassian account email" + // CLUSTOX: 'new-password', not 'off' -- Chrome ignores 'off' near + // a password-type input. During live testing the password manager + // silently replaced field contents with saved logins; the dots + // looked right and the pair failed as "invalid credentials". + autoComplete="new-password" + InputLabelProps={{ shrink: true }} + /> + + {emailError.value} + + + + + { + if (e.key === 'Enter') { + e.preventDefault(); + e.stopPropagation(); + e.nativeEvent.stopImmediatePropagation(); + handleSubmission(); + return; + } + }} + error={!!tokenError.value} + sx={{ width: '100%' }} + value={token.value} + onChange={(e) => handleTokenChange(e.currentTarget.value)} + label="Atlassian API Token" + type="password" + autoComplete="new-password" + // Browsers apply their password-autofill heuristics to any + // type="password" field, token or not. Keeping the label + // permanently shrunk removes any race between that and React + // learning the value. + InputLabelProps={{ shrink: true }} + /> + + {tokenError.value} + + + + + + Create an API token with account, repository and pullrequest + read scopes + + + {' ->'} + + + + + + + Learn more about Atlassian + + API tokens + + here + + + + + + Confirm + + + + + + ); +}; diff --git a/web-server/src/content/Dashboards/ConfigureGithubModalBody.tsx b/web-server/src/content/Dashboards/ConfigureGithubModalBody.tsx index e1c36692c..5763d79a7 100644 --- a/web-server/src/content/Dashboards/ConfigureGithubModalBody.tsx +++ b/web-server/src/content/Dashboards/ConfigureGithubModalBody.tsx @@ -8,6 +8,8 @@ import { FlexBox } from '@/components/FlexBox'; import { Line } from '@/components/Text'; import { Integration } from '@/constants/integrations'; import { useAuth } from '@/hooks/useAuth'; +// CLUSTOX: server-resolved workspace, immune to stale persisted redux state. +import { useClustoxUser } from '@/hooks/useClustoxUser'; import { useBoolState, useEasyState } from '@/hooks/useEasyState'; import { fetchCurrentOrg } from '@/slices/auth'; import { fetchTeams } from '@/slices/team'; @@ -24,7 +26,12 @@ export const ConfigureGithubModalBody: FC<{ onClose: () => void; }> = ({ onClose }) => { const token = useEasyState(''); - const { orgId } = useAuth(); + const { orgId: contextOrgId } = useAuth(); + // CLUSTOX: prefer the server-resolved workspace. The redux copy is persisted + // by redux-persist and can rehydrate empty from a previous session, which + // previously produced a POST to /orgs/undefined/integration. + const { orgId: sessionOrgId } = useClustoxUser(); + const orgId = sessionOrgId ?? contextOrgId; const { enqueueSnackbar } = useSnackbar(); const customDomain = useEasyState(''); const dispatch = useDispatch(); @@ -69,6 +76,11 @@ export const ConfigureGithubModalBody: FC<{ return; } + if (!orgId) { + setError('No workspace selected. Reload the page and try again.'); + return; + } + isLoading.true(); try { const isValid = await checkGitHubValidity( @@ -151,6 +163,11 @@ export const ConfigureGithubModalBody: FC<{ }} label="Github Personal Access Token" type="password" + // Browsers apply their password-autofill heuristics to any + // type="password" field, token or not. Keeping the label + // permanently shrunk removes any race between that and React + // learning the value. + InputLabelProps={{ shrink: true }} /> {showError.value} diff --git a/web-server/src/content/Dashboards/ConfigureGitlabModalBody.tsx b/web-server/src/content/Dashboards/ConfigureGitlabModalBody.tsx index eef8a0942..82ec76cbc 100644 --- a/web-server/src/content/Dashboards/ConfigureGitlabModalBody.tsx +++ b/web-server/src/content/Dashboards/ConfigureGitlabModalBody.tsx @@ -8,6 +8,8 @@ import { FlexBox } from '@/components/FlexBox'; import { Line } from '@/components/Text'; import { Integration } from '@/constants/integrations'; import { useAuth } from '@/hooks/useAuth'; +// CLUSTOX: server-resolved workspace, immune to stale persisted redux state. +import { useClustoxUser } from '@/hooks/useClustoxUser'; import { useBoolState, useEasyState } from '@/hooks/useEasyState'; import { fetchCurrentOrg } from '@/slices/auth'; import { fetchTeams } from '@/slices/team'; @@ -25,7 +27,12 @@ export const ConfigureGitlabModalBody: FC<{ }> = ({ onClose }) => { const token = useEasyState(''); const customDomain = useEasyState(''); - const { orgId } = useAuth(); + const { orgId: contextOrgId } = useAuth(); + // CLUSTOX: prefer the server-resolved workspace. The redux copy is persisted + // by redux-persist and can rehydrate empty from a previous session, which + // previously produced a POST to /orgs/undefined/integration. + const { orgId: sessionOrgId } = useClustoxUser(); + const orgId = sessionOrgId ?? contextOrgId; const { enqueueSnackbar } = useSnackbar(); const dispatch = useDispatch(); const isLoading = useBoolState(); @@ -68,6 +75,12 @@ export const ConfigureGitlabModalBody: FC<{ setDomainError('Please enter a valid domain'); throw Error('Invalid domain'); } + + // CLUSTOX: refuse rather than POST to /orgs/undefined/integration. + if (!orgId) { + setScopeError('No workspace selected. Reload the page and try again.'); + throw Error('No workspace'); + } } catch (e) { console.error(e); return; @@ -163,6 +176,11 @@ export const ConfigureGitlabModalBody: FC<{ }} label="Gitlab Personal Access Token" type="password" + // Browsers apply their password-autofill heuristics to any + // type="password" field, token or not. Keeping the label + // permanently shrunk removes any race between that and React + // learning the value. + InputLabelProps={{ shrink: true }} /> {showScopeError.value} diff --git a/web-server/src/content/Dashboards/ConfigureJiraModalBody.tsx b/web-server/src/content/Dashboards/ConfigureJiraModalBody.tsx new file mode 100644 index 000000000..b3daaa2c5 --- /dev/null +++ b/web-server/src/content/Dashboards/ConfigureJiraModalBody.tsx @@ -0,0 +1,170 @@ +import { Link, TextField } from '@mui/material'; +import { LoadingButton } from '@mui/lab'; +import { useSnackbar } from 'notistack'; +import { FC, useCallback } from 'react'; + +import { FlexBox } from '@/components/FlexBox'; +import { Line } from '@/components/Text'; +import { Integration } from '@/constants/integrations'; +import { useAuth } from '@/hooks/useAuth'; +// CLUSTOX: server-resolved workspace, immune to stale persisted redux state. +import { useClustoxUser } from '@/hooks/useClustoxUser'; +import { useBoolState, useEasyState } from '@/hooks/useEasyState'; +import { fetchCurrentOrg } from '@/slices/auth'; +import { useDispatch } from '@/store'; +import { + checkJiraValidity, + linkProvider, + normalizeJiraSiteUrl +} from '@/utils/auth'; +import { depFn } from '@/utils/fn'; + +export const ConfigureJiraModalBody: FC<{ + onClose: () => void; +}> = ({ onClose }) => { + const siteUrl = useEasyState(''); + const email = useEasyState(''); + const token = useEasyState(''); + const { orgId: contextOrgId } = useAuth(); + // CLUSTOX: prefer the server-resolved workspace. The redux copy is persisted + // by redux-persist and can rehydrate empty from a previous session, which + // previously produced a POST to /orgs/undefined/integration. + const { orgId: sessionOrgId } = useClustoxUser(); + const orgId = sessionOrgId ?? contextOrgId; + const { enqueueSnackbar } = useSnackbar(); + const dispatch = useDispatch(); + const isLoading = useBoolState(); + + const showError = useEasyState(''); + + const setError = useCallback( + (error: string) => { + depFn(showError.set, error); + }, + [showError.set] + ); + + const handleSubmission = useCallback(async () => { + if (!siteUrl.value || !email.value || !token.value) { + setError('Please fill in all three fields'); + return; + } + if (!orgId) { + setError('No workspace selected. Reload the page and try again.'); + return; + } + + isLoading.true(); + try { + const result = await checkJiraValidity( + siteUrl.value, + email.value, + token.value + ); + + if (!result.valid) { + setError( + result.reason === 'unauthorized' + ? 'That email/API token combination was rejected by Jira.' + : result.reason === 'unreachable' + ? "Couldn't reach that Jira site — check the URL." + : 'Could not verify this Jira account. Please try again.' + ); + return; + } + + await linkProvider(token.value, orgId, Integration.JIRA, { + site_url: normalizeJiraSiteUrl(siteUrl.value), + email: email.value + }); + + // Await, don't fire-and-forget: closing the modal before this + // resolves would let the Jira card render on the pre-link ("not + // linked") state for a moment -- same race already fixed for + // GitHub/GitLab's own link flow. + await dispatch(fetchCurrentOrg()); + + enqueueSnackbar( + result.displayName + ? `Jira linked — signed in as ${result.displayName}` + : 'Jira linked successfully', + { variant: 'success', autoHideDuration: 3000 } + ); + onClose(); + } catch (e: any) { + setError(e.message || 'Unknown error'); + console.error(e); + } finally { + isLoading.false(); + } + }, [ + siteUrl.value, + email.value, + token.value, + dispatch, + enqueueSnackbar, + isLoading.false, + isLoading.true, + onClose, + orgId, + setError + ]); + + return ( + + Enter your Jira workspace details below. + + siteUrl.set(e.target.value)} + InputLabelProps={{ shrink: true }} + /> + email.set(e.target.value)} + InputLabelProps={{ shrink: true }} + /> + + token.set(e.target.value)} + // Browsers apply their password-autofill heuristics to any + // type="password" field, token or not. Keeping the label + // permanently shrunk removes any race between that and React + // learning the value. + InputLabelProps={{ shrink: true }} + /> + + Generate one at{' '} + + id.atlassian.com/manage-profile/security + + + + + + {showError.value} + + + + + Verify & link + + + + ); +}; diff --git a/web-server/src/content/Dashboards/JiraIntegrationCard.tsx b/web-server/src/content/Dashboards/JiraIntegrationCard.tsx new file mode 100644 index 000000000..5eb54cab1 --- /dev/null +++ b/web-server/src/content/Dashboards/JiraIntegrationCard.tsx @@ -0,0 +1,329 @@ +import { + ArrowForwardIosRounded, + ChevronRightRounded, + SettingsRounded +} from '@mui/icons-material'; +import { Button, useTheme } from '@mui/material'; +import CircularProgress from '@mui/material/CircularProgress'; +import { useSnackbar } from 'notistack'; +import { FC, ReactNode, useEffect } from 'react'; + +import { FlexBox } from '@/components/FlexBox'; +import { JiraConnectionsManager } from '@/components/JiraConnectionsManager'; +import { Line } from '@/components/Text'; +import { track } from '@/constants/events'; +import { FetchState } from '@/constants/ui-states'; +import { jiraIntegrationDisplay } from '@/content/Dashboards/githubIntegration'; +import { useIntegrationHandlers } from '@/content/Dashboards/useIntegrationHandlers'; +import { useModal } from '@/contexts/ModalContext'; +import { useAuth } from '@/hooks/useAuth'; +import { useBoolState } from '@/hooks/useEasyState'; +import { useFeature } from '@/hooks/useFeature'; +import { fetchCurrentOrg } from '@/slices/auth'; +import { useDispatch, useSelector } from '@/store'; + +const cardRadius = 10.5; +const cardBorder = 1.5; +const getRadiusWithPadding = (radius: number, padding: number) => + `${radius + padding}px`; + +// CLUSTOX: Jira multi-account support -- see +// docs/JIRA_MULTI_ACCOUNT_PLAN.md. One Jira card, not two: this used to be +// a standalone JiraConnectionsCard living beside this one on the +// integrations page, which read as two unrelated "Jira" tiles to manage. +// While the flag is on (the default), this card's whole job is opening the +// connections manager -- the legacy single-account link/unlink flow below +// only renders while it's off, which is the fallback for whoever disables +// the flag, not a live second path. +export const JiraIntegrationCard = () => { + const showMultiAccount = useFeature('show_jira_multi_account'); + if (showMultiAccount) return ; + return ; +}; + +const JiraMultiAccountCard = () => { + const theme = useTheme(); + const { addModal } = useModal(); + + const openManager = () => + addModal({ + title: 'Jira connections', + body: , + showCloseIcon: true + }); + + return ( + + + + + + + {jiraIntegrationDisplay.icon} + + + {jiraIntegrationDisplay.name} + + + + Connect one or more Jira sites or accounts + + + + + + + + ); +}; + +// CLUSTOX: mirrors GithubIntegrationCard/GitlabIntegrationCard exactly -- +// same sliceLoading source, same layout, same unlink confirmation flow. +// Jira Phase 1 (see docs/JIRA_INTEGRATION_PROPOSAL.md) is link-only: no +// sync, no project picker yet, so there's nothing else for this card to do +// beyond reflect integrations.jira. Rendered only while +// show_jira_multi_account is off -- see JiraIntegrationCard above. +const LegacyJiraIntegrationCard = () => { + const theme = useTheme(); + const { integrations } = useAuth(); + const isJiraIntegrated = integrations.jira; + const sliceLoading = useSelector( + (s) => s.auth.requests.org === FetchState.REQUEST + ); + const { link, unlink } = useIntegrationHandlers(); + + const localLoading = useBoolState(false); + + const isLoading = sliceLoading || localLoading.value; + + const dispatch = useDispatch(); + + const { enqueueSnackbar } = useSnackbar(); + + return ( + + {isJiraIntegrated && ( + + + + )} + + + + + + {jiraIntegrationDisplay.icon} + + + {jiraIntegrationDisplay.name} + + + + { + track( + isJiraIntegrated + ? 'INTEGRATION_UNLINK_TRIGGERED' + : 'INTEGRATION_LINK_TRIGGERED', + { integration_name: jiraIntegrationDisplay.name } + ); + if (!isJiraIntegrated) { + link.jira(); + return; + } + const shouldExecute = window.confirm( + 'Are you sure you want to unlink?' + ); + if (shouldExecute) { + localLoading.true(); + await unlink + .jira() + .then(() => { + enqueueSnackbar('Jira unlinked successfully', { + variant: 'success' + }); + }) + .then(async () => dispatch(fetchCurrentOrg())) + .catch((e) => { + console.error('Failed to unlink Jira', e); + enqueueSnackbar('Failed to unlink Jira', { + variant: 'error' + }); + }) + .finally(localLoading.false); + } + }} + label={!isJiraIntegrated ? 'Link' : 'Unlink'} + bgOpacity={!isJiraIntegrated ? 0.45 : 0.25} + endIcon={ + isLoading ? ( + + ) : ( + + ) + } + minWidth="72px" + /> + + + + + ); +}; + +const IntegrationActionsButton: FC<{ + onClick: AnyFunction; + label: ReactNode; + bgOpacity?: number; + startIcon?: ReactNode; + endIcon?: ReactNode; + minWidth?: string; +}> = ({ + label, + onClick, + bgOpacity = 0.45, + endIcon = ( + + ), + startIcon = , + minWidth = '80px' +}) => { + const theme = useTheme(); + + return ( + + ); +}; + +const LinkedIcon = () => { + const isVisible = useBoolState(false); + useEffect(() => { + setTimeout(isVisible.true, 200); + }, [isVisible.true]); + return ( + + + + + + + + + + + ); +}; diff --git a/web-server/src/content/Dashboards/__tests__/BitbucketIntegrationCard.test.tsx b/web-server/src/content/Dashboards/__tests__/BitbucketIntegrationCard.test.tsx new file mode 100644 index 000000000..92063c9a6 --- /dev/null +++ b/web-server/src/content/Dashboards/__tests__/BitbucketIntegrationCard.test.tsx @@ -0,0 +1,60 @@ +jest.mock('@/hooks/useAuth', () => ({ useAuth: jest.fn() })); +jest.mock('@/content/Dashboards/useIntegrationHandlers', () => ({ + useIntegrationHandlers: jest.fn() +})); +jest.mock('@/store', () => ({ + useDispatch: jest.fn(), + useSelector: jest.fn() +})); +jest.mock('notistack', () => ({ useSnackbar: jest.fn() })); +jest.mock('@/slices/auth', () => ({ + fetchCurrentOrg: jest.fn(() => ({ type: 'mock' })) +})); +jest.mock('@/constants/events', () => ({ track: jest.fn() })); + +import { screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { useSnackbar } from 'notistack'; + +import { BitbucketIntegrationCard } from '@/content/Dashboards/BitbucketIntegrationCard'; +import { useIntegrationHandlers } from '@/content/Dashboards/useIntegrationHandlers'; +import { useAuth } from '@/hooks/useAuth'; +import { useDispatch, useSelector } from '@/store'; +import { renderWithTheme as render } from '@/utils/testUtils'; + +const link = { bitbucket: jest.fn() }; +const unlink = { bitbucket: jest.fn().mockResolvedValue(undefined) }; + +describe('BitbucketIntegrationCard', () => { + beforeEach(() => { + jest.clearAllMocks(); + (useIntegrationHandlers as jest.Mock).mockReturnValue({ link, unlink }); + (useSelector as jest.Mock).mockReturnValue(false); + (useDispatch as jest.Mock).mockReturnValue(jest.fn()); + (useSnackbar as jest.Mock).mockReturnValue({ enqueueSnackbar: jest.fn() }); + }); + + it('offers Link when bitbucket is not integrated, and opens the modal', async () => { + (useAuth as jest.Mock).mockReturnValue({ integrations: {} }); + render(); + + expect(screen.getByText('Link')).toBeInTheDocument(); + await userEvent.setup().click(screen.getByText('Link')); + expect(link.bitbucket).toHaveBeenCalledTimes(1); + }); + + it('renders alongside an existing setup untouched: linked state reads from integrations.bitbucket only', () => { + // CLUSTOX: provider isolation on the frontend -- a workspace with GitHub + // and GitLab linked but not Bitbucket must see this card in its unlinked + // state, not inherit anyone else's. + (useAuth as jest.Mock).mockReturnValue({ + integrations: { + github: { integrated: true }, + gitlab: { integrated: true } + } + }); + render(); + + expect(screen.getByText('Link')).toBeInTheDocument(); + }); +}); diff --git a/web-server/src/content/Dashboards/__tests__/ConfigureBitbucketModalBody.test.tsx b/web-server/src/content/Dashboards/__tests__/ConfigureBitbucketModalBody.test.tsx new file mode 100644 index 000000000..d8d112e6a --- /dev/null +++ b/web-server/src/content/Dashboards/__tests__/ConfigureBitbucketModalBody.test.tsx @@ -0,0 +1,137 @@ +jest.mock('@/hooks/useAuth', () => ({ useAuth: jest.fn() })); +jest.mock('@/hooks/useClustoxUser', () => ({ useClustoxUser: jest.fn() })); +jest.mock('@/store', () => ({ useDispatch: jest.fn() })); +jest.mock('notistack', () => ({ useSnackbar: jest.fn() })); +jest.mock('@/slices/auth', () => ({ + fetchCurrentOrg: jest.fn(() => ({ type: 'mock' })) +})); +jest.mock('@/slices/team', () => ({ + fetchTeams: jest.fn(() => ({ type: 'mock' })) +})); +jest.mock('@/utils/auth', () => ({ + checkBitbucketValidity: jest.fn(), + linkProvider: jest.fn() +})); + +import { render, screen, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { useSnackbar } from 'notistack'; + +import { Integration } from '@/constants/integrations'; +import { ConfigureBitbucketModalBody } from '@/content/Dashboards/ConfigureBitbucketModalBody'; +import { useAuth } from '@/hooks/useAuth'; +import { useClustoxUser } from '@/hooks/useClustoxUser'; +import { useDispatch } from '@/store'; +import { checkBitbucketValidity, linkProvider } from '@/utils/auth'; + +const fillForm = async ( + email = 'hamad@clustox.com', + token = 'an-api-token' +) => { + const user = userEvent.setup(); + if (email) + await user.type(screen.getByLabelText('Atlassian account email'), email); + if (token) + await user.type(screen.getByLabelText('Atlassian API Token'), token); + await user.click(screen.getByRole('button', { name: /confirm/i })); +}; + +describe('ConfigureBitbucketModalBody', () => { + const onClose = jest.fn(); + const dispatch = jest.fn(() => Promise.resolve()); + const enqueueSnackbar = jest.fn(); + + beforeEach(() => { + jest.clearAllMocks(); + (useAuth as jest.Mock).mockReturnValue({ orgId: 'org-1' }); + (useClustoxUser as jest.Mock).mockReturnValue({ orgId: null }); + (useDispatch as jest.Mock).mockReturnValue(dispatch); + (useSnackbar as jest.Mock).mockReturnValue({ enqueueSnackbar }); + dispatch.mockReturnValue(Promise.resolve()); + }); + + it('rejects a malformed email before any network call', async () => { + render(); + await fillForm('not-an-email', 'a-token'); + + expect( + screen.getByText('Enter the Atlassian account email for this token') + ).toBeInTheDocument(); + expect(checkBitbucketValidity).not.toHaveBeenCalled(); + expect(linkProvider).not.toHaveBeenCalled(); + }); + + it('shows the broad credentials error and does not link on a failed check', async () => { + // CLUSTOX: the API cannot tell a revoked token from an expired one, and + // Atlassian tokens expose no scope header -- the copy stays broad on + // purpose. + (checkBitbucketValidity as jest.Mock).mockResolvedValue({ + valid: false, + reason: 'invalid_credentials' + }); + render(); + await fillForm(); + + await waitFor(() => + expect(screen.getByText('Invalid email or API token')).toBeInTheDocument() + ); + expect(linkProvider).not.toHaveBeenCalled(); + expect(onClose).not.toHaveBeenCalled(); + }); + + it('names the missing account scope when the server could tell', async () => { + (checkBitbucketValidity as jest.Mock).mockResolvedValue({ + valid: false, + reason: 'missing_account_scope' + }); + render(); + await fillForm(); + + await waitFor(() => + expect( + screen.getByText(/missing the Account read scope/i) + ).toBeInTheDocument() + ); + expect(linkProvider).not.toHaveBeenCalled(); + }); + + it('trims pasted whitespace before validating and linking', async () => { + // CLUSTOX: a token copied from Atlassian's UI often carries a trailing + // newline; untrimmed it corrupts the Basic auth pair into a 401 that + // reads exactly like a wrong token. + (checkBitbucketValidity as jest.Mock).mockResolvedValue({ valid: true }); + (linkProvider as jest.Mock).mockResolvedValue({}); + render(); + await fillForm(' hamad@clustox.com ', 'an-api-token '); + + await waitFor(() => expect(linkProvider).toHaveBeenCalledTimes(1)); + expect(checkBitbucketValidity).toHaveBeenCalledWith( + 'hamad@clustox.com', + 'an-api-token' + ); + expect(linkProvider).toHaveBeenCalledWith( + 'an-api-token', + 'org-1', + expect.anything(), + { email: 'hamad@clustox.com' } + ); + }); + + it('links with the email in meta on a passed check', async () => { + (checkBitbucketValidity as jest.Mock).mockResolvedValue({ valid: true }); + (linkProvider as jest.Mock).mockResolvedValue({}); + render(); + await fillForm(); + + await waitFor(() => expect(linkProvider).toHaveBeenCalledTimes(1)); + // CLUSTOX: the email rides in provider_meta -- the sync factory reads it + // back as the Basic auth username, so this exact shape is the contract. + expect(linkProvider).toHaveBeenCalledWith( + 'an-api-token', + 'org-1', + Integration.BITBUCKET, + { email: 'hamad@clustox.com' } + ); + await waitFor(() => expect(onClose).toHaveBeenCalled()); + }); +}); diff --git a/web-server/src/content/Dashboards/__tests__/ConfigureJiraModalBody.test.tsx b/web-server/src/content/Dashboards/__tests__/ConfigureJiraModalBody.test.tsx new file mode 100644 index 000000000..f3ec7ce1d --- /dev/null +++ b/web-server/src/content/Dashboards/__tests__/ConfigureJiraModalBody.test.tsx @@ -0,0 +1,188 @@ +jest.mock('@/hooks/useAuth', () => ({ useAuth: jest.fn() })); +jest.mock('@/hooks/useClustoxUser', () => ({ useClustoxUser: jest.fn() })); +jest.mock('@/store', () => ({ useDispatch: jest.fn() })); +jest.mock('notistack', () => ({ useSnackbar: jest.fn() })); +jest.mock('@/slices/auth', () => ({ fetchCurrentOrg: jest.fn(() => ({ type: 'mock' })) })); +jest.mock('@/utils/auth', () => ({ + checkJiraValidity: jest.fn(), + linkProvider: jest.fn(), + normalizeJiraSiteUrl: jest.requireActual('@/utils/auth').normalizeJiraSiteUrl +})); + +import { render, screen, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { useSnackbar } from 'notistack'; + +import { ConfigureJiraModalBody } from '@/content/Dashboards/ConfigureJiraModalBody'; +import { useAuth } from '@/hooks/useAuth'; +import { useClustoxUser } from '@/hooks/useClustoxUser'; +import { useDispatch } from '@/store'; +import { checkJiraValidity, linkProvider } from '@/utils/auth'; + +const fillForm = async ( + siteUrl = 'mycompany.atlassian.net', + email = 'jordan@mycompany.com', + token = 'a-token' +) => { + const user = userEvent.setup(); + if (siteUrl) await user.type(screen.getByLabelText('Jira Site URL'), siteUrl); + if (email) await user.type(screen.getByLabelText('Email'), email); + if (token) await user.type(screen.getByLabelText('API Token'), token); + await user.click(screen.getByRole('button', { name: /verify.*link/i })); +}; + +describe('ConfigureJiraModalBody', () => { + const onClose = jest.fn(); + const dispatch = jest.fn(() => Promise.resolve()); + const enqueueSnackbar = jest.fn(); + + beforeEach(() => { + jest.clearAllMocks(); + (useAuth as jest.Mock).mockReturnValue({ orgId: 'org-1' }); + (useClustoxUser as jest.Mock).mockReturnValue({ orgId: null }); + (useDispatch as jest.Mock).mockReturnValue(dispatch); + (useSnackbar as jest.Mock).mockReturnValue({ enqueueSnackbar }); + dispatch.mockReturnValue(Promise.resolve()); + }); + + it('requires all three fields before attempting to verify anything', async () => { + render(); + await fillForm('', '', ''); + + expect( + screen.getByText('Please fill in all three fields') + ).toBeInTheDocument(); + expect(checkJiraValidity).not.toHaveBeenCalled(); + }); + + it('refuses to submit with no workspace resolved, rather than link an undefined org', async () => { + (useAuth as jest.Mock).mockReturnValue({ orgId: null }); + (useClustoxUser as jest.Mock).mockReturnValue({ orgId: null }); + render(); + await fillForm(); + + expect( + screen.getByText('No workspace selected. Reload the page and try again.') + ).toBeInTheDocument(); + expect(checkJiraValidity).not.toHaveBeenCalled(); + }); + + it("prefers the server-resolved workspace (useClustoxUser) over the redux-persisted one", async () => { + (useAuth as jest.Mock).mockReturnValue({ orgId: 'stale-redux-org' }); + (useClustoxUser as jest.Mock).mockReturnValue({ orgId: 'fresh-server-org' }); + (checkJiraValidity as jest.Mock).mockResolvedValue({ valid: true }); + (linkProvider as jest.Mock).mockResolvedValue(undefined); + render(); + + await fillForm(); + + await waitFor(() => + expect(linkProvider).toHaveBeenCalledWith( + 'a-token', + 'fresh-server-org', + 'jira', + expect.anything() + ) + ); + }); + + it.each([ + ['unauthorized', 'That email/API token combination was rejected by Jira.'], + ['unreachable', "Couldn't reach that Jira site — check the URL."], + ['unknown', 'Could not verify this Jira account. Please try again.'] + ])('shows the right message for reason=%s, and never links', async (reason, message) => { + (checkJiraValidity as jest.Mock).mockResolvedValue({ valid: false, reason }); + render(); + + await fillForm(); + + expect(await screen.findByText(message)).toBeInTheDocument(); + expect(linkProvider).not.toHaveBeenCalled(); + expect(onClose).not.toHaveBeenCalled(); + }); + + it('normalizes the site URL before storing it, even though it validates the raw input', async () => { + (checkJiraValidity as jest.Mock).mockResolvedValue({ valid: true }); + (linkProvider as jest.Mock).mockResolvedValue(undefined); + render(); + + await fillForm('https://mycompany.atlassian.net/', 'jordan@mycompany.com', 'tok'); + + await waitFor(() => + expect(checkJiraValidity).toHaveBeenCalledWith( + 'https://mycompany.atlassian.net/', // raw input, unmodified + 'jordan@mycompany.com', + 'tok' + ) + ); + await waitFor(() => + expect(linkProvider).toHaveBeenCalledWith( + 'tok', + 'org-1', + 'jira', + { site_url: 'mycompany.atlassian.net', email: 'jordan@mycompany.com' } // normalized + ) + ); + }); + + it('awaits fetchCurrentOrg before closing -- the card behind this modal reads Redux directly', async () => { + const order: string[] = []; + (checkJiraValidity as jest.Mock).mockResolvedValue({ valid: true }); + (linkProvider as jest.Mock).mockResolvedValue(undefined); + dispatch.mockImplementation(() => { + order.push('fetchCurrentOrg'); + return Promise.resolve(); + }); + onClose.mockImplementation(() => order.push('onClose')); + render(); + + await fillForm(); + + await waitFor(() => expect(onClose).toHaveBeenCalled()); + expect(order).toEqual(['fetchCurrentOrg', 'onClose']); + }); + + it('shows the signed-in-as name on success when Jira returned one', async () => { + (checkJiraValidity as jest.Mock).mockResolvedValue({ + valid: true, + displayName: 'Jordan Diaz' + }); + (linkProvider as jest.Mock).mockResolvedValue(undefined); + render(); + + await fillForm(); + + await waitFor(() => + expect(enqueueSnackbar).toHaveBeenCalledWith( + 'Jira linked — signed in as Jordan Diaz', + expect.objectContaining({ variant: 'success' }) + ) + ); + }); + + it('falls back to a generic success message when Jira did not return a display name', async () => { + (checkJiraValidity as jest.Mock).mockResolvedValue({ valid: true }); + (linkProvider as jest.Mock).mockResolvedValue(undefined); + render(); + + await fillForm(); + + await waitFor(() => + expect(enqueueSnackbar).toHaveBeenCalledWith( + 'Jira linked successfully', + expect.objectContaining({ variant: 'success' }) + ) + ); + }); + + it('shows the error message and does not close when linkProvider itself throws', async () => { + (checkJiraValidity as jest.Mock).mockResolvedValue({ valid: true }); + (linkProvider as jest.Mock).mockRejectedValue(new Error('org already has jira')); + render(); + + await fillForm(); + + expect(await screen.findByText('org already has jira')).toBeInTheDocument(); + expect(onClose).not.toHaveBeenCalled(); + }); +}); diff --git a/web-server/src/content/Dashboards/__tests__/JiraIntegrationCard.test.tsx b/web-server/src/content/Dashboards/__tests__/JiraIntegrationCard.test.tsx new file mode 100644 index 000000000..38c473f69 --- /dev/null +++ b/web-server/src/content/Dashboards/__tests__/JiraIntegrationCard.test.tsx @@ -0,0 +1,165 @@ +jest.mock('@/hooks/useAuth', () => ({ useAuth: jest.fn() })); +jest.mock('@/hooks/useFeature', () => ({ useFeature: jest.fn() })); +jest.mock('@/contexts/ModalContext', () => ({ useModal: jest.fn() })); +jest.mock('@/components/JiraConnectionsManager', () => ({ + JiraConnectionsManager: () =>
jira-connections-manager-stub
+})); +jest.mock('@/content/Dashboards/useIntegrationHandlers', () => ({ + useIntegrationHandlers: jest.fn() +})); +jest.mock('@/store', () => ({ + useDispatch: jest.fn(), + useSelector: jest.fn() +})); +jest.mock('notistack', () => ({ useSnackbar: jest.fn() })); +jest.mock('@/slices/auth', () => ({ fetchCurrentOrg: jest.fn(() => ({ type: 'mock' })) })); + +import { screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { useSnackbar } from 'notistack'; + +import { JiraIntegrationCard } from '@/content/Dashboards/JiraIntegrationCard'; +import { useIntegrationHandlers } from '@/content/Dashboards/useIntegrationHandlers'; +import { useModal } from '@/contexts/ModalContext'; +import { useAuth } from '@/hooks/useAuth'; +import { useFeature } from '@/hooks/useFeature'; +import { useDispatch, useSelector } from '@/store'; +import { renderWithTheme as render } from '@/utils/testUtils'; + +const link = { jira: jest.fn() }; +const unlink = { jira: jest.fn().mockResolvedValue(undefined) }; + +// CLUSTOX: docs/JIRA_MULTI_ACCOUNT_PLAN.md -- this card now branches on +// show_jira_multi_account. These tests exercise the legacy link/unlink +// flow specifically, so the flag is pinned off here; see the +// "multi-account mode" describe block below for the other branch. +describe('JiraIntegrationCard', () => { + beforeEach(() => { + jest.clearAllMocks(); + (useFeature as jest.Mock).mockReturnValue(false); + (useModal as jest.Mock).mockReturnValue({ addModal: jest.fn() }); + (useIntegrationHandlers as jest.Mock).mockReturnValue({ link, unlink }); + (useSelector as jest.Mock).mockReturnValue(false); // requests.org !== REQUEST + (useDispatch as jest.Mock).mockReturnValue(jest.fn()); + (useSnackbar as jest.Mock).mockReturnValue({ enqueueSnackbar: jest.fn() }); + }); + + it('shows "Link" and no Linked badge when integrations.jira is falsy', () => { + (useAuth as jest.Mock).mockReturnValue({ integrations: {} }); + render(); + + expect(screen.getByText('Link')).toBeInTheDocument(); + expect(screen.queryByTestId('jira-linked-badge')).not.toBeInTheDocument(); + }); + + it('shows "Unlink" and the Linked badge when integrations.jira is truthy', () => { + (useAuth as jest.Mock).mockReturnValue({ + integrations: { jira: { integrated: true } } + }); + render(); + + expect(screen.getByText('Unlink')).toBeInTheDocument(); + expect(screen.getByTestId('jira-linked-badge')).toBeInTheDocument(); + }); + + it('opens the Configure Jira modal via link.jira() when not yet linked', async () => { + (useAuth as jest.Mock).mockReturnValue({ integrations: {} }); + render(); + + await userEvent.click(screen.getByText('Link')); + + expect(link.jira).toHaveBeenCalledTimes(1); + expect(unlink.jira).not.toHaveBeenCalled(); + }); + + it('asks for confirmation before unlinking, and does nothing if declined', async () => { + (useAuth as jest.Mock).mockReturnValue({ + integrations: { jira: { integrated: true } } + }); + const confirmSpy = jest.spyOn(window, 'confirm').mockReturnValue(false); + render(); + + await userEvent.click(screen.getByText('Unlink')); + + expect(confirmSpy).toHaveBeenCalled(); + expect(unlink.jira).not.toHaveBeenCalled(); + confirmSpy.mockRestore(); + }); + + it('unlinks and shows a success snackbar when confirmed', async () => { + (useAuth as jest.Mock).mockReturnValue({ + integrations: { jira: { integrated: true } } + }); + const enqueueSnackbar = jest.fn(); + (useSnackbar as jest.Mock).mockReturnValue({ enqueueSnackbar }); + jest.spyOn(window, 'confirm').mockReturnValue(true); + render(); + + await userEvent.click(screen.getByText('Unlink')); + + expect(unlink.jira).toHaveBeenCalledTimes(1); + expect(enqueueSnackbar).toHaveBeenCalledWith( + 'Jira unlinked successfully', + expect.objectContaining({ variant: 'success' }) + ); + }); + + it('shows a failure snackbar, not a thrown error, when unlink rejects', async () => { + (useAuth as jest.Mock).mockReturnValue({ + integrations: { jira: { integrated: true } } + }); + unlink.jira.mockRejectedValueOnce(new Error('network down')); + const enqueueSnackbar = jest.fn(); + (useSnackbar as jest.Mock).mockReturnValue({ enqueueSnackbar }); + jest.spyOn(window, 'confirm').mockReturnValue(true); + render(); + + await userEvent.click(screen.getByText('Unlink')); + + expect( + await screen.findByText((t) => t === 'Unlink') + ).toBeInTheDocument(); + expect(enqueueSnackbar).toHaveBeenCalledWith( + 'Failed to unlink Jira', + expect.objectContaining({ variant: 'error' }) + ); + }); +}); + +describe('JiraIntegrationCard in multi-account mode', () => { + beforeEach(() => { + jest.clearAllMocks(); + (useFeature as jest.Mock).mockReturnValue(true); + (useIntegrationHandlers as jest.Mock).mockReturnValue({ link, unlink }); + (useSelector as jest.Mock).mockReturnValue(false); + (useDispatch as jest.Mock).mockReturnValue(jest.fn()); + (useSnackbar as jest.Mock).mockReturnValue({ enqueueSnackbar: jest.fn() }); + (useAuth as jest.Mock).mockReturnValue({ integrations: {} }); + }); + + it('shows a single "Manage connections" action, not Link/Unlink', () => { + (useModal as jest.Mock).mockReturnValue({ addModal: jest.fn() }); + render(); + + expect( + screen.getByRole('button', { name: 'Manage connections' }) + ).toBeInTheDocument(); + expect(screen.queryByText('Link')).not.toBeInTheDocument(); + expect(screen.queryByText('Unlink')).not.toBeInTheDocument(); + }); + + it('opens the connections manager modal on click', async () => { + const addModal = jest.fn(); + (useModal as jest.Mock).mockReturnValue({ addModal }); + render(); + + await userEvent.click( + screen.getByRole('button', { name: 'Manage connections' }) + ); + + expect(addModal).toHaveBeenCalledTimes(1); + expect(addModal).toHaveBeenCalledWith( + expect.objectContaining({ title: 'Jira connections' }) + ); + }); +}); diff --git a/web-server/src/content/Dashboards/githubIntegration.tsx b/web-server/src/content/Dashboards/githubIntegration.tsx index 4a6cad422..2141b7e6a 100644 --- a/web-server/src/content/Dashboards/githubIntegration.tsx +++ b/web-server/src/content/Dashboards/githubIntegration.tsx @@ -1,7 +1,9 @@ import faker from '@faker-js/faker'; import { GitHub } from '@mui/icons-material'; +import BitbucketIcon from '@/mocks/icons/bitbucket.svg'; import GitlabIcon from '@/mocks/icons/gitlab.svg'; +import JiraIcon from '@/mocks/icons/jira-icon.svg'; export const githubIntegrationsDisplay = { id: faker.datatype.uuid(), @@ -23,4 +25,27 @@ export const gitLabIntegrationDisplay = { icon: } as IntegrationItem; +export const bitbucketIntegrationDisplay = { + id: '5b3f7e1a-92c4-4d6e-8f0a-1c9d2e3f4a5b', + type: 'bitbucket', + name: 'Bitbucket', + description: 'Code insights & blockers', + color: '#2684FF', + bg: 'linear-gradient(135deg, hsla(214, 90%, 52%, 0.6) 0%, hsla(221, 83%, 33%, 0.6) 100%)', + icon: +} as IntegrationItem; + +// CLUSTOX: Jira -- a project-tracker integration, not a code provider (see +// docs/JIRA_INTEGRATION_PROPOSAL.md). Phase 1 only links and stores the +// token; ticket sync and PR correlation are later phases. +export const jiraIntegrationDisplay = { + id: '8f0a3f2e-2f1d-4b1a-9b6b-6a2b8f6e9c4d', + type: 'jira', + name: 'Jira', + description: 'Ticket-linked cycle time', + color: '#2684FF', + bg: 'linear-gradient(135deg, hsla(212, 96%, 55%, 0.6) 0%, hsla(217, 100%, 40%, 0.6) 100%)', + icon: +} as IntegrationItem; + export type IntegrationItem = typeof githubIntegrationsDisplay; diff --git a/web-server/src/content/Dashboards/useIntegrationHandlers.tsx b/web-server/src/content/Dashboards/useIntegrationHandlers.tsx index 0ded2063c..6c70523a6 100644 --- a/web-server/src/content/Dashboards/useIntegrationHandlers.tsx +++ b/web-server/src/content/Dashboards/useIntegrationHandlers.tsx @@ -1,7 +1,9 @@ import { useMemo } from 'react'; import { Integration } from '@/constants/integrations'; +import { ConfigureBitbucketModalBody } from '@/content/Dashboards/ConfigureBitbucketModalBody'; import { ConfigureGitlabModalBody } from '@/content/Dashboards/ConfigureGitlabModalBody'; +import { ConfigureJiraModalBody } from '@/content/Dashboards/ConfigureJiraModalBody'; import { useModal } from '@/contexts/ModalContext'; import { useAuth } from '@/hooks/useAuth'; import { unlinkProvider } from '@/utils/auth'; @@ -27,11 +29,25 @@ export const useIntegrationHandlers = () => { title: 'Configure Gitlab', body: , showCloseIcon: true + }), + bitbucket: () => + addModal({ + title: 'Configure Bitbucket', + body: , + showCloseIcon: true + }), + jira: () => + addModal({ + title: 'Configure Jira', + body: , + showCloseIcon: true }) }, unlink: { github: () => unlinkProvider(orgId, Integration.GITHUB), - gitlab: () => unlinkProvider(orgId, Integration.GITLAB) + gitlab: () => unlinkProvider(orgId, Integration.GITLAB), + bitbucket: () => unlinkProvider(orgId, Integration.BITBUCKET), + jira: () => unlinkProvider(orgId, Integration.JIRA) } }; diff --git a/web-server/src/content/DoraMetrics/DoraCards/ChangeFailureRateCard.tsx b/web-server/src/content/DoraMetrics/DoraCards/ChangeFailureRateCard.tsx index 27d6cee1f..c92dfcd2b 100644 --- a/web-server/src/content/DoraMetrics/DoraCards/ChangeFailureRateCard.tsx +++ b/web-server/src/content/DoraMetrics/DoraCards/ChangeFailureRateCard.tsx @@ -1,14 +1,16 @@ -import { Chip, alpha } from '@mui/material'; +import { Chip } from '@mui/material'; import pluralize from 'pluralize'; import { head } from 'ramda'; -import { useMemo } from 'react'; +import { useCallback, useMemo } from 'react'; -import { Chart2, ChartOptions } from '@/components/Chart2'; +import { Chart2 } from '@/components/Chart2'; +import { useSelectedContributors } from '@/components/ContributorFilter'; import { FlexBox } from '@/components/FlexBox'; import { useOverlayPage } from '@/components/OverlayPageContext'; import { Line } from '@/components/Text'; import { track } from '@/constants/events'; import { + BenchmarkVerdictPill, CardRoot, NoDataImg } from '@/content/DoraMetrics/DoraCards/sharedComponents'; @@ -21,41 +23,24 @@ import { } from '@/hooks/useStateTeamConfig'; import { useSelector } from '@/store'; import { IntegrationGroup } from '@/types/resources'; +import { benchmarkCaption } from '@/utils/benchmarks'; import { NoIncidentsLabel } from './NoIncidentsLabel'; -import { useChangeFailureRateProps } from './sharedHooks'; +import { + doraCardTrendSeries, + useChangeFailureRateProps, + useDoraCardChartOptions +} from './sharedHooks'; import { DoraMetricsComparisonPill } from '../DoraMetricsComparisonPill'; import { getDoraLink } from '../getDoraLink'; import { MetricExternalRead } from '../MetricExternalRead'; import { MissingDORAProviderLink } from '../MissingDORAProviderLink'; -const chartOptions = { - options: { - scales: { - x: { - display: false - }, - y: { - display: false - } - }, - events: [], - plugins: { - zoom: { - zoom: { - drag: { - enabled: false - } - } - } - } - } -} as ChartOptions; - export const ChangeFailureRateCard = () => { const { integrationSet } = useAuth(); const dateRangeLabel = useCurrentDateRangeLabel(); + const selectedContributors = useSelectedContributors(); const { trendsSeriesMap } = useDoraMetricsGraph(); const isCodeProviderIntegrationEnabled = integrationSet.has( @@ -83,23 +68,36 @@ export const ChangeFailureRateCard = () => { const changeFailureRateCount = useCountUp(changeFailureRateProps.count || 0); + const changeFailureRateBenchmark = useSelector( + (s) => s.doraMetrics.metrics_summary?.benchmarks?.change_failure_rate + ); + + const changeFailureRateValues = useMemo( + () => + head(trendsSeriesMap?.changeFailureRateTrends || [])?.data.map( + (s) => s.y + ) || [], + [trendsSeriesMap?.changeFailureRateTrends] + ); + const series = useMemo( - () => [ - { - label: 'Change Failure rate', - fill: 'start', - data: head(trendsSeriesMap?.changeFailureRateTrends || [])?.data.map( - (s) => s.y - ), - backgroundColor: alpha(changeFailureRateProps.backgroundColor, 0.2), - lineTension: 0.2 - } - ], - [ - changeFailureRateProps.backgroundColor, - trendsSeriesMap?.changeFailureRateTrends - ] + () => + doraCardTrendSeries( + 'Change Failure rate', + changeFailureRateValues, + changeFailureRateProps.backgroundColor + ), + [changeFailureRateProps.backgroundColor, changeFailureRateValues] + ); + + const weekLabels = useMemo( + () => + head(trendsSeriesMap?.changeFailureRateTrends || [])?.data.map((s) => + String(s.x) + ) || [], + [trendsSeriesMap?.changeFailureRateTrends] ); + const formatCfr = useCallback((value: number) => `${value}%`, []); const { weeksCovered, daysCovered } = useStateDateConfig(); const isCfrDataAvailable = Boolean( @@ -107,6 +105,56 @@ export const ChangeFailureRateCard = () => { (changeFailureRateProps.count || prevChangeFailureRate) ); + // CLUSTOX: change failure rate is incidents / deployments, so the two ways + // of having "no data" are not the same thing and must not be treated alike: + // + // deployments, no incidents -> a genuine 0%, the best possible result, + // and it beats every target + // no deployments at all -> no denominator, so the rate is undefined + // + // Only the first is a score. Reporting 0% for a team that shipped nothing + // would congratulate them for it, which is worse than showing nothing -- + // the card already says "Due to no deployments" in that state. + const hasDeployments = Boolean( + changeFailureRateProps.avgWeeklyDeploymentFrequency + ); + // CLUSTOX: `count` is already `Number((cfr || 0).toFixed(2))`, so it is 0 + // rather than undefined when nothing failed. No substitution needed -- an + // earlier `isCfrDataAvailable ? count : 0` here implied one that could + // never happen. + const cfrActual = changeFailureRateProps.count; + + // CLUSTOX: gated on the *target* existing rather than on incidents existing, + // for the reason above. canShowIncidentsData stays in the gate because + // without it the card shows a missing-provider link instead of a chart, and + // 0% there would be a claim about a team we have no data for. + const canCompareCfr = canShowIncidentsData && hasDeployments; + + const changeFailureRateBenchmarkCaption = useMemo( + () => + canCompareCfr && changeFailureRateBenchmark?.target != null + ? benchmarkCaption( + 'change_failure_rate', + cfrActual, + changeFailureRateBenchmark.target, + changeFailureRateBenchmark.source + ) + : null, + [canCompareCfr, changeFailureRateBenchmark, cfrActual] + ); + + const changeFailureRateChartOptions = useDoraCardChartOptions( + canCompareCfr + ? { + metric: 'change_failure_rate', + target: changeFailureRateBenchmark?.target, + actual: cfrActual, + values: changeFailureRateValues + } + : null, + { labels: weekLabels, format: formatCfr } + ); + const { addPage } = useOverlayPage(); return ( { )}
+ {Boolean(selectedContributors.length) && ( + // CLUSTOX: unlike Lead Time and Deployment Frequency, this card + // has no per-contributor breakdown yet -- say so explicitly, or a + // selected filter with an unmoving number reads as broken. + + team-wide — per-contributor arrives with Jira + + )} + {changeFailureRateBenchmarkCaption && ( + + )} {canShowIncidentsData ? ( @@ -178,13 +237,25 @@ export const ChangeFailureRateCard = () => { id="cfr-frequency" type="line" series={series} - options={chartOptions} + options={changeFailureRateChartOptions} /> ) : ( )} - + {/* CLUSTOX: pointer events pass through to the canvas so the + chart's tooltip can fire; the content column re-enables them so + its own pills, links and tooltips keep working. Card click is + unaffected -- it lives on CardRoot, above both. */} + *': { pointerEvents: 'auto' } }} + > {canShowIncidentsData ? ( diff --git a/web-server/src/content/DoraMetrics/DoraCards/ChangeTimeCard.tsx b/web-server/src/content/DoraMetrics/DoraCards/ChangeTimeCard.tsx index ac9acaab7..27d7ccd84 100644 --- a/web-server/src/content/DoraMetrics/DoraCards/ChangeTimeCard.tsx +++ b/web-server/src/content/DoraMetrics/DoraCards/ChangeTimeCard.tsx @@ -1,12 +1,13 @@ import { ArrowForwardRounded, WarningAmberRounded } from '@mui/icons-material'; import CheckCircleOutlineRoundedIcon from '@mui/icons-material/CheckCircleOutlineRounded'; import ErrorOutlineRoundedIcon from '@mui/icons-material/ErrorOutlineRounded'; -import { alpha, Button, Chip, darken, List, ListItem } from '@mui/material'; +import { Button, Chip, darken, List, ListItem } from '@mui/material'; import Link from 'next/link'; import pluralize from 'pluralize'; -import { useMemo } from 'react'; +import { useCallback, useMemo } from 'react'; -import { Chart2, ChartOptions } from '@/components/Chart2'; +import { Chart2 } from '@/components/Chart2'; +import { useSelectedContributors } from '@/components/ContributorFilter'; import { FlexBox } from '@/components/FlexBox'; import { useOverlayPage } from '@/components/OverlayPageContext'; import { Line } from '@/components/Text'; @@ -14,14 +15,20 @@ import { track } from '@/constants/events'; import { ROUTES } from '@/constants/routes'; import { isRoleLessThanEM } from '@/constants/useRoute'; import { + BenchmarkVerdictPill, CardRoot, NoDataImg } from '@/content/DoraMetrics/DoraCards/sharedComponents'; -import { usePropsForChangeTimeCard } from '@/content/DoraMetrics/DoraCards/sharedHooks'; +import { + doraCardTrendSeries, + useDoraCardChartOptions, + usePropsForChangeTimeCard +} from '@/content/DoraMetrics/DoraCards/sharedHooks'; import { useAuth } from '@/hooks/useAuth'; import { useCountUp } from '@/hooks/useCountUp'; import { useSelector } from '@/store'; import { ChangeTimeModes } from '@/types/resources'; +import { benchmarkCaption } from '@/utils/benchmarks'; import { merge } from '@/utils/datatype'; import { getDurationString, getSortedDatesAsArrayFromMap } from '@/utils/date'; @@ -30,33 +37,11 @@ import { DoraMetricsComparisonPill } from '../DoraMetricsComparisonPill'; import { MetricExternalRead } from '../MetricExternalRead'; import { MissingDORAProviderLink } from '../MissingDORAProviderLink'; -const chartOptions = { - options: { - scales: { - x: { - display: false - }, - y: { - display: false - } - }, - events: [], - plugins: { - zoom: { - zoom: { - drag: { - enabled: false - } - } - } - } - } -} as ChartOptions; - export const ChangeTimeCard = () => { const { addPage } = useOverlayPage(); const { role } = useAuth(); const isEng = isRoleLessThanEM(role); + const selectedContributors = useSelectedContributors(); const { reposCountWithWorkflowConfigured, @@ -75,6 +60,10 @@ export const ChangeTimeCard = () => { ] ); + const leadTimeBenchmark = useSelector( + (s) => s.doraMetrics.metrics_summary?.benchmarks?.lead_time + ); + const isCodeProviderIntegrationEnabled = true; const showClassificationBadge = @@ -85,24 +74,71 @@ export const ChangeTimeCard = () => { prevLeadTimeTrendsData ); + const leadTimeValues = useMemo( + () => + getSortedDatesAsArrayFromMap(mergedLeadTimeTrends).map( + (key) => mergedLeadTimeTrends[key].lead_time + ), + [mergedLeadTimeTrends] + ); + const series = useMemo( - () => [ - { - label: 'Lead Time', - fill: 'start', - data: getSortedDatesAsArrayFromMap(mergedLeadTimeTrends).map( - (key) => mergedLeadTimeTrends[key].lead_time - ), - backgroundColor: activeModeProps.backgroundColor, - borderColor: alpha(activeModeProps.backgroundColor, 0.5), - lineTension: 0.2 - } - ], - [activeModeProps.backgroundColor, mergedLeadTimeTrends] + () => + doraCardTrendSeries( + 'Lead Time', + leadTimeValues, + activeModeProps.backgroundColor + ), + [activeModeProps.backgroundColor, leadTimeValues] + ); + + const weekLabels = useMemo( + () => getSortedDatesAsArrayFromMap(mergedLeadTimeTrends), + [mergedLeadTimeTrends] + ); + const formatLeadTime = useCallback( + (value: number) => getDurationString(value) || '0', + [] + ); + + // CLUSTOX: the band is only built when the chart itself is on screen. With + // insufficient data the card renders NoDataImg instead, and a band drawn + // over a placeholder would be comparing a real target against nothing. + const chartOptions = useDoraCardChartOptions( + isSufficientDataAvailable + ? { + metric: 'lead_time', + target: leadTimeBenchmark?.target, + // CLUSTOX: seconds on both sides -- `activeModeProps.count` and the + // plotted `lead_time` trend are both raw seconds, and the target is + // stored in seconds too. Nothing to convert here, unlike + // Deployment Frequency. + actual: activeModeProps.count, + values: leadTimeValues + } + : null, + { labels: weekLabels, format: formatLeadTime } ); const leadTimeDuration = useCountUp(activeModeProps.count || 0); + // CLUSTOX: gated on isSufficientDataAvailable -- same guard the card + // already uses to decide between the chart and "Insufficient data". A + // target line/caption over a placeholder state would compare a real + // benchmark against a meaningless zero. + const leadTimeBenchmarkCaption = useMemo( + () => + isSufficientDataAvailable && leadTimeBenchmark + ? benchmarkCaption( + 'lead_time', + activeModeProps.count, + leadTimeBenchmark.target, + leadTimeBenchmark.source + ) + : null, + [isSufficientDataAvailable, leadTimeBenchmark, activeModeProps.count] + ); + return ( { @@ -270,6 +306,18 @@ export const ChangeTimeCard = () => { )} + {Boolean(selectedContributors.length) && ( + // CLUSTOX: this card's number is filtered by PR author, while + // Deployment Frequency is filtered by deploy actor -- often a + // different person. Naming the relationship here is what keeps + // the two cards disagreeing from reading as a bug. + + authored by {selectedContributors.join(', ')} + + )} + {leadTimeBenchmarkCaption && ( + + )} {isSufficientDataAvailable ? ( @@ -284,7 +332,19 @@ export const ChangeTimeCard = () => { )} - + {/* CLUSTOX: pointer events pass through to the canvas so the + chart's tooltip can fire; the content column re-enables them so + its own pills, links and tooltips keep working. Card click is + unaffected -- it lives on CardRoot, above both. */} + *': { pointerEvents: 'auto' } }} + > {isSufficientDataAvailable ? ( diff --git a/web-server/src/content/DoraMetrics/DoraCards/LeadTimeBreakdownCard.tsx b/web-server/src/content/DoraMetrics/DoraCards/LeadTimeBreakdownCard.tsx new file mode 100644 index 000000000..39762af54 --- /dev/null +++ b/web-server/src/content/DoraMetrics/DoraCards/LeadTimeBreakdownCard.tsx @@ -0,0 +1,83 @@ +import { Chip } from '@mui/material'; +import { FC } from 'react'; + +import { FlexBox } from '@/components/FlexBox'; +import { Line } from '@/components/Text'; +import { CardRoot } from '@/content/DoraMetrics/DoraCards/sharedComponents'; +import { useLeadTimePipeline } from '@/content/PullRequests/useChangeTimePipeline'; +import { LeadTimeStatsCore } from '@/content/PullRequests/LeadTimeStatsCore'; +import { useTicketLeadTimeSegment } from '@/content/PullRequests/useTicketLeadTimeSegment'; +import { useAuth } from '@/hooks/useAuth'; +import { getDurationString } from '@/utils/date'; + +// CLUSTOX: Jira integration -- the extended Lead Time breakdown, +// promoted from a click-through-only drill-down to its own +// always-visible card on the main DORA Metrics page, matching the +// design reference. See docs/JIRA_INTEGRATION_PROPOSAL.md §6A. +// +// ChangeTimeCard.tsx (the actual, protected "Lead Time for Changes" +// tile above this one on the page) is completely untouched -- this is +// a new, adjacent card, not a change to it. Reuses the same +// LeadTimeStatsCore/useTicketLeadTimeSegment already built for +// TeamInsightsBody's drill-down, and reads the team-wide 5-segment +// breakdown already sitting in the dora_metrics redux slice this page +// itself populates -- no new fetch for that part. +// +// Renders nothing when Jira isn't linked, or when there's no +// ticket-matched PR data for this team/period -- there's no honest +// "idea to production" number to show without at least one real +// matched PR, and the plain 5-segment breakdown already has its own +// place (the existing drill-down), so this card would be pure +// duplication otherwise. +export const LeadTimeBreakdownCard: FC = () => { + const { integrations } = useAuth(); + const isJiraLinked = Boolean(integrations?.jira?.integrated); + const { leadTimeDetailsArray } = useLeadTimePipeline(); + const { ticketSegment, comparison } = useTicketLeadTimeSegment(); + + if (!isJiraLinked || !ticketSegment) return null; + + return ( + + + + + Lead Time for Changes + + + Extended with a leading phase from Jira — ticket creation to + first commit — so the number reflects idea-to-production, not + just commit-to-production. + + + {/* CLUSTOX: matches the design reference -- the "idea to + production vs. commit-only" comparison as a single header + badge, not an inline line above the bar (LeadTimeStatsCore no + longer renders that line itself, see its own note). */} + {comparison && ( + + {getDurationString(comparison.extendedSeconds, { + segments: 2 + })}{' '} + avg, up from{' '} + {getDurationString(comparison.commitOnlySeconds, { + segments: 2 + })}{' '} + commit-only + + } + /> + )} + + + + ); +}; diff --git a/web-server/src/content/DoraMetrics/DoraCards/LinesOfCodeCard.tsx b/web-server/src/content/DoraMetrics/DoraCards/LinesOfCodeCard.tsx new file mode 100644 index 000000000..954228ad3 --- /dev/null +++ b/web-server/src/content/DoraMetrics/DoraCards/LinesOfCodeCard.tsx @@ -0,0 +1,283 @@ +import ErrorOutlineRoundedIcon from '@mui/icons-material/ErrorOutlineRounded'; +import HelpOutlineRounded from '@mui/icons-material/HelpOutlineRounded'; +import { useCallback, useMemo } from 'react'; + +import { Chart2 } from '@/components/Chart2'; +import { useSelectedContributors } from '@/components/ContributorFilter'; +import { FlexBox } from '@/components/FlexBox'; +import { Line } from '@/components/Text'; +import { + BenchmarkVerdictPill, + CardRoot, + NoDataImg +} from '@/content/DoraMetrics/DoraCards/sharedComponents'; +import { + doraCardTrendSeries, + useDoraCardChartOptions +} from '@/content/DoraMetrics/DoraCards/sharedHooks'; +import { useCountUp } from '@/hooks/useCountUp'; +import { useSelector } from '@/store'; +import { benchmarkCaption } from '@/utils/benchmarks'; +import { buildLocCardModel } from '@/utils/locCard'; + +import { DoraMetricsComparisonPill } from '../DoraMetricsComparisonPill'; + +// CLUSTOX: lines of code has no DORA classification -- there is no +// elite/high/medium/low threshold table for it, and there should not be one. +// So the card borrows no `commonProps` entry and shows no classification chip; +// a "Medium" badge here would be an invented judgement. The only colour that +// carries meaning on this card is the benchmark caption's, exactly as on the +// other four. +// +// It must also not be any of the four classification colours. This was +// `#a4d3d3`, which is character-for-character `commonProps.medium.color` -- +// and `` renders as a legend directly beneath this same +// grid, so the headline would have silently read as a "Medium" rating: the +// exact judgement the omitted chip was avoiding. `info` is outside that +// palette. +const LOC_ACCENT = '#33C2FF'; + +const { format: compact } = Intl.NumberFormat('en', { + notation: 'compact', + maximumFractionDigits: 1 +}); +const { format: exact } = Intl.NumberFormat('en'); + +export const LinesOfCodeCard = () => { + const selectedContributors = useSelectedContributors(); + + // CLUSTOX: both keys are optional on the response -- an older backend, or a + // LOC route that failed inside the BFF's Promise.all, leaves them + // `undefined`. Read them raw and let `buildLocCardModel` do every access, so + // there is exactly one place that has to tolerate absence. + const locStats = useSelector((s) => s.doraMetrics.metrics_summary?.loc_stats); + const locTrends = useSelector( + (s) => s.doraMetrics.metrics_summary?.loc_trends + ); + const locBenchmark = useSelector( + (s) => s.doraMetrics.metrics_summary?.benchmarks?.lines_of_code + ); + + const model = useMemo( + () => buildLocCardModel(locStats, locTrends), + [locStats, locTrends] + ); + + // CLUSTOX: the plotted series is average PR size, NOT the weekly totals the + // headline shows. The band is drawn in this chart's own data space, so the + // series behind the number has to be in the same unit as the target -- lines + // per PR. Plotting totals here and drawing a 200-line target over them would + // put the band on the axis floor of a ~70,000-tall scale: correct + // arithmetic, and a wrong answer that looks entirely plausible. + const series = useMemo( + () => + doraCardTrendSeries('Avg. PR size', model.avgPrSizeValues, LOC_ACCENT), + [model.avgPrSizeValues] + ); + + const formatPrSize = useCallback((value: number) => `${value} lines/PR`, []); + + // CLUSTOX: gated on `canComparePrSize`, which is false both when LOC was + // never measured and when nothing was merged. Passing `actual: 0` in the + // latter case would paint a success band for a team that shipped nothing -- + // the same trap the Change Failure Rate card has with no deployments. + const chartOptions = useDoraCardChartOptions( + model.canComparePrSize + ? { + metric: 'lines_of_code', + target: locBenchmark?.target, + // CLUSTOX: lines per PR on both sides. `avg_pr_size` is what the + // benchmark form stores in lines, what `values` plots, and what this + // compares -- no conversion anywhere, unlike Deployment Frequency. + actual: model.avgPrSize, + values: model.avgPrSizeValues + } + : null, + { labels: model.weeks, format: formatPrSize } + ); + + const benchmarkLine = useMemo( + () => + model.canComparePrSize && locBenchmark + ? benchmarkCaption( + 'lines_of_code', + model.avgPrSize, + locBenchmark.target, + locBenchmark.source + ) + : null, + [model.canComparePrSize, model.avgPrSize, locBenchmark] + ); + + // CLUSTOX: the headline counts up the DAILY RATE, not the period total. A + // total is not comparable between two ranges of different lengths -- widen + // the date picker and the same pace of work reads as a bigger number. The + // total is still on the card, one line below, where it reads as context + // rather than as the metric. + const linesPerDay = useCountUp(model.avgDaily); + + return ( + // CLUSTOX: no `onClick` and no pointer cursor. There is no LOC drill-down + // overlay, and CardRoot's default hover-brighten plus pointer cursor would + // promise one -- the four other cards each open a real page. + + + + + + Lines of Code + + + + Lines added and deleted across pull requests merged in this + period. Unmerged work is not counted. + + + The benchmark tracks average PR size, not weekly volume: + volume has no good direction, while smaller PRs review + faster and merge sooner. The trend behind the number is + average PR size. + + + } + > + + + + + {Boolean(selectedContributors.length) && ( + // CLUSTOX: LOC goes through the same `pr_filter` the Lead Time metric + // gets, so it is filtered by PR author -- the same wording, because + // it is genuinely the same filter and not the deploy actor. + + authored by {selectedContributors.join(', ')} + + )} + {benchmarkLine && } + + + {model.isMeasured ? ( + + ) : ( + + )} + + + {/* CLUSTOX: pointer events pass through to the canvas so the + chart's tooltip can fire; the content column re-enables them so + its own pills, links and tooltips keep working. Card click is + unaffected -- it lives on CardRoot, above both. */} + *': { pointerEvents: 'auto' } }} + > + {model.isMeasured ? ( + + + Lines changed / day + + + + {compact(linesPerDay)} + + + {/* CLUSTOX: the period total, kept as context under the rate. + Dropping it would lose the only absolute figure on the card + -- "9 lines/day" alone gives no sense of whether that is 60 + lines over a week or 280 over a month. */} + + {exact(model.total)} lines over the selected range + + + + +{compact(model.additions)} + + + / + + {/* CLUSTOX: deletions are `warning`, never `error`. Deleted + code is not a failure -- often the opposite -- and red + here would read as one. */} + + −{compact(model.deletions)} + + + + + Avg. PR size {exact(model.avgPrSize)}{' '} + {model.avgPrSize === 1 ? 'line' : 'lines'} + + {/* CLUSTOX: the delta rides on average PR size, not on the + headline. Weekly volume has no good direction -- higher + could be productive or bloated -- so a green up-arrow on + it would be a judgement the metric cannot support. PR size + is unambiguously lower-is-better, hence `positive={false}`. + The headline's previous-period figure is in its tooltip, + uncoloured. */} + {/* CLUSTOX: `canComparePrSize` is in this gate, not just on + the band and caption. Without it, a team that merged + nothing this period (avg_pr_size 0) against 640 last + period computes -100% and, because `positive={false}` + inverts the tone, renders it GREEN -- congratulating them + for shipping nothing. Same trap as the change failure + rate card, which reads 0% as a perfect score unless a + denominator exists. */} + {model.canComparePrSize && + Boolean(model.avgPrSize || model.prevAvgPrSize) && ( + + `${exact(Math.round(val))} lines/PR` + } + positive={false} + boxed + light + size="1.2em" + lineProps={{ bold: false, fontWeight: 600 }} + /> + )} + + + ) : ( + // CLUSTOX: "not measured", deliberately not "no lines changed". + // A team that changed nothing gets a real zero from the backend + // and lands in the branch above; this branch only happens when + // the response carried no `loc_stats` at all. + + Lines of code unavailable + + )} + + + + + ); +}; diff --git a/web-server/src/content/DoraMetrics/DoraCards/MeanTimeToRestoreCard.tsx b/web-server/src/content/DoraMetrics/DoraCards/MeanTimeToRestoreCard.tsx index 672987849..1bd6df011 100644 --- a/web-server/src/content/DoraMetrics/DoraCards/MeanTimeToRestoreCard.tsx +++ b/web-server/src/content/DoraMetrics/DoraCards/MeanTimeToRestoreCard.tsx @@ -1,55 +1,45 @@ import ErrorOutlineRoundedIcon from '@mui/icons-material/ErrorOutlineRounded'; import { Chip } from '@mui/material'; import { head } from 'ramda'; -import { useMemo } from 'react'; +import { useCallback, useMemo } from 'react'; -import { Chart2, ChartOptions } from '@/components/Chart2'; +import { Chart2 } from '@/components/Chart2'; +import { useSelectedContributors } from '@/components/ContributorFilter'; import { FlexBox } from '@/components/FlexBox'; import { useOverlayPage } from '@/components/OverlayPageContext'; import { Line } from '@/components/Text'; import { track } from '@/constants/events'; import { + BenchmarkVerdictPill, CardRoot, NoDataImg } from '@/content/DoraMetrics/DoraCards/sharedComponents'; import { useCountUp } from '@/hooks/useCountUp'; import { useDoraMetricsGraph } from '@/hooks/useDoraMetricsGraph'; +import { useSelector } from '@/store'; +import { + benchmarkCaption, + benchmarkSourceLabel, + formatBenchmarkValue +} from '@/utils/benchmarks'; import { getDurationString } from '@/utils/date'; import { NoIncidentsLabel } from './NoIncidentsLabel'; -import { useMeanTimeToRestoreProps } from './sharedHooks'; +import { + doraCardTrendSeries, + useDoraCardChartOptions, + useMeanTimeToRestoreProps +} from './sharedHooks'; import { DoraMetricsComparisonPill } from '../DoraMetricsComparisonPill'; import { getDoraLink } from '../getDoraLink'; import { MetricExternalRead } from '../MetricExternalRead'; import { MissingDORAProviderLink } from '../MissingDORAProviderLink'; -const chartOptions = { - options: { - scales: { - x: { - display: false - }, - y: { - display: false - } - }, - events: [], - plugins: { - zoom: { - zoom: { - drag: { - enabled: false - } - } - } - } - } -} as ChartOptions; - export const MeanTimeToRestoreCard = () => { const { isNoDataAvailable, ...meanTimeToRestoreProps } = useMeanTimeToRestoreProps(); + const selectedContributors = useSelectedContributors(); const { trendsSeriesMap } = useDoraMetricsGraph(); const isIncidentProviderIntegrationEnabled = true; @@ -60,25 +50,89 @@ export const MeanTimeToRestoreCard = () => { const showClassificationBadge = isIncidentProviderIntegrationEnabled && !isNoDataAvailable; + const meanTimeToRestoreValues = useMemo( + () => + head(trendsSeriesMap?.meanTimeToRestoreTrends || [])?.data.map( + (s) => s.y + ) || [], + [trendsSeriesMap?.meanTimeToRestoreTrends] + ); + const series = useMemo( - () => [ - { - label: 'Mean time to restore', - fill: 'start', - data: head(trendsSeriesMap?.meanTimeToRestoreTrends || [])?.data.map( - (s) => s.y - ), - backgroundColor: meanTimeToRestoreProps.backgroundColor - } - ], - [ - trendsSeriesMap?.meanTimeToRestoreTrends, - meanTimeToRestoreProps.backgroundColor - ] + () => + doraCardTrendSeries( + 'Mean time to restore', + meanTimeToRestoreValues, + meanTimeToRestoreProps.backgroundColor + ), + [meanTimeToRestoreValues, meanTimeToRestoreProps.backgroundColor] + ); + + const weekLabels = useMemo( + () => + head(trendsSeriesMap?.meanTimeToRestoreTrends || [])?.data.map((s) => + String(s.x) + ) || [], + [trendsSeriesMap?.meanTimeToRestoreTrends] + ); + const formatMttr = useCallback( + (value: number) => getDurationString(value) || '0s', + [] ); const meanTimeToRestoreCount = useCountUp(meanTimeToRestoreProps.count || 0); + const meanTimeToRecoveryBenchmark = useSelector( + (s) => s.doraMetrics.metrics_summary?.benchmarks?.mean_time_to_recovery + ); + + // CLUSTOX: `canShowMTRData` is NOT sufficient to gate a comparison. It comes + // from `isNoDataAvailable`, which mixes periods -- + // `!incidents && !prevAvgTimeToRestore && !currAvgTimeToRestore` -- so a team + // with zero incidents THIS period but any recovery time LAST period passes + // it. `count` is then null, `null <= target` is true in JS, and the card + // rendered a green "0s is under target" beside its own "No incidents + // reported" headline: two contradictory claims in one card, and a + // congratulation for a period with nothing measured. + // + // A recovery time exists only if something was recovered, so the comparison + // needs a measured value, not merely a card that is not blank. + const hasMeasuredRecovery = + meanTimeToRestoreProps.count != null && meanTimeToRestoreProps.count > 0; + const canCompareMTTR = canShowMTRData && hasMeasuredRecovery; + + const meanTimeToRecoveryBenchmarkCaption = useMemo( + () => + canCompareMTTR && meanTimeToRecoveryBenchmark + ? benchmarkCaption( + 'mean_time_to_recovery', + meanTimeToRestoreProps.count, + meanTimeToRecoveryBenchmark.target, + meanTimeToRecoveryBenchmark.source + ) + : null, + [canCompareMTTR, meanTimeToRecoveryBenchmark, meanTimeToRestoreProps.count] + ); + + // CLUSTOX: the opposite call to Change Failure Rate's, deliberately. Zero + // incidents there is a real 0% -- here it means there is no recovery time to + // measure at all, so there is nothing to shade a band against and no side of + // the target the team can be said to be on. The card names the target + // instead (below), so an admin can still see one is configured. + const meanTimeToRestoreChartOptions = useDoraCardChartOptions( + canCompareMTTR + ? { + metric: 'mean_time_to_recovery', + target: meanTimeToRecoveryBenchmark?.target, + // CLUSTOX: seconds throughout -- the stat, the plotted trend and the + // stored target are all raw seconds. + actual: meanTimeToRestoreProps.count, + values: meanTimeToRestoreValues + } + : null, + { labels: weekLabels, format: formatMttr } + ); + const { addPage } = useOverlayPage(); return ( @@ -144,6 +198,16 @@ export const MeanTimeToRestoreCard = () => { )} + {Boolean(selectedContributors.length) && ( + // CLUSTOX: see the matching note on ChangeFailureRateCard -- MTTR + // has no per-contributor breakdown yet either. + + team-wide — per-contributor arrives with Jira + + )} + {meanTimeToRecoveryBenchmarkCaption && ( + + )} {canShowMTRData ? ( @@ -151,13 +215,25 @@ export const MeanTimeToRestoreCard = () => { id="mttr-frequency" type="line" series={series} - options={chartOptions} + options={meanTimeToRestoreChartOptions} /> ) : ( )} - + {/* CLUSTOX: pointer events pass through to the canvas so the + chart's tooltip can fire; the content column re-enables them so + its own pills, links and tooltips keep working. Card click is + unaffected -- it lives on CardRoot, above both. */} + *': { pointerEvents: 'auto' } }} + > {canShowMTRData ? ( @@ -176,17 +252,45 @@ export const MeanTimeToRestoreCard = () => { )} - `${getDurationString(val) || 0}`} - positive={false} - boxed - light - size="1.2em" - lineProps={{ bold: false, fontWeight: 600 }} - sx={{ marginBottom: '-8px' }} - /> + {/* CLUSTOX: the target-only line also has to be reachable + HERE, not just in the `!canShowMTRData` branch below. + That branch is unreachable when the previous period had + incidents and this one did not -- exactly the state where + a target exists and nothing can be compared to it. */} + {!hasMeasuredRecovery && + meanTimeToRecoveryBenchmark?.target != null && ( + + target{' '} + {formatBenchmarkValue( + 'mean_time_to_recovery', + meanTimeToRecoveryBenchmark.target + )} + {' — '} + {benchmarkSourceLabel( + meanTimeToRecoveryBenchmark.source + )} + + )} + {/* CLUSTOX: same gate as the band and caption. With no + incidents this period against a recovery time last + period, this computed -100% and -- because MTTR is + lower-is-better, so `positive={false}` inverts the tone -- + painted it green, next to a headline reading "No + incidents reported". Nothing was recovered faster; there + was nothing to recover. */} + {hasMeasuredRecovery && ( + `${getDurationString(val) || 0}`} + positive={false} + boxed + light + size="1.2em" + lineProps={{ bold: false, fontWeight: 600 }} + sx={{ marginBottom: '-8px' }} + /> + )} { No incidents reported Hence Time to Recovery is unavailable + {/* CLUSTOX: grey and target-only -- it states that a + benchmark exists without claiming the team met or missed + it. There is no measurement here to compare against, so + any tone at all would be inventing a verdict. */} + {meanTimeToRecoveryBenchmark?.target != null && ( + + target{' '} + {formatBenchmarkValue( + 'mean_time_to_recovery', + meanTimeToRecoveryBenchmark.target + )} + {' — '} + {benchmarkSourceLabel(meanTimeToRecoveryBenchmark.source)} + + )} ) : ( diff --git a/web-server/src/content/DoraMetrics/DoraCards/SprintRollupCard.tsx b/web-server/src/content/DoraMetrics/DoraCards/SprintRollupCard.tsx new file mode 100644 index 000000000..c351b0019 --- /dev/null +++ b/web-server/src/content/DoraMetrics/DoraCards/SprintRollupCard.tsx @@ -0,0 +1,156 @@ +import axios from 'axios'; +import { FC, useEffect } from 'react'; + +import { FlexBox } from '@/components/FlexBox'; +import { Line } from '@/components/Text'; +import { CardRoot } from '@/content/DoraMetrics/DoraCards/sharedComponents'; +import { useAuth } from '@/hooks/useAuth'; +import { useBoolState, useEasyState } from '@/hooks/useEasyState'; +import { useSingleTeamConfig } from '@/hooks/useStateTeamConfig'; +import { depFn } from '@/utils/fn'; + +// CLUSTOX: Jira integration -- the Sprint rollup chart (docs/ +// JIRA_INTEGRATION_PROPOSAL.md §6D). Its own card with its own fetch, +// same additive pattern as TicketCycleTimeCard/LeadTimeBreakdownCard -- +// no date-range dependency, since a sprint's own start/end dates are its +// natural window, not the page's selected period. +type Sprint = { + name: string; + state: string; + start_date: string | null; + end_date: string | null; + planned_count: number; + completed_count: number; +}; + +const BAR_MAX_HEIGHT_PX = 120; + +const useSprints = () => { + const { integrations } = useAuth(); + const isJiraLinked = Boolean(integrations?.jira?.integrated); + const { singleTeamId } = useSingleTeamConfig(); + + const sprints = useEasyState([]); + const isLoading = useBoolState(false); + + useEffect(() => { + if (!isJiraLinked || !singleTeamId) return; + depFn(isLoading.true); + axios(`/api/internal/team/${singleTeamId}/sprints`) + .then((res) => depFn(sprints.set, res.data)) + .catch((error) => console.error(error)) + .finally(isLoading.false); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [isJiraLinked, singleTeamId]); + + return { isJiraLinked, isLoading: isLoading.value, sprints: sprints.value }; +}; + +export const SprintRollupCard: FC = () => { + const { isJiraLinked, isLoading, sprints } = useSprints(); + + if (!isJiraLinked || isLoading || !sprints.length) return null; + + // Shared scale across every bar in the chart, not per-sprint -- so a + // sprint's bar height is comparable against every other sprint's, the + // same way the design reference's own bars read (Sprint 42's taller + // bars mean literally more tickets, not "more relative to its own + // sprint"). + const maxCount = Math.max( + ...sprints.flatMap((sprint) => [sprint.planned_count, sprint.completed_count]), + 1 + ); + + return ( + + + + Sprint rollup + + + Planned vs. shipped, per sprint. + + + {/* alignItems="flex-start", not flex-end -- every SprintColumn's + bar area is the same fixed BAR_MAX_HEIGHT_PX, but a longer + sprint name (e.g. "PZDA Sprint 3 &4") wraps to 2 lines while a + shorter one stays on 1, making that whole column taller. + Aligning by the bottom edge would push the taller column's + bars upward relative to the others, so their baselines + (0 count) wouldn't line up even though the bar heights + themselves are correct -- aligning by the top instead keeps + every column's fixed-height bar area starting at the same Y, + regardless of how its label wraps below it. + + flexWrap="nowrap" + overflowX="auto", not "wrap": a team with + more than a handful of sprints wrapping to a second row left a + lone last column stranded far below the others, under a mostly + empty row -- same "wide content scrolls in its own container" + treatment as TicketCycleTimeCard's list, rather than reflowing + the whole card's height around however many sprints exist. */} + + {sprints.map((sprint) => ( + + ))} + + + + + + Planned + + + + + + Shipped + + + + + ); +}; + +const SprintColumn: FC<{ sprint: Sprint; maxCount: number }> = ({ + sprint, + maxCount +}) => ( + + + + + + + {sprint.name} + + +); diff --git a/web-server/src/content/DoraMetrics/DoraCards/TicketCycleTimeCard.tsx b/web-server/src/content/DoraMetrics/DoraCards/TicketCycleTimeCard.tsx new file mode 100644 index 000000000..78b54b79e --- /dev/null +++ b/web-server/src/content/DoraMetrics/DoraCards/TicketCycleTimeCard.tsx @@ -0,0 +1,374 @@ +import { OpenInNew, WarningAmberRounded } from '@mui/icons-material'; +import { Button, CircularProgress } from '@mui/material'; +import axios from 'axios'; +import { format } from 'date-fns'; +import { FC, useEffect } from 'react'; + +import { CardRoot } from '@/content/DoraMetrics/DoraCards/sharedComponents'; +import { FlexBox } from '@/components/FlexBox'; +import { Line } from '@/components/Text'; +import { useModal } from '@/contexts/ModalContext'; +import { useAuth } from '@/hooks/useAuth'; +import { useBoolState, useEasyState } from '@/hooks/useEasyState'; +import { useSingleTeamConfig } from '@/hooks/useStateTeamConfig'; +import { getDurationString } from '@/utils/date'; +import { depFn } from '@/utils/fn'; + +// CLUSTOX: Jira integration, Phase 4 (§6C/§6E) -- see +// docs/JIRA_INTEGRATION_PROPOSAL.md. Deliberately its own fetch, not +// wired into useDoraStats/the dora_metrics redux slice the 4 existing +// cards share -- this is purely additive to the DORA Metrics page, not +// a change to any of the existing 4 cards' own data or code. +// +// One fetch backs two cards that now live in different places on the +// page (Ticket Cycle Time beside Sprint Rollup; Data Hygiene as its +// own full-width row below both, matching the design reference) -- +// useTicketInsights is exported so DoraMetricsBody can call it once +// and pass the result into both, rather than each card fetching (and +// duplicating the request for) the same data independently. +const CATEGORY_COLOR: Record = { + 'To Do': 'info.main', + 'In Progress': 'warning.main', + Done: 'success.main' +}; +const CATEGORY_ORDER = ['To Do', 'In Progress', 'Done']; + +// A category that took a genuinely tiny share of a ticket's total life +// (e.g. real data: a project closed its "Done" segment in ~37 minutes +// against a ~13 day total -- 0.2%) rounds to a sub-pixel sliver at its +// true percentage, which reads as missing data rather than "this really +// happened, it was just fast". Every category below this floor is +// bumped up to it; the tooltip and the numbers everywhere else stay the +// real, unadjusted values -- only the rendered width is touched. +const MIN_SEGMENT_PCT = 4; + +/** + * Segment widths as percentages that always sum to 100 across every + * category with real (nonzero) time in it, each at least + * MIN_SEGMENT_PCT wide. Categories with room above the floor donate + * width proportionally to the ones below it, so the bar never overflows + * (or underfills) its container. + */ +export const segmentWidths = ( + avgSecondsByCategory: Record, + avgTotalSeconds: number +): { category: string; seconds: number; pct: number }[] => { + const present = CATEGORY_ORDER.filter( + (category) => (avgSecondsByCategory[category] || 0) > 0 + ); + if (!present.length || !avgTotalSeconds) return []; + + const rawPct = present.map( + (category) => (avgSecondsByCategory[category] / avgTotalSeconds) * 100 + ); + const deficit = rawPct.reduce( + (sum, pct) => sum + Math.max(0, MIN_SEGMENT_PCT - pct), + 0 + ); + const donorPool = rawPct.reduce( + (sum, pct) => sum + Math.max(0, pct - MIN_SEGMENT_PCT), + 0 + ); + + return present.map((category, i) => { + const pct = rawPct[i]; + let flooredPct = pct; + if (deficit > 0) { + flooredPct = + pct <= MIN_SEGMENT_PCT + ? MIN_SEGMENT_PCT + : pct - ((pct - MIN_SEGMENT_PCT) / donorPool) * deficit; + } + return { + category, + seconds: avgSecondsByCategory[category], + pct: flooredPct + }; + }); +}; + +type ProjectCycleTime = { + project_key: string; + project_name: string; + ticket_count: number; + avg_total_seconds: number; + avg_seconds_by_category: Record; +}; + +export type TicketInsights = { + cycle_time_by_project: ProjectCycleTime[]; + prs_without_ticket_count: number; +}; + +export const useTicketInsights = () => { + const { integrations } = useAuth(); + const isJiraLinked = Boolean(integrations?.jira?.integrated); + const { singleTeamId, dates } = useSingleTeamConfig(); + + const insights = useEasyState(null); + const isLoading = useBoolState(false); + + useEffect(() => { + if (!isJiraLinked || !singleTeamId) return; + depFn(isLoading.true); + axios(`/api/internal/team/${singleTeamId}/ticket_insights`, { + params: { + from_date: dates.start.toISOString(), + to_date: dates.end.toISOString() + } + }) + .then((res) => depFn(insights.set, res.data)) + .catch((error) => console.error(error)) + .finally(isLoading.false); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [isJiraLinked, singleTeamId, dates.start, dates.end]); + + return { + isJiraLinked, + isLoading: isLoading.value, + insights: insights.value, + singleTeamId, + dates + }; +}; + +// CLUSTOX: presentational, not self-fetching -- takes useTicketInsights's +// result as props so DoraMetricsBody (which now places this card and +// DataHygieneCard in different spots on the page) can call the hook +// once and hand the same result to both. +export const TicketCycleTimeCard: FC<{ + isJiraLinked: boolean; + isLoading: boolean; + insights: TicketInsights | null; +}> = ({ isJiraLinked, isLoading, insights }) => { + if (!isJiraLinked || isLoading) return null; + if (!insights?.cycle_time_by_project.length) return null; + + return ; +}; + +const ProjectCycleTimeCard: FC<{ projects: ProjectCycleTime[] }> = ({ + projects +}) => ( + + + + Ticket Cycle Time + + + Time in each Jira status, independent of code — useful even before + any ticket is matched to a PR. + + + + {projects.map((project) => ( + + ))} + + + {CATEGORY_ORDER.map((category) => ( + + + + {category} + + + ))} + + +); + +const ProjectCycleTimeRow: FC<{ project: ProjectCycleTime }> = ({ project }) => ( + + + + + {project.project_key} + {' '} + — {project.project_name} + + + {getDurationString(project.avg_total_seconds)} avg + + + + {segmentWidths(project.avg_seconds_by_category, project.avg_total_seconds).map( + ({ category, seconds, pct }) => ( + + ) + )} + + +); + +// CLUSTOX: its own full-width row on the page now, not stacked below +// Ticket Cycle Time in the same column -- matches the design reference. +// Takes useTicketInsights's result as props, same reasoning as +// TicketCycleTimeCard above -- DoraMetricsBody calls the hook once for +// both. +export const DataHygieneCard: FC<{ + isJiraLinked: boolean; + isLoading: boolean; + insights: TicketInsights | null; + teamId: ID; + dates: { start: Date; end: Date }; +}> = ({ isJiraLinked, isLoading, insights, teamId, dates }) => { + const { addModal } = useModal(); + + if (!isJiraLinked || isLoading) return null; + if (!insights?.prs_without_ticket_count) return null; + + const count = insights.prs_without_ticket_count; + const projectKeys = insights.cycle_time_by_project.map((p) => p.project_key); + + const openUnlinkedPrsModal = () => { + addModal({ + title: 'PRs with no linked Jira ticket', + body: , + showCloseIcon: true + }); + }; + + // Matches the design reference's own example ("...against the PAY-123 + // convention") but with this team's real project key, not a + // placeholder -- a hint is only useful if it's the actual convention + // to check against. + const exampleKey = `${projectKeys[0] ?? 'PROJ'}-123`; + + return ( + + + + Data Hygiene + + + Surfaces process gaps rather than silently excluding unmatched PRs + from ticket-aware metrics. + + + + + + + {count} + {' '} + PR{count === 1 ? '' : 's'} merged this period with{' '} + + no linked ticket + {' '} + — check branch naming against the {exampleKey} convention. + + + + + ); +}; + +type UnlinkedPr = { + id: string; + title: string; + url: string; + head_branch: string; + author: string; + merged_at: string; +}; + +// CLUSTOX: Jira integration, Phase 4 (§6E) -- the Data Hygiene +// drill-down. Its own lazy fetch, only issued when the modal actually +// opens -- the count above already answers "how many", this answers +// "which ones", so it isn't worth loading on every DORA Metrics page +// view. See docs/JIRA_INTEGRATION_PROPOSAL.md. +export const UnlinkedPrsModalBody: FC<{ + teamId: ID; + dates: { start: Date; end: Date }; +}> = ({ teamId, dates }) => { + const prs = useEasyState(null); + const isLoading = useBoolState(true); + const hasError = useBoolState(false); + + useEffect(() => { + axios(`/api/internal/team/${teamId}/unlinked_prs`, { + params: { + from_date: dates.start.toISOString(), + to_date: dates.end.toISOString() + } + }) + .then((res) => depFn(prs.set, res.data)) + .catch((error) => { + console.error(error); + depFn(hasError.true); + }) + .finally(isLoading.false); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [teamId, dates.start, dates.end]); + + if (isLoading.value) + return ( + + + Loading... + + ); + + if (hasError.value) + return Could not load PRs. Please try again.; + + return ( + + + Check each branch/title against your ticket-key convention (e.g. + PROJ-123) -- a PR here either never referenced a ticket, or + referenced one this regex didn't recognize. + + + {prs.value.map((pr) => ( + + + + {pr.title} + + + + + {pr.head_branch} · {pr.author} · merged{' '} + {format(new Date(pr.merged_at), 'MMM d, yyyy')} + + + ))} + + + ); +}; diff --git a/web-server/src/content/DoraMetrics/DoraCards/WeeklyDeliveryVolumeCard.tsx b/web-server/src/content/DoraMetrics/DoraCards/WeeklyDeliveryVolumeCard.tsx index a5d1eee58..08b30d227 100644 --- a/web-server/src/content/DoraMetrics/DoraCards/WeeklyDeliveryVolumeCard.tsx +++ b/web-server/src/content/DoraMetrics/DoraCards/WeeklyDeliveryVolumeCard.tsx @@ -1,13 +1,15 @@ -import { alpha, Chip } from '@mui/material'; +import { Chip } from '@mui/material'; import pluralize from 'pluralize'; -import { useMemo } from 'react'; +import { useCallback, useMemo } from 'react'; -import { Chart2, ChartOptions } from '@/components/Chart2'; +import { Chart2 } from '@/components/Chart2'; +import { useSelectedContributors } from '@/components/ContributorFilter'; import { FlexBox } from '@/components/FlexBox'; import { useOverlayPage } from '@/components/OverlayPageContext'; import { Line } from '@/components/Text'; import { track } from '@/constants/events'; import { + BenchmarkVerdictPill, CardRoot, NoDataImg } from '@/content/DoraMetrics/DoraCards/sharedComponents'; @@ -18,42 +20,25 @@ import { } from '@/hooks/useStateTeamConfig'; import { useSelector } from '@/store'; import { IntegrationGroup } from '@/types/resources'; +import { benchmarkCaption } from '@/utils/benchmarks'; import { getSortedDatesAsArrayFromMap } from '@/utils/date'; -import { useAvgIntervalBasedDeploymentFrequency } from './sharedHooks'; +import { + doraCardTrendSeries, + useAvgIntervalBasedDeploymentFrequency, + useDoraCardChartOptions +} from './sharedHooks'; import { DoraMetricsComparisonPill } from '../DoraMetricsComparisonPill'; import { getDoraLink } from '../getDoraLink'; import { MetricExternalRead } from '../MetricExternalRead'; import { MissingDORAProviderLink } from '../MissingDORAProviderLink'; -const chartOptions = { - options: { - scales: { - x: { - display: false - }, - y: { - display: false - } - }, - events: [], - plugins: { - zoom: { - zoom: { - drag: { - enabled: false - } - } - } - } - } -} as ChartOptions; - export const WeeklyDeliveryVolumeCard = () => { const { integrationSet } = useAuth(); const dateRangeLabel = useCurrentDateRangeLabel(); const deploymentFrequencyProps = useAvgIntervalBasedDeploymentFrequency(); + const selectedContributors = useSelectedContributors(); const { addPage } = useOverlayPage(); const deploymentsConfigured = true; @@ -82,20 +67,95 @@ export const WeeklyDeliveryVolumeCard = () => { .total_deployments || 0 ); + const deploymentFrequencyBenchmark = useSelector( + (s) => s.doraMetrics.metrics_summary?.benchmarks?.deployment_frequency + ); + + // CLUSTOX: the benchmark target is always "deployments/week" (see + // task-5-brief), but the card's headline count adapts to the selected + // interval (day/week/month) via useAvgIntervalBasedDeploymentFrequency. + // Comparing against that would silently mix units, so the weekly-specific + // stat is used here instead, regardless of which interval is displayed. + const avgWeeklyDeploymentFrequency = useSelector( + (s) => + s.doraMetrics.metrics_summary?.deployment_frequency_stats.current + .avg_weekly_deployment_frequency + ); + + // CLUSTOX: these buckets are weekly counts -- the trends endpoint is + // `get_weekly_deployment_frequency_trends`, which buckets deployments + // "weekly" regardless of the interval the headline is displayed in. That is + // what makes it safe to draw a deployments/week target in this chart's own + // data space. + const weeklyDeploymentCounts = useMemo( + () => + getSortedDatesAsArrayFromMap(weekDeliveryVolumeData).map( + (date) => weekDeliveryVolumeData[date].count + ), + [weekDeliveryVolumeData] + ); + const series = useMemo( - () => [ - { - label: 'Deployments', - fill: 'start', - data: getSortedDatesAsArrayFromMap(weekDeliveryVolumeData).map( - (date) => weekDeliveryVolumeData[date].count - ), - backgroundColor: deploymentFrequencyProps?.backgroundColor, - borderColor: alpha(deploymentFrequencyProps?.backgroundColor, 0.5), - lineTension: 0.2 - } - ], - [deploymentFrequencyProps?.backgroundColor, weekDeliveryVolumeData] + () => + doraCardTrendSeries( + 'Deployments', + weeklyDeploymentCounts, + deploymentFrequencyProps?.backgroundColor + ), + [deploymentFrequencyProps?.backgroundColor, weeklyDeploymentCounts] + ); + + const weekLabels = useMemo( + () => getSortedDatesAsArrayFromMap(weekDeliveryVolumeData), + [weekDeliveryVolumeData] + ); + const formatDeployments = useCallback( + (value: number) => `${value} ${value === 1 ? 'deployment' : 'deployments'}`, + [] + ); + + // CLUSTOX: gated on isCodeProviderIntegrationEnabled -- the same guard the + // card uses to decide between the chart and NoDataImg. Zero deployments is + // still a meaningful comparison against a weekly target (it's the "below + // target" case), so no additional zero-count guard here. + const deploymentFrequencyBenchmarkCaption = useMemo( + () => + isCodeProviderIntegrationEnabled && + deploymentFrequencyBenchmark?.target != null + ? benchmarkCaption( + 'deployment_frequency', + avgWeeklyDeploymentFrequency || 0, + deploymentFrequencyBenchmark.target, + deploymentFrequencyBenchmark.source + ) + : null, + [ + isCodeProviderIntegrationEnabled, + deploymentFrequencyBenchmark, + avgWeeklyDeploymentFrequency + ] + ); + + // CLUSTOX: `avgWeeklyDeploymentFrequency`, never + // `deploymentFrequencyProps.count`. The headline switches unit on its own -- + // getBadgeDetails picks day / week / month by which average first clears 1 -- + // so an active team's headline is a *per-day* figure while the target is + // per week. Feeding the headline in would compare 2.5/day against a 5/week + // target and paint the band warning for a team deploying 17 times a week. + // + // This is also the only card whose band covers *upward*: deployment + // frequency is absent from LOWER_IS_BETTER, so benchmarkBandOptions shades + // target -> top of the axis rather than target -> 0. + const deploymentFrequencyChartOptions = useDoraCardChartOptions( + isCodeProviderIntegrationEnabled + ? { + metric: 'deployment_frequency', + target: deploymentFrequencyBenchmark?.target, + actual: avgWeeklyDeploymentFrequency || 0, + values: weeklyDeploymentCounts + } + : null, + { labels: weekLabels, format: formatDeployments } ); const { weeksCovered, daysCovered } = useStateDateConfig(); @@ -172,6 +232,16 @@ export const WeeklyDeliveryVolumeCard = () => { )} + {Boolean(selectedContributors.length) && ( + // CLUSTOX: filtered by deploy actor, not PR author -- see the note + // on ChangeTimeCard for why that distinction has to be visible. + + deployed by {selectedContributors.join(', ')} + + )} + {deploymentFrequencyBenchmarkCaption && ( + + )} {isCodeProviderIntegrationEnabled ? ( @@ -179,7 +249,7 @@ export const WeeklyDeliveryVolumeCard = () => { id="weekly-delivery-frequency" type="line" series={series} - options={chartOptions} + options={deploymentFrequencyChartOptions} /> ) : ( @@ -193,6 +263,10 @@ export const WeeklyDeliveryVolumeCard = () => { paddingX={2} gap1 justifyCenter + sx={{ + pointerEvents: 'none', + '& > *': { pointerEvents: 'auto' } + }} > diff --git a/web-server/src/content/DoraMetrics/DoraCards/__tests__/LeadTimeBreakdownCard.test.tsx b/web-server/src/content/DoraMetrics/DoraCards/__tests__/LeadTimeBreakdownCard.test.tsx new file mode 100644 index 000000000..fe363f1b1 --- /dev/null +++ b/web-server/src/content/DoraMetrics/DoraCards/__tests__/LeadTimeBreakdownCard.test.tsx @@ -0,0 +1,146 @@ +jest.mock('@/hooks/useAuth', () => ({ useAuth: jest.fn() })); +jest.mock('@/store', () => ({ useSelector: jest.fn() })); +jest.mock('@/components/OverlayPageContext', () => ({ + useOverlayPage: jest.fn() +})); +jest.mock('@/content/PullRequests/useChangeTimePipeline', () => ({ + ...jest.requireActual('@/content/PullRequests/useChangeTimePipeline'), + useLeadTimePipeline: jest.fn(), + usePrChangeTimePipeline: jest.fn() +})); +jest.mock('@/content/PullRequests/useTicketLeadTimeSegment', () => ({ + useTicketLeadTimeSegment: jest.fn() +})); + +import { screen } from '@testing-library/react'; + +import { useOverlayPage } from '@/components/OverlayPageContext'; +import { + useLeadTimePipeline, + usePrChangeTimePipeline +} from '@/content/PullRequests/useChangeTimePipeline'; +import { useTicketLeadTimeSegment } from '@/content/PullRequests/useTicketLeadTimeSegment'; +import { useAuth } from '@/hooks/useAuth'; +import { useSelector } from '@/store'; +import { renderWithTheme as render } from '@/utils/testUtils'; + +import { LeadTimeBreakdownCard } from '../LeadTimeBreakdownCard'; + +// CLUSTOX: Jira integration -- the extended Lead Time breakdown, +// promoted to its own always-visible card on the main DORA Metrics +// page. See docs/JIRA_INTEGRATION_PROPOSAL.md §6A. +const fiveSegments = () => [ + { duration: 3600, bgColor: '#000', color: '#fff', clipPath: '', title: 'Commit', description: '', legendLabel: 'First commit → PR opened' }, + { duration: 3600, bgColor: '#000', color: '#fff', clipPath: '', title: 'Response', description: '', legendLabel: 'First response' }, + { duration: 3600, bgColor: '#000', color: '#fff', clipPath: '', title: 'Rework', description: '', legendLabel: 'Rework' }, + { duration: 3600, bgColor: '#000', color: '#fff', clipPath: '', title: 'Merge', description: '', legendLabel: 'Merge' }, + { duration: 3600, bgColor: '#000', color: '#fff', clipPath: '', title: 'Deploy', description: '', legendLabel: 'Merge → deploy' } +]; + +describe('LeadTimeBreakdownCard', () => { + beforeEach(() => { + jest.clearAllMocks(); + (useSelector as jest.Mock).mockReturnValue([]); + (useOverlayPage as jest.Mock).mockReturnValue({ addPage: jest.fn() }); + (usePrChangeTimePipeline as jest.Mock).mockReturnValue({ + reposWithNoDeploymentsConfigured: [], + reposCountWithWorkflowConfigured: 0 + }); + (useLeadTimePipeline as jest.Mock).mockReturnValue({ + leadTimeDetailsArray: fiveSegments(), + totalLeadTime: 18000 + }); + }); + + it('renders nothing when Jira is not linked', () => { + (useAuth as jest.Mock).mockReturnValue({ integrations: {} }); + (useTicketLeadTimeSegment as jest.Mock).mockReturnValue({ + ticketSegment: null, + comparison: null + }); + + const { container } = render(); + + expect(container).toBeEmptyDOMElement(); + }); + + it('renders nothing when Jira is linked but there are no ticket-matched PRs this period', () => { + (useAuth as jest.Mock).mockReturnValue({ + integrations: { jira: { integrated: true } } + }); + (useTicketLeadTimeSegment as jest.Mock).mockReturnValue({ + ticketSegment: null, + comparison: null + }); + + const { container } = render(); + + expect(container).toBeEmptyDOMElement(); + }); + + it('renders the card with the ticket segment and comparison when both are present', () => { + (useAuth as jest.Mock).mockReturnValue({ + integrations: { jira: { integrated: true } } + }); + (useTicketLeadTimeSegment as jest.Mock).mockReturnValue({ + ticketSegment: { + duration: 172800, + bgColor: '#111', + color: '#eee', + title: 'Idea', + description: 'Ticket to first commit', + legendLabel: 'Ticket created → first commit', + isNew: true + }, + comparison: { + extendedSeconds: 190800, + commitOnlySeconds: 18000, + matchedPrCount: 200 + } + }); + + render(); + + expect(screen.getByText('Lead Time for Changes')).toBeInTheDocument(); + // Full design-reference labels in the legend, not the short in-bar + // titles ("Idea"/"Commit") LeadTimeStatsCore uses in its default, + // non-legend mode. + expect( + screen.getByText('Ticket created → first commit') + ).toBeInTheDocument(); + expect(screen.getByText('First commit → PR opened')).toBeInTheDocument(); + expect(screen.getByText('New')).toBeInTheDocument(); + // The comparison now renders once, as its own header badge (design + // reference), not as an inline line above the bar -- see + // LeadTimeStatsCore's own test for the assertion that it no longer + // renders that line itself. + expect(screen.getByText(/avg, up from/)).toBeInTheDocument(); + expect(screen.getByText(/commit-only/)).toBeInTheDocument(); + }); + + it('does not render a redundant "Total" footer -- the comparison line already covers it', () => { + (useAuth as jest.Mock).mockReturnValue({ + integrations: { jira: { integrated: true } } + }); + (useTicketLeadTimeSegment as jest.Mock).mockReturnValue({ + ticketSegment: { + duration: 172800, + bgColor: '#111', + color: '#eee', + title: 'Idea', + description: '', + legendLabel: 'Ticket created → first commit', + isNew: true + }, + comparison: { + extendedSeconds: 190800, + commitOnlySeconds: 18000, + matchedPrCount: 200 + } + }); + + render(); + + expect(screen.queryByText(/^Total:/)).not.toBeInTheDocument(); + }); +}); diff --git a/web-server/src/content/DoraMetrics/DoraCards/__tests__/LinesOfCodeCard.test.tsx b/web-server/src/content/DoraMetrics/DoraCards/__tests__/LinesOfCodeCard.test.tsx new file mode 100644 index 000000000..0ec2f46a1 --- /dev/null +++ b/web-server/src/content/DoraMetrics/DoraCards/__tests__/LinesOfCodeCard.test.tsx @@ -0,0 +1,237 @@ +// CLUSTOX: this project has repeatedly shipped defects where data was +// produced correctly and dropped one layer downstream, so the model unit test +// alone is not enough -- it proves the numbers are derived, not that they are +// rendered. This renders the real component and asserts on its output. +// +// The mocks below replace exactly three things and nothing else: the redux +// store (so a payload can be posed), the contributor filter (a router-backed +// hook), and `Chart2` (a `next/dynamic` client-only chart). Chart2's stand-in +// records the props it was handed, which is how the series/band unit pairing +// is checked at the point it actually matters. +import { ThemeProvider } from '@mui/material'; +import { renderToStaticMarkup } from 'react-dom/server'; + +import { MainTheme } from '@/theme/schemes/theme'; +import type { TeamDoraMetricsApiResponseType } from '@/types/resources'; + +const chartCalls: { series: any; options: any }[] = []; + +let metricsSummary: Partial | undefined; + +jest.mock('@/store', () => ({ + useSelector: (selector: (s: any) => unknown) => + selector({ doraMetrics: { metrics_summary: metricsSummary } }), + useDispatch: () => () => undefined +})); + +// CLUSTOX: the comparison pill is shared with all four existing cards and +// reaches for the team/date config through redux. Stubbed to the two numbers +// it was handed, which is the only part of it this card is responsible for. +jest.mock('../../DoraMetricsComparisonPill', () => ({ + DoraMetricsComparisonPill: ({ val, against }: any) => + `[pill ${val} vs ${against}]` +})); + +jest.mock('@/components/ContributorFilter', () => ({ + useSelectedContributors: () => [] as string[] +})); + +jest.mock('@/components/Chart2', () => ({ + Chart2: (props: any) => { + chartCalls.push({ series: props.series, options: props.options }); + return null; + } +})); + +import { LinesOfCodeCard } from '../LinesOfCodeCard'; + +const LOC_STATS = { + current: { + additions: 149332, + deletions: 20650, + total: 169982, + avg_pr_size: 654, + avg_daily: 12142 + }, + previous: { + additions: 63033, + deletions: 7407, + total: 70440, + avg_pr_size: 640, + avg_daily: 5031 + } +}; + +const LOC_TRENDS = { + current: { + '2026-07-20T00:00:00+00:00': { + additions: 63033, + deletions: 7407, + total: 70440, + avg_pr_size: 640 + }, + '2026-07-27T00:00:00+00:00': { + additions: 65867, + deletions: 10086, + total: 75953, + avg_pr_size: 575 + }, + '2026-08-03T00:00:00+00:00': { + additions: 20432, + deletions: 3157, + total: 23589, + avg_pr_size: 1310 + } + }, + previous: {} +}; + +// CLUSTOX: the real app theme, not a stub. `benchmarkBandOptions` reads +// `theme.colors.success.main` off it, and the card's tooltips are styled from +// it too -- a stub would pass while the app crashed on the same code path. +const render = () => + renderToStaticMarkup( + + + + ); + +beforeEach(() => { + chartCalls.length = 0; + metricsSummary = undefined; +}); + +describe('LinesOfCodeCard', () => { + it('renders the measured numbers, not just derives them', () => { + metricsSummary = { loc_stats: LOC_STATS, loc_trends: LOC_TRENDS } as any; + + const html = render(); + + // CLUSTOX: the headline itself runs through `useCountUp`, which animates + // from 0 on an interval and so is still 0 in a single server render -- + // exactly as it is on the first painted frame in the browser. Its exact + // figure is asserted through the tooltip it also renders, which is not + // animated. + // + // The headline is the DAILY RATE, not the period total: a total is not + // comparable between two date ranges of different lengths. Both numbers + // are asserted here precisely because they are easy to swap -- 12,142 and + // 169,982 would each look plausible in the other's place. + expect(html).toContain('12,142 lines per day'); + expect(html).toContain('5,031 in the previous period'); + expect(html).toContain('169,982 lines over the selected range'); + expect(html).toContain('+149.3K'); + expect(html).toContain('20.7K'); + expect(html).toContain('149,332 added, 20,650 deleted'); + // The benchmarked figure, spelled out rather than compacted. + expect(html).toContain('Avg. PR size 654 lines'); + // Previous-period delta rides on PR size, not on the directionless total. + expect(html).toContain('[pill 654 vs 640]'); + }); + + it('draws the band over the avg_pr_size series, in lines-per-PR', () => { + metricsSummary = { + loc_stats: LOC_STATS, + loc_trends: LOC_TRENDS, + benchmarks: { lines_of_code: { target: 200, source: 'team' } } + } as any; + + const html = render(); + + // CLUSTOX: the whole point. A band whose target is 200 lines/PR drawn over + // a series of weekly totals (70440, 75953, 23589) would be a hairline on + // the axis floor -- correct arithmetic, unreadable, and unquestioned. + expect(chartCalls).toHaveLength(1); + expect(chartCalls[0].series[0].data).toEqual([640, 575, 1310]); + + const annotations = + chartCalls[0].options.options.plugins.annotation.annotations; + expect(annotations.targetLine.yMin).toBe(200); + // The axis is sized to the avg_pr_size series (max 1310), not to 75953. + expect(chartCalls[0].options.options.scales.y.suggestedMax).toBeLessThan( + 2000 + ); + + // Caption and band agree: 654 lines/PR against a 200 lines/PR target. + expect(html).toContain('654 lines is over target (200 lines)'); + }); + + it('renders with loc_stats absent instead of throwing', () => { + // CLUSTOX: the old-backend case, and the reason it matters: this card + // renders inside the same tree as the other four, so an exception here + // would unmount all of them. "Does not throw" is the regression guarantee. + metricsSummary = { + lead_time_stats: { current: { lead_time: 3600 } } + } as any; + + const html = render(); + + expect(html).toContain('Lines of code unavailable'); + expect(html).not.toContain('Avg. PR size'); + // No chart at all, so nothing for a band to be drawn over. + expect(chartCalls).toHaveLength(0); + }); + + it('renders with the entire metrics_summary absent', () => { + metricsSummary = undefined; + + expect(() => render()).not.toThrow(); + expect(render()).toContain('Lines of code unavailable'); + }); + + it('shows a measured zero as data, with no band and no caption', () => { + const zero = { additions: 0, deletions: 0, total: 0, avg_pr_size: 0 }; + metricsSummary = { + loc_stats: { current: zero, previous: zero }, + loc_trends: { current: {}, previous: {} }, + benchmarks: { lines_of_code: { target: 200, source: 'team' } } + } as any; + + const html = render(); + + expect(html).toContain('Lines changed'); + expect(html).not.toContain('Lines of code unavailable'); + // CLUSTOX: 0 beats every lower-is-better target. Congratulating a team for + // shipping nothing is the failure mode this asserts against. + expect(html).not.toContain('under target'); + expect(chartCalls[0].options.options.plugins?.annotation).toBeUndefined(); + }); + + it('shows no comparison pill when this period merged nothing', () => { + // CLUSTOX: the zero case above uses 0 for BOTH periods, so the pill is + // hidden by its own `avgPrSize || prevAvgPrSize` guard and this path never + // ran. With 0 now against 640 previously, the pill computes -100% and -- + // because PR size is lower-is-better, so `positive={false}` inverts the + // tone -- would render that drop GREEN. A team that merged nothing must + // not be told it improved. + const previous = { + additions: 60000, + deletions: 4000, + total: 64000, + avg_pr_size: 640 + }; + const benchmarks = { lines_of_code: { target: 200, source: 'team' } }; + + // Control: a real current period DOES render the pill, so the assertion + // below is about this case and not about the marker having moved. + metricsSummary = { + loc_stats: { + current: { additions: 500, deletions: 100, total: 600, avg_pr_size: 300 }, + previous + }, + loc_trends: { current: {}, previous: {} }, + benchmarks + } as any; + expect(render()).toContain('[pill 300 vs 640]'); + + metricsSummary = { + loc_stats: { + current: { additions: 0, deletions: 0, total: 0, avg_pr_size: 0 }, + previous + }, + loc_trends: { current: {}, previous: {} }, + benchmarks + } as any; + expect(render()).not.toContain('[pill'); + }); +}); diff --git a/web-server/src/content/DoraMetrics/DoraCards/__tests__/SprintRollupCard.test.tsx b/web-server/src/content/DoraMetrics/DoraCards/__tests__/SprintRollupCard.test.tsx new file mode 100644 index 000000000..83f2a0d44 --- /dev/null +++ b/web-server/src/content/DoraMetrics/DoraCards/__tests__/SprintRollupCard.test.tsx @@ -0,0 +1,105 @@ +jest.mock('@/hooks/useAuth', () => ({ useAuth: jest.fn() })); +jest.mock('@/hooks/useStateTeamConfig', () => ({ + useSingleTeamConfig: jest.fn() +})); +jest.mock('axios'); + +import { screen, waitFor } from '@testing-library/react'; +import axios from 'axios'; + +import { useAuth } from '@/hooks/useAuth'; +import { useSingleTeamConfig } from '@/hooks/useStateTeamConfig'; +import { renderWithTheme as render } from '@/utils/testUtils'; + +import { SprintRollupCard } from '../SprintRollupCard'; + +const TEAM_ID = 'team-1'; + +const sprint = (overrides = {}) => ({ + name: 'PZDA Sprint 1', + state: 'closed', + start_date: '2026-07-20T07:55:00.130Z', + end_date: '2026-08-03T05:00:00.000Z', + planned_count: 355, + completed_count: 272, + ...overrides +}); + +// CLUSTOX: Jira integration -- the Sprint rollup chart. See +// docs/JIRA_INTEGRATION_PROPOSAL.md §6D. +describe('SprintRollupCard', () => { + beforeEach(() => { + jest.clearAllMocks(); + (useSingleTeamConfig as jest.Mock).mockReturnValue({ singleTeamId: TEAM_ID }); + }); + + it('renders nothing when Jira is not linked for this org', () => { + (useAuth as jest.Mock).mockReturnValue({ integrations: {} }); + const { container } = render(); + + expect(container).toBeEmptyDOMElement(); + expect(axios).not.toHaveBeenCalled(); + }); + + it('renders nothing when there is no team selected yet', () => { + (useAuth as jest.Mock).mockReturnValue({ + integrations: { jira: { integrated: true } } + }); + (useSingleTeamConfig as jest.Mock).mockReturnValue({ singleTeamId: undefined }); + const { container } = render(); + + expect(container).toBeEmptyDOMElement(); + expect(axios).not.toHaveBeenCalled(); + }); + + it('renders nothing when the team has no sprints', async () => { + (useAuth as jest.Mock).mockReturnValue({ + integrations: { jira: { integrated: true } } + }); + (axios as unknown as jest.Mock).mockResolvedValue({ data: [] }); + const { container } = render(); + + await waitFor(() => expect(axios).toHaveBeenCalled()); + expect(container).toBeEmptyDOMElement(); + }); + + it('fetches from the team-scoped sprints endpoint, with no date-range params', async () => { + (useAuth as jest.Mock).mockReturnValue({ + integrations: { jira: { integrated: true } } + }); + (axios as unknown as jest.Mock).mockResolvedValue({ data: [sprint()] }); + render(); + + await waitFor(() => + expect(axios).toHaveBeenCalledWith(`/api/internal/team/${TEAM_ID}/sprints`) + ); + }); + + it('renders one column per sprint, in the order the backend returned them', async () => { + (useAuth as jest.Mock).mockReturnValue({ + integrations: { jira: { integrated: true } } + }); + (axios as unknown as jest.Mock).mockResolvedValue({ + data: [ + sprint({ name: 'Sprint 1' }), + sprint({ name: 'Sprint 2' }) + ] + }); + render(); + + expect(await screen.findByText('Sprint 1')).toBeInTheDocument(); + expect(screen.getByText('Sprint 2')).toBeInTheDocument(); + }); + + it('shows the legend and card title once sprints are present', async () => { + (useAuth as jest.Mock).mockReturnValue({ + integrations: { jira: { integrated: true } } + }); + (axios as unknown as jest.Mock).mockResolvedValue({ data: [sprint()] }); + render(); + + expect(await screen.findByText('Sprint rollup')).toBeInTheDocument(); + expect(screen.getByText('Planned')).toBeInTheDocument(); + expect(screen.getByText('Shipped')).toBeInTheDocument(); + }); +}); diff --git a/web-server/src/content/DoraMetrics/DoraCards/__tests__/TicketCycleTimeCard.test.tsx b/web-server/src/content/DoraMetrics/DoraCards/__tests__/TicketCycleTimeCard.test.tsx new file mode 100644 index 000000000..ca97e7615 --- /dev/null +++ b/web-server/src/content/DoraMetrics/DoraCards/__tests__/TicketCycleTimeCard.test.tsx @@ -0,0 +1,393 @@ +jest.mock('@/hooks/useAuth', () => ({ useAuth: jest.fn() })); +jest.mock('@/hooks/useStateTeamConfig', () => ({ + useSingleTeamConfig: jest.fn() +})); +jest.mock('@/contexts/ModalContext', () => ({ useModal: jest.fn() })); +jest.mock('axios'); + +import { renderHook, screen, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import axios from 'axios'; + +import { useModal } from '@/contexts/ModalContext'; +import { useAuth } from '@/hooks/useAuth'; +import { useSingleTeamConfig } from '@/hooks/useStateTeamConfig'; +import { renderWithTheme as render } from '@/utils/testUtils'; + +import { + DataHygieneCard, + segmentWidths, + TicketCycleTimeCard, + UnlinkedPrsModalBody, + useTicketInsights +} from '../TicketCycleTimeCard'; + +const TEAM_ID = 'team-1'; +const DATES = { start: new Date('2026-05-01'), end: new Date('2026-08-01') }; + +const project = (overrides = {}) => ({ + project_key: 'PZDA', + project_name: 'Project Zero Deposit Africa', + ticket_count: 222, + avg_total_seconds: 1128947, + avg_seconds_by_category: { + 'To Do': 495562, + 'In Progress': 631158, + Done: 2227 + }, + ...overrides +}); + +const insights = (overrides = {}) => ({ + cycle_time_by_project: [project()], + prs_without_ticket_count: 0, + ...overrides +}); + +// CLUSTOX: Jira integration, Phase 4 (§6C/§6E). See +// docs/JIRA_INTEGRATION_PROPOSAL.md. useTicketInsights is the one fetch +// backing both TicketCycleTimeCard and DataHygieneCard -- now tested on +// its own (see the "fetches exactly once" test below), since the two +// cards it feeds are presentational and no longer fetch independently. +describe('useTicketInsights', () => { + beforeEach(() => { + jest.clearAllMocks(); + (useSingleTeamConfig as jest.Mock).mockReturnValue({ + singleTeamId: TEAM_ID, + dates: DATES + }); + }); + + it('does not fetch when Jira is not linked for this org', () => { + (useAuth as jest.Mock).mockReturnValue({ integrations: {} }); + const { result } = renderHook(() => useTicketInsights()); + + expect(result.current.isJiraLinked).toBe(false); + expect(axios).not.toHaveBeenCalled(); + }); + + it('does not fetch when there is no team selected yet', () => { + (useAuth as jest.Mock).mockReturnValue({ + integrations: { jira: { integrated: true } } + }); + (useSingleTeamConfig as jest.Mock).mockReturnValue({ + singleTeamId: undefined, + dates: DATES + }); + renderHook(() => useTicketInsights()); + + expect(axios).not.toHaveBeenCalled(); + }); + + it('fetches this team\'s ticket insights for the given date range, exactly once', async () => { + (useAuth as jest.Mock).mockReturnValue({ + integrations: { jira: { integrated: true } } + }); + (axios as unknown as jest.Mock).mockResolvedValue({ data: insights() }); + + const { result } = renderHook(() => useTicketInsights()); + + await waitFor(() => expect(result.current.insights).not.toBeNull()); + expect(axios).toHaveBeenCalledTimes(1); + expect(axios).toHaveBeenCalledWith( + `/api/internal/team/${TEAM_ID}/ticket_insights`, + { + params: { + from_date: DATES.start.toISOString(), + to_date: DATES.end.toISOString() + } + } + ); + }); +}); + +describe('TicketCycleTimeCard', () => { + it('renders nothing when Jira is not linked', () => { + const { container } = render( + + ); + expect(container).toBeEmptyDOMElement(); + }); + + it('renders nothing while still loading', () => { + const { container } = render( + + ); + expect(container).toBeEmptyDOMElement(); + }); + + it('renders nothing when no project has ticket data for this period', () => { + const { container } = render( + + ); + expect(container).toBeEmptyDOMElement(); + }); + + it('renders one row per tracked project, with its own key, name, and avg', () => { + render( + + ); + + expect(screen.getByText('PZDA')).toBeInTheDocument(); + expect(screen.getByText('PAY')).toBeInTheDocument(); + expect(screen.getByText(/Payments/)).toBeInTheDocument(); + }); + + it('still renders a tiny-but-real category as a visible segment, not a sub-pixel sliver', () => { + // Real production data: "Done" was ~0.2% of the total (closed in + // ~37 minutes against a ~13 day ticket life) -- at that literal + // percentage the bar rendered no visible green at all, reading as + // missing data. This is the regression the Done segment must not + // repeat. + render( + + ); + + // FlexBox's `title` prop renders a MUI Tooltip, not a native `title` + // attribute -- MUI clones the child with an `aria-label` mirroring + // the tooltip text instead, which is what's actually queryable + // without opening the tooltip. + expect(screen.getByLabelText(/Done: 37m 7s/)).toBeInTheDocument(); + }); +}); + +// CLUSTOX: covers the visual bug found comparing the live-rendered +// widget against real data -- a category real enough to have its own +// tooltip could still render with ~0 width. segmentWidths is the pure +// function behind the bar, tested in isolation so the floor/donation +// math doesn't need a full render to verify. +describe('segmentWidths', () => { + it('returns each category at its true percentage when none needs flooring', () => { + const widths = segmentWidths( + { 'To Do': 100, 'In Progress': 100, Done: 100 }, + 300 + ); + + // toBeCloseTo, not toEqual -- (100/300)*100 and 100/3 are the same + // value mathematically but land a ULP apart as IEEE-754 floats. + widths.forEach((w) => expect(w.pct).toBeCloseTo(100 / 3, 10)); + }); + + it('floors a near-zero category to the minimum and still sums to 100', () => { + const widths = segmentWidths( + { 'To Do': 495562, 'In Progress': 631158, Done: 2227 }, + 1128947 + ); + + const done = widths.find((w) => w.category === 'Done'); + expect(done.pct).toBeGreaterThanOrEqual(4); + expect(widths.reduce((sum, w) => sum + w.pct, 0)).toBeCloseTo(100, 5); + }); + + it('excludes a category with zero seconds entirely', () => { + const widths = segmentWidths({ 'To Do': 100, Done: 100 }, 200); + expect(widths.map((w) => w.category)).toEqual(['To Do', 'Done']); + }); + + it('returns an empty list when the total is zero, instead of dividing by it', () => { + expect(segmentWidths({ 'To Do': 0 }, 0)).toEqual([]); + }); + + it('preserves each segment\'s real seconds for the tooltip, unadjusted', () => { + const widths = segmentWidths( + { 'To Do': 495562, 'In Progress': 631158, Done: 2227 }, + 1128947 + ); + + expect(widths.find((w) => w.category === 'Done').seconds).toBe(2227); + }); +}); + +// CLUSTOX: Jira integration, Phase 4 (§6E) -- now its own full-width +// card on the page (was stacked below Ticket Cycle Time in the same +// column), fed by the same useTicketInsights result as +// TicketCycleTimeCard rather than fetching it again. See +// docs/JIRA_INTEGRATION_PROPOSAL.md. +describe('DataHygieneCard', () => { + beforeEach(() => { + jest.clearAllMocks(); + (useModal as jest.Mock).mockReturnValue({ addModal: jest.fn() }); + }); + + it('renders nothing when Jira is not linked', () => { + const { container } = render( + + ); + expect(container).toBeEmptyDOMElement(); + }); + + it('renders nothing while still loading', () => { + const { container } = render( + + ); + expect(container).toBeEmptyDOMElement(); + }); + + it('renders nothing when every merged PR is linked', () => { + const { container } = render( + + ); + expect(container).toBeEmptyDOMElement(); + }); + + it('shows the unmatched-PR count and the real project key in the convention hint', () => { + render( + + ); + + expect(screen.getByText(/PRs merged this period with/)).toBeInTheDocument(); + expect(screen.getByText('no linked ticket')).toBeInTheDocument(); + // Real project key, not a hardcoded placeholder -- matches the + // design reference's own "check branch naming against PAY-123" + // convention hint, just with this team's actual key. + expect(screen.getByText(/PZDA-123 convention/)).toBeInTheDocument(); + }); + + it('opens the unlinked-PRs drill-down modal, scoped to the current team and date range, on "View PRs"', async () => { + const addModal = jest.fn(); + (useModal as jest.Mock).mockReturnValue({ addModal }); + render( + + ); + + await userEvent.click(screen.getByRole('button', { name: /view prs/i })); + + expect(addModal).toHaveBeenCalledTimes(1); + const [modalArgs] = addModal.mock.calls[0]; + expect(modalArgs.title).toBe('PRs with no linked Jira ticket'); + expect(modalArgs.body.props).toMatchObject({ teamId: TEAM_ID, dates: DATES }); + }); +}); + +const unlinkedPr = (overrides = {}) => ({ + id: 'pr-1', + title: 'feat(payments): add refund flow', + url: 'https://github.com/org/repo/pull/42', + head_branch: 'feat/refund-flow', + author: 'jordan', + merged_at: '2026-07-15T10:00:00+00:00', + ...overrides +}); + +// CLUSTOX: Jira integration, Phase 4 (§6E) -- the Data Hygiene +// drill-down's own lazy fetch. See docs/JIRA_INTEGRATION_PROPOSAL.md. +describe('UnlinkedPrsModalBody', () => { + beforeEach(() => jest.clearAllMocks()); + + it('fetches this team\'s unlinked PRs for the given date range', async () => { + (axios as unknown as jest.Mock).mockResolvedValue({ data: [unlinkedPr()] }); + + render(); + + await screen.findByText(/feat\(payments\): add refund flow/); + expect(axios).toHaveBeenCalledWith( + `/api/internal/team/${TEAM_ID}/unlinked_prs`, + { + params: { + from_date: DATES.start.toISOString(), + to_date: DATES.end.toISOString() + } + } + ); + }); + + it('renders branch, author, and merge date for each PR', async () => { + (axios as unknown as jest.Mock).mockResolvedValue({ data: [unlinkedPr()] }); + + render(); + + expect( + await screen.findByText(/feat\/refund-flow/) + ).toBeInTheDocument(); + expect(screen.getByText(/jordan/)).toBeInTheDocument(); + expect(screen.getByText(/Jul 15, 2026/)).toBeInTheDocument(); + }); + + it('renders one entry per PR, in the order the backend returned them', async () => { + (axios as unknown as jest.Mock).mockResolvedValue({ + data: [ + unlinkedPr({ id: 'pr-new', title: 'newer PR' }), + unlinkedPr({ id: 'pr-old', title: 'older PR' }) + ] + }); + + render(); + + expect(await screen.findByText('newer PR')).toBeInTheDocument(); + expect(screen.getByText('older PR')).toBeInTheDocument(); + }); + + it('shows an error message instead of crashing when the fetch fails', async () => { + (axios as unknown as jest.Mock).mockRejectedValue(new Error('network error')); + jest.spyOn(console, 'error').mockImplementation(() => {}); + + render(); + + expect(await screen.findByText(/Could not load PRs/)).toBeInTheDocument(); + }); +}); diff --git a/web-server/src/content/DoraMetrics/DoraCards/__tests__/noBenchmarksRegression.test.tsx b/web-server/src/content/DoraMetrics/DoraCards/__tests__/noBenchmarksRegression.test.tsx new file mode 100644 index 000000000..d69f8f9a9 --- /dev/null +++ b/web-server/src/content/DoraMetrics/DoraCards/__tests__/noBenchmarksRegression.test.tsx @@ -0,0 +1,389 @@ +/** + * The regression guarantee for the benchmark-bands + lines-of-code feature. + * + * With no benchmark configured anywhere -- no team row, no global baseline, so + * `metrics_summary.benchmarks` is absent entirely -- every one of the five + * cards must render exactly as it did before this feature: its measured + * numbers, no shaded target band, no benchmark caption. This is the state every + * installation is in until an admin sets a target, and the four cards people + * already rely on must not move. + * + * CLUSTOX: `LinesOfCodeCard.test.tsx` covers the LOC card's own empty states, + * and `benchmarks.test.ts` / `benchmarkBand.test.ts` cover the two utils + * returning null for an absent target. Neither renders the four ORIGINAL + * cards, which is where the regression would actually show up -- a card that + * threw on `benchmarks` being undefined, or drew an annotation block against + * `target: undefined`, would take the whole grid down and no existing test + * would see it. That gap is what this file closes. + * + * Every mock below stands in for something the card reaches for that has + * nothing to do with benchmarks -- redux, the router-backed contributor + * filter, the auth context, the overlay router, and the `next/dynamic` chart. + * Nothing on the benchmark path is mocked: `useDoraCardChartOptions`, + * `benchmarkBandOptions` and `benchmarkCaption` all run for real. + */ +import { ThemeProvider } from '@mui/material'; +import { renderToStaticMarkup } from 'react-dom/server'; + +import { MainTheme } from '@/theme/schemes/theme'; + +const chartCalls: { name: string; options: any }[] = []; + +let metricsSummary: any; +let currentCardName = ''; + +jest.mock('@/store', () => ({ + useSelector: (selector: (s: any) => unknown) => + selector({ + doraMetrics: { + metrics_summary: metricsSummary, + allReposAssignedToTeam: [] + } + }), + useDispatch: () => () => undefined +})); + +jest.mock('@/components/ContributorFilter', () => ({ + useSelectedContributors: () => [] as string[] +})); + +jest.mock('@/components/Chart2', () => ({ + Chart2: (props: any) => { + chartCalls.push({ name: currentCardName, options: props.options }); + return null; + } +})); + +jest.mock('@/hooks/useAuth', () => ({ + useAuth: () => ({ + // Both integration groups present, so every card takes its + // data-available branch rather than short-circuiting to a placeholder -- + // a card showing NoDataImg would trivially have no band. + integrationSet: new Set(['CODE', 'INCIDENT']), + orgId: 'org-1', + role: 'ADMIN' + }) +})); + +jest.mock('@/components/OverlayPageContext', () => ({ + useOverlayPage: () => ({ addPage: () => undefined }) +})); + +jest.mock('@/hooks/useStateTeamConfig', () => ({ + useStateDateConfig: () => ({ weeksCovered: 4, daysCovered: 0 }), + useCurrentDateRangeLabel: () => 'last 4 weeks', + useSingleTeamConfig: () => ({ singleTeamId: 'team-1' }), + useStateTeamConfig: () => ({ singleTeamId: 'team-1' }) +})); + +jest.mock('@/hooks/useDoraMetricsGraph', () => ({ + useDoraMetricsGraph: () => ({ + trendsSeriesMap: { + changeFailureRateTrends: [{ data: [{ y: 10 }, { y: 14 }, { y: 12 }] }], + meanTimeToRestoreTrends: [ + { data: [{ y: 3600 }, { y: 5400 }, { y: 4200 }] } + ] + } + }) +})); + +jest.mock('@/constants/events', () => ({ + track: () => undefined, + TrackEvents: {} +})); + +jest.mock('../../DoraMetricsComparisonPill', () => ({ + DoraMetricsComparisonPill: () => null +})); + +import { ChangeFailureRateCard } from '../ChangeFailureRateCard'; +import { ChangeTimeCard } from '../ChangeTimeCard'; +import { LinesOfCodeCard } from '../LinesOfCodeCard'; +import { MeanTimeToRestoreCard } from '../MeanTimeToRestoreCard'; +import { WeeklyDeliveryVolumeCard } from '../WeeklyDeliveryVolumeCard'; + +/** + * A fully-measured dashboard: every metric has real current and previous + * numbers and real trends. The only thing missing is `benchmarks`. + */ +const MEASURED_SUMMARY = { + lead_time_stats: { + current: { lead_time: 86400, pr_count: 12 }, + previous: { lead_time: 93600, pr_count: 10 } + }, + lead_time_trends: { + current: { + '2026-07-20T00:00:00+00:00': { lead_time: 90000 }, + '2026-07-27T00:00:00+00:00': { lead_time: 86400 } + }, + previous: {} + }, + deployment_frequency_stats: { + current: { + avg_deployment_frequency: 4, + avg_weekly_deployment_frequency: 4, + total_deployments: 16, + duration: 'week' + }, + previous: { + avg_deployment_frequency: 3, + avg_weekly_deployment_frequency: 3, + total_deployments: 12, + duration: 'week' + } + }, + deployment_frequency_trends: { + current: { + '2026-07-20T00:00:00+00:00': { count: 5 }, + '2026-07-27T00:00:00+00:00': { count: 4 } + }, + previous: {} + }, + change_failure_rate_stats: { + current: { change_failure_rate: 12, total_deployments: 16 }, + previous: { change_failure_rate: 15, total_deployments: 12 } + }, + change_failure_rate_trends: { current: {}, previous: {} }, + mean_time_to_restore_stats: { + current: { mean_time_to_recovery: 4200, incident_count: 3 }, + previous: { mean_time_to_recovery: 5400, incident_count: 4 } + }, + mean_time_to_restore_trends: { current: {}, previous: {} }, + loc_stats: { + current: { + additions: 149332, + deletions: 20650, + total: 169982, + avg_pr_size: 654 + }, + previous: { + additions: 63033, + deletions: 7407, + total: 70440, + avg_pr_size: 640 + } + }, + loc_trends: { + current: { + '2026-07-20T00:00:00+00:00': { + additions: 63033, + deletions: 7407, + total: 70440, + avg_pr_size: 640 + }, + '2026-07-27T00:00:00+00:00': { + additions: 65867, + deletions: 10086, + total: 75953, + avg_pr_size: 575 + } + }, + previous: {} + } +}; + +const CARDS = [ + ['Change Time', ChangeTimeCard], + ['Weekly Delivery Volume', WeeklyDeliveryVolumeCard], + ['Change Failure Rate', ChangeFailureRateCard], + ['Mean Time to Restore', MeanTimeToRestoreCard], + ['Lines of Code', LinesOfCodeCard] +] as const; + +// CLUSTOX: the real app theme, not a stub -- `benchmarkBandOptions` reads +// `theme.colors.success.main` off it, so a stub could pass here while the app +// threw on the same code path. +const renderCard = (name: string, Card: () => JSX.Element) => { + currentCardName = name; + return renderToStaticMarkup( + + + + ); +}; + +const optionsFor = (name: string) => + chartCalls.filter((call) => call.name === name).map((call) => call.options); + +beforeEach(() => { + chartCalls.length = 0; + currentCardName = ''; + metricsSummary = { ...MEASURED_SUMMARY }; +}); + +describe('with no benchmark configured anywhere', () => { + it.each(CARDS)('%s renders without throwing', (name, Card) => { + expect(() => renderCard(name, Card)).not.toThrow(); + }); + + it.each(CARDS)('%s draws no target band', (name, Card) => { + renderCard(name, Card); + + const optionsList = optionsFor(name); + // Guards the assertion below: a card that rendered no chart at all would + // vacuously have no annotations, and would not be proving anything. + expect(optionsList.length).toBeGreaterThan(0); + + for (const options of optionsList) { + expect(options.options.plugins.annotation).toBeUndefined(); + // The axis is untouched too. `suggestedMax` exists only to make room + // for a target, so its presence means a band was sized in. + expect(options.options.scales.y.suggestedMax).toBeUndefined(); + } + }); + + it.each(CARDS)('%s shows no benchmark caption', (name, Card) => { + const html = renderCard(name, Card); + + // Every caption `benchmarkCaption` can produce contains "target (", and + // every one names its source as a "benchmark". Neither wording can appear + // when no target exists. + // + // CLUSTOX: this path is gated twice -- each card checks its benchmark + // object before calling, and `benchmarkCaption` returns null for a null + // target -- so no single-util mutation makes this assertion fire. Its + // sensitivity comes from the control below, which asserts the very same + // 'target (' string DOES appear once a benchmark is configured. + expect(html).not.toContain('target ('); + expect(html).not.toContain('benchmark'); + // The specific way this breaks if a null target leaks into the caption. + expect(html).not.toContain('undefined'); + expect(html).not.toContain('NaN'); + }); + + it('keeps every no-benchmark chart on the shared base contract', () => { + // CLUSTOX: this used to assert reference identity with the module-level + // base constant. That contract ended when tooltips arrived: each card now + // closes over its own labels and formatter, so identity across cards is + // impossible by construction. What the identity assertion was actually + // protecting is pinned structurally instead -- nothing benchmark-shaped + // on the no-target path, axes hidden, drag-zoom off, and 'click' absent + // from the event list so the canvas never competes with CardRoot's own + // onClick. + for (const [name, Card] of CARDS) renderCard(name, Card); + + const allOptions = chartCalls.map((call) => call.options); + expect(allOptions.length).toBeGreaterThanOrEqual(4); + for (const options of allOptions) { + const inner = options.options; + expect(inner.plugins?.annotation).toBeUndefined(); + expect(inner.scales?.y?.suggestedMax).toBeUndefined(); + expect(inner.scales.x.display).toBe(false); + expect(inner.scales.y.display).toBe(false); + expect(inner.plugins.zoom.zoom.drag.enabled).toBe(false); + expect(inner.events).not.toContain('click'); + // The tooltip is the one deliberate addition on this path -- it + // describes measured data, which exists with or without a target. + expect(inner.plugins.tooltip.callbacks.label).toBeInstanceOf(Function); + } + }); + + it.each(CARDS)('%s still renders its measured numbers', (name, Card) => { + // The guarantee is "unchanged", not "blank". A card that dropped its data + // would satisfy every no-band assertion above. + const html = renderCard(name, Card); + expect(html).not.toContain('Insufficient data'); + expect(html.length).toBeGreaterThan(200); + }); +}); + +describe('control: a configured benchmark does reach the same cards', () => { + // CLUSTOX: without this, every assertion above would still pass if the band + // had been removed from the cards entirely, or if these renders were somehow + // no longer reaching the real chart options. Same fixtures, targets added. + const BENCHMARKS = { + lead_time: { target: 72000, source: 'team' }, + deployment_frequency: { target: 5, source: 'team' }, + change_failure_rate: { target: 5, source: 'team' }, + mean_time_to_recovery: { target: 1800, source: 'team' }, + lines_of_code: { target: 200, source: 'team' } + }; + + it.each(CARDS)('%s draws a band and a caption', (name, Card) => { + metricsSummary = { ...MEASURED_SUMMARY, benchmarks: BENCHMARKS }; + + const html = renderCard(name, Card); + + const optionsList = optionsFor(name); + expect(optionsList.length).toBeGreaterThan(0); + for (const options of optionsList) { + expect(options.options.plugins.annotation.annotations.band).toBeDefined(); + } + expect(html).toContain('target ('); + }); +}); + +describe('a target of zero is still a target', () => { + // CLUSTOX: `0` is a real benchmark (zero failed changes), never "absent". + // A truthiness check anywhere on the path from the payload to the band + // would drop it and make the strictest target behave like no target at all + // -- indistinguishable, on screen, from the regression this file guards. + it('draws a band for a change failure rate target of 0', () => { + metricsSummary = { + ...MEASURED_SUMMARY, + benchmarks: { change_failure_rate: { target: 0, source: 'team' } } + }; + + const html = renderCard('Change Failure Rate', ChangeFailureRateCard); + + const [options] = optionsFor('Change Failure Rate'); + expect(options.options.plugins.annotation.annotations.band).toBeDefined(); + expect(html).toContain('target (0%)'); + }); +}); + +describe('no incidents this period is not a fast recovery', () => { + // CLUSTOX: the mixed-period fixture. `isNoDataAvailable` is + // `!incidents && !prevAvgTimeToRestore && !currAvgTimeToRestore`, so a team + // with zero incidents THIS period but a recovery time LAST period does not + // trip it -- the card takes its data-available branch with a null count. + // `null <= target` is true in JS and `Math.round(null)` is 0, so the card + // rendered a green "0s is under target" and a success-tinted band beside its + // own "No incidents reported" headline. Nothing was recovered quickly; there + // was nothing to recover. + const MIXED_PERIOD = { + current: { + mean_time_to_recovery: null as number | null, + incident_count: 0 + }, + previous: { mean_time_to_recovery: 5400, incident_count: 4 } + }; + + it('shows no band, no caption and no comparison pill', () => { + metricsSummary = { + ...MEASURED_SUMMARY, + mean_time_to_restore_stats: MIXED_PERIOD, + benchmarks: { + mean_time_to_recovery: { target: 3600, source: 'global' } + } + }; + + const html = renderCard('Mean Time to Recovery', MeanTimeToRestoreCard); + + expect(html).not.toContain('under target'); + expect(html).not.toContain('over target'); + const [options] = optionsFor('Mean Time to Recovery'); + expect(options.options.plugins?.annotation).toBeUndefined(); + }); + + it('still names the target, without claiming a verdict', () => { + metricsSummary = { + ...MEASURED_SUMMARY, + mean_time_to_restore_stats: MIXED_PERIOD, + benchmarks: { + mean_time_to_recovery: { target: 3600, source: 'global' } + } + }; + + // The grey target-only line used to live solely in the `!canShowMTRData` + // branch, which this state never reaches -- so the one state that most + // needs "a target exists, but nothing to compare" showed nothing at all. + const html = renderCard('Mean Time to Recovery', MeanTimeToRestoreCard); + + // `toContain('target')` alone would be vacuous: the buggy caption + // ("0s is under target (1h)") contains it too. The grey line reads + // "target 1h"; the caption reads "target (1h)". Only the first matches. + expect(html).toContain('target 1h'); + expect(html).not.toContain('under target'); + }); +}); diff --git a/web-server/src/content/DoraMetrics/DoraCards/sharedComponents.tsx b/web-server/src/content/DoraMetrics/DoraCards/sharedComponents.tsx index 8bde83b58..4997bcbf9 100644 --- a/web-server/src/content/DoraMetrics/DoraCards/sharedComponents.tsx +++ b/web-server/src/content/DoraMetrics/DoraCards/sharedComponents.tsx @@ -1,7 +1,10 @@ -import { Paper, useTheme } from '@mui/material'; +import CheckCircleOutlineRoundedIcon from '@mui/icons-material/CheckCircleOutlineRounded'; +import ErrorOutlineRoundedIcon from '@mui/icons-material/ErrorOutlineRounded'; +import { alpha, Paper, useTheme } from '@mui/material'; import Img from 'next/image'; import { FlexBox, FlexBoxProps } from '@/components/FlexBox'; +import { Line } from '@/components/Text'; export const CardRoot = (props: FlexBoxProps) => ( { /> ); }; + +// CLUSTOX: the benchmark verdict as a pill rather than a small text line. The +// plain line was easy to miss beside the gold classification chip; the pill +// puts the two judgements at equal visual weight -- industry rating on the +// right of the header, your own target here. Gap leads ("43% under target"), +// source trails, and the full sentence with both raw values lives in the +// hover tooltip, so the glanceable form and the exact form are both there. +// +// Tones stay success/warning -- never red. A missed internal goal is not an +// error, and a red chip would read as one. +export const BenchmarkVerdictPill = ({ + caption +}: { + caption: { + text: string; + headline: string; + sourceShort: string; + tone: 'good' | 'warn'; + }; +}) => { + const theme = useTheme(); + const tone = + caption.tone === 'good' + ? theme.colors.success.main + : theme.colors.warning.main; + return ( + + + {caption.tone === 'good' ? ( + + ) : ( + + )} + + {caption.headline} + + + · {caption.sourceShort} + + + + ); +}; diff --git a/web-server/src/content/DoraMetrics/DoraCards/sharedHooks.tsx b/web-server/src/content/DoraMetrics/DoraCards/sharedHooks.tsx index 0ca6d78ac..a1bdb93d2 100644 --- a/web-server/src/content/DoraMetrics/DoraCards/sharedHooks.tsx +++ b/web-server/src/content/DoraMetrics/DoraCards/sharedHooks.tsx @@ -1,5 +1,8 @@ +import { alpha, useTheme } from '@mui/material'; +import { mergeDeepRight } from 'ramda'; import { useCallback, useEffect, useMemo } from 'react'; +import { ChartOptions } from '@/components/Chart2'; import { Row } from '@/constants/db'; import { changeTimeThresholds, @@ -9,6 +12,10 @@ import { useAuth } from '@/hooks/useAuth'; import { doraMetricsSlice } from '@/slices/dora_metrics'; import { useDispatch, useSelector } from '@/store'; import { ChangeTimeModes, IntegrationGroup } from '@/types/resources'; +import { + BenchmarkBandInput, + benchmarkBandOptions +} from '@/utils/benchmarkBand'; import { getDoraScore } from '@/utils/dora'; import { @@ -295,3 +302,170 @@ export const useChangeFailureRateProps = () => { }; }, [cfrProps, changeFailureRate]); }; + +// CLUSTOX: all four DORA cards declared this same object as a module-level +// `chartOptions` constant. The benchmark band depends on props, so the +// constant has to become a memo -- and rather than copy that memo into four +// files where the four copies can silently drift apart, the shared base and +// the merge live here once. +const BASE_DORA_CARD_CHART_OPTIONS = { + options: { + scales: { + x: { + display: false + }, + y: { + display: false + } + }, + events: [], + plugins: { + zoom: { + zoom: { + drag: { + enabled: false + } + } + } + } + } +} as ChartOptions; + +/** + * The chart options for a DORA card, with the benchmark target band merged in + * when there is a target to draw. + * + * Pass `null` when the card should show no band at all -- distinct from + * passing an input whose `target` is null, though both render the card exactly + * as it looked before this feature. + */ +export type DoraCardChartDisplay = { + /** ISO week keys, one per plotted point, oldest first. */ + labels?: string[]; + /** Formats a plotted value for the tooltip, in the metric's own words. */ + format?: (value: number) => string; +}; + +// CLUSTOX: "Week of Jul 20" rather than the raw ISO key the API buckets by. +// Falls back to the raw label so a malformed key degrades to ugly-but-true. +const weekLabel = (iso: string): string => { + const date = new Date(iso); + if (Number.isNaN(date.getTime())) return iso; + return `Week of ${date.toLocaleDateString('en-US', { + month: 'short', + day: 'numeric' + })}`; +}; + +export const useDoraCardChartOptions = ( + band: Omit | null, + display?: DoraCardChartDisplay +): ChartOptions => { + // CLUSTOX: the tone colours come from the live theme rather than from + // benchmarkBand's hex fallbacks; those exist only so the util stays + // importable from a unit test that can't load MUI. + const theme = useTheme(); + const labels = display?.labels; + const format = display?.format; + + return useMemo(() => { + const bandOptions = band ? benchmarkBandOptions({ ...band, theme }) : null; + const target = bandOptions?.benchmarkBand?.target; + + // CLUSTOX: hover only -- 'click' stays out of the list so the canvas never + // competes with CardRoot's own onClick (the whole card opens the details + // overlay). The tooltip and crosshair configs already exist in + // InternalChart2; `events: []` was the single switch keeping all of it + // dead, which is why these cards read as decoration. + // Typed against ChartOptions here so ramda's MergeDeep literal type + // collapses to something assignable at the return casts below. + const interactive: ChartOptions = { + options: { + events: ['mousemove', 'mouseout'], + plugins: { + tooltip: { + displayColors: false, + callbacks: { + title: (items: { dataIndex: number }[]) => { + const iso = labels?.[items[0]?.dataIndex]; + return iso ? weekLabel(iso) : ''; + }, + label: (item: { parsed: { y: number } }) => { + const value = item.parsed.y; + const text = format ? format(value) : String(value); + // `!= null`: a target of 0 still gets a verdict. Direction + // words only, no glyphs -- the caption below the chart already + // carries the judgement, the tooltip just states the fact. + if (target != null) { + return `${text} \u00b7 ${ + value <= target ? 'under' : 'over' + } target`; + } + return text; + } + } + } + } + } + }; + + const withInteraction = mergeDeepRight( + BASE_DORA_CARD_CHART_OPTIONS, + interactive + ); + + // CLUSTOX: no target configured is the state every card is in until an + // admin sets one -- nothing benchmark-related is drawn. The tooltip stays: + // it describes measured data, which exists with or without a target. + if (!bandOptions) return withInteraction as ChartOptions; + + // CLUSTOX: the band goes on the *right* of the merge. Both sides define + // `scales.y`, and it is the band's `suggestedMax` that has to survive -- + // the base only sets `display: false`, which mergeDeepRight preserves from + // the left because the keys don't collide. + return mergeDeepRight(withInteraction, { + options: bandOptions + }) as unknown as ChartOptions; + }, [band, theme, labels, format]); +}; + +/** + * The one visual style every DORA card's trend shares: a real 2px stroke in + * the card's accent, a fill that fades to transparent toward the axis, and a + * dot on the newest point. + * + * CLUSTOX: before this, the trend was `borderWidth: 0` + a flat grey fill -- + * a shape with no line, which is most of why the graphs read as decoration. + * The gradient rides on chartjs-plugin-gradient (registered in + * InternalChart2, previously unused); `backgroundColor` stays set as the + * flat fallback for the brief window before that plugin's async registration + * lands, so the card is never fill-less. + */ +export const doraCardTrendSeries = ( + label: string, + values: number[], + accent: string +) => { + const dataMax = Math.max(0, ...values.filter(Number.isFinite)); + return [ + { + label, + fill: 'start', + data: values, + backgroundColor: alpha(accent, 0.14), + borderColor: accent, + borderWidth: 2, + lineTension: 0.35, + pointRadius: values.map((_, i) => (i === values.length - 1 ? 3 : 0)), + pointBackgroundColor: accent, + ...(dataMax > 0 && { + gradient: { + backgroundColor: { + axis: 'y' as const, + colors: { 0: alpha(accent, 0.02), [dataMax]: alpha(accent, 0.28) } + } + } + }) + } + ]; +}; diff --git a/web-server/src/content/DoraMetrics/DoraMetricsBody.tsx b/web-server/src/content/DoraMetrics/DoraMetricsBody.tsx index 76e01ba69..875c64f5c 100644 --- a/web-server/src/content/DoraMetrics/DoraMetricsBody.tsx +++ b/web-server/src/content/DoraMetrics/DoraMetricsBody.tsx @@ -17,6 +17,8 @@ import { ROUTES } from '@/constants/routes'; import { FetchState } from '@/constants/ui-states'; import { useDoraStats } from '@/content/DoraMetrics/DoraCards/sharedHooks'; import { useAuth } from '@/hooks/useAuth'; +// CLUSTOX: contributor filter -- shared fetch arguments, see the hook's note. +import { useDoraMetricsFetchArgs } from '@/hooks/useDoraMetricsFetchArgs'; import { useBoolState } from '@/hooks/useEasyState'; import { usePageRefreshCallback } from '@/hooks/usePageRefreshCallback'; import { @@ -31,19 +33,29 @@ import { getRandomLoadMsg } from '@/utils/loading-messages'; import { ClassificationPills } from './ClassificationPills'; import { ChangeFailureRateCard } from './DoraCards/ChangeFailureRateCard'; import { ChangeTimeCard } from './DoraCards/ChangeTimeCard'; +import { LeadTimeBreakdownCard } from './DoraCards/LeadTimeBreakdownCard'; +import { LinesOfCodeCard } from './DoraCards/LinesOfCodeCard'; import { MeanTimeToRestoreCard } from './DoraCards/MeanTimeToRestoreCard'; import { DataStillSyncing } from './DoraCards/SkeletalCard'; +import { SprintRollupCard } from './DoraCards/SprintRollupCard'; +import { + DataHygieneCard, + TicketCycleTimeCard, + useTicketInsights +} from './DoraCards/TicketCycleTimeCard'; import { WeeklyDeliveryVolumeCard } from './DoraCards/WeeklyDeliveryVolumeCard'; export const DoraMetricsBody = () => { const dispatch = useDispatch(); - const { orgId, integrationList } = useAuth(); + const { integrationList } = useAuth(); const isCodeProviderIntegrated = useMemo( () => integrationList.length > 0, [integrationList.length] ); - const { singleTeamId, dates } = useSingleTeamConfig(); const branchPayloadForPrFilters = useBranchesForPrFilters(); + // CLUSTOX: contributor filter -- org/team/date/contributor arguments come + // from one shared hook so every refetch path carries the selection. + const doraMetricsFetchArgs = useDoraMetricsFetchArgs(); const isLoading = useSelector( (s) => s.doraMetrics.requests?.metrics_summary === FetchState.REQUEST ); @@ -61,7 +73,16 @@ export const DoraMetricsBody = () => { .incident_count && !s.doraMetrics.metrics_summary?.lead_time_stats.current.lead_time && !s.doraMetrics.metrics_summary?.deployment_frequency_stats.current - .avg_deployment_frequency + .avg_deployment_frequency && + // CLUSTOX: lines of code counts as insight too. A team that merges PRs + // but has no deployment or incident provider wired up has real, non-zero + // LOC and would otherwise have the whole grid -- including its own + // populated card -- replaced by "nothing to show yet". + // + // Additive: one more `&&` can only make this predicate FALSE more often, + // so it can only show the grid where it was previously hidden. No team + // that sees the dashboard today can start seeing the empty state. + !s.doraMetrics.metrics_summary?.loc_stats?.current.total ); const { addPage } = useOverlayPage(); @@ -69,19 +90,13 @@ export const DoraMetricsBody = () => { useEffect(() => { dispatch( fetchTeamDoraMetrics({ - orgId, - teamId: singleTeamId, - fromDate: dates.start, - toDate: dates.end, + ...doraMetricsFetchArgs, ...branchPayloadForPrFilters }) ); }, [ - dates.end, - dates.start, dispatch, - orgId, - singleTeamId, + doraMetricsFetchArgs, isCodeProviderIntegrated, branchPayloadForPrFilters ]); @@ -90,6 +105,26 @@ export const DoraMetricsBody = () => { const { isSyncing } = useSyncedRepos(); + // CLUSTOX: Jira integration -- one fetch, shared as props between + // TicketCycleTimeCard and DataHygieneCard, which now live in + // different spots on the page (see the layout below) instead of one + // stacked inside the other. See TicketCycleTimeCard.tsx's own note. + const ticketInsights = useTicketInsights(); + + // CLUSTOX: isTeamInsightsEmpty (above) only ever looks at Git-based + // stats -- a repo-less, Jira-only team (see + // docs/JIRA_MULTI_ACCOUNT_PLAN.md's Test Scenario 4: a Jira project + // with no repo at all) has none of those by construction, and used to + // fall into the "configure team repos" empty state below even with + // real ticket-cycle-time data ready to show. Not shown while + // ticketInsights is still loading, or a team whose Jira fetch hasn't + // resolved yet would flash this empty state before its own data + // arrives. + const showEmptyState = + isTeamInsightsEmpty && + !ticketInsights.isLoading && + !ticketInsights.insights?.cycle_time_by_project?.length; + if (isErrored) return ( { /> ); if (!firstLoadDone) return ; - if (isTeamInsightsEmpty) + if (showEmptyState) if (isSyncing) return ; else return ( @@ -155,8 +190,56 @@ export const DoraMetricsBody = () => { + {/* CLUSTOX: slot five. Slot six is deliberately left empty -- an empty + cell reads as room for more, whereas a "coming soon" tile reads as + unfinished. At `md` and below the container is already + single-column, so nothing above this reflows. */} + + +
+ {/* CLUSTOX: Jira integration, Phase 4 -- additive only, doesn't + touch the 4 cards above. Renders nothing of its own accord when + Jira isn't linked or there's no ticket data for this + team/period. See docs/JIRA_INTEGRATION_PROPOSAL.md. */} + + {/* Side by side, not stacked -- matches the design reference's own + two-col layout: Ticket Cycle Time on the left, Sprint rollup on + the right. Each wrapped in its own flex-sized FlexBox rather + than passing a prop into either card: both already + independently render null on their own gating, and neither + needed a props-API change for a purely external layout + concern. + + alignItems="flex-start", not the default stretch: CardRoot + (shared by every DORA card) is height="100%" of its immediate + wrapper, so without this, whichever column is naturally taller + stretches the other one's wrapper to match, and that shorter + card's CardRoot fills the extra height with blank space at the + bottom instead of ending where its own content does. */} + + + + + + + + + {/* Its own full-width row, not stacked inside Ticket Cycle Time's + column above -- matches the design reference. Renders nothing + of its own accord when every merged PR is linked. */} + diff --git a/web-server/src/content/PullRequests/LeadTimeStatsCore.tsx b/web-server/src/content/PullRequests/LeadTimeStatsCore.tsx index 831d40082..000e8cd4b 100644 --- a/web-server/src/content/PullRequests/LeadTimeStatsCore.tsx +++ b/web-server/src/content/PullRequests/LeadTimeStatsCore.tsx @@ -22,7 +22,14 @@ import { Line } from '@/components/Text'; import { track } from '@/constants/events'; import { ROUTES } from '@/constants/routes'; import { isRoleLessThanEM } from '@/constants/useRoute'; -import { usePrChangeTimePipeline } from '@/content/PullRequests/useChangeTimePipeline'; +import { + ClipPathEnum, + usePrChangeTimePipeline +} from '@/content/PullRequests/useChangeTimePipeline'; +import { + TicketLeadTimeComparison, + TicketLeadTimeSegment +} from '@/content/PullRequests/useTicketLeadTimeSegment'; import { useAuth } from '@/hooks/useAuth'; import { useSelector } from '@/store'; import { ChangeTimeSegment } from '@/types/resources'; @@ -40,13 +47,46 @@ export const LeadTimeStatsCore: FC< cycle?: number; changeTimeSegments: ChangeTimeSegment[]; showTotal?: boolean; + // CLUSTOX: Jira integration -- the extended Lead Time breakdown's + // leading "ticket created -> first commit" segment (docs/ + // JIRA_INTEGRATION_PROPOSAL.md §6A). Both undefined for an org + // without Jira linked (or with no ticket-matched PRs this period), + // in which case this renders exactly the original 5-segment bar -- + // see ticketSegment's every usage below, each guarded by its + // presence. + ticketSegment?: TicketLeadTimeSegment; + comparison?: TicketLeadTimeComparison; + // CLUSTOX: Jira integration -- LeadTimeBreakdownCard's rendering, + // matching the design reference: a plain color bar (hover for + // detail) plus a full-label legend below, instead of this + // component's original title/duration text baked into each + // segment. False/undefined leaves every existing caller (the + // TeamInsightsBody drill-down) rendering exactly as it always has + // -- see the branch below, which returns before any of the + // original JSX runs. + showLegend?: boolean; } & BoxProps -> = ({ cycle = 0, changeTimeSegments, showTotal, ...props }) => { +> = ({ + cycle = 0, + changeTimeSegments, + showTotal, + ticketSegment, + comparison, + showLegend, + ...props +}) => { const theme = useTheme(); const { role } = useAuth(); const isEng = isRoleLessThanEM(role); const { addPage } = useOverlayPage(); const [initiation, response, rework, merge, deployment] = changeTimeSegments; + // The existing 5 segments' own math (calcCycleTime/calcLeadTime/the + // "Total" footer below) is untouched -- ticketSegment is purely + // prepended visually, never folded into that total, since it's the + // org-wide, unmodified lead_time this card has always shown. + const initiationClipPath = ticketSegment + ? ClipPathEnum.DEFAULT + : ClipPathEnum.FIRST; const allAssignedRepos = useSelector( (s) => s.doraMetrics.allReposAssignedToTeam @@ -84,8 +124,108 @@ export const LeadTimeStatsCore: FC< }); }, [addPage]); + if (showLegend) { + const legendSegments = [ + ...(ticketSegment ? [ticketSegment] : []), + initiation, + response, + rework, + merge, + deployment + ]; + + return ( + + {/* CLUSTOX: the comparison line used to render here, inline + above the bar. It's now LeadTimeBreakdownCard's own header + badge instead (matching the design reference, which shows it + once, top-right of the card title) -- comparison is still + accepted as a prop so that caller can size/format it here if + it ever needs to, but this component no longer renders it + itself. */} + + {legendSegments.map((segment) => ( + + + + ))} + + + {legendSegments.map((segment) => ( + + + + {segment.legendLabel} + + {segment.isNew && ( + + New + + )} + + ))} + + + ); + } + return ( + {comparison && ( + + + Idea to production: + + + {getDurationString(comparison.extendedSeconds, { segments: 2 })} + + + avg, up from{' '} + + {getDurationString(comparison.commitOnlySeconds, { + segments: 2 + })} + {' '} + commit-only, over {comparison.matchedPrCount} ticket-matched PR + {comparison.matchedPrCount === 1 ? '' : 's'} + + + )} track('HOVER_ON_CHANGE_TIME_QUICK_STATS_CHART')} > + {ticketSegment && ( + + {ticketSegment.title} + + {getDurationString(ticketSegment.duration, { + segments: 1 + }) || '-'}{' '} + + + + + + )} diff --git a/web-server/src/content/PullRequests/TeamInsightsBody.tsx b/web-server/src/content/PullRequests/TeamInsightsBody.tsx index 83f8e0192..ca8e4ff47 100644 --- a/web-server/src/content/PullRequests/TeamInsightsBody.tsx +++ b/web-server/src/content/PullRequests/TeamInsightsBody.tsx @@ -27,6 +27,7 @@ import { Line } from '@/components/Text'; import { TrendsLineChart } from '@/components/TrendsLineChart'; import { track } from '@/constants/events'; import { ClipPathEnum } from '@/content/PullRequests/useChangeTimePipeline'; +import { useTicketLeadTimeSegment } from '@/content/PullRequests/useTicketLeadTimeSegment'; import { useDoraMetricsGraph } from '@/hooks/useDoraMetricsGraph'; import { useBoolState } from '@/hooks/useEasyState'; import { usePageRefreshCallback } from '@/hooks/usePageRefreshCallback'; @@ -172,6 +173,12 @@ export const PrBreakdownAndInsights: FC<{ prs: PR[]; }> = ({ prs }) => { const { changeTimeDetailsArray } = useComputedPrChangeTime(prs); + // CLUSTOX: Jira integration -- the extended Lead Time breakdown's + // leading "ticket created -> first commit" segment (docs/ + // JIRA_INTEGRATION_PROPOSAL.md §6A). Both null for an org without + // Jira linked (or with no ticket-matched PRs this period) -- see + // useTicketLeadTimeSegment. + const { ticketSegment, comparison } = useTicketLeadTimeSegment(); if (!prs.length) return null; @@ -183,13 +190,17 @@ export const PrBreakdownAndInsights: FC<{ Average LT breakdown - Commit to deploy + + {ticketSegment ? 'Idea to deploy' : 'Commit to deploy'} + diff --git a/web-server/src/content/PullRequests/__tests__/LeadTimeStatsCore.test.tsx b/web-server/src/content/PullRequests/__tests__/LeadTimeStatsCore.test.tsx new file mode 100644 index 000000000..45bbb9ea3 --- /dev/null +++ b/web-server/src/content/PullRequests/__tests__/LeadTimeStatsCore.test.tsx @@ -0,0 +1,214 @@ +jest.mock('@/hooks/useAuth', () => ({ useAuth: jest.fn() })); +jest.mock('@/store', () => ({ useSelector: jest.fn() })); +jest.mock('@/components/OverlayPageContext', () => ({ + useOverlayPage: jest.fn() +})); +jest.mock('@/content/PullRequests/useChangeTimePipeline', () => ({ + ...jest.requireActual('@/content/PullRequests/useChangeTimePipeline'), + usePrChangeTimePipeline: jest.fn() +})); + +import { screen } from '@testing-library/react'; + +import { useOverlayPage } from '@/components/OverlayPageContext'; +import { usePrChangeTimePipeline } from '@/content/PullRequests/useChangeTimePipeline'; +import { useAuth } from '@/hooks/useAuth'; +import { useSelector } from '@/store'; +import { renderWithTheme as render } from '@/utils/testUtils'; + +import { LeadTimeStatsCore } from '../LeadTimeStatsCore'; + +// CLUSTOX: Jira integration -- the extended Lead Time breakdown's +// leading "ticket created -> first commit" segment (docs/ +// JIRA_INTEGRATION_PROPOSAL.md §6A). First test file for this +// component -- it had none before this change -- so it also covers the +// pre-existing 5-segment rendering, to make sure the new, optional +// segment never disturbs it. +const segment = (overrides = {}) => ({ + duration: 3600, + bgColor: '#000', + color: '#fff', + clipPath: 'polygon(0 0)', + title: 'Segment', + description: 'A segment', + legendLabel: 'A segment', + ...overrides +}); + +const ticketSegment = (overrides = {}) => ({ + duration: 7200, + bgColor: '#111', + color: '#eee', + title: 'Idea', + description: 'Ticket to first commit', + legendLabel: 'Ticket created → first commit', + isNew: true, + ...overrides +}); + +describe('LeadTimeStatsCore', () => { + beforeEach(() => { + jest.clearAllMocks(); + (useAuth as jest.Mock).mockReturnValue({ role: 'CONTRIBUTOR' }); + (useSelector as jest.Mock).mockReturnValue([]); + (useOverlayPage as jest.Mock).mockReturnValue({ addPage: jest.fn() }); + (usePrChangeTimePipeline as jest.Mock).mockReturnValue({ + reposWithNoDeploymentsConfigured: [], + reposCountWithWorkflowConfigured: 0 + }); + }); + + const segments = [ + segment({ title: 'Commit', legendLabel: 'First commit → PR opened' }), + segment({ title: 'Response', legendLabel: 'First response' }), + segment({ title: 'Rework', legendLabel: 'Rework' }), + segment({ title: 'Merge', legendLabel: 'Merge' }), + segment({ title: 'Deploy', legendLabel: 'Merge → deploy' }) + ]; + + it('renders exactly the original 5 segments when no ticket data is given', () => { + render(); + + expect(screen.getByText('Commit')).toBeInTheDocument(); + expect(screen.getByText('Deploy')).toBeInTheDocument(); + expect(screen.queryByText('Idea')).not.toBeInTheDocument(); + expect(screen.queryByText(/Idea to production/)).not.toBeInTheDocument(); + }); + + it('renders exactly the original 5 segments when ticketSegment/comparison are explicitly undefined', () => { + render( + + ); + + expect(screen.queryByText('Idea')).not.toBeInTheDocument(); + }); + + it('prepends the ticket segment and shows the comparison line when both are given', () => { + render( + + ); + + expect(screen.getByText('Idea')).toBeInTheDocument(); + expect(screen.getByText('Commit')).toBeInTheDocument(); + expect(screen.getByText(/Idea to production/)).toBeInTheDocument(); + expect(screen.getByText(/commit-only/)).toBeInTheDocument(); + expect(screen.getByText(/42 ticket-matched PRs/)).toBeInTheDocument(); + }); + + it('singularizes "PR" for exactly one matched PR', () => { + render( + + ); + + expect(screen.getByText(/1 ticket-matched PR$/)).toBeInTheDocument(); + }); + + it('still shows the unmodified "Total" footer when showTotal is set, regardless of the ticket segment', () => { + render( + + ); + + // The "Total" footer sums only the original 5 segments (3600 * 3 + + // 3600 + 3600 = 5h) -- ticketSegment's 7200s must not leak into it. + expect(screen.getByText(/^Total:/)).toBeInTheDocument(); + }); + + // CLUSTOX: LeadTimeBreakdownCard's rendering, matching the design + // reference exactly -- full phase names in a legend below a plain + // bar, not the short in-bar titles ("Commit"/"Idea") the default mode + // above uses. showLegend is opt-in and unused by every existing + // caller (TeamInsightsBody's drill-down), so the tests above must + // keep passing unchanged -- these are additional, not replacements. + describe('showLegend mode', () => { + it('shows the full legend label, not the short in-bar title', () => { + render(); + + expect(screen.getByText('First commit → PR opened')).toBeInTheDocument(); + expect(screen.getByText('Merge → deploy')).toBeInTheDocument(); + expect(screen.queryByText('Commit')).not.toBeInTheDocument(); + expect(screen.queryByText('Deploy')).not.toBeInTheDocument(); + }); + + it('prepends the ticket segment\'s legend label and a "New" pill when present', () => { + render( + + ); + + expect( + screen.getByText('Ticket created → first commit') + ).toBeInTheDocument(); + expect(screen.getByText('New')).toBeInTheDocument(); + }); + + it('does not show a "New" pill on any of the original 5 segments', () => { + render(); + + expect(screen.queryByText('New')).not.toBeInTheDocument(); + }); + + it('accepts a comparison prop without rendering it -- LeadTimeBreakdownCard shows it as its own header badge now', () => { + render( + + ); + + expect(screen.queryByText(/Idea to production/)).not.toBeInTheDocument(); + expect(screen.queryByText(/commit-only/)).not.toBeInTheDocument(); + }); + + it('never renders the "Total" footer, even when showTotal is set', () => { + // showLegend's own comparison line is the total; the plain-mode + // "Total:" footer would be a second, redundant total. + render( + + ); + + expect(screen.queryByText(/^Total:/)).not.toBeInTheDocument(); + }); + }); +}); diff --git a/web-server/src/content/PullRequests/__tests__/useChangeTimePipeline.test.ts b/web-server/src/content/PullRequests/__tests__/useChangeTimePipeline.test.ts new file mode 100644 index 000000000..908fecb97 --- /dev/null +++ b/web-server/src/content/PullRequests/__tests__/useChangeTimePipeline.test.ts @@ -0,0 +1,79 @@ +jest.mock('@/store', () => ({ useSelector: jest.fn() })); + +import { renderHook } from '@testing-library/react'; + +import { useSelector } from '@/store'; + +import { useLeadTimePipeline } from '../useChangeTimePipeline'; + +// CLUSTOX: regression test for a real crash found live -- metrics_summary +// is null until fetchTeamDoraMetrics resolves (see the dora_metrics +// redux slice's initialState), and useLeadTimePipeline read +// .lead_time_stats off it with no guard. Every sibling read of +// metrics_summary elsewhere in this codebase already uses +// metrics_summary?.field -- this hook was the one exception, and it +// only stayed hidden because its only prior caller (LeadTimeStatsCore, +// via usePrChangeTimePipeline) always mounted after the fetch had +// already resolved. LeadTimeBreakdownCard mounting it inline, on the +// same page as the fetch itself, hit the null window immediately and +// blanked the whole page. +describe('useLeadTimePipeline', () => { + it('does not throw and returns zeroed segments when metrics_summary is null', () => { + (useSelector as jest.Mock).mockImplementation((selector) => + selector({ doraMetrics: { metrics_summary: null } }) + ); + + const { result } = renderHook(() => useLeadTimePipeline()); + + expect(result.current.totalLeadTime).toBe(0); + expect(result.current.leadTimeDetailsArray.every((s) => s.duration === 0)).toBe( + true + ); + }); + + it('reads the real averages once metrics_summary is populated', () => { + (useSelector as jest.Mock).mockImplementation((selector) => + selector({ + doraMetrics: { + metrics_summary: { + lead_time_stats: { + current: { + first_commit_to_open: 100, + first_response_time: 200, + rework_time: 300, + merge_time: 400, + merge_to_deploy: 500 + } + } + } + } + }) + ); + + const { result } = renderHook(() => useLeadTimePipeline()); + + expect(result.current.totalLeadTime).toBe(1500); + expect(result.current.leadTimeDetailsArray.map((s) => s.duration)).toEqual([ + 100, 200, 300, 400, 500 + ]); + }); + + // CLUSTOX: LeadTimeStatsCore's legend-mode rendering (LeadTimeBreakdownCard) + // needs the design reference's own full phase names, distinct from + // each segment's short in-bar `title`. + it('includes the design reference\'s full phase name on every segment', () => { + (useSelector as jest.Mock).mockImplementation((selector) => + selector({ doraMetrics: { metrics_summary: null } }) + ); + + const { result } = renderHook(() => useLeadTimePipeline()); + + expect(result.current.leadTimeDetailsArray.map((s) => s.legendLabel)).toEqual([ + 'First commit → PR opened', + 'First response', + 'Rework', + 'Merge', + 'Merge → deploy' + ]); + }); +}); diff --git a/web-server/src/content/PullRequests/__tests__/useTicketLeadTimeSegment.test.ts b/web-server/src/content/PullRequests/__tests__/useTicketLeadTimeSegment.test.ts new file mode 100644 index 000000000..d7175210a --- /dev/null +++ b/web-server/src/content/PullRequests/__tests__/useTicketLeadTimeSegment.test.ts @@ -0,0 +1,132 @@ +jest.mock('@/hooks/useAuth', () => ({ useAuth: jest.fn() })); +jest.mock('@/hooks/useStateTeamConfig', () => ({ + useSingleTeamConfig: jest.fn() +})); +jest.mock('axios'); + +import { renderHook, waitFor } from '@testing-library/react'; +import axios from 'axios'; + +import { useAuth } from '@/hooks/useAuth'; +import { useSingleTeamConfig } from '@/hooks/useStateTeamConfig'; + +import { useTicketLeadTimeSegment } from '../useTicketLeadTimeSegment'; + +const TEAM_ID = 'team-1'; +const DATES = { start: new Date('2026-05-01'), end: new Date('2026-08-01') }; + +// CLUSTOX: Jira integration -- the extended Lead Time breakdown's +// leading "ticket created -> first commit" segment (docs/ +// JIRA_INTEGRATION_PROPOSAL.md §6A). +describe('useTicketLeadTimeSegment', () => { + beforeEach(() => { + jest.clearAllMocks(); + (useSingleTeamConfig as jest.Mock).mockReturnValue({ + singleTeamId: TEAM_ID, + dates: DATES + }); + }); + + it('returns nulls and never fetches when Jira is not linked', () => { + (useAuth as jest.Mock).mockReturnValue({ integrations: {} }); + + const { result } = renderHook(() => useTicketLeadTimeSegment()); + + expect(result.current).toEqual({ ticketSegment: null, comparison: null }); + expect(axios).not.toHaveBeenCalled(); + }); + + it('returns nulls and never fetches when no team is selected', () => { + (useAuth as jest.Mock).mockReturnValue({ + integrations: { jira: { integrated: true } } + }); + (useSingleTeamConfig as jest.Mock).mockReturnValue({ + singleTeamId: undefined, + dates: DATES + }); + + const { result } = renderHook(() => useTicketLeadTimeSegment()); + + expect(result.current).toEqual({ ticketSegment: null, comparison: null }); + expect(axios).not.toHaveBeenCalled(); + }); + + it('returns nulls when there are no ticket-matched PRs this period, even with real data returned', async () => { + (useAuth as jest.Mock).mockReturnValue({ + integrations: { jira: { integrated: true } } + }); + (axios as unknown as jest.Mock).mockResolvedValue({ + data: { + matched_pr_count: 0, + avg_ticket_to_first_commit_seconds: 0, + avg_commit_only_lead_time_seconds: 0, + avg_extended_lead_time_seconds: 0 + } + }); + + const { result } = renderHook(() => useTicketLeadTimeSegment()); + + await waitFor(() => expect(axios).toHaveBeenCalled()); + expect(result.current).toEqual({ ticketSegment: null, comparison: null }); + }); + + it('maps a real response to a segment and comparison, unrounded fields preserved', async () => { + (useAuth as jest.Mock).mockReturnValue({ + integrations: { jira: { integrated: true } } + }); + (axios as unknown as jest.Mock).mockResolvedValue({ + data: { + matched_pr_count: 200, + avg_ticket_to_first_commit_seconds: 759519, + avg_commit_only_lead_time_seconds: 12853, + avg_extended_lead_time_seconds: 772372 + } + }); + + const { result } = renderHook(() => useTicketLeadTimeSegment()); + + await waitFor(() => expect(result.current.ticketSegment).not.toBeNull()); + + expect(result.current.ticketSegment.duration).toBe(759519); + expect(result.current.ticketSegment.title).toBe('Idea'); + // Full design-reference phase name + the "New" pill flag, for + // LeadTimeStatsCore's legend-mode rendering. + expect(result.current.ticketSegment.legendLabel).toBe( + 'Ticket created → first commit' + ); + expect(result.current.ticketSegment.isNew).toBe(true); + expect(result.current.comparison).toEqual({ + extendedSeconds: 772372, + commitOnlySeconds: 12853, + matchedPrCount: 200 + }); + }); + + it('fetches with isoDateString-formatted dates, scoped to the selected team', async () => { + (useAuth as jest.Mock).mockReturnValue({ + integrations: { jira: { integrated: true } } + }); + (axios as unknown as jest.Mock).mockResolvedValue({ + data: { + matched_pr_count: 1, + avg_ticket_to_first_commit_seconds: 100, + avg_commit_only_lead_time_seconds: 50, + avg_extended_lead_time_seconds: 150 + } + }); + + renderHook(() => useTicketLeadTimeSegment()); + + await waitFor(() => + expect(axios).toHaveBeenCalledWith( + `/api/internal/team/${TEAM_ID}/ticket_lead_time`, + { + params: { + from_date: DATES.start.toISOString(), + to_date: DATES.end.toISOString() + } + } + ) + ); + }); +}); diff --git a/web-server/src/content/PullRequests/useChangeTimePipeline.ts b/web-server/src/content/PullRequests/useChangeTimePipeline.ts index 85e9ae216..d957d5e87 100644 --- a/web-server/src/content/PullRequests/useChangeTimePipeline.ts +++ b/web-server/src/content/PullRequests/useChangeTimePipeline.ts @@ -5,7 +5,7 @@ import { track } from '@/constants/events'; import { useEasyState } from '@/hooks/useEasyState'; import { useSelector } from '@/store'; import { brandColors } from '@/theme/schemes/theme'; -import { ChangeTimeModes } from '@/types/resources'; +import { ChangeTimeModes, LeadTimeApiResponse } from '@/types/resources'; export enum ClipPathEnum { 'FIRST' = 'polygon(0% 0%, calc(100% - 15px) 0%, 100% 50%, calc(100% - 15px) 100%, 0% 100%)', @@ -19,9 +19,19 @@ export enum StatMode { } export const useLeadTimePipeline = () => { - const averageSummary = useSelector( - (s) => s.doraMetrics.metrics_summary.lead_time_stats.current - ); + // CLUSTOX: metrics_summary is null until fetchTeamDoraMetrics resolves + // (see the redux slice's initialState) -- reading .lead_time_stats + // off it unguarded threw on first render for any caller mounted + // before that fetch completes, not just after. This only stayed + // hidden as long as every caller (LeadTimeStatsCore via + // usePrChangeTimePipeline) happened to mount lazily, after the fetch + // was already done -- LeadTimeBreakdownCard, mounted inline on the + // same page as the fetch itself, hit it immediately and blanked the + // whole page. strictNullChecks is off in this project, so tsc never + // caught it. + const averageSummary: Partial = + useSelector((s) => s.doraMetrics.metrics_summary?.lead_time_stats.current) || + {}; const firstCommitToPrDetails = useMemo( () => ({ @@ -30,7 +40,8 @@ export const useLeadTimePipeline = () => { color: darken(brandColors.ticketState.todo, 0.9), clipPath: ClipPathEnum.FIRST, title: 'Commit', - description: 'Time taken to create PR since the first commit' + description: 'Time taken to create PR since the first commit', + legendLabel: 'First commit → PR opened' }), [averageSummary.first_commit_to_open] ); @@ -42,7 +53,8 @@ export const useLeadTimePipeline = () => { color: darken(brandColors.pr.firstResponseTime, 0.9), clipPath: ClipPathEnum.DEFAULT, title: 'Response', - description: 'Time taken to submit the first review on a PR' + description: 'Time taken to submit the first review on a PR', + legendLabel: 'First response' }), [averageSummary.first_response_time] ); @@ -54,7 +66,8 @@ export const useLeadTimePipeline = () => { color: darken(brandColors.pr.reworkTime, 0.9), clipPath: ClipPathEnum.DEFAULT, title: 'Rework', - description: 'Time spent in reviewing the PR, and making changes (if any)' + description: 'Time spent in reviewing the PR, and making changes (if any)', + legendLabel: 'Rework' }), [averageSummary.rework_time] ); @@ -67,7 +80,8 @@ export const useLeadTimePipeline = () => { clipPath: ClipPathEnum.DEFAULT, title: 'Merge', description: - 'Time waited to finally merge the PR after approval was provided' + 'Time waited to finally merge the PR after approval was provided', + legendLabel: 'Merge' }), [averageSummary.merge_time] ); @@ -79,7 +93,8 @@ export const useLeadTimePipeline = () => { color: darken(brandColors.ticketState.done, 0.9), clipPath: ClipPathEnum.LAST, title: 'Deploy', - description: 'Time taken to deploy the PR once its merged' + description: 'Time taken to deploy the PR once its merged', + legendLabel: 'Merge → deploy' }), [averageSummary.merge_to_deploy] ); diff --git a/web-server/src/content/PullRequests/useTicketLeadTimeSegment.ts b/web-server/src/content/PullRequests/useTicketLeadTimeSegment.ts new file mode 100644 index 000000000..a7d695142 --- /dev/null +++ b/web-server/src/content/PullRequests/useTicketLeadTimeSegment.ts @@ -0,0 +1,100 @@ +import { darken, lighten } from '@mui/material'; +import axios from 'axios'; +import { useEffect } from 'react'; + +import { useAuth } from '@/hooks/useAuth'; +import { useBoolState, useEasyState } from '@/hooks/useEasyState'; +import { useSingleTeamConfig } from '@/hooks/useStateTeamConfig'; +import { brandColors } from '@/theme/schemes/theme'; +import { depFn } from '@/utils/fn'; + +type TicketLeadTimeMetrics = { + matched_pr_count: number; + avg_ticket_to_first_commit_seconds: number; + avg_commit_only_lead_time_seconds: number; + avg_extended_lead_time_seconds: number; +}; + +export type TicketLeadTimeSegment = { + duration: number; + bgColor: string; + color: string; + title: string; + description: string; + legendLabel: string; + // Always true in practice (this segment only ever exists because + // it's the new one) -- typed as `boolean`, not the literal `true`, + // so object-literal-returning helpers (including this file's own + // return value, and every test fixture) don't get inference-widened + // into a type error. + isNew: boolean; +}; + +export type TicketLeadTimeComparison = { + extendedSeconds: number; + commitOnlySeconds: number; + matchedPrCount: number; +}; + +/** + * CLUSTOX: Jira integration -- the extended Lead Time breakdown's + * leading "ticket created -> first commit" segment (docs/ + * JIRA_INTEGRATION_PROPOSAL.md §6A). + * + * Its own fetch, gated on Jira being linked: an org without Jira issues + * no extra request at all and gets exactly the unmodified 5-segment + * breakdown LeadTimeStatsCore has always rendered. Even with Jira + * linked, a window with no ticket-matched merged PRs (matched_pr_count + * === 0) also renders nothing new -- there's no honest "idea to + * production" number to show without at least one real matched PR. + */ +export const useTicketLeadTimeSegment = (): { + ticketSegment: TicketLeadTimeSegment | null; + comparison: TicketLeadTimeComparison | null; +} => { + const { integrations } = useAuth(); + const isJiraLinked = Boolean(integrations?.jira?.integrated); + const { singleTeamId, dates } = useSingleTeamConfig(); + + const metrics = useEasyState(null); + const isLoading = useBoolState(false); + + useEffect(() => { + if (!isJiraLinked || !singleTeamId) return; + depFn(isLoading.true); + axios(`/api/internal/team/${singleTeamId}/ticket_lead_time`, { + params: { + from_date: dates.start.toISOString(), + to_date: dates.end.toISOString() + } + }) + .then((res) => depFn(metrics.set, res.data)) + .catch((error) => console.error(error)) + .finally(isLoading.false); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [isJiraLinked, singleTeamId, dates.start, dates.end]); + + if (!isJiraLinked || isLoading.value || !metrics.value?.matched_pr_count) { + return { ticketSegment: null, comparison: null }; + } + + const m = metrics.value; + + return { + ticketSegment: { + duration: m.avg_ticket_to_first_commit_seconds, + bgColor: lighten(brandColors.ticketState.default, 0.1), + color: darken(brandColors.ticketState.default, 0.9), + title: 'Idea', + description: + 'Time from the Jira ticket being created to the first commit against it, averaged over merged PRs matched to a ticket this period', + legendLabel: 'Ticket created → first commit', + isNew: true + }, + comparison: { + extendedSeconds: m.avg_extended_lead_time_seconds, + commitOnlySeconds: m.avg_commit_only_lead_time_seconds, + matchedPrCount: m.matched_pr_count + } + }; +}; diff --git a/web-server/src/contexts/ThirdPartyAuthContext.tsx b/web-server/src/contexts/ThirdPartyAuthContext.tsx index fe3dffafa..b07846e11 100644 --- a/web-server/src/contexts/ThirdPartyAuthContext.tsx +++ b/web-server/src/contexts/ThirdPartyAuthContext.tsx @@ -3,6 +3,8 @@ import { Session } from 'next-auth'; import { FC, createContext, useEffect, useCallback } from 'react'; import { useMemo } from 'react'; +import { CODE_PROVIDER_INTEGRATIONS_MAP } from '@/constants/api'; +import { CODE_PROVIDERS } from '@/constants/codeProviders'; import { Integration } from '@/constants/integrations'; import { useBoolState, useEasyState } from '@/hooks/useEasyState'; import { useRefMounted } from '@/hooks/useRefMounted'; @@ -27,6 +29,13 @@ export interface AuthContextValue extends AuthState { integrations: Org['integrations']; onboardingState: OnboardingStep[]; integrationList: Integration[]; + // CLUSTOX: integrationList filtered to code-hosting providers only -- + // see docs/JIRA_INTEGRATION_PROPOSAL.md. This is the ONE place that + // filter is applied; every consumer that means "is a code provider + // linked" (not "is anything linked") should read hasCodeProviderLinked + // or this list, never integrationList.length directly. + codeProviderIntegrationList: Integration[]; + hasCodeProviderLinked: boolean; integrationSet: Set; activeCodeProvider: CodeProviderIntegrations | null; } @@ -39,6 +48,8 @@ export const AuthContext = createContext({ integrations: {}, onboardingState: [], integrationList: [], + codeProviderIntegrationList: [], + hasCodeProviderLinked: false, integrationSet: new Set(), activeCodeProvider: null }); @@ -53,10 +64,17 @@ export const AuthProvider: FC = (props) => { const loadingState = useBoolState(true); const fetchSession = useCallback(() => { - return axios - .get('/api/auth/session') - .then((r) => r.data) - .then(sessionState.set); + return ( + axios + .get('/api/auth/session') + .then((r) => r.data) + .then(sessionState.set) + // CLUSTOX: a 401 here is the expected unauthenticated case, not an + // error. Without this the rejection is unhandled and the login page + // is buried under a runtime error overlay. Falling through to a null + // session lets initialize() take its existing onUnauthenticated path. + .catch(() => sessionState.set(null)) + ); }, [sessionState.set]); useEffect(() => { @@ -117,26 +135,64 @@ export const AuthProvider: FC = (props) => { (integrations.github || integrations.gitlab) && IntegrationGroup.CODE ) + // CLUSTOX: Jira is a project-tracker integration, not a code + // provider -- see docs/JIRA_INTEGRATION_PROPOSAL.md. Kept as its + // own concat term rather than folded into the CODE check above, + // same shape as that check, so the two groups stay independently + // readable as the set of integration kinds grows. + .concat(integrations.jira && IntegrationGroup.PROJECT) .filter(Boolean) ), - [integrations.github, integrations.gitlab] + [integrations.github, integrations.gitlab, integrations.jira] + ); + + // CLUSTOX: consumers of integrationList/activeCodeProvider (DORA metrics + // gating, the team repo search, the sidebar's onboarding check) all assume + // every entry is a code host whose repos can be listed. `integrations` now + // also carries deployment-only providers like Jenkins, which are stored in + // the same table -- without this filter, linking only Jenkins would make a + // workspace with zero code providers look onboarded, and 'jenkins' would + // leak into API calls that expect a git host name. + const isCodeProvider = useCallback( + (name: string) => + Object.prototype.hasOwnProperty.call( + CODE_PROVIDER_INTEGRATIONS_MAP, + name + ), + [] ); const integrationList = useMemo( () => Object.entries(integrations) - .filter(([_, value]) => value.integrated) + .filter(([key, value]) => value.integrated && isCodeProvider(key)) .map(([key, _]) => key) as Integration[], - [integrations] + [integrations, isCodeProvider] + ); + + // CLUSTOX: the one place integrationList is filtered down to code + // providers -- every other file that used to read integrationList.length + // (or iterate integrationList directly) to mean "is a code provider + // linked" now reads this instead, so a future non-code integration + // doesn't silently break the same way Jira did. See + // docs/JIRA_INTEGRATION_PROPOSAL.md and pages/integrations.tsx's history. + const codeProviderIntegrationList = useMemo( + () => + integrationList.filter((item) => + (CODE_PROVIDERS as readonly string[]).includes(item) + ), + [integrationList] ); + const hasCodeProviderLinked = codeProviderIntegrationList.length > 0; const activeCodeProvider = useMemo( () => - Object.keys(state?.org?.integrations || {}).find( + (Object.keys(state?.org?.integrations || {}).find( (integrationName) => + (CODE_PROVIDERS as readonly string[]).includes(integrationName) && state?.org?.integrations[integrationName as keyof IntegrationsMap] ?.integrated - ) as CodeProviderIntegrations | null, + ) ?? null) as CodeProviderIntegrations | null, [state?.org?.integrations] ); @@ -148,6 +204,8 @@ export const AuthProvider: FC = (props) => { orgId: state.org?.id, role, integrations, + codeProviderIntegrationList, + hasCodeProviderLinked, integrationSet, integrationList, activeCodeProvider, diff --git a/web-server/src/hooks/useAutofillSync.ts b/web-server/src/hooks/useAutofillSync.ts new file mode 100644 index 000000000..e4adda04f --- /dev/null +++ b/web-server/src/hooks/useAutofillSync.ts @@ -0,0 +1,47 @@ +import { useEffect, useRef } from 'react'; + +/** + * Browser autofill can finish filling a field before React has finished + * hydrating and attached its event listeners, so the very first + * `input`/`change` event it fires is silently missed. The field then sits + * fully rendered with the browser's autofilled text, but React's + * controlled state -- and therefore MUI's label-shrink/border-notch, which + * key off that state -- still think it's empty. That's what makes the + * label render on top of the value until some *unrelated* interaction + * (any click, any keypress) finally gives React an event to catch up on. + * + * Polls the real input value directly for a brief window right after + * mount and, the moment it finds a value React doesn't know about yet, + * feeds it back through the normal onChange path -- so the state catches + * up before the user ever has to interact with the page. + */ +export const useAutofillSync = ( + currentValue: string, + onDetected: (value: string) => void +) => { + const ref = useRef(null); + + useEffect(() => { + if (currentValue) return; // already has a value via the normal path + + let attempts = 0; + const id = setInterval(() => { + attempts += 1; + const domValue = ref.current?.value; + if (domValue) { + onDetected(domValue); + clearInterval(id); + } else if (attempts > 20) { + // ~1s: autofill either isn't happening or has already been caught + // by a real event by now. + clearInterval(id); + } + }, 50); + + return () => clearInterval(id); + // Intentionally mount-only: this is racing hydration, not `value`. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + return ref; +}; diff --git a/web-server/src/hooks/useClustoxUser.ts b/web-server/src/hooks/useClustoxUser.ts new file mode 100644 index 000000000..8677696a5 --- /dev/null +++ b/web-server/src/hooks/useClustoxUser.ts @@ -0,0 +1,46 @@ +import { useCallback, useEffect, useState } from 'react'; + +import { ClustoxRole } from '@/auth/types'; + +export type ClustoxUser = { + user_id: string; + email: string; + name: string; + role: ClustoxRole; + /** Workspace to act on. Server-resolved, never persisted client-side. */ + org_id: string | null; +}; + +/** + * The signed-in user's identity and current role. + * + * Read from /api/clustox/me rather than from AuthContext, whose `role` field + * upstream hardcodes to UserRole.MOM. Use it for presentation decisions only; + * authorization is enforced server-side on every route. + */ +export const useClustoxUser = () => { + const [user, setUser] = useState(null); + const [loading, setLoading] = useState(true); + + const load = useCallback(async () => { + try { + const res = await fetch('/api/clustox/me'); + setUser(res.ok ? await res.json() : null); + } catch { + setUser(null); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + load(); + }, [load]); + + return { + user, + loading, + isSuperadmin: user?.role === 'SUPERADMIN', + orgId: user?.org_id ?? null + }; +}; diff --git a/web-server/src/hooks/useDoraMetricsFetchArgs.ts b/web-server/src/hooks/useDoraMetricsFetchArgs.ts new file mode 100644 index 000000000..efa01ac43 --- /dev/null +++ b/web-server/src/hooks/useDoraMetricsFetchArgs.ts @@ -0,0 +1,40 @@ +import { useMemo } from 'react'; + +import { useSelectedContributors } from '@/components/ContributorFilter'; +import { useAuth } from '@/hooks/useAuth'; +import { useSingleTeamConfig } from '@/hooks/useStateTeamConfig'; + +/** + * The org / team / date-window / contributor arguments that every + * `fetchTeamDoraMetrics` dispatch needs, built in exactly one place. + * + * There are four call sites (the DORA body's own effect, the page refresh + * callback, and the two settings dialogs that refetch after saving), and they + * had each hand-rolled this object. When the contributor filter was added only + * one of them learned about it, so any refetch triggered by a repo finishing + * its sync, or by saving a production-branch/incident-PR setting, silently + * dropped the selection: Lead Time and Deployment Frequency reverted to + * team-wide while the dropdown still showed the chip and the cards still read + * "authored by alice". + * + * Branch arguments stay with the caller on purpose -- the body and the refresh + * callback send `useBranchesForPrFilters()` (branches *and* branch_mode) while + * the two dialogs send only `branches`, and unifying that would change what + * unfiltered dashboards request. + */ +export const useDoraMetricsFetchArgs = () => { + const { orgId } = useAuth(); + const { singleTeamId, dates } = useSingleTeamConfig(); + const authors = useSelectedContributors(); + + return useMemo( + () => ({ + orgId, + teamId: singleTeamId, + fromDate: dates.start, + toDate: dates.end, + authors + }), + [orgId, singleTeamId, dates.start, dates.end, authors] + ); +}; diff --git a/web-server/src/hooks/useJiraConnections.ts b/web-server/src/hooks/useJiraConnections.ts new file mode 100644 index 000000000..6e693982e --- /dev/null +++ b/web-server/src/hooks/useJiraConnections.ts @@ -0,0 +1,97 @@ +import { useCallback, useEffect, useState } from 'react'; + +import { handleApi } from '@/api-helpers/axios-api-instance'; + +export type JiraConnection = { + id: string; + site_url: string; + email: string; + is_default: boolean; + provider_meta: Record; + created_at: string | null; +}; + +export type NewJiraConnection = { + site_url: string; + email: string; + access_token: string; +}; + +// CLUSTOX: no manual feature_flags stamping here -- middleware.ts already +// appends one `feature_flags` query param to every request (page and API +// alike), built from the resolved flags (defaults + cookie overrides). A +// second, hand-added `feature_flags` param used to collide with it: two +// query params of the same name parse as an array, and JSON.parse coerces +// that array to a comma-joined string, corrupting the JSON right at the +// join -- the "Unexpected non-whitespace character..." bug this comment +// replaces. See docs/JIRA_MULTI_ACCOUNT_PLAN.md Task 6. +const basePath = (orgId: string) => `/resources/orgs/${orgId}/jira-connections`; + +export const useJiraConnections = (orgId: string | null) => { + const [connections, setConnections] = useState([]); + const [loading, setLoading] = useState(true); + const [failed, setFailed] = useState(false); + + const load = useCallback(async () => { + if (!orgId) return; + setLoading(true); + setFailed(false); + try { + const data = await handleApi(basePath(orgId)); + setConnections(data || []); + } catch (e) { + console.error('Failed to load Jira connections', e); + setFailed(true); + } finally { + setLoading(false); + } + }, [orgId]); + + useEffect(() => { + load(); + }, [load]); + + const create = useCallback( + async (payload: NewJiraConnection) => { + if (!orgId) throw new Error('No workspace selected'); + await handleApi(basePath(orgId), { + method: 'post', + data: payload + }); + await load(); + }, + [orgId, load] + ); + + const remove = useCallback( + async (connectionId: string) => { + if (!orgId) throw new Error('No workspace selected'); + await handleApi(`${basePath(orgId)}/${connectionId}`, { + method: 'delete' + }); + await load(); + }, + [orgId, load] + ); + + const setDefault = useCallback( + async (connectionId: string) => { + if (!orgId) throw new Error('No workspace selected'); + await handleApi(`${basePath(orgId)}/${connectionId}`, { + method: 'patch' + }); + await load(); + }, + [orgId, load] + ); + + return { + connections, + loading, + failed, + reload: load, + create, + remove, + setDefault + }; +}; diff --git a/web-server/src/hooks/usePageRefreshCallback.ts b/web-server/src/hooks/usePageRefreshCallback.ts index 0ed8e5e1f..d2d464b85 100644 --- a/web-server/src/hooks/usePageRefreshCallback.ts +++ b/web-server/src/hooks/usePageRefreshCallback.ts @@ -1,19 +1,18 @@ import { useRouter } from 'next/router'; import { ROUTES } from '@/constants/routes'; -import { useAuth } from '@/hooks/useAuth'; -import { - useBranchesForPrFilters, - useSingleTeamConfig -} from '@/hooks/useStateTeamConfig'; +// CLUSTOX: contributor filter -- the refetch this callback triggers (a repo +// finishing its sync) must keep the selection, so the arguments come from the +// shared hook instead of being rebuilt here. +import { useDoraMetricsFetchArgs } from '@/hooks/useDoraMetricsFetchArgs'; +import { useBranchesForPrFilters } from '@/hooks/useStateTeamConfig'; import { fetchTeamDoraMetrics } from '@/slices/dora_metrics'; import { useDispatch } from '@/store'; export const usePageRefreshCallback = () => { const router = useRouter(); const dispatch = useDispatch(); - const { orgId } = useAuth(); - const { dates, singleTeamId } = useSingleTeamConfig(); + const doraMetricsFetchArgs = useDoraMetricsFetchArgs(); const branchPayloadForPrFilters = useBranchesForPrFilters(); switch (router.pathname) { @@ -21,10 +20,7 @@ export const usePageRefreshCallback = () => { return () => dispatch( fetchTeamDoraMetrics({ - orgId, - teamId: singleTeamId, - fromDate: dates.start, - toDate: dates.end, + ...doraMetricsFetchArgs, ...branchPayloadForPrFilters }) ); diff --git a/web-server/src/layouts/ExtendedSidebarLayout/Sidebar/SidebarMenu/items.tsx b/web-server/src/layouts/ExtendedSidebarLayout/Sidebar/SidebarMenu/items.tsx index 8102d0832..66d571e2d 100644 --- a/web-server/src/layouts/ExtendedSidebarLayout/Sidebar/SidebarMenu/items.tsx +++ b/web-server/src/layouts/ExtendedSidebarLayout/Sidebar/SidebarMenu/items.tsx @@ -3,7 +3,10 @@ import { GroupsTwoTone, Analytics, Settings, - Dns + Dns, + // CLUSTOX: icons for user management and workspace health + ManageAccounts, + Workspaces } from '@mui/icons-material'; import { ROUTES } from '@/constants/routes'; @@ -40,7 +43,11 @@ export enum SideBarItems { DORA_METRICS = 'DORA Metrics', MANAGE_TEAMS = 'Manage Teams', MANAGE_INTEGRATIONS = 'Manage Integrations', - SERVER_ADMIN = 'Server Admin' + SERVER_ADMIN = 'Server Admin', + // CLUSTOX: superadmin-only, filtered in useFilteredSidebarItems + MANAGE_USERS = 'Manage Users', + // CLUSTOX: visible to admins too, scoped to their own workspace + WORKSPACES = 'Workspaces' } const menuItems = (): MenuItems[] => [ @@ -71,6 +78,20 @@ const menuItems = (): MenuItems[] => [ name: 'System Logs', icon: Dns, link: ROUTES.SYSTEM_LOGS.PATH + }, + // CLUSTOX: sync health. Admins see their own workspace, superadmins all. + { + name: SideBarItems.WORKSPACES, + icon: Workspaces, + link: ROUTES.WORKSPACES.PATH + }, + // CLUSTOX: only rendered for superadmins. + { + name: SideBarItems.MANAGE_USERS, + icon: ManageAccounts, + link: ROUTES.USERS.PATH, + badge: 'NEW', + badgeTooltip: 'Invite teammates and manage roles' } ] } diff --git a/web-server/src/layouts/ExtendedSidebarLayout/Sidebar/SidebarMenu/useFilteredSidebarItems.tsx b/web-server/src/layouts/ExtendedSidebarLayout/Sidebar/SidebarMenu/useFilteredSidebarItems.tsx index 44c6aafef..763be32fc 100644 --- a/web-server/src/layouts/ExtendedSidebarLayout/Sidebar/SidebarMenu/useFilteredSidebarItems.tsx +++ b/web-server/src/layouts/ExtendedSidebarLayout/Sidebar/SidebarMenu/useFilteredSidebarItems.tsx @@ -1,6 +1,8 @@ import { useMemo } from 'react'; import { useAuth } from '@/hooks/useAuth'; +// CLUSTOX: real Clustox role for sidebar visibility +import { useClustoxUser } from '@/hooks/useClustoxUser'; import { ItemTags, SideBarItems @@ -16,7 +18,9 @@ const checkTag = (tag: string | number[] | number, check: string | number) => { }; export const useFilteredSidebarItems = () => { - const { integrationList } = useAuth(); + const { hasCodeProviderLinked } = useAuth(); + // CLUSTOX: the real role, since useAuth().role is hardcoded upstream. + const { isSuperadmin } = useClustoxUser(); const flagFilteredMenuItems = useMemo(() => { return menuItems(); @@ -25,8 +29,20 @@ export const useFilteredSidebarItems = () => { const sidebarItems = useMemo(() => { const filterCheck = (item: MenuItem): boolean => { if (checkTag(item.tag, ItemTags.HideItem)) return false; + // CLUSTOX: hide user management from non-superadmins. Presentation only + // -- the API returns 403 regardless of what the sidebar shows. + if (item.name === SideBarItems.MANAGE_USERS && !isSuperadmin) return false; + // CLUSTOX: a SuperAdmin owns no workspace, so the workspace they are + // viewing may legitimately have no integration yet. Collapsing their nav + // to just "Manage Integrations" would strand them, since user management + // and the workspace switcher are how they administer the instance. + // Gated on a code provider specifically, not "any integration" -- + // linking only Jira shouldn't unlock Teams/DORA/Settings, which have + // nothing to show without repo data behind them. See + // docs/JIRA_INTEGRATION_PROPOSAL.md. if ( - !integrationList.length && + !isSuperadmin && + !hasCodeProviderLinked && item.name !== SideBarItems.MANAGE_INTEGRATIONS ) return false; @@ -46,7 +62,7 @@ export const useFilteredSidebarItems = () => { items: itemsFilter(section.items) })) .filter((section) => section.items?.length); - }, [flagFilteredMenuItems, integrationList]); + }, [flagFilteredMenuItems, hasCodeProviderLinked, isSuperadmin]); return sidebarItems; }; diff --git a/web-server/src/layouts/ExtendedSidebarLayout/Sidebar/index.tsx b/web-server/src/layouts/ExtendedSidebarLayout/Sidebar/index.tsx index b08f10077..bc969fd12 100644 --- a/web-server/src/layouts/ExtendedSidebarLayout/Sidebar/index.tsx +++ b/web-server/src/layouts/ExtendedSidebarLayout/Sidebar/index.tsx @@ -10,8 +10,9 @@ import { import { format, isValid } from 'date-fns'; import { useContext, useMemo } from 'react'; +// CLUSTOX: signed-in user + sign out +import { ClustoxUserFooter } from '@/components/ClustoxUserFooter'; import { FlexBox } from '@/components/FlexBox'; -import Scrollbar from '@/components/Scrollbar'; import { Line } from '@/components/Text'; import { SidebarContext } from '@/contexts/SidebarContext'; import { useSelector } from '@/store'; @@ -27,7 +28,41 @@ const SidebarWrapper = styled(Box)( position: relative; z-index: 7; height: 100%; - padding-bottom: 61px; + display: flex; + flex-direction: column; +` +); + +// CLUSTOX FIX: react-custom-scrollbars-2 (still on 4.4.0, peer dep caps at +// React 17) hides the native scrollbar with a negative-margin trick that +// depends on ref/DOM timing that React 18 breaks. When it fails, the raw +// native scrollbar -- OS-drawn arrows, its own gutter color -- shows through +// instead of the library's thin styled thumb, and the container it applies +// that trick to is hard-coded to overflow:scroll (always-on) rather than +// auto, so it renders even when the menu doesn't overflow. A plain +// overflow-y:auto region with a themed ::-webkit-scrollbar sidesteps both: +// no legacy scrolling layer, and a scrollbar only when content truly +// overflows. +const ScrollableMenu = styled(Box)( + ({ theme }) => ` + flex: 1; + min-height: 0; + overflow-y: auto; + overflow-x: hidden; + + &::-webkit-scrollbar { + width: 5px; + } + &::-webkit-scrollbar-track { + background: transparent; + } + &::-webkit-scrollbar-thumb { + background: ${theme.colors.alpha.black[10]}; + border-radius: ${theme.general.borderRadiusLg}; + } + &::-webkit-scrollbar-thumb:hover { + background: ${theme.colors.alpha.black[30]}; + } ` ); @@ -74,7 +109,7 @@ const SidebarContent = () => { return ( <> - + { }} /> - + + + {/* CLUSTOX: signed-in identity and the only sign-out affordance. */} + diff --git a/web-server/src/mocks/__mocks__/svg.js b/web-server/src/mocks/__mocks__/svg.js new file mode 100644 index 000000000..21c147586 --- /dev/null +++ b/web-server/src/mocks/__mocks__/svg.js @@ -0,0 +1,7 @@ +// Next's build treats .svg imports as React components via an SVGR-style +// loader; Jest has no such transform and chokes trying to parse raw SVG/XML +// as JS. Stubbed to a trivial component -- nothing under test asserts on +// icon markup itself. +const React = require('react'); +module.exports = (props) => React.createElement('svg', props); +module.exports.default = module.exports; diff --git a/web-server/src/slices/dora_metrics.ts b/web-server/src/slices/dora_metrics.ts index bfb5a5ef1..36d2ef2fd 100644 --- a/web-server/src/slices/dora_metrics.ts +++ b/web-server/src/slices/dora_metrics.ts @@ -179,6 +179,11 @@ export const fetchTeamDoraMetrics = createAsyncThunk( toDate: Date; branches: string; branch_mode?: ActiveBranchMode; + // CLUSTOX: contributor filter -- git usernames. Sent under the + // bracketed key so the BFF's array-param transform (`key[]` -> + // `key: value[]`) picks it up; a plain `authors` key would arrive + // server-side as a lone string instead of an array. + authors?: string[]; }) => { return await handleApi( `internal/team/${params.teamId}/dora_metrics`, @@ -188,7 +193,8 @@ export const fetchTeamDoraMetrics = createAsyncThunk( from_date: params.fromDate, to_date: params.toDate, branches: params.branches, - branch_mode: params.branch_mode || ActiveBranchMode.ALL + branch_mode: params.branch_mode || ActiveBranchMode.ALL, + 'authors[]': params.authors?.length ? params.authors : undefined } } ); diff --git a/web-server/src/theme/schemes/theme.ts b/web-server/src/theme/schemes/theme.ts index 8c4838529..3d33265d3 100644 --- a/web-server/src/theme/schemes/theme.ts +++ b/web-server/src/theme/schemes/theme.ts @@ -534,6 +534,26 @@ export const MainTheme = createTheme({ } } }, + MuiInputLabel: { + styleOverrides: { + outlined: { + // A shrunk outlined label always straddles the border by design + // -- its bottom few pixels sit inside the input's own box, with + // only a notch cut into the *border stroke* so it isn't crossed + // out. That's invisible against a transparent input background, + // but browser autofill paints the input a solid opaque color + // right up to that same line, so the same normal overlap reads + // as text sitting on a hard block. Giving the shrunk label its + // own backing (matching the card) keeps it legible regardless of + // what's behind it -- autofill fill or otherwise. + '&.MuiInputLabel-shrink': { + backgroundColor: colors.alpha.white[100], + padding: '0 6px', + marginLeft: -6 + } + } + } + }, MuiListSubheader: { styleOverrides: { colorPrimary: { diff --git a/web-server/src/types/next-auth.d.ts b/web-server/src/types/next-auth.d.ts new file mode 100644 index 000000000..484064bdd --- /dev/null +++ b/web-server/src/types/next-auth.d.ts @@ -0,0 +1,25 @@ +import { ClustoxRole } from '@/auth/types'; + +declare module 'next-auth' { + interface Session { + user: { + id: string; + email: string; + name: string; + role: ClustoxRole; + }; + } + interface User { + id: string; + email: string; + name: string; + role: ClustoxRole; + } +} + +declare module 'next-auth/jwt' { + interface JWT { + userId: string; + role: ClustoxRole; + } +} diff --git a/web-server/src/types/request.ts b/web-server/src/types/request.ts index 608a7bb93..0d8f2ad43 100644 --- a/web-server/src/types/request.ts +++ b/web-server/src/types/request.ts @@ -1,5 +1,6 @@ import { Features } from '@/constants/feature'; +import type { AuthSession } from '@/auth/types'; import type { NextApiRequest, NextApiResponse } from 'next/types'; export type HttpMethods = @@ -23,6 +24,11 @@ export type ApiRequest = Omit< meta?: { features: Partial; }; + // CLUSTOX: stamped by Endpoint.serve() on every authenticated route. Typed + // here (rather than left as the `(req as any).session` every call site + // previously had to repeat) because jira-connections/index.ts is the + // first route to actually read it, for the connection's generated_by. + session?: AuthSession; method: HttpMethods; }; diff --git a/web-server/src/types/resources.ts b/web-server/src/types/resources.ts index ba65b0b2b..67352700c 100644 --- a/web-server/src/types/resources.ts +++ b/web-server/src/types/resources.ts @@ -5,6 +5,7 @@ import { ReactChild, ReactFragment, ReactPortal } from 'react'; import { Row } from '@/constants/db'; import { CIProvider, Integration } from '@/constants/integrations'; import { Team } from '@/types/api/teams'; +import { Benchmarks } from '@/utils/benchmarks'; export type DeepPartial = { [P in keyof T]?: T[P] extends object ? DeepPartial : T[P]; @@ -556,6 +557,23 @@ export type ChangeFailureRateTrendsApiResponse = Record< ChangeFailureRateApiResponse >; +// CLUSTOX: lines of code, the fifth metric on the DORA dashboard. Shape is +// `GET /teams//loc` verbatim. Every field is always a number -- the +// analytics server sends `0`, never null, for a window with no merged PRs, so +// a consumer must render `0` as the real measurement it is and not as +// "no data". `total` is `additions + deletions`; `avg_pr_size` is those gross +// lines per merged PR and is the figure a benchmark target is compared against. +export type LOCApiResponse = { + additions: number; + deletions: number; + total: number; + avg_pr_size: number; + /** Gross lines per calendar day over the selected range. */ + avg_daily: number; +}; + +export type LOCTrendsApiResponse = Record; + export type TeamDoraMetricsApiResponseType = { lead_time_stats: { current: LeadTimeApiResponse; @@ -592,6 +610,26 @@ export type TeamDoraMetricsApiResponseType = { lead_time_prs: PR[]; assigned_repos: (Row<'TeamRepos'> & Row<'OrgRepo'>)[]; unsynced_repos: ID[]; + // CLUSTOX: populated by the dora_metrics BFF route from + // `GET /teams//benchmarks`. Optional because the mock response and any + // cached older response omit the key -- the four cards must treat a missing + // `benchmarks` exactly like every target being null: no line, no caption, + // unchanged from before this feature. + benchmarks?: Benchmarks; + // CLUSTOX: populated by the dora_metrics BFF route from Task 1's two LOC + // routes. Optional for the same reason `benchmarks` is: the mock response + // omits it, and the route degrades these two keys to `undefined` rather than + // failing the whole response when the LOC routes are unavailable. `undefined` + // here means "could not be measured" and is not the same thing as a measured + // zero, which arrives as `{ additions: 0, ... }`. + loc_stats?: { + current: LOCApiResponse; + previous: LOCApiResponse; + }; + loc_trends?: { + current: LOCTrendsApiResponse; + previous: LOCTrendsApiResponse; + }; }; export enum ActiveBranchMode { @@ -787,6 +825,15 @@ export type ChangeTimeSegment = { clipPath: string; title: string; description: string; + // CLUSTOX: Jira integration -- LeadTimeStatsCore's legend-mode + // rendering (LeadTimeBreakdownCard) needs the design reference's own + // full phase name ("First commit → PR opened"), distinct from + // `title`'s short in-bar label ("Commit") and `description`'s + // tooltip sentence. Optional and unused by every other consumer of + // this type (e.g. useComputedPrChangeTime's segments, which only + // feed the existing, unchanged in-bar rendering). + legendLabel?: string; + isNew?: boolean; }; export type CockpitProjectFlows = { diff --git a/web-server/src/utils/__tests__/auth-jira.test.ts b/web-server/src/utils/__tests__/auth-jira.test.ts new file mode 100644 index 000000000..f13ada49a --- /dev/null +++ b/web-server/src/utils/__tests__/auth-jira.test.ts @@ -0,0 +1,119 @@ +jest.mock('axios'); + +import axios from 'axios'; + +import { checkJiraValidity, normalizeJiraSiteUrl } from '@/utils/auth'; + +describe('normalizeJiraSiteUrl', () => { + it('strips a leading https:// and trailing slash', () => { + expect(normalizeJiraSiteUrl('https://mycompany.atlassian.net/')).toBe( + 'mycompany.atlassian.net' + ); + }); + + it('strips a leading http:// too', () => { + expect(normalizeJiraSiteUrl('http://mycompany.atlassian.net')).toBe( + 'mycompany.atlassian.net' + ); + }); + + it('leaves a bare domain unchanged', () => { + expect(normalizeJiraSiteUrl('mycompany.atlassian.net')).toBe( + 'mycompany.atlassian.net' + ); + }); + + it('trims surrounding whitespace', () => { + expect(normalizeJiraSiteUrl(' mycompany.atlassian.net ')).toBe( + 'mycompany.atlassian.net' + ); + }); + + it('strips multiple trailing slashes', () => { + expect(normalizeJiraSiteUrl('https://mycompany.atlassian.net///')).toBe( + 'mycompany.atlassian.net' + ); + }); + + // CLUSTOX: regression test for a real bug found in code review. A regex + // that only stripped a leading protocol and trailing slash left any + // path/query intact -- pasting a full address copied from the browser + // while looking at a board (a very plausible thing for a user to do, + // more plausible than typing the bare domain) broke the very next + // request even with valid credentials. + it('reduces a full URL with a path to just the hostname', () => { + expect( + normalizeJiraSiteUrl( + 'https://mycompany.atlassian.net/jira/software/projects/ABC/boards/1' + ) + ).toBe('mycompany.atlassian.net'); + }); + + it('drops a port and query string too', () => { + expect( + normalizeJiraSiteUrl('https://mycompany.atlassian.net:443/?foo=bar') + ).toBe('mycompany.atlassian.net'); + }); +}); + +describe('checkJiraValidity', () => { + beforeEach(() => jest.clearAllMocks()); + + it('posts the three fields to the server-side validate endpoint', async () => { + (axios.post as jest.Mock).mockResolvedValue({ + data: { valid: true, display_name: 'Jordan Diaz' } + }); + + await checkJiraValidity('mycompany.atlassian.net', 'jordan@mycompany.com', 'tok'); + + expect(axios.post).toHaveBeenCalledWith( + '/api/integrations/jira/validate', + { + site_url: 'mycompany.atlassian.net', + email: 'jordan@mycompany.com', + api_token: 'tok' + } + ); + }); + + it('maps a valid response through, including the display name', async () => { + (axios.post as jest.Mock).mockResolvedValue({ + data: { valid: true, display_name: 'Jordan Diaz' } + }); + + await expect( + checkJiraValidity('mycompany.atlassian.net', 'a@b.com', 'tok') + ).resolves.toEqual({ + valid: true, + displayName: 'Jordan Diaz', + reason: undefined + }); + }); + + it('maps an invalid response through, including the reason', async () => { + (axios.post as jest.Mock).mockResolvedValue({ + data: { valid: false, reason: 'unauthorized' } + }); + + await expect( + checkJiraValidity('mycompany.atlassian.net', 'a@b.com', 'wrong') + ).resolves.toEqual({ + valid: false, + displayName: undefined, + reason: 'unauthorized' + }); + }); + + // CLUSTOX: unlike checkGitHubValidity/checkGitLabValidity (which hit the + // provider's API directly from the browser), this call goes through our + // own backend -- so a thrown error here means *our* server was + // unreachable, not Jira. Must still resolve, not throw, so the modal's + // calling code doesn't need a second try/catch layer on top of its own. + it('resolves to an unknown-reason failure, rather than throwing, if our own backend request fails', async () => { + (axios.post as jest.Mock).mockRejectedValue(new Error('network down')); + + await expect( + checkJiraValidity('mycompany.atlassian.net', 'a@b.com', 'tok') + ).resolves.toEqual({ valid: false, reason: 'unknown' }); + }); +}); diff --git a/web-server/src/utils/__tests__/benchmarkBand.test.ts b/web-server/src/utils/__tests__/benchmarkBand.test.ts new file mode 100644 index 000000000..1e1e58d99 --- /dev/null +++ b/web-server/src/utils/__tests__/benchmarkBand.test.ts @@ -0,0 +1,305 @@ +import { + benchmarkBandOptions, + BenchmarkBandInput, + BenchmarkBandOptions +} from '../benchmarkBand'; + +// CLUSTOX: `benchmarkBandOptions` returns null for an absent target, which is +// a state none of these cases exercise. Unwrapping here keeps every assertion +// below identical to the spec's without sprinkling `!` through the file. +const bandOptions = (input: BenchmarkBandInput): BenchmarkBandOptions => { + const opts = benchmarkBandOptions(input); + if (opts == null) throw new Error('expected band options, got null'); + return opts; +}; + +const boxOf = (opts: BenchmarkBandOptions) => + opts.plugins.annotation.annotations.band; + +const lineOf = (opts: BenchmarkBandOptions) => + opts.plugins.annotation.annotations.targetLine; + +// CLUSTOX: asserts on the tone *token*, not on the resolved rgba string. The +// rgba string is whatever theme fixture the test hands in, so matching it +// would only restate the fixture; the token is the decision under test. That +// the token is what actually reaches the canvas is covered separately by +// 'derives the band colours from the theme tone'. +const colorOf = (opts: BenchmarkBandOptions) => opts.benchmarkBand.tone; + +const theme = { + colors: { + success: { main: '#57CA22' }, + warning: { main: '#FFA319' } + } +}; + +describe('benchmarkBandOptions', () => { + it('extends the y scale so a target above the data is not clipped', () => { + const opts = bandOptions({ + metric: 'lead_time', + target: 100, + actual: 20, + values: [10, 20, 30] + }); + // Without this the band is drawn off-canvas and the card looks unchanged. + expect(opts.scales.y.suggestedMax).toBeGreaterThanOrEqual(100); + }); + + it('refuses to stretch the axis more than the limit past the data', () => { + // A fat-fingered target must not flatten the real series into a hairline. + const opts = bandOptions({ + metric: 'lines_of_code', + target: 20000, + actual: 200, + values: [180, 200, 220] + }); + expect(opts.scales.y.suggestedMax).toBeLessThan(1000); + }); + + it('reports a target it could not fit on the axis', () => { + // The caption has to carry the number when the band runs off the edge. + const fits = bandOptions({ + metric: 'lead_time', + target: 100, + actual: 20, + values: [10, 20, 30] + }); + const doesNotFit = bandOptions({ + metric: 'lines_of_code', + target: 20000, + actual: 200, + values: [180, 200, 220] + }); + expect(fits.benchmarkBand.targetIsOffScale).toBe(false); + expect(doesNotFit.benchmarkBand.targetIsOffScale).toBe(true); + }); + + it('keeps a usable band when a higher-is-better target is off scale', () => { + // CLUSTOX: the off-scale cases above are all lower-is-better, where the + // band runs [0, target] and a clipped target only shortens it. Deployment + // frequency runs [target, suggestedMax], so an off-scale target is the one + // direction where the two edges can cross and produce an inverted -- + // therefore invisible -- box. Everything else about this feature can look + // right while that one card silently draws nothing. + const opts = bandOptions({ + metric: 'deployment_frequency', + target: 900, + actual: 5, + values: [4, 5, 6] + }); + const { yMin, yMax } = boxOf(opts); + expect(opts.benchmarkBand.targetIsOffScale).toBe(true); + expect(yMax).toBeGreaterThanOrEqual(yMin); + expect(colorOf(opts)).toBe('warning'); + }); + + it('covers target down to zero when lower is better', () => { + const { yMin, yMax } = boxOf( + bandOptions({ + metric: 'lead_time', + target: 100, + actual: 20, + values: [10, 20, 30] + }) + ); + expect(yMin).toBe(0); + expect(yMax).toBe(100); + }); + + it('covers target upward for deployment frequency', () => { + const { yMin } = boxOf( + bandOptions({ + metric: 'deployment_frequency', + target: 5, + actual: 9, + values: [4, 9] + }) + ); + expect(yMin).toBe(5); + }); + + it('shades the whole plot for a deployment-frequency target under the data', () => { + // The bug this replaces: target 1/week against a series topping 35 put a + // CSS-positioned line at 3% of the card, indistinguishable from its + // bottom border. In data space the good region is nearly the whole plot. + const { yMin, yMax } = boxOf( + bandOptions({ + metric: 'deployment_frequency', + target: 1, + actual: 30, + values: [12, 35, 30] + }) + ); + expect(yMin).toBe(1); + expect(yMax).toBeGreaterThanOrEqual(35); + }); + + it('tints success when the actual value is on the good side', () => { + const opts = bandOptions({ + metric: 'lead_time', + target: 100, + actual: 20, + values: [20] + }); + expect(colorOf(opts)).toMatch(/success/); + }); + + it('never returns red', () => { + const opts = bandOptions({ + metric: 'lead_time', + target: 10, + actual: 900, + values: [900] + }); + expect(colorOf(opts)).not.toMatch(/error|red/); + }); + + it('counts hitting the target exactly as the good side', () => { + // Matches benchmarkCaption's `<=`/`>=`, so the band and the caption under + // it can never disagree about which side of the target a team is on. + expect( + colorOf( + bandOptions({ + metric: 'lead_time', + target: 100, + actual: 100, + values: [100] + }) + ) + ).toBe('success'); + expect( + colorOf( + bandOptions({ + metric: 'deployment_frequency', + target: 5, + actual: 5, + values: [5] + }) + ) + ).toBe('success'); + }); + + it('inverts the good side for deployment frequency', () => { + // Same numbers, opposite verdicts. Getting the direction backwards is + // invisible on any single card. + expect( + colorOf( + bandOptions({ + metric: 'deployment_frequency', + target: 10, + actual: 2, + values: [2] + }) + ) + ).toBe('warning'); + expect( + colorOf( + bandOptions({ + metric: 'lead_time', + target: 10, + actual: 2, + values: [2] + }) + ) + ).toBe('success'); + }); + + it('derives the band colours from the theme tone', () => { + const opts = bandOptions({ + metric: 'lead_time', + target: 100, + actual: 20, + values: [10, 20, 30], + theme + }); + // 87/202/34 is #57CA22, the theme's success main. + expect(boxOf(opts).backgroundColor).toBe('rgba(87, 202, 34, 0.12)'); + expect(lineOf(opts).borderColor).toBe('rgba(87, 202, 34, 0.6)'); + }); + + it('draws the band behind the series, not over it', () => { + // A box drawn at the default draw time paints on top of the line it is + // meant to sit behind. + expect( + boxOf( + bandOptions({ + metric: 'lead_time', + target: 100, + actual: 20, + values: [10, 20, 30] + }) + ).drawTime + ).toBe('beforeDatasetsDraw'); + }); + + it('marks the target itself with a line at the target value', () => { + const line = lineOf( + bandOptions({ + metric: 'mean_time_to_recovery', + target: 3600, + actual: 1800, + values: [1200, 1800] + }) + ); + expect(line.yMin).toBe(3600); + expect(line.yMax).toBe(3600); + }); + + it('treats a target of zero as a real target, not an absent one', () => { + // `0` is a legitimate benchmark (zero failed changes). Any truthiness + // check here silently drops the band for the strictest target there is. + const opts = bandOptions({ + metric: 'change_failure_rate', + target: 0, + actual: 0, + values: [0, 2, 4] + }); + expect(boxOf(opts).yMax).toBe(0); + expect(colorOf(opts)).toBe('success'); + }); + + it('returns null when there is no target to draw', () => { + expect( + benchmarkBandOptions({ + metric: 'lead_time', + target: null, + actual: 20, + values: [10, 20, 30] + }) + ).toBeNull(); + expect( + benchmarkBandOptions({ + metric: 'lead_time', + target: undefined, + actual: 20, + values: [10, 20, 30] + }) + ).toBeNull(); + }); + + it('survives a series with no usable numbers', () => { + // The Change Failure Rate series is built from an optional-chained + // `head(...)?.data.map(...)`, so undefined/NaN entries are reachable. One + // of them turning the axis max into NaN would take the whole chart down. + const opts = bandOptions({ + metric: 'change_failure_rate', + target: 15, + actual: 20, + values: [NaN, undefined, null] as unknown as number[] + }); + expect(Number.isFinite(opts.scales.y.suggestedMax)).toBe(true); + expect(opts.scales.y.suggestedMax).toBeGreaterThan(0); + }); + + it('never produces a zero-height axis', () => { + // A target of 0 with an all-zero series would otherwise ask chart.js for + // a scale running 0 to 0. + const opts = bandOptions({ + metric: 'change_failure_rate', + target: 0, + actual: 0, + values: [0, 0] + }); + expect(opts.scales.y.suggestedMax).toBeGreaterThan(0); + }); +}); diff --git a/web-server/src/utils/__tests__/benchmarks.test.ts b/web-server/src/utils/__tests__/benchmarks.test.ts new file mode 100644 index 000000000..16743ae2b --- /dev/null +++ b/web-server/src/utils/__tests__/benchmarks.test.ts @@ -0,0 +1,89 @@ +import { benchmarkCaption } from '../benchmarks'; + +describe('benchmarkCaption', () => { + it('treats being under target as good for lead time', () => { + const r = benchmarkCaption('lead_time', 3600, 7200, 'team'); + expect(r.tone).toBe('good'); + expect(r.text).toContain('under target'); + }); + + it('treats being under target as bad for deployment frequency', () => { + // Higher is better here -- the same direction means the opposite thing. + const r = benchmarkCaption('deployment_frequency', 2, 5, 'team'); + expect(r.tone).toBe('warn'); + expect(r.text).toContain('below target'); + }); + + it('treats being under target as good for CFR and MTTR too', () => { + // The other two members of LOWER_IS_BETTER. Deployment frequency is the + // single metric in the set where the direction inverts, so an entry + // wrongly added to or dropped from that set would silently reverse the + // meaning of a card with nothing else to catch it. + expect(benchmarkCaption('change_failure_rate', 5, 15, 'team').tone).toBe( + 'good' + ); + expect( + benchmarkCaption('mean_time_to_recovery', 900, 3600, 'team').tone + ).toBe('good'); + }); + + it('treats a smaller average PR as good, and reports it in lines', () => { + // Lower is better: small PRs review faster and merge sooner. And the + // unit is lines, not seconds -- 150 must not render as "2.5m". + const r = benchmarkCaption('lines_of_code', 150, 200, 'team'); + expect(r.tone).toBe('good'); + expect(r.text).toContain('150 lines'); + expect(r.text).toContain('under target (200 lines)'); + }); + + it('treats a target of zero lines as a real target', () => { + // `0` is a deliberate target, never "absent" -- a truthiness check here + // would drop the caption entirely. + expect(benchmarkCaption('lines_of_code', 10, 0, 'team')).not.toBeNull(); + expect(benchmarkCaption('lines_of_code', 10, 0, 'team').tone).toBe('warn'); + }); + + it('names the source so a team can see its setting did not save', () => { + expect(benchmarkCaption('lead_time', 3600, 7200, 'global').text).toContain( + 'default' + ); + expect(benchmarkCaption('lead_time', 3600, 7200, 'team').text).toContain( + 'team' + ); + }); + + it('returns null when there is no target', () => { + expect(benchmarkCaption('lead_time', 3600, null, null)).toBeNull(); + }); +}); + +describe('benchmarkCaption headline', () => { + it('leads with the gap as a percentage', () => { + // Shahzad's own example: 1.2d actual vs 2.1d target. The sentence form + // makes the reader do this subtraction; the headline does it for them. + const r = benchmarkCaption('lead_time', 103680, 181440, 'team'); + expect(r.headline).toBe('43% under target'); + expect(r.sourceShort).toBe('team benchmark'); + }); + + it('switches to a multiplier once the gap passes 2x', () => { + // 5/week against a 1/week target: "400% above target" is the same number + // and nobody parses it. + const r = benchmarkCaption('deployment_frequency', 5, 1, 'global'); + expect(r.headline).toBe('5x above target'); + expect(r.tone).toBe('good'); + }); + + it('falls back to the absolute value when the target is 0', () => { + // A CFR target of 0 is legitimate and makes a percentage undefined. + const r = benchmarkCaption('change_failure_rate', 2.5, 0, 'team'); + expect(r.headline).toBe('2.5% over target'); + expect(r.tone).toBe('warn'); + }); + + it('says on target when the target is met exactly', () => { + const r = benchmarkCaption('change_failure_rate', 0, 0, 'team'); + expect(r.headline).toBe('on target'); + expect(r.tone).toBe('good'); + }); +}); diff --git a/web-server/src/utils/__tests__/bitbucketRepos.test.ts b/web-server/src/utils/__tests__/bitbucketRepos.test.ts new file mode 100644 index 000000000..54abb0689 --- /dev/null +++ b/web-server/src/utils/__tests__/bitbucketRepos.test.ts @@ -0,0 +1,57 @@ +import { adaptBitbucketRepo } from '@/utils/bitbucketRepos'; +import { Integration } from '@/constants/integrations'; + +// CLUSTOX: the same realistic v2 payload shape the backend fixtures use -- +// asserting a simplified shape here would let the adapter and the test agree +// while both misread the real API. +const BB_REPO = { + uuid: '{c3d4e5f6-0000-4000-8000-000000000003}', + name: 'middleware', + slug: 'middleware', + full_name: 'clustox/middleware', + description: 'DORA metrics', + mainbranch: { name: 'main' }, + workspace: { slug: 'clustox' }, + links: { html: { href: 'https://bitbucket.org/clustox/middleware' } } +}; + +describe('adaptBitbucketRepo', () => { + it('returns the exact shape the GitHub and GitLab branches return', () => { + // Key-for-key: the repo-selection UI reads all three providers through + // one code path, so a missing or renamed key here surfaces as a blank + // column in the picker, not an error. + expect(adaptBitbucketRepo(BB_REPO, 'clustox')).toEqual({ + id: '{c3d4e5f6-0000-4000-8000-000000000003}', + name: 'middleware', + desc: 'DORA metrics', + slug: 'middleware', + web_url: 'https://bitbucket.org/clustox/middleware', + branch: 'main', + parent: 'clustox', + provider: Integration.BITBUCKET + }); + }); + + it('tolerates a repo with no mainbranch', () => { + // A freshly created empty repo has mainbranch: null. + const empty = { ...BB_REPO, mainbranch: null as any }; + expect(adaptBitbucketRepo(empty, 'clustox').branch).toBeNull(); + }); +}); + +describe('isBitbucketApiUrl', () => { + const { isBitbucketApiUrl } = require('@/utils/bitbucketRepos'); + + it('accepts only Bitbucket API URLs, prefix-anchored', () => { + expect( + isBitbucketApiUrl('https://api.bitbucket.org/2.0/repositories/x?page=2') + ).toBe(true); + // CLUSTOX: the cursor is client-supplied and fetched with the org's + // Basic auth header -- each of these is a token-exfiltration attempt. + expect(isBitbucketApiUrl('https://evil.example/steal')).toBe(false); + expect( + isBitbucketApiUrl('https://evil.example/https://api.bitbucket.org/2.0/') + ).toBe(false); + expect(isBitbucketApiUrl('http://api.bitbucket.org/2.0/x')).toBe(false); + }); +}); diff --git a/web-server/src/utils/__tests__/contributorFilters.test.ts b/web-server/src/utils/__tests__/contributorFilters.test.ts new file mode 100644 index 000000000..2aae9a71b --- /dev/null +++ b/web-server/src/utils/__tests__/contributorFilters.test.ts @@ -0,0 +1,91 @@ +import { stripContributorFilters } from '@/utils/contributorFilters'; + +// CLUSTOX: pins the fix for a bug where Change Failure Rate and MTTR were +// silently filtered by contributor even though their cards say "team-wide". +// `stripContributorFilters` is what keeps CFR/MTTR's request payload free of +// the contributor keys while leaving every other filter (base branches, repo +// filters, prod-branch workflow config) untouched -- exactly what the two +// unaffected fetches (Lead Time, Deployment Frequency) still receive +// unfiltered-of-nothing, i.e. with the contributor keys intact. +describe('stripContributorFilters', () => { + it('drops only `authors` from the pr filter, keeping everything else', () => { + const prFilters = { + pr_filter: { + authors: ['alice', 'bob'], + base_branches: ['main'], + repo_filters: { 'repo-1': { base_branches: ['main'] } } + } + }; + + const { prFilter } = stripContributorFilters(prFilters, null); + + expect(prFilter).toEqual({ + pr_filter: { + base_branches: ['main'], + repo_filters: { 'repo-1': { base_branches: ['main'] } } + } + }); + expect(prFilter.pr_filter).not.toHaveProperty('authors'); + }); + + // The shape below is the one the backend actually parses: `event_actors` + // lives *inside* the `workflow_filter` blob, alongside `head_branches`, + // because ParseWorkflowFilterProcessor.apply + // (mhq/service/workflows/workflow_filter.py) reads + // `workflow_filter.get("event_actors")` off the parsed blob. Stripping the + // top level would leave the key in place and CFR would stay filtered. + it('drops only `event_actors` from inside workflow_filter, keeping everything else', () => { + const workflowFilters = { + workflow_filter: { + head_branches: ['^main$'], + event_actors: ['alice'] + } + }; + + const { workflowFilter } = stripContributorFilters( + { pr_filter: null }, + workflowFilters + ); + + expect(workflowFilter).toEqual({ + workflow_filter: { head_branches: ['^main$'] } + }); + expect(workflowFilter.workflow_filter).not.toHaveProperty('event_actors'); + }); + + it('leaves a null pr_filter as-is when no other filters are set', () => { + const prFilters: { pr_filter: Record | null } = { + pr_filter: null + }; + + const { prFilter } = stripContributorFilters(prFilters, undefined); + + expect(prFilter).toBe(prFilters); + }); + + it('leaves an unset workflow filter as-is', () => { + const { workflowFilter } = stripContributorFilters( + { pr_filter: null }, + undefined + ); + + expect(workflowFilter).toBeUndefined(); + }); + + it('is a no-op when no contributor was selected', () => { + const prFilters = { + pr_filter: { base_branches: ['main'] } + }; + const workflowFilters = { + workflow_filter: { head_branches: ['^main$'] } + }; + + const { prFilter, workflowFilter } = stripContributorFilters( + prFilters, + workflowFilters + ); + + expect(prFilter).toEqual(prFilters); + expect(workflowFilter).toEqual(workflowFilters); + }); +}); diff --git a/web-server/src/utils/__tests__/contributorQuery.test.ts b/web-server/src/utils/__tests__/contributorQuery.test.ts new file mode 100644 index 000000000..e4e1ccb1e --- /dev/null +++ b/web-server/src/utils/__tests__/contributorQuery.test.ts @@ -0,0 +1,61 @@ +import { + contributorSelectionKey, + parseSelectedFromQuery +} from '@/utils/contributorQuery'; + +// CLUSTOX: the contributor selection lives in the URL, so this parser decides +// what a shared or hand-edited link actually filters on. Next types a query +// value as `string | string[]`: `?contributors=a,b` arrives as a string, but a +// repeated `?contributors=a&contributors=b` arrives as an array. The original +// took `raw[0]` and silently dropped everything after the first, so a +// two-person link quietly filtered on one person. +describe('parseSelectedFromQuery', () => { + it('parses the comma-separated form the component itself writes', () => { + expect(parseSelectedFromQuery('alice,bob')).toEqual(['alice', 'bob']); + }); + + it('keeps every value when the param repeats, not just the first', () => { + // `?contributors=alice&contributors=bob` + expect(parseSelectedFromQuery(['alice', 'bob'])).toEqual(['alice', 'bob']); + }); + + it('handles a repeated param whose entries are themselves comma-separated', () => { + expect(parseSelectedFromQuery(['alice,bob', 'carol'])).toEqual([ + 'alice', + 'bob', + 'carol' + ]); + }); + + it('de-duplicates, so a name repeated across forms is only sent once', () => { + expect(parseSelectedFromQuery(['alice', 'alice,bob'])).toEqual([ + 'alice', + 'bob' + ]); + }); + + it('drops empty segments from a malformed value', () => { + expect(parseSelectedFromQuery(',alice,,bob,')).toEqual(['alice', 'bob']); + }); + + it('returns an empty selection when the param is absent or blank', () => { + expect(parseSelectedFromQuery(undefined as unknown as string)).toEqual([]); + expect(parseSelectedFromQuery('')).toEqual([]); + expect(parseSelectedFromQuery([])).toEqual([]); + }); +}); + +// The memo key has to be a primitive, and it has to compare equal for two URL +// spellings that mean the same selection -- otherwise the DORA summary refetches +// on every render. +describe('contributorSelectionKey', () => { + it('collapses the array form to the same string as the comma form', () => { + expect(contributorSelectionKey(['alice', 'bob'])).toBe( + contributorSelectionKey('alice,bob') + ); + }); + + it('is a stable primitive for an absent param', () => { + expect(contributorSelectionKey(undefined as unknown as string)).toBe(''); + }); +}); diff --git a/web-server/src/utils/__tests__/fetchLocStats.test.ts b/web-server/src/utils/__tests__/fetchLocStats.test.ts new file mode 100644 index 000000000..2f5aeac13 --- /dev/null +++ b/web-server/src/utils/__tests__/fetchLocStats.test.ts @@ -0,0 +1,106 @@ +jest.mock('@/api-helpers/axios', () => ({ + handleRequest: jest.fn() +})); + +import { handleRequest } from '@/api-helpers/axios'; + +import { fetchLocStats } from '../cockpitMetricUtils'; + +const AGGREGATE = { + additions: 149332, + deletions: 20650, + total: 169982, + avg_pr_size: 654 +}; + +const TRENDS = { + '2026-07-20T00:00:00+00:00': { + additions: 63033, + deletions: 7407, + total: 70440, + avg_pr_size: 640 + } +}; + +const PR_FILTER = { pr_filter: { authors: ['hamad'] } } as any; + +const callArgs = () => + fetchLocStats({ + teamId: 'team-1' as any, + currStatsTimeObject: { + from_time: '2026-07-01T00:00:00+00:00', + to_time: '2026-08-01T00:00:00+00:00' + } as any, + prevStatsTimeObject: { + from_time: '2026-06-01T00:00:00+00:00', + to_time: '2026-07-01T00:00:00+00:00' + } as any, + currTrendsTimeObject: { + from_time: '2026-07-01T00:00:00+00:00', + to_time: '2026-08-01T00:00:00+00:00' + } as any, + prevTrendsTimeObject: { + from_time: '2026-06-01T00:00:00+00:00', + to_time: '2026-07-01T00:00:00+00:00' + } as any, + prFilter: PR_FILTER + }); + +describe('fetchLocStats', () => { + beforeEach(() => { + (handleRequest as jest.Mock).mockReset(); + (handleRequest as jest.Mock) + .mockResolvedValueOnce(AGGREGATE) + .mockResolvedValueOnce(AGGREGATE) + .mockResolvedValueOnce(TRENDS) + .mockResolvedValueOnce(TRENDS); + }); + + it('returns the shape the dora_metrics route spreads onto its response', async () => { + // CLUSTOX: the route does `loc_stats: locResponse?.loc_stats`, and the card + // reads `metrics_summary.loc_stats.current.avg_pr_size`. This pins the + // nesting between those two, which is exactly the contract that broke on + // the predecessor branch -- a key attached one level off produced a payload + // both the producer and its own test agreed on, and the consumer could not + // read. + const result = await callArgs(); + + expect(result.loc_stats.current.avg_pr_size).toBe(654); + expect(result.loc_stats.previous.total).toBe(169982); + expect(result.loc_trends.current).toEqual(TRENDS); + expect(result.loc_trends.previous).toEqual(TRENDS); + }); + + it('hits both the aggregate and the trends route, for both periods', async () => { + await callArgs(); + + const urls = (handleRequest as jest.Mock).mock.calls.map(([url]) => url); + expect(urls.filter((u) => u === '/teams/team-1/loc')).toHaveLength(2); + expect(urls.filter((u) => u === '/teams/team-1/loc/trends')).toHaveLength(2); + }); + + it('passes the pr_filter through on every call', async () => { + // CLUSTOX: dropping the filter here would not fail any other assertion -- + // the call still succeeds and the numbers still look plausible. It would + // just leave LOC reporting team-wide totals beside four contributor-filtered + // cards, which reads as a data bug rather than a missing argument. + await callArgs(); + + for (const [, config] of (handleRequest as jest.Mock).mock.calls) { + expect(config.params.pr_filter).toEqual(PR_FILTER.pr_filter); + } + }); + + it('sends timestamps carrying a timezone offset', async () => { + // CLUSTOX: the analytics route rejects a naive timestamp outright -- + // 400 "Timestamp passed without tz info". A stripped offset would take the + // whole LOC fetch down, and the soft `.catch` in the route would then hide + // it as a silently missing card. + await callArgs(); + + for (const [, config] of (handleRequest as jest.Mock).mock.calls) { + expect(config.params.from_time).toMatch(/[+-]\d{2}:\d{2}$/); + expect(config.params.to_time).toMatch(/[+-]\d{2}:\d{2}$/); + } + }); +}); diff --git a/web-server/src/utils/__tests__/filterUtils.test.ts b/web-server/src/utils/__tests__/filterUtils.test.ts index 3ffbd9445..a86900335 100644 --- a/web-server/src/utils/__tests__/filterUtils.test.ts +++ b/web-server/src/utils/__tests__/filterUtils.test.ts @@ -270,4 +270,43 @@ describe('getWorkFlowFiltersAsPayloadForSingleTeam', () => { expect(result).toEqual({ workflow_filter: { head_branches: ['^main$'] } }); }); + + // CLUSTOX: contributor filter. These two cases pin the payload against the + // *backend's* parser rather than against this module's own output. + // `ParseWorkflowFilterProcessor.apply` in + // backend/analytics_server/mhq/service/workflows/workflow_filter.py reads + // the JSON blob it is handed and does `workflow_filter.get("event_actors")` + // on it -- the same level it reads `head_branches` from. That blob is the + // value of the `workflow_filter` query param, so `event_actors` must be a + // sibling of `head_branches`, never a sibling of `workflow_filter`. A + // top-level key would also be an extra query param, which the PREVENT_EXTRA + // voluptuous schema on /teams//deployment_frequency rejects with a + // 400. + it('should nest event_actors inside workflow_filter, where the backend parser reads it', async () => { + const result = await getWorkFlowFiltersAsPayloadForSingleTeam({ + orgId: 'f48d7cce-25d4-41d1-903e-e09166677d92', + teamId: '18d934c1-2699-41bd-af64-c0394ba32fdf', + eventActors: ['octocat'] + }); + + expect(result).toEqual({ + workflow_filter: { + head_branches: ['^main$'], + event_actors: ['octocat'] + } + }); + // Guard the exact failure that shipped: a top-level sibling key is an + // unrecognised query param and 400s the whole deployment frequency call. + expect(result).not.toHaveProperty('event_actors'); + }); + + it('should leave the payload unchanged when eventActors is omitted', async () => { + const result = await getWorkFlowFiltersAsPayloadForSingleTeam({ + orgId: 'f48d7cce-25d4-41d1-903e-e09166677d92', + teamId: '18d934c1-2699-41bd-af64-c0394ba32fdf', + eventActors: [] + }); + + expect(result).toEqual({ workflow_filter: { head_branches: ['^main$'] } }); + }); }); diff --git a/web-server/src/utils/__tests__/locCard.test.ts b/web-server/src/utils/__tests__/locCard.test.ts new file mode 100644 index 000000000..27bc13fe9 --- /dev/null +++ b/web-server/src/utils/__tests__/locCard.test.ts @@ -0,0 +1,148 @@ +import { buildLocCardModel } from '../locCard'; + +// CLUSTOX: not invented numbers. This is the live response from +// `GET /teams//loc` and `/loc/trends` on the running container, copied +// verbatim, so these assertions prove real values survive the trip from the +// wire to the card rather than proving the builder agrees with itself. +const CURRENT = { + additions: 149332, + deletions: 20650, + total: 169982, + avg_pr_size: 654, + avg_daily: 1215 +}; + +const PREVIOUS = { + additions: 63033, + deletions: 7407, + total: 70440, + avg_pr_size: 640, + avg_daily: 1205 +}; + +const TRENDS = { + '2026-07-20T00:00:00+00:00': { + additions: 63033, + deletions: 7407, + total: 70440, + avg_pr_size: 640, + avg_daily: 1205 + }, + '2026-07-27T00:00:00+00:00': { + additions: 65867, + deletions: 10086, + total: 75953, + avg_pr_size: 575, + avg_daily: 1085 + }, + '2026-08-03T00:00:00+00:00': { + additions: 20432, + deletions: 3157, + total: 23589, + avg_pr_size: 1310, + avg_daily: 337 + }, + '2026-08-10T00:00:00+00:00': { + additions: 0, + deletions: 0, + total: 0, + avg_pr_size: 0, + avg_daily: 0 + } +}; + +const stats = { current: CURRENT, previous: PREVIOUS }; +const trends = { current: TRENDS, previous: {} }; + +describe('buildLocCardModel', () => { + it('carries the measured numbers through to the card', () => { + const model = buildLocCardModel(stats, trends); + + expect(model.isMeasured).toBe(true); + expect(model.total).toBe(169982); + expect(model.additions).toBe(149332); + expect(model.deletions).toBe(20650); + expect(model.avgPrSize).toBe(654); + expect(model.prevTotal).toBe(70440); + expect(model.prevAvgPrSize).toBe(640); + }); + + it('feeds the band the avg_pr_size series, never the totals series', () => { + // CLUSTOX: the defect this whole test file exists for. Both series are + // arrays of plausible numbers off the same objects, so a mix-up type-checks, + // renders, and reads as data. Pinning the values means swapping the two + // fields fails here rather than shipping a band drawn in lines-per-PR over + // an axis of weekly totals. + const model = buildLocCardModel(stats, trends); + + expect(model.avgPrSizeValues).toEqual([640, 575, 1310, 0]); + expect(Math.max(...model.avgPrSizeValues)).toBe(1310); + expect(model.avgPrSizeValues).not.toContain(75953); + }); + + it('orders the weekly series oldest first regardless of key order', () => { + const model = buildLocCardModel(stats, { + current: { + '2026-08-03T00:00:00+00:00': TRENDS['2026-08-03T00:00:00+00:00'], + '2026-07-20T00:00:00+00:00': TRENDS['2026-07-20T00:00:00+00:00'] + }, + previous: { + '2026-07-27T00:00:00+00:00': TRENDS['2026-07-27T00:00:00+00:00'] + } + }); + + expect(model.avgPrSizeValues).toEqual([640, 575, 1310]); + }); + + it('degrades to an empty, non-throwing model when loc_stats is absent', () => { + // CLUSTOX: the old-backend case -- `loc_stats` and `loc_trends` are + // optional on the response and the BFF route degrades them to `undefined` + // rather than failing the whole payload. Every one of these must be a + // usable value, not a throw and not a null the card would then have to + // guard again in JSX. + const model = buildLocCardModel(undefined, undefined); + + expect(model.isMeasured).toBe(false); + expect(model.canComparePrSize).toBe(false); + expect(model.total).toBe(0); + expect(model.avgPrSizeValues).toEqual([]); + }); + + it('survives loc_stats arriving without loc_trends', () => { + const model = buildLocCardModel(stats, undefined); + + expect(model.isMeasured).toBe(true); + expect(model.total).toBe(169982); + expect(model.avgPrSizeValues).toEqual([]); + }); + + it('treats a measured zero as data, not as absence', () => { + // CLUSTOX: a team that merged nothing gets `{additions: 0, ...}` from the + // backend, which is a real measurement. It must still render the card -- + // only a missing `loc_stats` is "not measured". + const zeroed = { + additions: 0, + deletions: 0, + total: 0, + avg_pr_size: 0, + avg_daily: 0 + }; + const model = buildLocCardModel( + { current: zeroed, previous: zeroed }, + { current: {}, previous: {} } + ); + + expect(model.isMeasured).toBe(true); + expect(model.total).toBe(0); + // ...but there is no PR size to benchmark, so no band and no caption. + // 0 beats every lower-is-better target and would otherwise congratulate a + // team for shipping nothing. + expect(model.canComparePrSize).toBe(false); + }); + + it('allows the comparison as soon as anything was merged', () => { + const model = buildLocCardModel(stats, trends); + + expect(model.canComparePrSize).toBe(true); + }); +}); diff --git a/web-server/src/utils/api-error.ts b/web-server/src/utils/api-error.ts new file mode 100644 index 000000000..a84e1c2f5 --- /dev/null +++ b/web-server/src/utils/api-error.ts @@ -0,0 +1,22 @@ +import { AxiosResponse } from 'axios'; + +/** + * CLUSTOX: reads the status and message off a rejected `handleApi` call. + * + * `handleApi` rejects with the AxiosResponse itself rather than an Error, and + * the BFF's error body is `{ message }` (see parseError). Both facts are easy + * to get wrong at each call site, and getting them wrong degrades silently to + * "no message", which is exactly the generic advice a caller reaching for this + * is trying to avoid. + */ +export const readApiError = ( + err: unknown +): { status: number | null; message: string | null } => { + const response = err as AxiosResponse<{ message?: string }> | undefined; + const message = response?.data?.message; + + return { + status: typeof response?.status === 'number' ? response.status : null, + message: typeof message === 'string' && message ? message : null + }; +}; diff --git a/web-server/src/utils/auth.ts b/web-server/src/utils/auth.ts index 22852b373..57a0fd3a5 100644 --- a/web-server/src/utils/auth.ts +++ b/web-server/src/utils/auth.ts @@ -71,6 +71,36 @@ export const getMissingPATScopes = async ( } }; +// Bitbucket functions + +// CLUSTOX: server-side, unlike checkGitHubValidity above -- +// api.bitbucket.org sends no CORS headers for Basic auth from foreign +// origins, so a browser-direct check dies in preflight. The internal +// endpoint calls /2.0/user and returns a boolean; the token goes over our +// own wire once and is never logged or echoed. +export type BitbucketValidity = { + valid: boolean; + reason?: 'invalid_credentials' | 'missing_account_scope'; +}; + +export const checkBitbucketValidity = async ( + email: string, + token: string +): Promise => { + try { + const response = await axios.post('/api/internal/bitbucket/token-check', { + email, + token + }); + return { + valid: Boolean(response.data?.valid), + reason: response.data?.reason + }; + } catch (error) { + return { valid: false, reason: 'invalid_credentials' }; + } +}; + // Gitlab functions export const checkGitLabValidity = async ( @@ -99,3 +129,66 @@ export const getMissingGitLabScopes = (scopes: string[]): string[] => { ); return missingScopes; }; + +// Jira functions + +// Reduces any of "mycompany.atlassian.net", "https://mycompany.atlassian.net/", +// or a full URL copied from the browser while looking at a board +// ("https://mycompany.atlassian.net/jira/software/projects/ABC/boards/1") +// to just the hostname. A regex that only stripped a leading protocol and +// trailing slash left any path/query intact, which broke the very next +// request (GET https://{that}/rest/api/3/myself 404s) for anyone who +// pasted a full address rather than typing the bare domain. Mirrors the +// normalizer in pages/api/integrations/jira/validate.ts -- kept as two +// small copies rather than a shared import across the client/server +// boundary, same as this file's other provider helpers. +export const normalizeJiraSiteUrl = (input: string) => { + const trimmed = input.trim(); + try { + const withProtocol = /^https?:\/\//i.test(trimmed) + ? trimmed + : `https://${trimmed}`; + return new URL(withProtocol).hostname; + } catch { + // Not parseable as a URL at all (e.g. empty string) -- fall through to + // the caller's own "fill in all three fields"/validity check rather + // than throw here. + return trimmed; + } +}; + +export type JiraValidityResult = { + valid: boolean; + displayName?: string; + reason?: 'unauthorized' | 'unreachable' | 'unknown'; +}; + +// CLUSTOX: unlike checkGitHubValidity/checkGitLabValidity, this cannot call +// Jira's API directly from the browser -- Jira Cloud doesn't send CORS +// headers permissive enough for a third-party origin's Basic-auth request, +// so it's proxied through our own backend instead (see +// pages/api/integrations/jira/validate.ts for why). +export const checkJiraValidity = async ( + siteUrl: string, + email: string, + apiToken: string +): Promise => { + try { + const { data } = await axios.post('/api/integrations/jira/validate', { + site_url: siteUrl, + email, + api_token: apiToken + }); + return { + valid: data.valid, + displayName: data.display_name, + reason: data.reason + }; + } catch (error) { + return { valid: false, reason: 'unknown' }; + } +}; + +// Jira API tokens inherit the full permissions of the account they belong +// to -- there's no OAuth-style scope grant to inspect the way there is for +// a GitHub/GitLab PAT, so there's no getMissingJiraScopes equivalent. diff --git a/web-server/src/utils/benchmarkBand.ts b/web-server/src/utils/benchmarkBand.ts new file mode 100644 index 000000000..95361d5c9 --- /dev/null +++ b/web-server/src/utils/benchmarkBand.ts @@ -0,0 +1,248 @@ +// CLUSTOX: replaces BenchmarkTargetLine, a CSS-positioned div whose height was +// target / max(target, ...values). A deployment-frequency target of 1 against +// a series topping 35 landed at 3% -- visually identical to the card's bottom +// border -- while a lead-time target of 8h larger than every plotted value +// pinned to the top. Correct arithmetic, unreadable result, in both +// directions. An annotation is positioned in data space by chart.js itself, so +// it cannot disagree with the series it is drawn over. +// +// `chartjs-plugin-annotation` is already registered in +// `components/Chart2/InternalChart2.tsx`. The four DORA cards hide both axes +// (`display: false`); that does not affect annotations, which key off the +// scale rather than off a rendered axis. +// +// Kept import-light for the same reason `benchmarks.ts` is: `@/utils/date` and +// `@/theme/schemes/theme` both chain into modules that blow up ts-jest, so the +// theme tones are passed in by the caller and only their hex fallbacks are +// duplicated here. +import { BenchmarkMetric, LOWER_IS_BETTER } from '@/utils/benchmarks'; + +export type BenchmarkBandTone = 'success' | 'warning'; + +/** + * The slice of the MUI theme this module needs. Structural, so the real + * `useTheme()` result satisfies it without the util importing MUI. + */ +export type BenchmarkBandTheme = { + colors: { + success: { main: string }; + warning: { main: string }; + }; +}; + +export type BenchmarkBandInput = { + metric: BenchmarkMetric; + /** The resolved benchmark target, in the same unit as `values`. */ + target: number | null | undefined; + /** The headline value the card displays, used only to pick the tone. */ + actual: number; + /** The same array the card's trend chart plots. */ + values: number[]; + theme?: BenchmarkBandTheme; +}; + +type BoxAnnotation = { + type: 'box'; + yMin: number; + yMax: number; + backgroundColor: string; + borderWidth: number; + drawTime: string; +}; + +type LineAnnotation = { + type: 'line'; + yMin: number; + yMax: number; + borderColor: string; + borderWidth: number; + borderDash: number[]; + drawTime: string; +}; + +export type BenchmarkBandOptions = { + scales: { y: { suggestedMax: number } }; + plugins: { + annotation: { + annotations: { + band: BoxAnnotation; + targetLine: LineAnnotation; + }; + }; + }; + // CLUSTOX: not a chart.js key -- chart.js ignores unknown option roots. It + // rides along so the card that merges these options can also read the two + // decisions made here without recomputing them: which tone the band was + // drawn in, and whether the target had to be left off the axis (in which + // case the caption is the only place the number appears). + benchmarkBand: { + tone: BenchmarkBandTone; + targetIsOffScale: boolean; + target: number; + }; +}; + +// CLUSTOX: mirrors `themeColors` in `@/theme/schemes/theme`, duplicated rather +// than imported because that module runs `createTheme` at import time and +// pulls all of MUI into any test that touches this one. Only reached when a +// caller omits `theme`; every card passes `useTheme()`. +const FALLBACK_TONE_COLORS: Record = { + success: '#57CA22', + warning: '#FFA319' +}; + +const BAND_OPACITY = 0.12; +const TARGET_LINE_OPACITY = 0.6; + +// CLUSTOX: matches Chart2's own `grace: '10%'` headroom, so the band's top +// edge sits where the card's other charts already put their ceiling. +const AXIS_HEADROOM = 1.1; + +// CLUSTOX: two opposite failures to avoid at once. Clamp the axis to the data +// and a target above it is clipped off-canvas, so the card looks exactly as it +// did before and the feature appears not to work. Extend the axis to the +// target unconditionally and a typo'd 20000-line target against a 200-line +// actual flattens the real series into a hairline at the bottom. So: include +// the target, but never stretch so far that the real series loses more than +// this share of the plot. Beyond that the axis stops, the band runs to the +// edge and the caption carries the number, because a scale nobody can read is +// worse than one that admits it is truncated. +// +// The value is 4 (the data keeps at least a quarter of the plot), NOT the 2 +// the brief's prose names. 2 cannot satisfy the brief's own first test case: +// target 100 against values [10, 20, 30] needs the axis to reach 100, which is +// 3.33x the data max of 30, so a 2x cap would clip exactly the target the test +// requires to be visible. The two test cases bound this constant to roughly +// [3.4, 4.5); 4 sits in the middle of that window and states a rule that is +// easy to hold in your head. +const AXIS_STRETCH_LIMIT = 4; + +const parseHex = (color: string): [number, number, number] | null => { + const hex = color.trim().replace(/^#/, ''); + const full = + hex.length === 3 + ? hex + .split('') + .map((c) => c + c) + .join('') + : hex; + if (!/^[0-9a-fA-F]{6}$/.test(full)) return null; + return [ + parseInt(full.slice(0, 2), 16), + parseInt(full.slice(2, 4), 16), + parseInt(full.slice(4, 6), 16) + ]; +}; + +// CLUSTOX: a local `alpha` rather than MUI's, to keep this module importable +// from a plain unit test. Handles the two forms the theme actually produces +// (hex, and the rgb/rgba that `alpha()` has already been applied to), and +// hands anything else back untouched -- an unrecognised colour should render +// opaque, not throw and take the whole chart down with it. +const withAlpha = (color: string, opacity: number): string => { + const rgb = parseHex(color); + if (rgb) return `rgba(${rgb[0]}, ${rgb[1]}, ${rgb[2]}, ${opacity})`; + + const parsed = color.match(/^rgba?\(([^)]+)\)$/i); + if (parsed) { + const [r, g, b] = parsed[1].split(',').map((part) => part.trim()); + if (r != null && g != null && b != null) + return `rgba(${r}, ${g}, ${b}, ${opacity})`; + } + + return color; +}; + +/** + * Builds the chart.js options that draw a benchmark target as a shaded band + * in the chart's own data space: a box covering the favourable side of the + * target, a hairline at the target itself, and the `suggestedMax` needed to + * keep both on canvas. + * + * Returns `null` when there is no target to draw -- the state every card is in + * before anyone configures a benchmark. + */ +export const benchmarkBandOptions = ({ + metric, + target, + actual, + values, + theme +}: BenchmarkBandInput): BenchmarkBandOptions | null => { + // CLUSTOX: `== null`, never truthiness. `0` is a real target -- zero failed + // changes is the strictest benchmark there is, and the one a truthiness + // check would silently drop. + if (target == null || !Number.isFinite(target)) return null; + + const lowerIsBetter = LOWER_IS_BETTER.has(metric); + + // CLUSTOX: filter to finite numbers first. The Change Failure Rate series is + // built from an optional-chained `head(...)?.data.map(...)`, so an + // undefined/NaN entry is reachable, and one of those turns the axis max into + // NaN -- which chart.js resolves by dropping the scale entirely. + const dataMax = Math.max(0, ...values.filter((v) => Number.isFinite(v))); + + const stretchCeiling = + dataMax > 0 ? dataMax * AXIS_STRETCH_LIMIT : Number.POSITIVE_INFINITY; + const suggestedMax = + Math.min(Math.max(dataMax, target, 0) * AXIS_HEADROOM, stretchCeiling) || + // CLUSTOX: an all-zero series with a target of 0 would otherwise ask + // chart.js for a scale running 0 to 0. + 1; + + const targetIsOffScale = target > suggestedMax; + + // CLUSTOX: the direction is per metric. Deployment frequency is the one + // metric in the set where the favourable side is *above* the target, so its + // band runs target -> top of the axis while every other band runs + // target -> 0. Getting this backwards silently reverses the feature's + // meaning on exactly one card. + const bandEdges = lowerIsBetter ? [0, target] : [target, suggestedMax]; + + // CLUSTOX: `<=`/`>=` matches benchmarkCaption, so the band and the caption + // printed directly under it can never disagree about which side of the + // target a team is on. + const onGoodSide = lowerIsBetter ? actual <= target : actual >= target; + + // CLUSTOX: never red -- a missed internal goal is not a system failure, and + // colouring it like one makes the dashboard punitive (spec, "The graph"). + const tone: BenchmarkBandTone = onGoodSide ? 'success' : 'warning'; + const toneColor = theme + ? theme.colors[tone].main + : FALLBACK_TONE_COLORS[tone]; + + return { + scales: { y: { suggestedMax } }, + plugins: { + annotation: { + annotations: { + band: { + type: 'box', + // CLUSTOX: sorted, so an unreachable deployment-frequency target + // (band bottom above the axis top) still describes a real + // interval for chart.js to clip rather than an inverted one. + yMin: Math.min(...bandEdges), + yMax: Math.max(...bandEdges), + backgroundColor: withAlpha(toneColor, BAND_OPACITY), + borderWidth: 0, + // CLUSTOX: at the default draw time the box paints over the + // series it is meant to sit behind. + drawTime: 'beforeDatasetsDraw' + }, + targetLine: { + type: 'line', + yMin: target, + yMax: target, + borderColor: withAlpha(toneColor, TARGET_LINE_OPACITY), + borderWidth: 1, + // CLUSTOX: dashed, carrying over the one thing the old CSS line + // got right -- it reads as a goal rather than as data. + borderDash: [4, 4], + drawTime: 'beforeDatasetsDraw' + } + } + } + }, + benchmarkBand: { tone, targetIsOffScale, target } + }; +}; diff --git a/web-server/src/utils/benchmarks.ts b/web-server/src/utils/benchmarks.ts new file mode 100644 index 000000000..bee8146f6 --- /dev/null +++ b/web-server/src/utils/benchmarks.ts @@ -0,0 +1,187 @@ +// CLUSTOX: per-team DORA benchmarks -- the caption shown under a metric card +// when a target (team-set or org-wide default) exists for it. +// +// Deliberately dependency-light (no imports at all) so `benchmarkCaption` is +// unit testable in isolation. `@/utils/date` -- the obvious place to borrow a +// duration formatter from -- chains into `@/utils/mock` -> `@faker-js/faker`, +// which ships ESM-only output and blows up ts-jest with "Cannot use import +// statement outside a module" the instant it's required (verified directly: +// a probe test importing `getDurationString` fails with that exact error). +// `contributorFilters.ts` hit the same wall for the same reason; this file +// stays equally self-contained. +export type BenchmarkMetric = + | 'lead_time' + | 'deployment_frequency' + | 'change_failure_rate' + | 'mean_time_to_recovery' + // CLUSTOX: average gross lines per merged PR, measured as `avg_pr_size` on + // the LOC response. Not weekly volume, and not a duration -- its unit is + // lines, entered and stored verbatim. + | 'lines_of_code'; + +export type BenchmarkSource = 'team' | 'global' | null; + +export type ResolvedBenchmark = { + target: number | null; + source: BenchmarkSource; +}; + +export type Benchmarks = Record; + +export type BenchmarkTone = 'good' | 'warn'; + +export type BenchmarkCaption = { + /** The full sentence -- pill tooltip, and anywhere space is not scarce. */ + text: string; + /** + * The verdict alone, gap first: "43% under target", "5x above target", + * "on target". The pill leads with this because the full sentence buries + * the verdict in the middle and makes the reader do the subtraction. + */ + headline: string; + /** "team benchmark" / "default benchmark" -- the pill's suffix. */ + sourceShort: string; + tone: BenchmarkTone; +}; + +// CLUSTOX: direction is per metric, not global. Lead time, change failure +// rate, MTTR and average PR size are all "smaller is better" -- deployment +// frequency is the one metric in the set where "smaller is better" is exactly +// backwards. Getting this list wrong silently inverts the whole feature's +// meaning, so it's the one thing under direct test coverage (see +// benchmarks.test.ts). +// +// Average PR size belongs here because small PRs get reviewed faster and +// merge sooner; a team beating a 200-line target is doing well, not badly. +export const LOWER_IS_BETTER: ReadonlySet = new Set([ + 'lead_time', + 'change_failure_rate', + 'mean_time_to_recovery', + 'lines_of_code' +]); + +const formatDuration = (seconds: number): string => { + const abs = Math.abs(seconds); + if (abs < 60) return `${Math.round(seconds)}s`; + if (abs < 3600) return `${Math.round(seconds / 60)}m`; + if (abs < 86400) return `${Math.round((seconds / 3600) * 10) / 10}h`; + return `${Math.round((seconds / 86400) * 10) / 10}d`; +}; + +// CLUSTOX: two decimals, matching what useChangeFailureRateProps already does +// to the *actual* value. Targets go in through a number input and come back +// as stored, so a baseline entered as 15 can arrive as 15.000000000000002 and +// would otherwise render verbatim next to a tidily rounded actual. +const roundForDisplay = (value: number): number => + Math.round(value * 100) / 100; + +// CLUSTOX: exported so a card can name a target on its own, without a +// comparison. Mean Time to Recovery with zero incidents is the case that needs +// it: there is no actual value to compare, but an admin still has to be able +// to see that a target exists. +export const formatBenchmarkValue = ( + metric: BenchmarkMetric, + value: number +): string => { + switch (metric) { + case 'lead_time': + case 'mean_time_to_recovery': + return formatDuration(value); + case 'change_failure_rate': + return `${roundForDisplay(value)}%`; + case 'deployment_frequency': { + const rounded = roundForDisplay(value); + return `${rounded} ${rounded === 1 ? 'deployment' : 'deployments'}/week`; + } + case 'lines_of_code': + return `${roundForDisplay(value)} lines`; + } +}; + +/** + * Names which benchmark applied, so a team that set a target but is being + * shown the org-wide default can tell that their setting didn't save rather + * than silently comparing against the wrong number. + * + * Exported for the same reason `formatBenchmarkValue` is -- the target-only + * line on a card with no measurement has to carry the same wording as the + * caption it stands in for. + */ +export const benchmarkSourceLabel = (source: BenchmarkSource): string => + source === 'team' ? "your team's benchmark" : 'the default benchmark'; + +/** + * Compares `actual` against `target` for `metric` and returns a caption + * stating the fact (over/under/above/below target) plus which side of the + * target is favourable for this particular metric. + * + * Returns `null` when there is no target to compare against -- that is the + * state every card is in before anyone configures a benchmark, and it must + * render nothing rather than a caption about a target that doesn't exist. + */ +export const benchmarkCaption = ( + metric: BenchmarkMetric, + actual: number, + target: number | null, + source: BenchmarkSource +): BenchmarkCaption | null => { + // CLUSTOX: `== null`, so `undefined` is treated as "no target" too. Every + // call site gates on the parent object today, but a `target: undefined` + // slipping through would otherwise build a caption comparing against + // `undefined` -- "3h is over target (NaN)". + if (target == null) return null; + + const lowerIsBetter = LOWER_IS_BETTER.has(metric); + // CLUSTOX: `<=`/`>=` so hitting the target exactly reads as "good", not as + // a coin flip between the two tones. + const favourable = lowerIsBetter ? actual <= target : actual >= target; + + let direction: string; + if (actual === target) { + direction = 'at'; + } else if (lowerIsBetter) { + direction = actual < target ? 'under' : 'over'; + } else { + direction = actual > target ? 'above' : 'below'; + } + + const sourceClause = benchmarkSourceLabel(source); + + const text = `${formatBenchmarkValue( + metric, + actual + )} is ${direction} target (${formatBenchmarkValue( + metric, + target + )}) — ${sourceClause}`; + + let headline: string; + if (actual === target) { + headline = 'on target'; + } else if (target > 0) { + const ratio = actual / target; + // CLUSTOX: "5x above target" once the gap passes 2x -- "400% above + // target" is technically the same number and nobody parses it. Under a + // target the percentage can never exceed 100, so the multiplier form + // only ever appears on the over/above side. + headline = + ratio >= 2 + ? `${Math.round(ratio * 10) / 10}x ${direction} target` + : `${Math.round( + (Math.abs(actual - target) / target) * 100 + )}% ${direction} target`; + } else { + // CLUSTOX: a target of 0 is legitimate (zero failed changes) and makes a + // percentage undefined -- fall back to the absolute value. + headline = `${formatBenchmarkValue(metric, actual)} ${direction} target`; + } + + return { + text, + headline, + sourceShort: source === 'team' ? 'team benchmark' : 'default benchmark', + // CLUSTOX: never 'error'/red -- a missed internal goal isn't a system + // failure, and colouring it like one makes the dashboard punitive. + tone: favourable ? 'good' : 'warn' + }; +}; diff --git a/web-server/src/utils/bitbucketRepos.ts b/web-server/src/utils/bitbucketRepos.ts new file mode 100644 index 000000000..e12e03197 --- /dev/null +++ b/web-server/src/utils/bitbucketRepos.ts @@ -0,0 +1,44 @@ +import { Integration } from '@/constants/integrations'; + +// CLUSTOX: lives here rather than in git_org_repos.ts so its test can import +// it without dragging Endpoint -> next-auth's ESM build into jest -- the same +// wall benchmarks.ts and contributorFilters.ts were written import-light to +// dodge. The route imports from here; one direction, no cycle. +export type AdaptedBitbucketRepo = { + id: string; + name: string; + desc: string | null; + slug: string; + web_url: string | undefined; + branch: string | null; + parent: string; + provider: Integration; +}; + +// The shape must match what the GitHub and GitLab branches return +// key-for-key: the repo-selection UI reads all three providers through one +// code path, so a missing or renamed key surfaces as a blank column in the +// picker, not an error. +export const adaptBitbucketRepo = ( + repo: Record, + org: string +): AdaptedBitbucketRepo => ({ + id: repo.uuid, + name: repo.name, + desc: repo.description ?? null, + slug: repo.slug, + web_url: repo.links?.html?.href, + branch: repo.mainbranch?.name || null, + // The canonical slug from the payload, not the user-typed workspace -- + // typed casing would otherwise propagate into org_name and every API path. + parent: repo.workspace?.slug ?? org, + provider: Integration.BITBUCKET +}); + +// CLUSTOX: the Bitbucket page cursor is an opaque URL the CLIENT sends back. +// Fetched unvalidated, it is an SSRF that carries the org's email:token Basic +// header to any host a workspace member names -- credential exfiltration in +// one request. Prefix-anchored, not includes(): a path like +// evil.example/https://api.bitbucket.org/2.0/ must fail. +export const isBitbucketApiUrl = (url: string): boolean => + url.startsWith('https://api.bitbucket.org/2.0/'); diff --git a/web-server/src/utils/cockpitMetricUtils.ts b/web-server/src/utils/cockpitMetricUtils.ts index ccd500e2a..6a27937b6 100644 --- a/web-server/src/utils/cockpitMetricUtils.ts +++ b/web-server/src/utils/cockpitMetricUtils.ts @@ -11,11 +11,29 @@ import { DeploymentFrequencyTrends, MeanTimeToRestoreApiTrendsResponse, ChangeFailureRateTrendsApiResponse, - UpdatedDeploymentFrequencyAnalyticsResponseV2 + UpdatedDeploymentFrequencyAnalyticsResponseV2, + // CLUSTOX: lines of code. + LOCApiResponse, + LOCTrendsApiResponse } from '@/types/resources'; +import { Benchmarks } from '@/utils/benchmarks'; import { adaptDeploymentFrequencyAndGetBadge } from './adapt_deployment_frequency'; +// CLUSTOX: the resolved benchmark targets for a team -- the team's own row +// falling back to the global baseline, per metric, resolved server-side. +// +// This has to be fetched on the dora_metrics path specifically. The four DORA +// cards read `metrics_summary.benchmarks`, and that slice key is written only +// by `fetchTeamDoraMetrics`, i.e. only by whatever this BFF route sends. A +// benchmark produced anywhere else -- it previously rode on +// /deployment_analytics -- lands in a different slice and no card ever sees +// it. It joins the Promise.all the route already runs, so it costs no extra +// round trip and the targets cannot disagree with the numbers drawn against +// them. +export const fetchTeamBenchmarks = (teamId: ID) => + handleRequest(`/teams/${teamId}/benchmarks`); + export const getFilters = (filtersArray: any[], teamIds: ID[]) => { return teamIds.reduce((objectSoFar, currentTeamId, currentIndex) => { return { @@ -85,6 +103,68 @@ export const fetchLeadTimeStats = async (params: { }; }; +// CLUSTOX: lines of code, the fifth card on the same dashboard. Deliberately +// built as a mirror of `fetchLeadTimeStats` above -- same four calls (current +// and previous aggregate, current and previous trends), same time objects, same +// `pr_filter`. Two consequences that matter: +// +// 1. The previous-period call is what makes the card able to show a delta at +// all. A single aggregate call would leave nothing to compare against and +// the card would silently lose the up/down arrow the other four have. +// 2. Passing the same `prFilter` the lead time metric gets is what makes the +// contributor filter, branch mode and the excluded-PRs setting apply to +// LOC too -- the analytics route already honours `pr_filter`, so this is +// free, and omitting it would give a filtered dashboard one card quietly +// reporting team-wide numbers next to four filtered ones. +// +// The time objects come from the caller already formatted by `isoDateString`, +// which emits an offset (`...+00:00`). That is load-bearing: `/teams//loc` +// rejects a naive timestamp with 400 `Timestamp passed without tz info`. +export const fetchLocStats = async (params: { + teamId: ID; + currStatsTimeObject: IntervalTimeMap; + prevStatsTimeObject: IntervalTimeMap; + prFilter: ReturnType; + currTrendsTimeObject: IntervalTimeMap; + prevTrendsTimeObject: IntervalTimeMap; +}) => { + const { + teamId, + currStatsTimeObject, + prevStatsTimeObject, + prFilter, + currTrendsTimeObject, + prevTrendsTimeObject + } = params; + + const [currLocResponse, prevLocResponse, currLocTrends, prevLocTrends] = + await Promise.all([ + handleRequest(`/teams/${teamId}/loc`, { + params: { ...currStatsTimeObject, ...prFilter } + }), + handleRequest(`/teams/${teamId}/loc`, { + params: { ...prevStatsTimeObject, ...prFilter } + }), + handleRequest(`/teams/${teamId}/loc/trends`, { + params: { ...currTrendsTimeObject, ...prFilter } + }), + handleRequest(`/teams/${teamId}/loc/trends`, { + params: { ...prevTrendsTimeObject, ...prFilter } + }) + ]); + + return { + loc_stats: { + current: currLocResponse, + previous: prevLocResponse + }, + loc_trends: { + current: currLocTrends, + previous: prevLocTrends + } + }; +}; + export const fetchMeanTimeToRestoreStats = async (params: { teamId: ID; currStatsTimeObject: IntervalTimeMap; diff --git a/web-server/src/utils/contributorFilters.ts b/web-server/src/utils/contributorFilters.ts new file mode 100644 index 000000000..cb2e5ae05 --- /dev/null +++ b/web-server/src/utils/contributorFilters.ts @@ -0,0 +1,37 @@ +import { omit } from 'ramda'; + +// CLUSTOX: Change Failure Rate and MTTR have no defensible per-contributor +// definition yet -- the nearest analog would be filtering incidents by +// GitHub issue assignee, which is a weak foundation to build on and unpick +// once Jira-based incident ownership lands. So these two keep every other +// filter (base branches, repo filters, prod-branch workflow config) but +// never see the contributor selection, unlike Lead Time and Deployment +// Frequency. +// +// Lives in its own file, deliberately import-light (just ramda), so the +// "only the contributor keys are stripped, everything else passes through" +// contract can be unit tested without dragging in the dora_metrics route's +// full dependency graph (db, auth, `@/utils/date` -> `@/utils/mock` -> +// `@faker-js/faker`, which is ESM-only and breaks under this repo's Jest +// config). +export const stripContributorFilters = < + PrFilters extends { pr_filter: Record | null }, + WorkflowFilters extends Record | null | undefined +>( + prFilters: PrFilters, + workflowFilters: WorkflowFilters +) => ({ + prFilter: prFilters.pr_filter + ? { pr_filter: omit(['authors'], prFilters.pr_filter) } + : prFilters, + // `event_actors` sits inside the `workflow_filter` blob, because that is the + // level the backend parses it at (ParseWorkflowFilterProcessor.apply reads + // `workflow_filter.get("event_actors")`). So it has to be stripped from + // there too -- stripping the top level would silently do nothing. + workflowFilter: workflowFilters?.workflow_filter + ? { + ...workflowFilters, + workflow_filter: omit(['event_actors'], workflowFilters.workflow_filter) + } + : workflowFilters +}); diff --git a/web-server/src/utils/contributorQuery.ts b/web-server/src/utils/contributorQuery.ts new file mode 100644 index 000000000..599c50517 --- /dev/null +++ b/web-server/src/utils/contributorQuery.ts @@ -0,0 +1,35 @@ +import { uniq } from 'ramda'; + +// CLUSTOX: URL <-> selection plumbing for the DORA contributor filter. +// +// The selection lives in the query string rather than redux, so a filtered +// dashboard survives a refresh and can be pasted to someone else. That makes +// this the only thing standing between a shared or hand-edited link and the +// filter the dashboard actually applies, which is worth testing directly. +// +// Lives here rather than in ContributorFilter.tsx, and deliberately imports +// nothing but ramda, because this repo's Jest config maps `.tsx` through +// ts-jest with Next's `jsx: preserve` -- so importing the component into a +// test fails on the first JSX token. Same reasoning as contributorFilters.ts. + +export const CONTRIBUTOR_QUERY_KEY = 'contributors'; + +/** + * Next types a query value as `string | string[]`: `?contributors=a,b` arrives + * as a string, but a *repeated* `?contributors=a&contributors=b` arrives as an + * array. Both forms mean the same thing to a user, so both are accepted. + */ +export const parseSelectedFromQuery = (raw: string | string[]): string[] => { + const values = Array.isArray(raw) ? raw : raw ? [raw] : []; + return uniq(values.flatMap((value) => value.split(',')).filter(Boolean)); +}; + +/** + * A stable primitive identity for a raw query value, for use as a memo + * dependency. The array form is a fresh object on every render, so depending on + * it directly would invalidate the memo forever and refetch the DORA summary in + * a loop. Collapsing to the same comma-separated string the component writes + * makes the two URL spellings compare equal, as they should. + */ +export const contributorSelectionKey = (raw: string | string[]): string => + Array.isArray(raw) ? raw.join(',') : raw || ''; diff --git a/web-server/src/utils/filterUtils.ts b/web-server/src/utils/filterUtils.ts index f0c8f6b74..87b3757eb 100644 --- a/web-server/src/utils/filterUtils.ts +++ b/web-server/src/utils/filterUtils.ts @@ -84,11 +84,28 @@ export const getWorkFlowFilters = (params: { export const getWorkFlowFiltersAsPayloadForSingleTeam = async (params: { orgId: ID; teamId: ID; + // CLUSTOX: contributor filter. Deployments have no author, so the nearest + // equivalent is the actor who triggered the run. + eventActors?: string[]; }) => { - const { orgId, teamId } = params; + const { orgId, teamId, eventActors } = params; const teamProdBranchesMap = await getAllTeamsReposProdBranchesForOrgAsMap(orgId); - return Object.fromEntries( + const filter = Object.fromEntries( Object.entries(workFlowFiltersFromTeamProdBranches(teamProdBranchesMap)) )[teamId]; + + if (!eventActors?.length) return filter; + // CLUSTOX: `event_actors` belongs *inside* the workflow_filter blob, next to + // `head_branches`. The backend reads it at that level -- see + // ParseWorkflowFilterProcessor.apply in + // mhq/service/workflows/workflow_filter.py, which does + // `workflow_filter.get("event_actors")` on the parsed blob. Hanging it off + // the top level instead makes it a sibling query param, and the + // PREVENT_EXTRA voluptuous schema on /teams//deployment_frequency + // rejects the whole request. + return { + ...filter, + workflow_filter: { ...filter?.workflow_filter, event_actors: eventActors } + }; }; diff --git a/web-server/src/utils/locCard.ts b/web-server/src/utils/locCard.ts new file mode 100644 index 000000000..a17947bbb --- /dev/null +++ b/web-server/src/utils/locCard.ts @@ -0,0 +1,153 @@ +// CLUSTOX: everything the Lines of Code card needs, derived in one pure +// function so the two states that matter can be asserted without rendering +// MUI: `loc_stats` present (real numbers reach the card) and `loc_stats` +// absent (an older backend behind this frontend, which must degrade rather +// than throw). +// +// Kept dependency-light for the same reason `benchmarks.ts` and +// `benchmarkBand.ts` are: `@/utils/date` -- the obvious place to borrow +// `getSortedDatesAsArrayFromMap` from -- chains into `@/utils/mock` -> +// `@faker-js/faker`, whose ESM-only output blows up ts-jest the instant it is +// required. The week sort is reimplemented below rather than imported, and is +// character-for-character the same comparator. +// +// The two `import type` lines are erased at compile time, so this module still +// pulls in nothing at runtime. +import type { LOCApiResponse, LOCTrendsApiResponse } from '@/types/resources'; + +export type LocStats = { + current: LOCApiResponse; + previous: LOCApiResponse; +}; + +export type LocTrends = { + current: LOCTrendsApiResponse; + previous: LOCTrendsApiResponse; +}; + +export type LocCardModel = { + /** + * Whether LOC was measured at all. `false` means the response carried no + * `loc_stats` -- not that the team changed nothing, which arrives as a real + * `{ additions: 0, ... }` and leaves this `true`. + */ + isMeasured: boolean; + additions: number; + deletions: number; + /** additions + deletions over the whole selected range. */ + total: number; + /** + * Gross lines per calendar day -- the card's headline figure. + * + * The headline is a rate rather than `total` because a total is not + * comparable between two ranges of different lengths: the same pace of work + * reads four times "better" over a month than over a week. Computed in the + * backend, where the interval is known, rather than re-derived here from a + * day count the card would have to guess. + */ + avgDaily: number; + /** Average gross lines per merged PR -- the benchmarked figure. */ + avgPrSize: number; + prevTotal: number; + prevAvgDaily: number; + prevAvgPrSize: number; + /** + * The weekly average-PR-size series, oldest week first. + * + * This is the ONLY series the benchmark band may be drawn over. The band's + * `yMin`/`yMax`/`suggestedMax` are in the plotted chart's own data space, so + * pairing a target in lines-per-PR (~200) with a series of weekly totals + * (~70,000) would draw a band that is arithmetically correct and visually a + * hairline on the axis floor -- a plausible wrong answer nobody questions. + */ + avgPrSizeValues: number[]; + /** ISO week keys for `avgPrSizeValues`, same order -- tooltip titles. */ + weeks: string[]; + /** + * Whether average PR size can be compared against a target at all. + * + * False when nothing was merged: `avg_pr_size` is 0 there (the backend + * returns 0 rather than dividing by zero), and 0 beats every lower-is-better + * target, so a team that shipped nothing would be congratulated for it. Same + * trap the Change Failure Rate card hits with no deployments. + */ + canComparePrSize: boolean; +}; + +const EMPTY_MODEL: LocCardModel = { + isMeasured: false, + additions: 0, + deletions: 0, + total: 0, + avgDaily: 0, + avgPrSize: 0, + prevTotal: 0, + prevAvgDaily: 0, + prevAvgPrSize: 0, + avgPrSizeValues: [], + weeks: [], + canComparePrSize: false +}; + +// CLUSTOX: an explicit absence check, never `value || 0`. The four LOC fields +// are always numbers on the wire, so this only ever substitutes for a +// malformed payload -- but `||` would also rewrite a genuine measured 0, which +// is the one value this whole card has to keep distinguishable from "absent". +// Returns 0 for a number that is not finite too, since NaN in the plotted +// series makes chart.js drop the whole y scale (see benchmarkBand.ts). +const num = (value: number | undefined | null): number => + typeof value === 'number' && Number.isFinite(value) ? value : 0; + +const sortedWeeks = ( + trends: LOCTrendsApiResponse | undefined | null +): string[] => + Object.keys(trends || {}).sort( + (a, b) => new Date(a).getTime() - new Date(b).getTime() + ); + +/** + * Builds the Lines of Code card's view model. + * + * Both arguments are optional on `TeamDoraMetricsApiResponseType` -- the BFF + * route degrades them to `undefined` rather than failing the whole response -- + * so every field access here tolerates `undefined` at any depth. + */ +export const buildLocCardModel = ( + locStats: LocStats | undefined | null, + locTrends: LocTrends | undefined | null +): LocCardModel => { + const current = locStats?.current; + + // CLUSTOX: the old-backend case. Returning a fully-formed zero model rather + // than nulls means the card's JSX reads the same fields in both states and + // cannot crash on one of them; `isMeasured` is what the card branches on. + if (current == null) return EMPTY_MODEL; + + const previous = locStats?.previous; + + // CLUSTOX: current and previous trend buckets are keyed by week start and + // cover disjoint periods, so this is a union, not an overwrite -- the same + // `merge(current, previous)` the Lead Time card does, spelled out to keep + // this module import-free. + const mergedTrends = { ...locTrends?.current, ...locTrends?.previous }; + + const total = num(current.total); + const weeks = sortedWeeks(mergedTrends); + + return { + isMeasured: true, + additions: num(current.additions), + deletions: num(current.deletions), + total, + avgDaily: num(current.avg_daily), + avgPrSize: num(current.avg_pr_size), + prevTotal: num(previous?.total), + prevAvgDaily: num(previous?.avg_daily), + prevAvgPrSize: num(previous?.avg_pr_size), + weeks, + avgPrSizeValues: weeks.map((week) => num(mergedTrends[week]?.avg_pr_size)), + // CLUSTOX: `> 0` is a real numeric predicate here, not an absence check -- + // zero lines changed genuinely means there is no PR size to benchmark. + canComparePrSize: total > 0 + }; +}; diff --git a/web-server/src/utils/mailer.ts b/web-server/src/utils/mailer.ts new file mode 100644 index 000000000..3c8254031 --- /dev/null +++ b/web-server/src/utils/mailer.ts @@ -0,0 +1,122 @@ +import nodemailer, { Transporter } from 'nodemailer'; + +/** + * SMTP is entirely optional. A self-hosted instance that hasn't configured + * it should keep working exactly as before -- invite links still get + * created and shown for manual copy/send, they just don't also get + * emailed. Callers check `mailerConfigured()` and treat a missing send as + * non-fatal, never as a reason to fail invite creation. + */ +export const mailerConfigured = () => Boolean(process.env.SMTP_HOST); + +let transporter: Transporter | null = null; + +const getTransporter = () => { + if (transporter) return transporter; + + // Port 465 is implicit TLS; anything else (587, 25) starts unencrypted + // and upgrades via STARTTLS, which nodemailer does automatically when + // `secure` is false. + const port = Number(process.env.SMTP_PORT) || 587; + + transporter = nodemailer.createTransport({ + host: process.env.SMTP_HOST, + port, + secure: port === 465, + auth: process.env.SMTP_USER + ? { user: process.env.SMTP_USER, pass: process.env.SMTP_PASSWORD } + : undefined + }); + + return transporter; +}; + +export type InviteEmailInput = { + to: string; + name: string; + role: 'SUPERADMIN' | 'ADMIN'; + orgName: string | null; + inviteUrl: string; + expiresAt: Date; +}; + +const escapeHtml = (s: string) => + s.replace( + /[&<>"']/g, + (c) => + ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' })[ + c + ] as string + ); + +const buildHtml = ({ + name, + role, + orgName, + inviteUrl, + expiresAt +}: InviteEmailInput) => { + const roleLine = + role === 'SUPERADMIN' + ? 'a superadmin, able to see every workspace and manage users' + : orgName + ? `an admin of the "${escapeHtml(orgName)}" workspace` + : 'an admin, with your own new workspace'; + + return ` +
+
+ + Middleware +
+

You've been invited, ${escapeHtml( + name + )}

+

+ You'll be ${roleLine}. Follow the link below to choose a password and + finish setting up your account. +

+

+ + Accept invitation + +

+

+ This link expires on ${expiresAt.toUTCString()} and can only be used + once. If you weren't expecting this, you can ignore it. +

+
+ `; +}; + +const buildText = ({ name, inviteUrl, expiresAt }: InviteEmailInput) => + `Hi ${name},\n\n` + + `You've been invited to Middleware. Set up your account here:\n${inviteUrl}\n\n` + + `This link expires on ${expiresAt.toUTCString()} and can only be used once.\n`; + +/** + * Best-effort. Returns whether the send actually succeeded so callers can + * surface that to the UI -- but a failure here is never a reason to undo + * invite creation; the link itself is already valid and copyable + * regardless of whether the email went out. + */ +export const sendInviteEmail = async ( + input: InviteEmailInput +): Promise<{ sent: boolean; error?: string }> => { + if (!mailerConfigured()) return { sent: false, error: 'SMTP not configured' }; + + try { + await getTransporter().sendMail({ + from: process.env.SMTP_FROM || process.env.SMTP_USER, + to: input.to, + subject: "You've been invited to Middleware", + text: buildText(input), + html: buildHtml(input) + }); + return { sent: true }; + } catch (err) { + // Logged, not thrown -- see module comment on why this stays non-fatal. + console.error('[mailer] failed to send invite email', err); + return { sent: false, error: (err as Error).message }; + } +}; diff --git a/web-server/src/utils/testUtils.tsx b/web-server/src/utils/testUtils.tsx new file mode 100644 index 000000000..101598544 --- /dev/null +++ b/web-server/src/utils/testUtils.tsx @@ -0,0 +1,17 @@ +import { render, RenderOptions } from '@testing-library/react'; +import { ReactElement } from 'react'; + +import ThemeProviderWrapper from '@/theme/ThemeProvider'; + +// CLUSTOX: the app always renders inside this ThemeProvider (see +// pages/_app.tsx) -- it's what supplies theme.colors, theme.header, etc. on +// top of bare MUI, which several components reach for even indirectly (a +// FlexBox's `title` prop renders a MUI Tooltip via Shared.tsx's +// TooltipWrapper, which reads theme.colors.alpha). Without this wrapper, +// useTheme() falls back to MUI's bare default theme and anything touching +// those custom fields throws. First hit by JiraIntegrationCard's Linked +// badge. +export const renderWithTheme = ( + ui: ReactElement, + options?: Omit +) => render(ui, { wrapper: ThemeProviderWrapper, ...options }); diff --git a/web-server/yarn.lock b/web-server/yarn.lock index 3f4bc1ae5..0e906942c 100644 --- a/web-server/yarn.lock +++ b/web-server/yarn.lock @@ -7,6 +7,11 @@ resolved "https://registry.yarnpkg.com/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz#bd9154aec9983f77b3a034ecaa015c2e4201f6cf" integrity sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA== +"@adobe/css-tools@^4.4.0": + version "4.5.0" + resolved "https://registry.yarnpkg.com/@adobe/css-tools/-/css-tools-4.5.0.tgz#b5b71a25a4d16afa2482592ddfa62fccc60bc7d1" + integrity sha512-6OzddxPio9UiWTCemp4N8cYLV2ZN1ncRnV1cVGtve7dhPOtRkleRyx32GQCYSwDYgaHU3USMm84tNsvKzRCa1Q== + "@ampproject/remapping@^2.2.0": version "2.2.1" resolved "https://registry.yarnpkg.com/@ampproject/remapping/-/remapping-2.2.1.tgz#99e8e11851128b8702cd57c33684f1d0f260b630" @@ -23,6 +28,15 @@ "@babel/highlight" "^7.22.13" chalk "^2.4.2" +"@babel/code-frame@^7.10.4": + version "7.29.7" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.29.7.tgz#f2fbbfea87c44a21590ec515b778b2c26d8866e7" + integrity sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw== + dependencies: + "@babel/helper-validator-identifier" "^7.29.7" + js-tokens "^4.0.0" + picocolors "^1.1.1" + "@babel/code-frame@^7.25.7": version "7.25.7" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.25.7.tgz#438f2c524071531d643c6f0188e1e28f130cebc7" @@ -263,6 +277,11 @@ resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.7.tgz#77b7f60c40b15c97df735b38a66ba1d7c3e93da5" integrity sha512-AM6TzwYqGChO45oiuPqwL2t20/HdMC1rTPAesnBCgPCSF1x3oN9MVUwQV2iyz4xqWrctwK5RNC8LV22kaQCNYg== +"@babel/helper-validator-identifier@^7.29.7": + version "7.29.7" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz#bd87084ced0c796ec46bda492de6e83d29e89fc2" + integrity sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg== + "@babel/helper-validator-option@^7.16.7", "@babel/helper-validator-option@^7.22.15": version "7.22.15" resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.22.15.tgz#694c30dfa1d09a6534cdfcafbe56789d36aba040" @@ -2468,6 +2487,46 @@ dependencies: tslib "^2.4.0" +"@testing-library/dom@^9.0.0": + version "9.3.4" + resolved "https://registry.yarnpkg.com/@testing-library/dom/-/dom-9.3.4.tgz#50696ec28376926fec0a1bf87d9dbac5e27f60ce" + integrity sha512-FlS4ZWlp97iiNWig0Muq8p+3rVDjRiYE+YKGbAqXOu9nwJFFOdL00kFpz42M+4huzYi86vAK1sOOfyOG45muIQ== + dependencies: + "@babel/code-frame" "^7.10.4" + "@babel/runtime" "^7.12.5" + "@types/aria-query" "^5.0.1" + aria-query "5.1.3" + chalk "^4.1.0" + dom-accessibility-api "^0.5.9" + lz-string "^1.5.0" + pretty-format "^27.0.2" + +"@testing-library/jest-dom@^6.4.2": + version "6.10.0" + resolved "https://registry.yarnpkg.com/@testing-library/jest-dom/-/jest-dom-6.10.0.tgz#8a76841e94b72d55d09d2a34b9db9d75da9cbc08" + integrity sha512-HQwu0KaB2zyT0iLzBL+8CLyZDL3KlZlZJ+2iyc9uCUnlJVskJU/UlPuVCyIPhtukjPQdT2QNoR5nCP5FqTmmDQ== + dependencies: + "@adobe/css-tools" "^4.4.0" + aria-query "^5.0.0" + css.escape "^1.5.1" + dom-accessibility-api "^0.6.3" + picocolors "^1.1.1" + redent "^3.0.0" + +"@testing-library/react@^14.2.1": + version "14.3.1" + resolved "https://registry.yarnpkg.com/@testing-library/react/-/react-14.3.1.tgz#29513fc3770d6fb75245c4e1245c470e4ffdd830" + integrity sha512-H99XjUhWQw0lTgyMN05W3xQG1Nh4lq574D8keFf1dDoNTJgp66VbJozRaczoF+wsiaPJNt/TcnfpLGufGxSrZQ== + dependencies: + "@babel/runtime" "^7.12.5" + "@testing-library/dom" "^9.0.0" + "@types/react-dom" "^18.0.0" + +"@testing-library/user-event@^14.5.2": + version "14.6.3" + resolved "https://registry.yarnpkg.com/@testing-library/user-event/-/user-event-14.6.3.tgz#360bd2e2f496ba738dfd59f114368dc479199a7f" + integrity sha512-6dBq67jT8lE+JTE8Exm02Kt6ze43hz1jdiSpSJwtTZiT1xQQ6b7nZYTTQ9njdArdU8XklOwaDp/AbT/eYSKF4g== + "@tootallnate/once@1": version "1.1.2" resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" @@ -2483,6 +2542,11 @@ resolved "https://registry.yarnpkg.com/@trysound/sax/-/sax-0.2.0.tgz#cccaab758af56761eb7bf37af6f03f326dd798ad" integrity sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA== +"@types/aria-query@^5.0.1": + version "5.0.4" + resolved "https://registry.yarnpkg.com/@types/aria-query/-/aria-query-5.0.4.tgz#1a31c3d378850d2778dabb6374d036dcba4ba708" + integrity sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw== + "@types/babel__core@^7.1.14": version "7.20.1" resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.20.1.tgz#916ecea274b0c776fec721e333e55762d3a9614b" @@ -2516,6 +2580,13 @@ dependencies: "@babel/types" "^7.20.7" +"@types/bcryptjs@^3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@types/bcryptjs/-/bcryptjs-3.0.0.tgz#d7be11653aa82cf17ffee4f3925f1f80cfc1add2" + integrity sha512-WRZOuCuaz8UcZZE4R5HXTco2goQSI2XxjGY3hbM/xDvwmqFWd4ivooImsMx65OKM6CtNKbnZ5YL+YwAwK7c1dg== + dependencies: + bcryptjs "*" + "@types/debug@^4.0.0": version "4.1.12" resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.12.tgz#a155f21690871953410df4b6b6f53187f0500917" @@ -2635,6 +2706,13 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-16.18.96.tgz#eb0012d23ff53d14d64ec8a352bf89792de6aade" integrity sha512-84iSqGXoO+Ha16j8pRZ/L90vDMKX04QTYMTfYeE1WrjWaZXuchBehGUZEpNgx7JnmlrIHdnABmpjrQjhCnNldQ== +"@types/nodemailer@^8.0.1": + version "8.0.1" + resolved "https://registry.yarnpkg.com/@types/nodemailer/-/nodemailer-8.0.1.tgz#4ef2b4e62c819225a0b8a6384bf750c0c664d465" + integrity sha512-PxpaInm8V1JQDd4j0ds5HfvWQk8JupS1C0Picb96QJsrrRDjBH+DlK7L4ZdNSqNULhiZRQHc40nLVShaGxXAMw== + dependencies: + "@types/node" "*" + "@types/nprogress@0.2.0": version "0.2.0" resolved "https://registry.yarnpkg.com/@types/nprogress/-/nprogress-0.2.0.tgz#86c593682d4199212a0509cc3c4d562bbbd6e45f" @@ -2676,6 +2754,11 @@ dependencies: "@types/react" "^17" +"@types/react-dom@^18.0.0": + version "18.3.7" + resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-18.3.7.tgz#b89ddf2cd83b4feafcc4e2ea41afdfb95a0d194f" + integrity sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ== + "@types/react-redux@7.1.20": version "7.1.20" resolved "https://registry.yarnpkg.com/@types/react-redux/-/react-redux-7.1.20.tgz#42f0e61ababb621e12c66c96dda94c58423bd7df" @@ -3070,7 +3153,14 @@ argparse@^2.0.1: resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== -aria-query@^5.3.2: +aria-query@5.1.3: + version "5.1.3" + resolved "https://registry.yarnpkg.com/aria-query/-/aria-query-5.1.3.tgz#19db27cd101152773631396f7a95a3b58c22c35e" + integrity sha512-R5iJ5lkuHybztUfuOAznmboyjWq8O6sqNqtK7CLOqdydi54VNbORp49mb14KbWgG1QD3JFO9hJdZ+y4KutfdOQ== + dependencies: + deep-equal "^2.0.5" + +aria-query@^5.0.0, aria-query@^5.3.2: version "5.3.2" resolved "https://registry.yarnpkg.com/aria-query/-/aria-query-5.3.2.tgz#93f81a43480e33a338f19163a3d10a50c01dcd59" integrity sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw== @@ -3413,6 +3503,11 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== +bcryptjs@*, bcryptjs@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/bcryptjs/-/bcryptjs-3.0.3.tgz#4b93d6a398c48bfc9f32ee65d301174a8a8ea56f" + integrity sha512-GlF5wPWnSa/X5LKM1o0wz0suXIINz1iHRLvTS+sLyi7XPbe5ycmYI3DlZqVGZZtDgl4DmasFg7gOB3JYbphV5g== + before-after-hook@^2.2.0: version "2.2.2" resolved "https://registry.yarnpkg.com/before-after-hook/-/before-after-hook-2.2.2.tgz#a6e8ca41028d90ee2c24222f201c90956091613e" @@ -3511,6 +3606,14 @@ bytes@3.1.2: resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5" integrity sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg== +call-bind-apply-helpers@^1.0.1, call-bind-apply-helpers@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz#4b5428c222be985d79c3d82657479dbe0b59b2d6" + integrity sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ== + dependencies: + es-errors "^1.3.0" + function-bind "^1.1.2" + call-bind@^1.0.0, call-bind@^1.0.2, call-bind@^1.0.4, call-bind@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.5.tgz#6fa2b7845ce0ea49bf4d8b9ef64727a2c2e2e513" @@ -3531,6 +3634,14 @@ call-bind@^1.0.6, call-bind@^1.0.7: get-intrinsic "^1.2.4" set-function-length "^1.2.1" +call-bound@^1.0.2: + version "1.0.4" + resolved "https://registry.yarnpkg.com/call-bound/-/call-bound-1.0.4.tgz#238de935d2a2a692928c538c7ccfa91067fd062a" + integrity sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg== + dependencies: + call-bind-apply-helpers "^1.0.2" + get-intrinsic "^1.3.0" + callsites@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" @@ -3922,6 +4033,11 @@ css-what@^6.0.1: resolved "https://registry.yarnpkg.com/css-what/-/css-what-6.1.0.tgz#fb5effcf76f1ddea2c81bdfaa4de44e79bac70f4" integrity sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw== +css.escape@^1.5.1: + version "1.5.1" + resolved "https://registry.yarnpkg.com/css.escape/-/css.escape-1.5.1.tgz#42e27d4fa04ae32f931a4b4d4191fa9cddee97cb" + integrity sha512-YUifsXXuknHlUsmlgyY0PKzgPOr7/FjCePfHNt0jxm83wHZi44VDMQ7/fGNkjY3/jV1MC+1CmZbaHzugyeRtpg== + csso@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/csso/-/csso-4.2.0.tgz#ea3a561346e8dc9f546d6febedd50187cf389529" @@ -4136,6 +4252,30 @@ dedent@^1.0.0: resolved "https://registry.yarnpkg.com/dedent/-/dedent-1.5.1.tgz#4f3fc94c8b711e9bb2800d185cd6ad20f2a90aff" integrity sha512-+LxW+KLWxu3HW3M2w2ympwtqPrqYRzU8fqi6Fhd18fBALe15blJPI/I4+UHveMVG6lJqB4JNd4UG0S5cnVHwIg== +deep-equal@^2.0.5: + version "2.2.3" + resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-2.2.3.tgz#af89dafb23a396c7da3e862abc0be27cf51d56e1" + integrity sha512-ZIwpnevOurS8bpT4192sqAowWM76JDKSHYzMLty3BZGSswgq6pBaH3DhCSW5xVAZICZyKdOBPjwww5wfgT/6PA== + dependencies: + array-buffer-byte-length "^1.0.0" + call-bind "^1.0.5" + es-get-iterator "^1.1.3" + get-intrinsic "^1.2.2" + is-arguments "^1.1.1" + is-array-buffer "^3.0.2" + is-date-object "^1.0.5" + is-regex "^1.1.4" + is-shared-array-buffer "^1.0.2" + isarray "^2.0.5" + object-is "^1.1.5" + object-keys "^1.1.1" + object.assign "^4.1.4" + regexp.prototype.flags "^1.5.1" + side-channel "^1.0.4" + which-boxed-primitive "^1.0.2" + which-collection "^1.0.1" + which-typed-array "^1.1.13" + deep-is@^0.1.3, deep-is@~0.1.3: version "0.1.4" resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.4.tgz#a6f2dce612fadd2ef1f519b73551f17e85199831" @@ -4256,6 +4396,16 @@ doctrine@^3.0.0: dependencies: esutils "^2.0.2" +dom-accessibility-api@^0.5.9: + version "0.5.16" + resolved "https://registry.yarnpkg.com/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz#5a7429e6066eb3664d911e33fb0e45de8eb08453" + integrity sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg== + +dom-accessibility-api@^0.6.3: + version "0.6.3" + resolved "https://registry.yarnpkg.com/dom-accessibility-api/-/dom-accessibility-api-0.6.3.tgz#993e925cc1d73f2c662e7d75dd5a5445259a8fd8" + integrity sha512-7ZgogeTnjuHbo+ct10G9Ffp0mif17idi0IyWNVA/wcwcm7NPOD/WEHVP3n7n3MhXqxoIYm8d6MuZohYWIZ4T3w== + dom-css@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/dom-css/-/dom-css-2.1.0.tgz#fdbc2d5a015d0a3e1872e11472bbd0e7b9e6a202" @@ -4315,6 +4465,15 @@ dotenv@^16.0.3: resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.0.3.tgz#115aec42bac5053db3c456db30cc243a5a836a07" integrity sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ== +dunder-proto@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/dunder-proto/-/dunder-proto-1.0.1.tgz#d7ae667e1dc83482f8b70fd0f6eefc50da30f58a" + integrity sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A== + dependencies: + call-bind-apply-helpers "^1.0.1" + es-errors "^1.3.0" + gopd "^1.2.0" + duplexer2@^0.1.2: version "0.1.4" resolved "https://registry.yarnpkg.com/duplexer2/-/duplexer2-0.1.4.tgz#8b12dab878c0d69e3e7891051662a32fc6bddcc1" @@ -4500,11 +4659,31 @@ es-define-property@^1.0.0: dependencies: get-intrinsic "^1.2.4" +es-define-property@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/es-define-property/-/es-define-property-1.0.1.tgz#983eb2f9a6724e9303f61addf011c72e09e0b0fa" + integrity sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g== + es-errors@^1.2.1, es-errors@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/es-errors/-/es-errors-1.3.0.tgz#05f75a25dab98e4fb1dcd5e1472c0546d5057c8f" integrity sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw== +es-get-iterator@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/es-get-iterator/-/es-get-iterator-1.1.3.tgz#3ef87523c5d464d41084b2c3c9c214f1199763d6" + integrity sha512-sPZmqHBe6JIiTfN5q2pEi//TwxmAFHwj/XEuYjTuse78i8KxaqMTTzxPoFKuzRpDpTJ+0NAbpfenkmH2rePtuw== + dependencies: + call-bind "^1.0.2" + get-intrinsic "^1.1.3" + has-symbols "^1.0.3" + is-arguments "^1.1.1" + is-map "^2.0.2" + is-set "^2.0.2" + is-string "^1.0.7" + isarray "^2.0.5" + stop-iteration-iterator "^1.0.0" + es-iterator-helpers@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/es-iterator-helpers/-/es-iterator-helpers-1.1.0.tgz#f6d745d342aea214fe09497e7152170dc333a7a6" @@ -4532,6 +4711,13 @@ es-object-atoms@^1.0.0: dependencies: es-errors "^1.3.0" +es-object-atoms@^1.1.1: + version "1.1.2" + resolved "https://registry.yarnpkg.com/es-object-atoms/-/es-object-atoms-1.1.2.tgz#a2d0b373205724dfa525d23b0c3e1b1ca582c99b" + integrity sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw== + dependencies: + es-errors "^1.3.0" + es-set-tostringtag@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/es-set-tostringtag/-/es-set-tostringtag-2.0.1.tgz#338d502f6f674301d710b80c8592de8a15f09cd8" @@ -5219,11 +5405,35 @@ get-intrinsic@^1.2.3, get-intrinsic@^1.2.4: has-symbols "^1.0.3" hasown "^2.0.0" +get-intrinsic@^1.2.5, get-intrinsic@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.3.0.tgz#743f0e3b6964a93a5491ed1bffaae054d7f98d01" + integrity sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ== + dependencies: + call-bind-apply-helpers "^1.0.2" + es-define-property "^1.0.1" + es-errors "^1.3.0" + es-object-atoms "^1.1.1" + function-bind "^1.1.2" + get-proto "^1.0.1" + gopd "^1.2.0" + has-symbols "^1.1.0" + hasown "^2.0.2" + math-intrinsics "^1.1.0" + get-package-type@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a" integrity sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q== +get-proto@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/get-proto/-/get-proto-1.0.1.tgz#150b3f2743869ef3e851ec0c49d15b1d14d00ee1" + integrity sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g== + dependencies: + dunder-proto "^1.0.1" + es-object-atoms "^1.0.0" + get-stream@^6.0.0: version "6.0.1" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-6.0.1.tgz#a262d8eef67aced57c2852ad6167526a43cbf7b7" @@ -5376,6 +5586,11 @@ gopd@^1.0.1: dependencies: get-intrinsic "^1.1.3" +gopd@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.2.0.tgz#89f56b8217bdbc8802bd299df6d7f1081d7e51a1" + integrity sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg== + graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.4, graceful-fs@^4.2.9: version "4.2.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" @@ -5442,6 +5657,11 @@ has-symbols@^1.0.2, has-symbols@^1.0.3: resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8" integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== +has-symbols@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.1.0.tgz#fc9c6a783a084951d0b971fe1018de813707a338" + integrity sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ== + has-tostringtag@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/has-tostringtag/-/has-tostringtag-1.0.0.tgz#7e133818a7d394734f941e73c3d3f9291e658b25" @@ -5642,6 +5862,11 @@ imurmurhash@^0.1.4: resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" integrity sha1-khi5srkoojixPcT7a21XbyMUU+o= +indent-string@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-4.0.0.tgz#624f8f4497d619b2d9768531d58f4122854d7251" + integrity sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg== + inflight@^1.0.4: version "1.0.6" resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" @@ -5683,6 +5908,15 @@ internal-slot@^1.0.7: hasown "^2.0.0" side-channel "^1.0.4" +internal-slot@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/internal-slot/-/internal-slot-1.1.0.tgz#1eac91762947d2f7056bc838d93e13b2e9604961" + integrity sha512-4gd7VpWNQNB4UKKCFFVcp1AVv+FMOgs9NKzjHKusc8jTMhd5eL1NqQqOpE0KzMds804/yHlglp3uxgluOqAPLw== + dependencies: + es-errors "^1.3.0" + hasown "^2.0.2" + side-channel "^1.1.0" + internmap@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/internmap/-/internmap-1.0.1.tgz#0017cc8a3b99605f0302f2b198d272e015e5df95" @@ -5716,6 +5950,14 @@ is-alphanumerical@^2.0.0: is-alphabetical "^2.0.0" is-decimal "^2.0.0" +is-arguments@^1.1.1: + version "1.2.0" + resolved "https://registry.yarnpkg.com/is-arguments/-/is-arguments-1.2.0.tgz#ad58c6aecf563b78ef2bf04df540da8f5d7d8e1b" + integrity sha512-7bVbi0huj/wrIAOzb8U1aszg9kdi3KN/CyU19CTI7tAoZYEZoL9yCDXpbXN+uPsuWnP02cyug1gleqq+TU+YCA== + dependencies: + call-bound "^1.0.2" + has-tostringtag "^1.0.2" + is-array-buffer@^3.0.1, is-array-buffer@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/is-array-buffer/-/is-array-buffer-3.0.2.tgz#f2653ced8412081638ecb0ebbd0c41c6e0aecbbe" @@ -5866,6 +6108,11 @@ is-map@^2.0.1: resolved "https://registry.yarnpkg.com/is-map/-/is-map-2.0.2.tgz#00922db8c9bf73e81b7a335827bc2a43f2b91127" integrity sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg== +is-map@^2.0.2: + version "2.0.3" + resolved "https://registry.yarnpkg.com/is-map/-/is-map-2.0.3.tgz#ede96b7fe1e270b3c4465e3a465658764926d62e" + integrity sha512-1Qed0/Hr2m+YqxnM09CjA2d/i6YZNfF6R2oRAOj36eUdS6qIV/huPJNSEpKbupewFs+ZsJlxsjjPbc0/afW6Lw== + is-negative-zero@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.2.tgz#7bf6f03a28003b8b3965de3ac26f664d765f3150" @@ -5926,6 +6173,11 @@ is-set@^2.0.1: resolved "https://registry.yarnpkg.com/is-set/-/is-set-2.0.2.tgz#90755fa4c2562dc1c5d4024760d6119b94ca18ec" integrity sha512-+2cnTEZeY5z/iXGbLhPrOAaK/Mau5k5eXq9j14CpRTftq0pAJu2MwVRSZhyZWBzx3o6X795Lz6Bpb6R0GKf37g== +is-set@^2.0.2: + version "2.0.3" + resolved "https://registry.yarnpkg.com/is-set/-/is-set-2.0.3.tgz#8ab209ea424608141372ded6e0cb200ef1d9d01d" + integrity sha512-iPAjerrse27/ygGLxw+EBR9agv9Y6uLeYVJMu+QNCoouJ1/1ri0mGrcWpfCqFZuzzx3WjtwxG098X+n4OuRkPg== + is-shared-array-buffer@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz#8f259c573b60b6a32d4058a1a07430c0a7344c79" @@ -6815,6 +7067,11 @@ lru-cache@^6.0.0: dependencies: yallist "^4.0.0" +lz-string@^1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.5.0.tgz#c1ab50f77887b712621201ba9fd4e3a6ed099941" + integrity sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ== + make-dir@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-4.0.0.tgz#c3c2307a771277cd9638305f915c29ae741b614e" @@ -6839,6 +7096,11 @@ markdown-table@^3.0.0: resolved "https://registry.yarnpkg.com/markdown-table/-/markdown-table-3.0.3.tgz#e6331d30e493127e031dd385488b5bd326e4a6bd" integrity sha512-Z1NL3Tb1M9wH4XESsCDEksWoKTdlUafKc4pt0GRwjUyXaCFZ+dc3g2erqB6zm3szA2IUSi7VnPI+o/9jnxh9hw== +math-intrinsics@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/math-intrinsics/-/math-intrinsics-1.1.0.tgz#a0dd74be81e2aa5c2f27e65ce283605ee4e2b7f9" + integrity sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g== + mdast-util-find-and-replace@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.1.tgz#a6fc7b62f0994e973490e45262e4bc07607b04e0" @@ -7332,6 +7594,11 @@ mimic-fn@^2.1.0: resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== +min-indent@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/min-indent/-/min-indent-1.0.1.tgz#a63f681673b30571fbe8bc25686ae746eefa9869" + integrity sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg== + minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" @@ -7486,6 +7753,11 @@ node-releases@^2.0.13: resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.13.tgz#d5ed1627c23e3461e819b02e57b75e4899b1c81d" integrity sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ== +nodemailer@^9.0.5: + version "9.0.5" + resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-9.0.5.tgz#edf488e2d25b1688614922d7e733f551114099fc" + integrity sha512-wvjiKvjczmsN7U/8006JOdXubgBk2XFAbioDMbT+sM7cPs0QrhJTa6KBRX7P5REGGkDcLUz/EarWidb8G8C1jQ== + normalize-path@^3.0.0, normalize-path@~3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65" @@ -7551,6 +7823,19 @@ object-inspect@^1.13.1, object-inspect@^1.9.0: resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.13.1.tgz#b96c6109324ccfef6b12216a956ca4dc2ff94bc2" integrity sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ== +object-inspect@^1.13.3, object-inspect@^1.13.4: + version "1.13.4" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.13.4.tgz#8375265e21bc20d0fa582c22e1b13485d6e00213" + integrity sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew== + +object-is@^1.1.5: + version "1.1.6" + resolved "https://registry.yarnpkg.com/object-is/-/object-is-1.1.6.tgz#1a6a53aed2dd8f7e6775ff870bea58545956ab07" + integrity sha512-F8cZ+KfGlSGi09lJT7/Nd6KJZ9ygtvYC0/UYYLI9nmQKLMnydpB9yvbv9K1uSkEu7FU9vYPmVwLg328tX+ot3Q== + dependencies: + call-bind "^1.0.7" + define-properties "^1.2.1" + object-keys@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e" @@ -7909,6 +8194,11 @@ picocolors@^1.0.0: resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ== +picocolors@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.1.1.tgz#3d321af3eab939b083c8f929a1d12cda81c26b6b" + integrity sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA== + picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.3, picomatch@^2.3.1: version "2.3.1" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" @@ -8115,6 +8405,15 @@ prettier@^3.0.3: resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.0.3.tgz#432a51f7ba422d1469096c0fdc28e235db8f9643" integrity sha512-L/4pUDMxcNa8R/EthV08Zt42WBO4h1rarVtK0K+QJG0X187OLo7l699jWw0GKuwzkPQ//jMFA/8Xm6Fh3J/DAg== +pretty-format@^27.0.2: + version "27.5.1" + resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-27.5.1.tgz#2181879fdea51a7a5851fb39d920faa63f01d88e" + integrity sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ== + dependencies: + ansi-regex "^5.0.1" + ansi-styles "^5.0.0" + react-is "^17.0.1" + pretty-format@^29.0.0, pretty-format@^29.6.3, pretty-format@^29.7.0: version "29.7.0" resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-29.7.0.tgz#ca42c758310f365bfa71a0bda0a807160b776812" @@ -8288,7 +8587,7 @@ react-is@^16.13.1, react-is@^16.7.0: resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.13.1.tgz#789729a4dc36de2999dc156dd6c1d9c18cea56a4" integrity sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ== -react-is@^17.0.2: +react-is@^17.0.1, react-is@^17.0.2: version "17.0.2" resolved "https://registry.yarnpkg.com/react-is/-/react-is-17.0.2.tgz#e691d4a8e9c789365655539ab372762b0efb54f0" integrity sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w== @@ -8402,6 +8701,14 @@ rechoir@^0.8.0: dependencies: resolve "^1.20.0" +redent@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/redent/-/redent-3.0.0.tgz#e557b7998316bb53c9f1f56fa626352c6963059f" + integrity sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg== + dependencies: + indent-string "^4.0.0" + strip-indent "^3.0.0" + redux-persist@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/redux-persist/-/redux-persist-6.0.0.tgz#b4d2972f9859597c130d40d4b146fecdab51b3a8" @@ -8850,6 +9157,35 @@ shimmer@^1.1.0, shimmer@^1.2.0: resolved "https://registry.yarnpkg.com/shimmer/-/shimmer-1.2.1.tgz#610859f7de327b587efebf501fb43117f9aff337" integrity sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw== +side-channel-list@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/side-channel-list/-/side-channel-list-1.0.1.tgz#c2e0b5a14a540aebee3bbc6c3f8666cc9b509127" + integrity sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w== + dependencies: + es-errors "^1.3.0" + object-inspect "^1.13.4" + +side-channel-map@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/side-channel-map/-/side-channel-map-1.0.1.tgz#d6bb6b37902c6fef5174e5f533fab4c732a26f42" + integrity sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA== + dependencies: + call-bound "^1.0.2" + es-errors "^1.3.0" + get-intrinsic "^1.2.5" + object-inspect "^1.13.3" + +side-channel-weakmap@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz#11dda19d5368e40ce9ec2bdc1fb0ecbc0790ecea" + integrity sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A== + dependencies: + call-bound "^1.0.2" + es-errors "^1.3.0" + get-intrinsic "^1.2.5" + object-inspect "^1.13.3" + side-channel-map "^1.0.1" + side-channel@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf" @@ -8869,6 +9205,17 @@ side-channel@^1.0.6: get-intrinsic "^1.2.4" object-inspect "^1.13.1" +side-channel@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.1.1.tgz#ea02c62e05dc4bea67d4442f0fb71ee192f8e0ab" + integrity sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ== + dependencies: + es-errors "^1.3.0" + object-inspect "^1.13.4" + side-channel-list "^1.0.1" + side-channel-map "^1.0.1" + side-channel-weakmap "^1.0.2" + signal-exit@^3.0.3, signal-exit@^3.0.7: version "3.0.7" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" @@ -9000,6 +9347,14 @@ statuses@2.0.1: resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63" integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== +stop-iteration-iterator@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/stop-iteration-iterator/-/stop-iteration-iterator-1.1.0.tgz#f481ff70a548f6124d0312c3aa14cbfa7aa542ad" + integrity sha512-eLoXW/DHyl62zxY4SCaIgnRhuMr6ri4juEYARS8E6sCEqzKpOiE521Ucofdx+KnDZl5xmvGYaaKCk5FEOxJCoQ== + dependencies: + es-errors "^1.3.0" + internal-slot "^1.1.0" + streamsearch@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-1.1.0.tgz#404dd1e2247ca94af554e841a8ef0eaa238da764" @@ -9154,6 +9509,13 @@ strip-final-newline@^2.0.0: resolved "https://registry.yarnpkg.com/strip-final-newline/-/strip-final-newline-2.0.0.tgz#89b852fb2fcbe936f6f4b3187afb0a12c1ab58ad" integrity sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA== +strip-indent@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/strip-indent/-/strip-indent-3.0.0.tgz#c32e1cee940b6b3432c771bc2c54bcce73cd3001" + integrity sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ== + dependencies: + min-indent "^1.0.0" + strip-json-comments@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006"