From f293eff19f25588ac4d2d222476c31bfc1552b80 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:01:18 +0400 Subject: [PATCH 01/12] Exempt the rules tool's unittest tests from pytest-style rules The shared rules tool needs only the standard library, so its tests use unittest on purpose. Registered as api.quality.unittest-ruff-exemptions. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 1 + docs/generated/rules.md | 5 +++++ pyproject.toml | 3 +++ rules/registry.toml | 19 +++++++++++++++++++ 4 files changed, 28 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 338132dd5..58059c9d8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -126,6 +126,7 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`. - `api.quality.runtime-evaluated-models-resolve`: Every package model is one ruff treats as runtime-evaluated (a base resolves, through the import in its own module, to a name in runtime-evaluated-base-classes, or is a same-module class that does), and every Pydantic model and SQLAlchemy mapper resolves its annotations once the package is imported. (blocking: pytest `tests/unit/test_runtime_evaluated_annotations.py`) - `api.quality.tests-pass`: The unit, contract and to-refactor test suites pass. (blocking: CI job `test`) - `api.quality.tests-ruff-exemptions`: Test files may use pytest's bare assert, need no docstrings, annotations or named constants for magic numbers, and may reach into private members, live outside packages and import inside a test (S101, D1, ANN, PLR2004, SLF001, INP001, PLC0415); every other ruff rule applies to tests. (blocking: ruff per-file ignore of ANN, D1, INP001, PLC0415, PLR2004, S101, SLF001 for `**/tests/**`) +- `api.quality.unittest-ruff-exemptions`: The shared rules tool's tests use unittest, because the tool needs only the standard library, so ruff's pytest-style rules (PT) don't apply to them. (blocking: ruff per-file ignore of PT for `scripts/rules/tests/**`) ### Observability diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 93914ea39..6c9f6b261 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -273,6 +273,7 @@ flowchart LR | `api.quality.runtime-evaluated-models-resolve` | Every package model is one ruff treats as runtime-evaluated (a base resolves, through the import in its own module, to a name in runtime-evaluated-base-classes, or is a same-module class that does), and every Pydantic model and SQLAlchemy mapper resolves its annotations once the package is imported. | mechanical | enforced | yes | pytest `tests/unit/test_runtime_evaluated_annotations.py` | `pr.yml:test` | **unproven** | | `api.quality.tests-pass` | The unit, contract and to-refactor test suites pass. | mechanical | enforced | yes | CI job `test` | `pr.yml:test` | — | | `api.quality.tests-ruff-exemptions` | Test files may use pytest's bare assert, need no docstrings, annotations or named constants for magic numbers, and may reach into private members, live outside packages and import inside a test (S101, D1, ANN, PLR2004, SLF001, INP001, PLC0415); every other ruff rule applies to tests. | mechanical | enforced | yes | ruff per-file ignore of ANN, D1, INP001, PLC0415, PLR2004, S101, SLF001 for `**/tests/**` | `pr.yml:lint` | — | +| `api.quality.unittest-ruff-exemptions` | The shared rules tool's tests use unittest, because the tool needs only the standard library, so ruff's pytest-style rules (PT) don't apply to them. | mechanical | enforced | yes | ruff per-file ignore of PT for `scripts/rules/tests/**` | `pr.yml:lint` | — | ```mermaid flowchart LR @@ -297,6 +298,8 @@ flowchart LR e_CI_job_test["CI job test"] r_api_quality_tests_ruff_exemptions("api.quality.tests-ruff-exemptions (enforced)") e_ruff_per_file_ignore_of_ANN__D1__INP001__PLC0415__PLR2004__S101__SLF001_for____tests___["ruff per-file ignore of ANN, D1, INP001, PLC0415, PLR2004, S101, SLF001 for **/tests/**"] + r_api_quality_unittest_ruff_exemptions("api.quality.unittest-ruff-exemptions (enforced)") + e_ruff_per_file_ignore_of_PT_for_scripts_rules_tests___["ruff per-file ignore of PT for scripts/rules/tests/**"] r_api_quality_alembic_revisions_unused_imports --> e_ruff_per_file_ignore_of_ALL_for_migrations___versions___py e_ruff_per_file_ignore_of_ALL_for_migrations___versions___py -.-> local_only r_api_quality_environment_variables_load --> e_CI_job_test_env_vars @@ -313,6 +316,8 @@ flowchart LR e_CI_job_test --> j_CI_pr_yml_test r_api_quality_tests_ruff_exemptions --> e_ruff_per_file_ignore_of_ANN__D1__INP001__PLC0415__PLR2004__S101__SLF001_for____tests___ e_ruff_per_file_ignore_of_ANN__D1__INP001__PLC0415__PLR2004__S101__SLF001_for____tests___ --> j_CI_pr_yml_lint + r_api_quality_unittest_ruff_exemptions --> e_ruff_per_file_ignore_of_PT_for_scripts_rules_tests___ + e_ruff_per_file_ignore_of_PT_for_scripts_rules_tests___ --> j_CI_pr_yml_lint ``` ## Observability diff --git a/pyproject.toml b/pyproject.toml index 4dd75c35c..75a2c2803 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -131,6 +131,9 @@ ignore = ["D203", "D213"] # live in directories that aren't packages, and import inside a test (often # after patching the environment). Every other rule applies to tests. "**/tests/**" = ["ANN", "D1", "INP001", "PLC0415", "PLR2004", "S101", "SLF001"] +# The shared rules tool needs only the standard library, so its tests use +# unittest on purpose; pytest-style rules don't apply to them. +"scripts/rules/tests/**" = ["PT"] [tool.mypy] python_version = "3.11" diff --git a/rules/registry.toml b/rules/registry.toml index b1b7a6771..12fb11e8c 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -742,6 +742,25 @@ workflow = ".github/workflows/pr.yml" job = "lint" invocation = "ruff check ." + +[[rule]] +id = "api.quality.unittest-ruff-exemptions" +category = "quality" +statement = "The shared rules tool's tests use unittest, because the tool needs only the standard library, so ruff's pytest-style rules (PT) don't apply to them." +rationale = "docs/engineering/skills/rules-registry.md#baselines" +scope = ["scripts/rules/tests/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "ruff" +mode = "per-file-ignore" +pattern = "scripts/rules/tests/**" +codes = ["PT"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "lint" +invocation = "ruff check ." [[rule]] id = "api.quality.mypy-v2" category = "quality" From cd591ee516a1bebac6764276a55e6f1ed72068b6 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:08:35 +0400 Subject: [PATCH 02/12] Use literals and comprehensions (C4, PERF) Rewrite keyword dict() calls as dict literals, generators passed to set() as set comprehensions and a constant-valued dict comprehension as dict.fromkeys. Replace append loops with list.extend or a comprehension, and the two conditional bundle-merging loops with dict.update. Key order, values and messages are unchanged. Two lists that mypy inferred from their first append are now annotated. Co-Authored-By: Claude Opus 5.5 --- policyengine_api/constants.py | 2 +- policyengine_api/country.py | 228 +++++------ policyengine_api/routes/household_routes.py | 8 +- policyengine_api/routes/policy_routes.py | 66 ++-- .../routes/report_output_routes.py | 30 +- policyengine_api/routes/simulation_routes.py | 30 +- .../routes/user_profile_routes.py | 30 +- .../services/economy_comparison.py | 357 +++++++++--------- policyengine_api/services/economy_service.py | 20 +- scripts/compare_migration_baseline.py | 6 +- scripts/guards/migration_contracts.py | 5 +- scripts/rules/baseline.py | 6 +- scripts/rules/ci_scripts.py | 25 +- scripts/rules/discover.py | 102 ++--- scripts/rules/resolve.py | 10 +- scripts/rules/validate.py | 16 +- tests/unit/test_ci_complete_gate.py | 6 +- tests/unit/test_country.py | 2 +- tests/unit/v2/test_household_routes.py | 4 +- tests/unit/v2/test_policy_routes.py | 2 +- 20 files changed, 484 insertions(+), 471 deletions(-) diff --git a/policyengine_api/constants.py b/policyengine_api/constants.py index 492bbde03..84dc5932d 100644 --- a/policyengine_api/constants.py +++ b/policyengine_api/constants.py @@ -166,7 +166,7 @@ def get_bundle_default_dataset_option(country_id: str) -> dict: } except Exception: # noqa: BLE001 # unreadable package metadata falls back to 0.0.0; import must not fail _dist_versions = {} - COUNTRY_PACKAGE_VERSIONS = {country: "0.0.0" for country in COUNTRIES} + COUNTRY_PACKAGE_VERSIONS = dict.fromkeys(COUNTRIES, "0.0.0") _policyengine_bundle = _load_policyengine_bundle() _bundle_package_versions = _resolve_bundle_package_versions(_policyengine_bundle) diff --git a/policyengine_api/country.py b/policyengine_api/country.py index 94aa6ff6a..06930d330 100644 --- a/policyengine_api/country.py +++ b/policyengine_api/country.py @@ -138,26 +138,26 @@ def __init__(self, country_package_name: str, country_id: str) -> None: def build_metadata(self) -> None: """Build the JSON-safe metadata document served by the metadata endpoints.""" self.metadata = self._json_safe( - dict( - variables=self.build_variables(), - parameters=self.build_parameters(), - entities=self.build_entities(), - variableModules=self.tax_benefit_system.variable_module_metadata, - economy_options=self.build_microsimulation_options(), - current_law_id={ + { + "variables": self.build_variables(), + "parameters": self.build_parameters(), + "entities": self.build_entities(), + "variableModules": self.tax_benefit_system.variable_module_metadata, + "economy_options": self.build_microsimulation_options(), + "current_law_id": { "uk": 1, "us": 2, "ca": 3, "ng": 4, "il": 5, }[self.country_id], - basicInputs=self.tax_benefit_system.basic_inputs, - modelled_policies=self.tax_benefit_system.modelled_policies, - spm=spm_metadata(self.country_id), - version=get_package_version( + "basicInputs": self.tax_benefit_system.basic_inputs, + "modelled_policies": self.tax_benefit_system.modelled_policies, + "spm": spm_metadata(self.country_id), + "version": get_package_version( self.country_package_name.replace("_", "-") ), - ) + } ) def _json_safe( @@ -181,7 +181,7 @@ def build_microsimulation_options(self) -> dict: """Build the regions, time periods and datasets offered for economy runs.""" # Shape: { region: [{ name: "uk", label: "the UK" }], # time_period: [{ name: 2022, label: "2022", ... }] } - options = dict() + options = {} if self.country_id == "uk": constituency_names_path = ( Path(__file__).parent / "data" / "constituencies_2024.csv" @@ -189,41 +189,41 @@ def build_microsimulation_options(self) -> dict: constituency_names = pd.read_csv(constituency_names_path) constituency_names = constituency_names.sort_values("name") region = [ - dict(name="uk", label="the UK", type="national"), - dict(name="country/england", label="England", type="country"), - dict(name="country/scotland", label="Scotland", type="country"), - dict(name="country/wales", label="Wales", type="country"), - dict(name="country/ni", label="Northern Ireland", type="country"), + {"name": "uk", "label": "the UK", "type": "national"}, + {"name": "country/england", "label": "England", "type": "country"}, + {"name": "country/scotland", "label": "Scotland", "type": "country"}, + {"name": "country/wales", "label": "Wales", "type": "country"}, + {"name": "country/ni", "label": "Northern Ireland", "type": "country"}, ] - for i in range(len(constituency_names)): - region.append( - dict( - name=f"constituency/{constituency_names.iloc[i]['name']}", - label=constituency_names.iloc[i]["name"], - type="constituency", - ) - ) + region.extend( + { + "name": f"constituency/{constituency_names.iloc[i]['name']}", + "label": constituency_names.iloc[i]["name"], + "type": "constituency", + } + for i in range(len(constituency_names)) + ) local_authority_names_path = ( Path(__file__).parent / "data" / "local_authorities_2021.csv" ) local_authority_names = pd.read_csv(local_authority_names_path) local_authority_names = local_authority_names.sort_values("name") - for i in range(len(local_authority_names)): - region.append( - dict( - name=f"local_authority/{local_authority_names.iloc[i]['name']}", - label=local_authority_names.iloc[i]["name"], - type="local_authority", - ) - ) + region.extend( + { + "name": f"local_authority/{local_authority_names.iloc[i]['name']}", + "label": local_authority_names.iloc[i]["name"], + "type": "local_authority", + } + for i in range(len(local_authority_names)) + ) time_period = [ - dict(name=2024, label="2024"), - dict(name=2025, label="2025"), - dict(name=2026, label="2026"), - dict(name=2027, label="2027"), - dict(name=2028, label="2028"), - dict(name=2029, label="2029"), - dict(name=2030, label="2030"), + {"name": 2024, "label": "2024"}, + {"name": 2025, "label": "2025"}, + {"name": 2026, "label": "2026"}, + {"name": 2027, "label": "2027"}, + {"name": 2028, "label": "2028"}, + {"name": 2029, "label": "2029"}, + {"name": 2030, "label": "2030"}, ] datasets = [get_bundle_default_dataset_option("uk")] options["region"] = region @@ -231,76 +231,76 @@ def build_microsimulation_options(self) -> dict: options["datasets"] = datasets elif self.country_id == "us": region = [ - dict(name="us", label="the US", type="national"), - dict(name="state/al", label="Alabama", type="state"), - dict(name="state/ak", label="Alaska", type="state"), - dict(name="state/az", label="Arizona", type="state"), - dict(name="state/ar", label="Arkansas", type="state"), - dict(name="state/ca", label="California", type="state"), - dict(name="state/co", label="Colorado", type="state"), - dict(name="state/ct", label="Connecticut", type="state"), - dict(name="state/de", label="Delaware", type="state"), - dict(name="state/dc", label="District of Columbia", type="state"), - dict(name="state/fl", label="Florida", type="state"), - dict(name="state/ga", label="Georgia", type="state"), - dict(name="state/hi", label="Hawaii", type="state"), - dict(name="state/id", label="Idaho", type="state"), - dict(name="state/il", label="Illinois", type="state"), - dict(name="state/in", label="Indiana", type="state"), - dict(name="state/ia", label="Iowa", type="state"), - dict(name="state/ks", label="Kansas", type="state"), - dict(name="state/ky", label="Kentucky", type="state"), - dict(name="state/la", label="Louisiana", type="state"), - dict(name="state/me", label="Maine", type="state"), - dict(name="state/md", label="Maryland", type="state"), - dict(name="state/ma", label="Massachusetts", type="state"), - dict(name="state/mi", label="Michigan", type="state"), - dict(name="state/mn", label="Minnesota", type="state"), - dict(name="state/ms", label="Mississippi", type="state"), - dict(name="state/mo", label="Missouri", type="state"), - dict(name="state/mt", label="Montana", type="state"), - dict(name="state/ne", label="Nebraska", type="state"), - dict(name="state/nv", label="Nevada", type="state"), - dict(name="state/nh", label="New Hampshire", type="state"), - dict(name="state/nj", label="New Jersey", type="state"), - dict(name="state/nm", label="New Mexico", type="state"), - dict(name="state/ny", label="New York", type="state"), - dict(name="state/nc", label="North Carolina", type="state"), - dict(name="state/nd", label="North Dakota", type="state"), - dict(name="state/oh", label="Ohio", type="state"), - dict(name="state/ok", label="Oklahoma", type="state"), - dict(name="state/or", label="Oregon", type="state"), - dict(name="state/pa", label="Pennsylvania", type="state"), - dict(name="state/ri", label="Rhode Island", type="state"), - dict(name="state/sc", label="South Carolina", type="state"), - dict(name="state/sd", label="South Dakota", type="state"), - dict(name="state/tn", label="Tennessee", type="state"), - dict(name="state/tx", label="Texas", type="state"), - dict(name="state/ut", label="Utah", type="state"), - dict(name="state/vt", label="Vermont", type="state"), - dict(name="state/va", label="Virginia", type="state"), - dict(name="state/wa", label="Washington", type="state"), - dict(name="state/wv", label="West Virginia", type="state"), - dict(name="state/wi", label="Wisconsin", type="state"), - dict(name="state/wy", label="Wyoming", type="state"), + {"name": "us", "label": "the US", "type": "national"}, + {"name": "state/al", "label": "Alabama", "type": "state"}, + {"name": "state/ak", "label": "Alaska", "type": "state"}, + {"name": "state/az", "label": "Arizona", "type": "state"}, + {"name": "state/ar", "label": "Arkansas", "type": "state"}, + {"name": "state/ca", "label": "California", "type": "state"}, + {"name": "state/co", "label": "Colorado", "type": "state"}, + {"name": "state/ct", "label": "Connecticut", "type": "state"}, + {"name": "state/de", "label": "Delaware", "type": "state"}, + {"name": "state/dc", "label": "District of Columbia", "type": "state"}, + {"name": "state/fl", "label": "Florida", "type": "state"}, + {"name": "state/ga", "label": "Georgia", "type": "state"}, + {"name": "state/hi", "label": "Hawaii", "type": "state"}, + {"name": "state/id", "label": "Idaho", "type": "state"}, + {"name": "state/il", "label": "Illinois", "type": "state"}, + {"name": "state/in", "label": "Indiana", "type": "state"}, + {"name": "state/ia", "label": "Iowa", "type": "state"}, + {"name": "state/ks", "label": "Kansas", "type": "state"}, + {"name": "state/ky", "label": "Kentucky", "type": "state"}, + {"name": "state/la", "label": "Louisiana", "type": "state"}, + {"name": "state/me", "label": "Maine", "type": "state"}, + {"name": "state/md", "label": "Maryland", "type": "state"}, + {"name": "state/ma", "label": "Massachusetts", "type": "state"}, + {"name": "state/mi", "label": "Michigan", "type": "state"}, + {"name": "state/mn", "label": "Minnesota", "type": "state"}, + {"name": "state/ms", "label": "Mississippi", "type": "state"}, + {"name": "state/mo", "label": "Missouri", "type": "state"}, + {"name": "state/mt", "label": "Montana", "type": "state"}, + {"name": "state/ne", "label": "Nebraska", "type": "state"}, + {"name": "state/nv", "label": "Nevada", "type": "state"}, + {"name": "state/nh", "label": "New Hampshire", "type": "state"}, + {"name": "state/nj", "label": "New Jersey", "type": "state"}, + {"name": "state/nm", "label": "New Mexico", "type": "state"}, + {"name": "state/ny", "label": "New York", "type": "state"}, + {"name": "state/nc", "label": "North Carolina", "type": "state"}, + {"name": "state/nd", "label": "North Dakota", "type": "state"}, + {"name": "state/oh", "label": "Ohio", "type": "state"}, + {"name": "state/ok", "label": "Oklahoma", "type": "state"}, + {"name": "state/or", "label": "Oregon", "type": "state"}, + {"name": "state/pa", "label": "Pennsylvania", "type": "state"}, + {"name": "state/ri", "label": "Rhode Island", "type": "state"}, + {"name": "state/sc", "label": "South Carolina", "type": "state"}, + {"name": "state/sd", "label": "South Dakota", "type": "state"}, + {"name": "state/tn", "label": "Tennessee", "type": "state"}, + {"name": "state/tx", "label": "Texas", "type": "state"}, + {"name": "state/ut", "label": "Utah", "type": "state"}, + {"name": "state/vt", "label": "Vermont", "type": "state"}, + {"name": "state/va", "label": "Virginia", "type": "state"}, + {"name": "state/wa", "label": "Washington", "type": "state"}, + {"name": "state/wv", "label": "West Virginia", "type": "state"}, + {"name": "state/wi", "label": "Wisconsin", "type": "state"}, + {"name": "state/wy", "label": "Wyoming", "type": "state"}, ] # Add all 436 congressional districts (435 voting + DC) region.extend(build_congressional_district_metadata()) time_period = [ - dict(name=2035, label="2035"), - dict(name=2034, label="2034"), - dict(name=2033, label="2033"), - dict(name=2032, label="2032"), - dict(name=2031, label="2031"), - dict(name=2030, label="2030"), - dict(name=2029, label="2029"), - dict(name=2028, label="2028"), - dict(name=2027, label="2027"), - dict(name=2026, label="2026"), - dict(name=2025, label="2025"), - dict(name=2024, label="2024"), - dict(name=2023, label="2023"), - dict(name=2022, label="2022"), + {"name": 2035, "label": "2035"}, + {"name": 2034, "label": "2034"}, + {"name": 2033, "label": "2033"}, + {"name": 2032, "label": "2032"}, + {"name": 2031, "label": "2031"}, + {"name": 2030, "label": "2030"}, + {"name": 2029, "label": "2029"}, + {"name": 2028, "label": "2028"}, + {"name": 2027, "label": "2027"}, + {"name": 2026, "label": "2026"}, + {"name": 2025, "label": "2025"}, + {"name": 2024, "label": "2024"}, + {"name": 2023, "label": "2023"}, + {"name": 2022, "label": "2022"}, ] datasets = [get_bundle_default_dataset_option("us")] options["region"] = region @@ -308,10 +308,10 @@ def build_microsimulation_options(self) -> dict: options["datasets"] = datasets elif self.country_id == "ca": region = [ - dict(name="ca", label="Canada"), + {"name": "ca", "label": "Canada"}, ] time_period = [ - dict(name=2023, label="2023"), + {"name": 2023, "label": "2023"}, ] datasets = [{}] options["region"] = region @@ -319,10 +319,10 @@ def build_microsimulation_options(self) -> dict: options["datasets"] = datasets elif self.country_id == "ng": region = [ - dict(name="ng", label="Nigeria"), + {"name": "ng", "label": "Nigeria"}, ] time_period = [ - dict(name=2023, label="2023"), + {"name": 2023, "label": "2023"}, ] datasets = [{}] options["region"] = region @@ -330,10 +330,10 @@ def build_microsimulation_options(self) -> dict: options["datasets"] = datasets elif self.country_id == "il": region = [ - dict(name="il", label="Israel"), + {"name": "il", "label": "Israel"}, ] time_period = [ - dict(name=2023, label="2023"), + {"name": 2023, "label": "2023"}, ] datasets = [{}] options["region"] = region @@ -369,7 +369,7 @@ def build_variables(self) -> dict: } if variable.value_type.__name__ == "Enum": variable_data[variable_name]["possibleValues"] = [ - dict(value=value.name, label=value.value) + {"value": value.name, "label": value.value} for value in variable.possible_values ] variable_data[variable_name]["defaultValue"] = ( diff --git a/policyengine_api/routes/household_routes.py b/policyengine_api/routes/household_routes.py index 26e7b6df0..d7c684c59 100644 --- a/policyengine_api/routes/household_routes.py +++ b/policyengine_api/routes/household_routes.py @@ -145,9 +145,11 @@ def _spm_error_response(error: Exception) -> Response | None: def _calculation_response(calculation: HouseholdCalculationResult) -> dict: - result: dict[str, object] = dict( - status="ok", message=None, result=calculation.household - ) + result: dict[str, object] = { + "status": "ok", + "message": None, + "result": calculation.household, + } if calculation.warnings: result["warnings"] = list(calculation.warnings) if calculation.spm_provenance is not None: diff --git a/policyengine_api/routes/policy_routes.py b/policyengine_api/routes/policy_routes.py index 524506bda..97c23fbac 100644 --- a/policyengine_api/routes/policy_routes.py +++ b/policyengine_api/routes/policy_routes.py @@ -221,13 +221,13 @@ def set_policy(country_id: str) -> Response: except PolicyMirrorUnavailableError: return _policy_mirror_unavailable() - response_body = dict( - status="ok", - message=message, - result=dict( - policy_id=policy_id, - ), - ) + response_body = { + "status": "ok", + "message": message, + "result": { + "policy_id": policy_id, + }, + } code = 200 if is_existing_policy else 201 return Response(json.dumps(response_body), status=code, mimetype="application/json") @@ -265,14 +265,14 @@ def get_policy_search(country_id: str) -> Response: 404, ) - policies = [dict(id=result.id, label=result.label) for result in results] + policies = [{"id": result.id, "label": result.label} for result in results] return Response( json.dumps( - dict( - status="ok", - message="Policies found", - result=policies, - ) + { + "status": "ok", + "message": "Policies found", + "result": policies, + } ), status=200, mimetype="application/json", @@ -365,14 +365,14 @@ def set_user_policy(country_id: str) -> Response: if not creation.created: return Response( json.dumps( - dict( - status="ok", - message=( + { + "status": "ok", + "message": ( f"The reform #{reform_id} / baseline #{baseline_id} pair " f"already exists for user {user_id}" ), - result=dict(id=user_policy.id), - ) + "result": {"id": user_policy.id}, + } ), status=200, mimetype="application/json", @@ -380,11 +380,11 @@ def set_user_policy(country_id: str) -> Response: return Response( json.dumps( - dict( - status="ok", - message="Record created successfully", - result=_serialize_user_policy(user_policy), - ) + { + "status": "ok", + "message": "Record created successfully", + "result": _serialize_user_policy(user_policy), + } ), status=201, mimetype="application/json", @@ -400,11 +400,11 @@ def get_user_policy(country_id: str, user_id: str) -> dict | Response: except ValueError: return _policy_configuration_unavailable() user_policies = user_policy_service.list_user_policies(country_id, user_id) - return dict( - status="ok", - message=None, - result=[_serialize_user_policy(row) for row in user_policies], - ) + return { + "status": "ok", + "message": None, + "result": [_serialize_user_policy(row) for row in user_policies], + } UPDATE_USER_POLICY_ALLOWED_FIELDS = USER_POLICY_MUTABLE_FIELDS @@ -491,11 +491,11 @@ def update_user_policy(country_id: str) -> Response: return Response( json.dumps( - dict( - status="ok", - message="Record updated successfully", - result=dict(id=user_policy_id), - ) + { + "status": "ok", + "message": "Record updated successfully", + "result": {"id": user_policy_id}, + } ), status=200, mimetype="application/json", diff --git a/policyengine_api/routes/report_output_routes.py b/policyengine_api/routes/report_output_routes.py index d1c588fed..0fd282916 100644 --- a/policyengine_api/routes/report_output_routes.py +++ b/policyengine_api/routes/report_output_routes.py @@ -105,11 +105,11 @@ def create_report_output(country_id: str) -> Response: ) status_code = 201 if creation.created else 200 - response_body = dict( - status="ok", - message=message, - result=result, - ) + response_body = { + "status": "ok", + "message": message, + "result": result, + } return Response( json.dumps(response_body), @@ -155,11 +155,11 @@ def get_report_output(country_id: str, report_id: int) -> Response: raise NotFound(msg) result = _serialize_v1_report_output(view) - response_body = dict( - status="ok", - message=None, - result=result, - ) + response_body = { + "status": "ok", + "message": None, + "result": result, + } return Response( json.dumps(response_body), @@ -223,11 +223,11 @@ def update_report_output(country_id: str) -> Response: ) result = _serialize_v1_report_output(view) - response_body = dict( - status="ok", - message="Report output updated successfully", - result=result, - ) + response_body = { + "status": "ok", + "message": "Report output updated successfully", + "result": result, + } return Response( json.dumps(response_body), diff --git a/policyengine_api/routes/simulation_routes.py b/policyengine_api/routes/simulation_routes.py index d3d127d9e..fc55c6a97 100644 --- a/policyengine_api/routes/simulation_routes.py +++ b/policyengine_api/routes/simulation_routes.py @@ -115,11 +115,11 @@ def create_simulation(country_id: str) -> Response: ) status_code = 201 if creation.created else 200 - response_body = dict( - status="ok", - message=message, - result=result, - ) + response_body = { + "status": "ok", + "message": message, + "result": result, + } return Response( json.dumps(response_body), @@ -168,11 +168,11 @@ def get_simulation(country_id: str, simulation_id: int) -> Response: msg = f"Simulation #{simulation_id} not found." raise NotFound(msg) - response_body = dict( - status="ok", - message=None, - result=result, - ) + response_body = { + "status": "ok", + "message": None, + "result": result, + } return Response( json.dumps(response_body), @@ -234,11 +234,11 @@ def update_simulation(country_id: str) -> Response: result = _serialize_v1_simulation(simulation) - response_body = dict( - status="ok", - message="Simulation updated successfully", - result=result, - ) + response_body = { + "status": "ok", + "message": "Simulation updated successfully", + "result": result, + } return Response( json.dumps(response_body), diff --git a/policyengine_api/routes/user_profile_routes.py b/policyengine_api/routes/user_profile_routes.py index 3cc175b5b..2a1e6a00e 100644 --- a/policyengine_api/routes/user_profile_routes.py +++ b/policyengine_api/routes/user_profile_routes.py @@ -54,11 +54,11 @@ def set_user_profile(country_id: str) -> Response: ) result = _serialize_user_profile(profile, include_auth0_id=False) - response = dict( - status="ok", - message="Record created successfully" if created else "Record exists", - result=result, - ) + response = { + "status": "ok", + "message": "Record created successfully" if created else "Record exists", + "result": result, + } return Response( json.dumps(response), status=201 if created else 200, @@ -99,11 +99,11 @@ def get_user_profile(country_id: str) -> Response: msg = "No such user" raise NotFound(msg) - response_body = dict( - status="ok", - message=f"User #{readable_row['user_id']} found successfully", - result=readable_row, - ) + response_body = { + "status": "ok", + "message": f"User #{readable_row['user_id']} found successfully", + "result": readable_row, + } return Response( json.dumps(response_body), @@ -149,11 +149,11 @@ def update_user_profile(country_id: str) -> Response: msg = "No such user id" raise NotFound(msg) - response_body = dict( - status="ok", - message=f"User profile #{user_id} updated successfully", - result=dict(user_id=user_id), - ) + response_body = { + "status": "ok", + "message": f"User profile #{user_id} updated successfully", + "result": {"user_id": user_id}, + } return Response( json.dumps(response_body), diff --git a/policyengine_api/services/economy_comparison.py b/policyengine_api/services/economy_comparison.py index eccf99f22..26c1c362f 100644 --- a/policyengine_api/services/economy_comparison.py +++ b/policyengine_api/services/economy_comparison.py @@ -22,14 +22,14 @@ def budgetary_impact(baseline: dict, reform: dict) -> dict: state_tax_revenue_impact = reform["total_state_tax"] - baseline["total_state_tax"] benefit_spending_impact = reform["total_benefits"] - baseline["total_benefits"] budgetary_impact = tax_revenue_impact - benefit_spending_impact - return dict( - budgetary_impact=budgetary_impact, - tax_revenue_impact=tax_revenue_impact, - state_tax_revenue_impact=state_tax_revenue_impact, - benefit_spending_impact=benefit_spending_impact, - households=sum(baseline["household_weight"]), - baseline_net_income=baseline["total_net_income"], - ) + return { + "budgetary_impact": budgetary_impact, + "tax_revenue_impact": tax_revenue_impact, + "state_tax_revenue_impact": state_tax_revenue_impact, + "benefit_spending_impact": benefit_spending_impact, + "households": sum(baseline["household_weight"]), + "baseline_net_income": baseline["total_net_income"], + } def labor_supply_response(baseline: dict, reform: dict) -> dict: @@ -63,53 +63,53 @@ def labor_supply_response(baseline: dict, reform: dict) -> dict: substitution_lsr_hh = MicroSeries(substitution_lsr_hh, weights=household_weight) income_lsr_hh = MicroSeries(income_lsr_hh, weights=household_weight) - decile_avg = dict( - income=income_lsr_hh.groupby(decile).mean().to_dict(), - substitution=substitution_lsr_hh.groupby(decile).mean().to_dict(), - ) - decile_rel = dict( - income=( + decile_avg = { + "income": income_lsr_hh.groupby(decile).mean().to_dict(), + "substitution": substitution_lsr_hh.groupby(decile).mean().to_dict(), + } + decile_rel = { + "income": ( income_lsr_hh.groupby(decile).sum() / original_earnings.groupby(decile).sum() ).to_dict(), - substitution=( + "substitution": ( substitution_lsr_hh.groupby(decile).sum() / original_earnings.groupby(decile).sum() ).to_dict(), - ) + } - relative_lsr = dict( - income=(income_lsr_hh.sum() / original_earnings.sum()), - substitution=(substitution_lsr_hh.sum() / original_earnings.sum()), - ) + relative_lsr = { + "income": (income_lsr_hh.sum() / original_earnings.sum()), + "substitution": (substitution_lsr_hh.sum() / original_earnings.sum()), + } decile_rel["income"] = {int(k): v for k, v in decile_rel["income"].items() if k > 0} decile_rel["substitution"] = { int(k): v for k, v in decile_rel["substitution"].items() if k > 0 } - hours = dict( - baseline=baseline["weekly_hours"], - reform=reform["weekly_hours"], - change=reform["weekly_hours"] - baseline["weekly_hours"], - income_effect=reform["weekly_hours_income_effect"] + hours = { + "baseline": baseline["weekly_hours"], + "reform": reform["weekly_hours"], + "change": reform["weekly_hours"] - baseline["weekly_hours"], + "income_effect": reform["weekly_hours_income_effect"] - baseline["weekly_hours_income_effect"], - substitution_effect=reform["weekly_hours_substitution_effect"] + "substitution_effect": reform["weekly_hours_substitution_effect"] - baseline["weekly_hours_substitution_effect"], - ) + } - return dict( - substitution_lsr=substitution_lsr, - income_lsr=income_lsr, - relative_lsr=relative_lsr, - total_change=total_change, - revenue_change=revenue_change, - decile=dict( - average=decile_avg, - relative=decile_rel, - ), - hours=hours, - ) + return { + "substitution_lsr": substitution_lsr, + "income_lsr": income_lsr, + "relative_lsr": relative_lsr, + "total_change": total_change, + "revenue_change": revenue_change, + "decile": { + "average": decile_avg, + "relative": decile_rel, + }, + "hours": hours, + } def detailed_budgetary_impact(baseline: dict, reform: dict, country_id: str) -> dict: @@ -121,11 +121,12 @@ def detailed_budgetary_impact(baseline: dict, reform: dict, country_id: str) -> if country_id == "uk": for program in baseline["programs"]: # baseline[programs][program] = total budgetary impact of program - result[program] = dict( - baseline=baseline["programs"][program], - reform=reform["programs"][program], - difference=reform["programs"][program] - baseline["programs"][program], - ) + result[program] = { + "baseline": baseline["programs"][program], + "reform": reform["programs"][program], + "difference": reform["programs"][program] + - baseline["programs"][program], + } return result @@ -164,10 +165,10 @@ def decile_impact(baseline: dict, reform: dict) -> dict: ) rel_decile_dict = rel_income_change_by_decile.to_dict() avg_decile_dict = avg_income_change_by_decile.to_dict() - return dict( - relative={int(k): v for k, v in rel_decile_dict.items()}, - average={int(k): v for k, v in avg_decile_dict.items()}, - ) + return { + "relative": {int(k): v for k, v in rel_decile_dict.items()}, + "average": {int(k): v for k, v in avg_decile_dict.items()}, + } def wealth_decile_impact(baseline: dict, reform: dict) -> dict: @@ -204,10 +205,10 @@ def wealth_decile_impact(baseline: dict, reform: dict) -> dict: ) rel_decile_dict = rel_income_change_by_decile.to_dict() avg_decile_dict = avg_income_change_by_decile.to_dict() - return dict( - relative={int(k): v for k, v in rel_decile_dict.items()}, - average={int(k): v for k, v in avg_decile_dict.items()}, - ) + return { + "relative": {int(k): v for k, v in rel_decile_dict.items()}, + "average": {int(k): v for k, v in avg_decile_dict.items()}, + } def inequality_impact(baseline: dict, reform: dict) -> dict: @@ -221,20 +222,20 @@ def inequality_impact(baseline: dict, reform: dict) -> dict: dict: The impact of the reform on inequality. """ - return dict( - gini=dict( - baseline=baseline["gini"], - reform=reform["gini"], - ), - top_10_pct_share=dict( - baseline=baseline["top_10_percent_share"], - reform=reform["top_10_percent_share"], - ), - top_1_pct_share=dict( - baseline=baseline["top_1_percent_share"], - reform=reform["top_1_percent_share"], - ), - ) + return { + "gini": { + "baseline": baseline["gini"], + "reform": reform["gini"], + }, + "top_10_pct_share": { + "baseline": baseline["top_10_percent_share"], + "reform": reform["top_10_percent_share"], + }, + "top_1_pct_share": { + "baseline": baseline["top_1_percent_share"], + "reform": reform["top_1_percent_share"], + }, + } def poverty_impact(baseline: dict, reform: dict) -> dict: @@ -262,48 +263,48 @@ def poverty_impact(baseline: dict, reform: dict) -> dict: ) age = MicroSeries(baseline["age"]) - poverty = dict( - child=dict( - baseline=float(baseline_poverty[age < 18].mean()), - reform=float(reform_poverty[age < 18].mean()), - ), - adult=dict( - baseline=float(baseline_poverty[(age >= 18) & (age < 65)].mean()), - reform=float(reform_poverty[(age >= 18) & (age < 65)].mean()), - ), - senior=dict( - baseline=float(baseline_poverty[age >= 65].mean()), - reform=float(reform_poverty[age >= 65].mean()), - ), - all=dict( - baseline=float(baseline_poverty.mean()), - reform=float(reform_poverty.mean()), - ), - ) + poverty = { + "child": { + "baseline": float(baseline_poverty[age < 18].mean()), + "reform": float(reform_poverty[age < 18].mean()), + }, + "adult": { + "baseline": float(baseline_poverty[(age >= 18) & (age < 65)].mean()), + "reform": float(reform_poverty[(age >= 18) & (age < 65)].mean()), + }, + "senior": { + "baseline": float(baseline_poverty[age >= 65].mean()), + "reform": float(reform_poverty[age >= 65].mean()), + }, + "all": { + "baseline": float(baseline_poverty.mean()), + "reform": float(reform_poverty.mean()), + }, + } - deep_poverty = dict( - child=dict( - baseline=float(baseline_deep_poverty[age < 18].mean()), - reform=float(reform_deep_poverty[age < 18].mean()), - ), - adult=dict( - baseline=float(baseline_deep_poverty[(age >= 18) & (age < 65)].mean()), - reform=float(reform_deep_poverty[(age >= 18) & (age < 65)].mean()), - ), - senior=dict( - baseline=float(baseline_deep_poverty[age >= 65].mean()), - reform=float(reform_deep_poverty[age >= 65].mean()), - ), - all=dict( - baseline=float(baseline_deep_poverty.mean()), - reform=float(reform_deep_poverty.mean()), - ), - ) + deep_poverty = { + "child": { + "baseline": float(baseline_deep_poverty[age < 18].mean()), + "reform": float(reform_deep_poverty[age < 18].mean()), + }, + "adult": { + "baseline": float(baseline_deep_poverty[(age >= 18) & (age < 65)].mean()), + "reform": float(reform_deep_poverty[(age >= 18) & (age < 65)].mean()), + }, + "senior": { + "baseline": float(baseline_deep_poverty[age >= 65].mean()), + "reform": float(reform_deep_poverty[age >= 65].mean()), + }, + "all": { + "baseline": float(baseline_deep_poverty.mean()), + "reform": float(reform_deep_poverty.mean()), + }, + } - return dict( - poverty=poverty, - deep_poverty=deep_poverty, - ) + return { + "poverty": poverty, + "deep_poverty": deep_poverty, + } def compute_income_change( @@ -374,7 +375,7 @@ def intra_decile_impact(baseline: dict, reform: dict) -> dict: outcome_groups[label].append(people_in_proportion) all_outcomes[label] = sum(outcome_groups[label]) / 10 - return dict(deciles=outcome_groups, all=all_outcomes) + return {"deciles": outcome_groups, "all": all_outcomes} def intra_wealth_decile_impact(baseline: dict, reform: dict) -> dict: @@ -433,7 +434,7 @@ def intra_wealth_decile_impact(baseline: dict, reform: dict) -> dict: outcome_groups[label].append(people_in_proportion) all_outcomes[label] = sum(outcome_groups[label]) / 10 - return dict(deciles=outcome_groups, all=all_outcomes) + return {"deciles": outcome_groups, "all": all_outcomes} def poverty_gender_breakdown(baseline: dict, reform: dict) -> dict: @@ -463,32 +464,32 @@ def poverty_gender_breakdown(baseline: dict, reform: dict) -> dict: ) is_male = MicroSeries(baseline["is_male"]) - poverty = dict( - male=dict( - baseline=float(baseline_poverty[is_male].mean()), - reform=float(reform_poverty[is_male].mean()), - ), - female=dict( - baseline=float(baseline_poverty[~is_male].mean()), - reform=float(reform_poverty[~is_male].mean()), - ), - ) + poverty = { + "male": { + "baseline": float(baseline_poverty[is_male].mean()), + "reform": float(reform_poverty[is_male].mean()), + }, + "female": { + "baseline": float(baseline_poverty[~is_male].mean()), + "reform": float(reform_poverty[~is_male].mean()), + }, + } - deep_poverty = dict( - male=dict( - baseline=float(baseline_deep_poverty[is_male].mean()), - reform=float(reform_deep_poverty[is_male].mean()), - ), - female=dict( - baseline=float(baseline_deep_poverty[~is_male].mean()), - reform=float(reform_deep_poverty[~is_male].mean()), - ), - ) + deep_poverty = { + "male": { + "baseline": float(baseline_deep_poverty[is_male].mean()), + "reform": float(reform_deep_poverty[is_male].mean()), + }, + "female": { + "baseline": float(baseline_deep_poverty[~is_male].mean()), + "reform": float(reform_deep_poverty[~is_male].mean()), + }, + } - return dict( - poverty=poverty, - deep_poverty=deep_poverty, - ) + return { + "poverty": poverty, + "deep_poverty": deep_poverty, + } def poverty_racial_breakdown(baseline: dict, reform: dict) -> dict: @@ -512,28 +513,28 @@ def poverty_racial_breakdown(baseline: dict, reform: dict) -> dict: ) race = MicroSeries(baseline["race"]) # Can be WHITE, BLACK, HISPANIC, or OTHER. - poverty = dict( - white=dict( - baseline=float(baseline_poverty[race == "WHITE"].mean()), - reform=float(reform_poverty[race == "WHITE"].mean()), - ), - black=dict( - baseline=float(baseline_poverty[race == "BLACK"].mean()), - reform=float(reform_poverty[race == "BLACK"].mean()), - ), - hispanic=dict( - baseline=float(baseline_poverty[race == "HISPANIC"].mean()), - reform=float(reform_poverty[race == "HISPANIC"].mean()), - ), - other=dict( - baseline=float(baseline_poverty[race == "OTHER"].mean()), - reform=float(reform_poverty[race == "OTHER"].mean()), - ), - ) + poverty = { + "white": { + "baseline": float(baseline_poverty[race == "WHITE"].mean()), + "reform": float(reform_poverty[race == "WHITE"].mean()), + }, + "black": { + "baseline": float(baseline_poverty[race == "BLACK"].mean()), + "reform": float(reform_poverty[race == "BLACK"].mean()), + }, + "hispanic": { + "baseline": float(baseline_poverty[race == "HISPANIC"].mean()), + "reform": float(reform_poverty[race == "HISPANIC"].mean()), + }, + "other": { + "baseline": float(baseline_poverty[race == "OTHER"].mean()), + "reform": float(reform_poverty[race == "OTHER"].mean()), + }, + } - return dict( - poverty=poverty, - ) + return { + "poverty": poverty, + } class UKConstituencyBreakdownByConstituency(BaseModel): @@ -888,30 +889,30 @@ def compare_economic_outputs( wealth_decile_impact_data = {} intra_wealth_decile_impact_data = {} - return dict( - budget=budgetary_impact_data, - detailed_budget=detailed_budgetary_impact_data, - decile=decile_impact_data, - inequality=inequality_impact_data, - poverty=poverty_impact_data, - poverty_by_gender=poverty_by_gender_data, - poverty_by_race=poverty_by_race_data, - intra_decile=intra_decile_impact_data, - wealth_decile=wealth_decile_impact_data, - intra_wealth_decile=intra_wealth_decile_impact_data, - labor_supply_response=labor_supply_response_data, - constituency_impact=constituency_impact_data, - local_authority_impact=local_authority_impact_data, - ) + return { + "budget": budgetary_impact_data, + "detailed_budget": detailed_budgetary_impact_data, + "decile": decile_impact_data, + "inequality": inequality_impact_data, + "poverty": poverty_impact_data, + "poverty_by_gender": poverty_by_gender_data, + "poverty_by_race": poverty_by_race_data, + "intra_decile": intra_decile_impact_data, + "wealth_decile": wealth_decile_impact_data, + "intra_wealth_decile": intra_wealth_decile_impact_data, + "labor_supply_response": labor_supply_response_data, + "constituency_impact": constituency_impact_data, + "local_authority_impact": local_authority_impact_data, + } if baseline.get("type") == "cliff": - return dict( - baseline=dict( - cliff_gap=baseline["cliff_gap"], - cliff_share=baseline["cliff_share"], - ), - reform=dict( - cliff_gap=reform["cliff_gap"], - cliff_share=reform["cliff_share"], - ), - ) + return { + "baseline": { + "cliff_gap": baseline["cliff_gap"], + "cliff_share": baseline["cliff_share"], + }, + "reform": { + "cliff_gap": reform["cliff_gap"], + "cliff_share": reform["cliff_share"], + }, + } return None diff --git a/policyengine_api/services/economy_service.py b/policyengine_api/services/economy_service.py index 552b9031d..1e8011a03 100644 --- a/policyengine_api/services/economy_service.py +++ b/policyengine_api/services/economy_service.py @@ -1668,18 +1668,26 @@ def _with_policyengine_bundle( "dataset": setup_options.dataset, } if isinstance(result.get("policyengine_bundle"), dict): - for key, value in result["policyengine_bundle"].items(): - if value is not None: - bundle[key] = value + bundle.update( + { + key: value + for key, value in result["policyengine_bundle"].items() + if value is not None + } + ) execution_bundle = ( getattr(execution, "policyengine_bundle", None) if execution is not None else None ) if isinstance(execution_bundle, dict): - for key, value in execution_bundle.items(): - if value is not None: - bundle[key] = value + bundle.update( + { + key: value + for key, value in execution_bundle.items() + if value is not None + } + ) response = { **result, "policyengine_bundle": bundle, diff --git a/scripts/compare_migration_baseline.py b/scripts/compare_migration_baseline.py index 74728e9e3..e3a6bc3df 100644 --- a/scripts/compare_migration_baseline.py +++ b/scripts/compare_migration_baseline.py @@ -136,8 +136,10 @@ def compare( f"(≤ {p95_limit:.2f}ms)" ) - for name in sorted(set(candidate_stats) - set(baseline_stats)): - lines.append(f"NOTE {name}: not in baseline; not gated") + lines.extend( + f"NOTE {name}: not in baseline; not gated" + for name in sorted(set(candidate_stats) - set(baseline_stats)) + ) return passed, lines diff --git a/scripts/guards/migration_contracts.py b/scripts/guards/migration_contracts.py index d058c5169..ceb7836a6 100644 --- a/scripts/guards/migration_contracts.py +++ b/scripts/guards/migration_contracts.py @@ -16,16 +16,13 @@ def _check_unique_values( *, label: str, ) -> list[str]: - violations = [] seen: set[str] = set() duplicates: set[str] = set() for value in values: if value in seen: duplicates.add(value) seen.add(value) - for value in sorted(duplicates): - violations.append(f"duplicate {label}: {value!r}") - return violations + return [f"duplicate {label}: {value!r}" for value in sorted(duplicates)] def _check_route_groups(payload: dict[str, Any]) -> list[str]: diff --git a/scripts/rules/baseline.py b/scripts/rules/baseline.py index b68a49f64..9b56d294b 100644 --- a/scripts/rules/baseline.py +++ b/scripts/rules/baseline.py @@ -85,6 +85,8 @@ def check_baselines( "explicit and reviewable", ) ) - for rule_id in sorted(current_ids - {rule.id for rule in base.rules}): - notes.append(f"rule {rule_id} is new") + notes.extend( + f"rule {rule_id} is new" + for rule_id in sorted(current_ids - {rule.id for rule in base.rules}) + ) return violations, notes diff --git a/scripts/rules/ci_scripts.py b/scripts/rules/ci_scripts.py index 7f04c46b6..e14be8aaf 100644 --- a/scripts/rules/ci_scripts.py +++ b/scripts/rules/ci_scripts.py @@ -177,24 +177,25 @@ def compare(registry: Registry, findings: Findings, root: Path) -> list[Violatio "fix what `shellcheck