From d70d956e5077fdd1e9a822c2648308a773f116b4 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Tue, 27 Oct 2026 13:32:35 +0530 Subject: [PATCH 1/4] FIX: Use pinned Rust releases with explicit dev-build overrides Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CONTRIBUTING.md | 44 +++ MANIFEST.in | 3 +- .../build-release-package-pipeline.yml | 7 + OneBranchPipelines/conda-release-pipeline.yml | 3 - OneBranchPipelines/dummy-release-pipeline.yml | 4 +- .../jobs/consolidate-artifacts-job.yml | 16 +- .../official-release-pipeline.yml | 4 +- .../scripts/validate-release-versions.ps1 | 73 +++-- .../scripts/validate-rs-wheel-dependency.ps1 | 10 +- .../stages/build-linux-single-stage.yml | 21 +- .../stages/build-macos-single-stage.yml | 11 +- .../stages/build-windows-single-stage.yml | 10 + .../steps/conda-publish-step.yml | 4 - .../steps/conda-release-step.yml | 4 - OneBranchPipelines/stress-test-pipeline.yml | 63 +++- conda/README.md | 28 +- eng/conda_tools/build.py | 16 +- eng/conda_tools/inputs.py | 33 +- eng/conda_tools/provenance.py | 12 +- eng/conda_tools/release.py | 5 +- .../mssql-odbc-daily-validation-pipeline.yml | 12 +- eng/pipelines/pr-validation-pipeline.yml | 41 +-- eng/pipelines/steps/install-mssql-py-core.yml | 4 +- .../download_mssql_python_rs_wheels.py | 179 ++++++----- eng/scripts/install-mssql-py-core.ps1 | 171 +---------- eng/scripts/install-mssql-py-core.sh | 203 +------------ eng/scripts/install_mssql_python_rs.py | 113 +++++++ eng/scripts/mssql_python_rs.py | 80 +++++ eng/versions/mssql-python-rs-nuget.version | 1 - setup.py | 7 +- tests/test_030_pe_machine_assert.py | 3 + tests/test_034_conda_verify_cwd.py | 43 +++ tests/test_036_conda_provenance.py | 144 ++++----- tests/test_037_rs_wheel_download.py | 261 +++++++--------- tests/test_038_mssql_odbc_daily_validation.py | 27 +- tests/test_039_rs_runtime_pipelines.py | 281 ++++++++++++++++++ tests/test_040_rs_runtime_selection.py | 181 +++++++++++ 37 files changed, 1285 insertions(+), 837 deletions(-) create mode 100644 eng/scripts/install_mssql_python_rs.py create mode 100644 eng/scripts/mssql_python_rs.py delete mode 100644 eng/versions/mssql-python-rs-nuget.version create mode 100644 tests/test_039_rs_runtime_pipelines.py create mode 100644 tests/test_040_rs_runtime_selection.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 83ff7dd67..37602e9b7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,6 +15,50 @@ or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any addi ## Before Contributing +### Selecting the Rust Runtime + +Normal builds and tests install the exact released `mssql-python-rs` version in +[`eng/versions/mssql-python-rs.version`](eng/versions/mssql-python-rs.version) +from public PyPI. The runtime is a separate dependency, not embedded in the +`mssql-python` wheel. No NuGet transport package is needed. + +To test a Python feature against merged but unreleased Rust changes, select the +exact version from a successful `mssql-rs` dev or nightly publishing run: + +```powershell +# Windows, with the development environment activated +$env:MSSQL_PYTHON_RS_VERSION = "0.4.0.dev20261028187000" # Replace with a published build +.\eng\scripts\install-mssql-py-core.ps1 +``` + +```bash +# Linux or macOS, with the development environment activated +export MSSQL_PYTHON_RS_VERSION=0.4.0.dev20261028187000 # Replace with a published build +bash eng/scripts/install-mssql-py-core.sh +``` + +Keep the environment variable set while building/installing `mssql-python` and +running tests. Its generated dependency metadata then uses the same selected +version. Development versions come from the anonymous Azure Artifacts PyPI +index; clean releases come from public PyPI. A missing requested wheel fails; +the installer never falls back to another version. Cross-architecture Windows +builds validate the target wheel without installing it into the host Python. + +Azure build/test pipelines expose the same selection as `rsVersionOverride`. +Leave it empty for the committed release pin. Record the exact dev version in +the feature's PR so another developer can reproduce the tests. Before running +tests after other package installations, verify the runtime has not changed: + +```console +python eng/scripts/mssql_python_rs.py --verify +``` + +Feature-test artifacts are not eligible for public release, even if an override +selects a stable version. Once the required Rust runtime is released on PyPI, +update the committed pin and validate with no override before releasing Python. +Unset `MSSQL_PYTHON_RS_VERSION` to return to the committed release pin and rerun +the installer. + ### Install Local Formatting Hooks From the repository root, with your Python 3.10+ development virtual environment diff --git a/MANIFEST.in b/MANIFEST.in index 8a680962e..43edbd917 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ -include eng/versions/mssql-python-rs.version \ No newline at end of file +include eng/versions/mssql-python-rs.version +include eng/scripts/mssql_python_rs.py \ No newline at end of file diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index bf0904658..68a579214 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -51,6 +51,11 @@ schedules: # PIPELINE PARAMETERS # ========================= parameters: + - name: rsVersionOverride + displayName: 'Exact mssql-python-rs test version (NonOfficial only)' + type: string + default: '' + # OneBranch build type determines compliance level # - Official: Production builds with full SDL compliance, all security scanning enabled # - NonOfficial: Development/test builds with reduced security scanning @@ -182,6 +187,8 @@ parameters: # PIPELINE VARIABLES # ========================= variables: + - name: MSSQL_PYTHON_RS_VERSION + value: ${{ parameters.rsVersionOverride }} # Determine effective build type: scheduled builds are Official, manual/PR builds use parameter # Build.Reason values: Schedule, Manual, IndividualCI, PullRequest, BatchedCI - name: effectiveOneBranchType diff --git a/OneBranchPipelines/conda-release-pipeline.yml b/OneBranchPipelines/conda-release-pipeline.yml index a2611e809..2b1da3947 100644 --- a/OneBranchPipelines/conda-release-pipeline.yml +++ b/OneBranchPipelines/conda-release-pipeline.yml @@ -165,7 +165,6 @@ extends: parameters: mssqlPythonVersion: '$(Provenance.mssqlPythonVersion)' releaseVersions: '$(Provenance.releaseVersions)' - rsTransportVersion: '$(Provenance.rsTransportVersion)' # Dry run (publishToConda=false): enumerate every package and print the exact # `anaconda upload` plan WITHOUT uploading, so a validate-only run shows precisely @@ -250,7 +249,6 @@ extends: ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' mssqlPythonVersion: $[ dependencies.ValidateConda.outputs['Provenance.mssqlPythonVersion'] ] releaseVersions: $[ dependencies.ValidateConda.outputs['Provenance.releaseVersions'] ] - rsTransportVersion: $[ dependencies.ValidateConda.outputs['Provenance.rsTransportVersion'] ] steps: - checkout: self fetchDepth: 1 @@ -260,4 +258,3 @@ extends: condaLabel: '${{ parameters.condaLabel }}' mssqlPythonVersion: '$(mssqlPythonVersion)' releaseVersions: '$(releaseVersions)' - rsTransportVersion: '$(rsTransportVersion)' diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index a7808aed7..a464d4bf3 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -202,14 +202,14 @@ extends: inputs: targetType: 'filePath' filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' - arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version"' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version" -ReceiptFile "$(Build.SourcesDirectory)\artifacts\dist\transport.json"' - task: PowerShell@2 displayName: '[TEST] Validate mssql-python-rs wheel dependency' inputs: targetType: 'filePath' filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-rs-wheel-dependency.ps1' - arguments: '-WheelsDir "$(Build.SourcesDirectory)\artifacts\dist" -VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version"' + arguments: '-WheelsDir "$(Build.SourcesDirectory)\artifacts\dist" -VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version" -ReceiptFile "$(Build.SourcesDirectory)\artifacts\dist\transport.json"' # Guard: if the released mssql-python wheel declares an # mssql-python-odbc dependency, it MUST be the exact pinned version. diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index d2b7b9ff1..19d7f0cb1 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -125,16 +125,22 @@ jobs: displayName: 'Consolidate Windows symbols (optional)' continueOnError: true - # Stage the actual RS payload using THIS producer source's immutable pins. - # Downstream Conda jobs consume this artifact, never their checkout's NuGet pin. + # Carry the effective runtime selection with the artifact, including test overrides. - bash: | set -e python3 "$(Build.SourcesDirectory)/eng/scripts/download_mssql_python_rs_wheels.py" \ --output-dir "$(ob_outputDirectory)/rs-wheels" - cp "$(Build.SourcesDirectory)/eng/versions/mssql-python-rs.version" "$(ob_outputDirectory)/dist/" - cp "$(Build.SourcesDirectory)/eng/versions/mssql-python-rs-nuget.version" "$(ob_outputDirectory)/dist/" + python3 -c "import json,pathlib; root=pathlib.Path('$(ob_outputDirectory)'); receipt=json.loads((root/'rs-wheels/transport.json').read_text()); (root/'dist/mssql-python-rs.version').write_text(receipt['distribution_version']+'\n'); (root/'dist/transport.json').write_bytes((root/'rs-wheels/transport.json').read_bytes())" echo "mssql-python-rs version: $(cat "$(ob_outputDirectory)/dist/mssql-python-rs.version")" - displayName: 'Stage source-pinned mssql-python-rs wheels and transport receipt' + displayName: 'Stage effective mssql-python-rs wheels, version and receipt' + + - task: PowerShell@2 + displayName: 'Validate every wheel uses the effective runtime version' + inputs: + pwsh: true + targetType: filePath + filePath: OneBranchPipelines/scripts/validate-rs-wheel-dependency.ps1 + arguments: '-WheelsDir "$(ob_outputDirectory)/dist" -VersionFile "$(ob_outputDirectory)/dist/mssql-python-rs.version" -ReceiptFile "$(ob_outputDirectory)/dist/transport.json"' # Verify consolidation - bash: | diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index f84240d23..262cc0944 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -202,14 +202,14 @@ extends: inputs: targetType: 'filePath' filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' - arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version"' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version" -ReceiptFile "$(Build.SourcesDirectory)\artifacts\dist\transport.json"' - task: PowerShell@2 displayName: 'Validate mssql-python-rs wheel dependency' inputs: targetType: 'filePath' filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-rs-wheel-dependency.ps1' - arguments: '-WheelsDir "$(Build.SourcesDirectory)\artifacts\dist" -VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version"' + arguments: '-WheelsDir "$(Build.SourcesDirectory)\artifacts\dist" -VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version" -ReceiptFile "$(Build.SourcesDirectory)\artifacts\dist\transport.json"' # Guard: if the released mssql-python wheel declares an # mssql-python-odbc dependency, it MUST be the exact pinned version. diff --git a/OneBranchPipelines/scripts/validate-release-versions.ps1 b/OneBranchPipelines/scripts/validate-release-versions.ps1 index ec77286dd..84d470776 100644 --- a/OneBranchPipelines/scripts/validate-release-versions.ps1 +++ b/OneBranchPipelines/scripts/validate-release-versions.ps1 @@ -3,47 +3,62 @@ <# .SYNOPSIS - Validates that a version file contains a stable (non-prerelease) version. - + Gates Official builds and releases on an exact public PyPI runtime release. .DESCRIPTION - Reads a single .version file and rejects any version containing - dev, alpha, beta, or rc tags. Intended to gate official releases. - -.PARAMETER VersionFile - Path to a .version file. Defaults to eng/versions/mssql-python-rs.version - relative to the repository root. - -.EXAMPLE - # Run from repo root (uses default path): - .\eng\scripts\validate-release-versions.ps1 - - # Explicit path: - .\OneBranchPipelines\scripts\validate-release-versions.ps1 -VersionFile C:\work\mssql-python\eng\versions\mssql-python-rs.version + BuildOnly rejects overrides and non-release source pins without network access. + Release validation additionally requires the selected artifact's transport + receipt and verifies the exact version exists on public PyPI before ESRP. #> - param( - [string]$VersionFile + [string]$VersionFile, + [string]$ReceiptFile, + [switch]$BuildOnly ) $ErrorActionPreference = 'Stop' if (-not $VersionFile) { - $repoRoot = (Resolve-Path "$PSScriptRoot\..\..").Path - $VersionFile = Join-Path $repoRoot 'eng\versions\mssql-python-rs.version' + $VersionFile = Join-Path $PSScriptRoot '../../eng/versions/mssql-python-rs.version' +} + +if (-not [string]::IsNullOrWhiteSpace($env:MSSQL_PYTHON_RS_VERSION)) { + throw 'Official builds and releases forbid MSSQL_PYTHON_RS_VERSION overrides, including stable versions.' } -if (-not (Test-Path $VersionFile)) { - Write-Error "Version file not found: $VersionFile" - exit 1 +$version = (Get-Content -LiteralPath $VersionFile -Raw).Trim() +if ($version -cnotmatch '^[0-9]+\.[0-9]+\.[0-9]+$') { + throw "Official builds and releases require an exact stable X.Y.Z runtime version, got '$version'." } -$version = (Get-Content $VersionFile -Raw).Trim() -$name = [System.IO.Path]::GetFileNameWithoutExtension($VersionFile) +if (-not $BuildOnly) { + if (-not $ReceiptFile) { + $ReceiptFile = Join-Path (Split-Path -Parent $VersionFile) 'transport.json' + } + $receipt = Get-Content -LiteralPath $ReceiptFile -Raw | ConvertFrom-Json + if ($receipt.version_override -isnot [bool] -or $receipt.version_override) { + throw 'Official releases require a receipt with version_override=false.' + } + if ($receipt.distribution_version -cne $version) { + throw 'The staged runtime version and transport receipt disagree.' + } + if ($receipt.index_url -cnotmatch '^https://pypi\.org/simple/?$') { + throw 'Official releases require the public PyPI runtime source.' + } + if (-not $receipt.wheel_sha256 -or @($receipt.wheel_sha256.PSObject.Properties).Count -eq 0) { + throw 'The runtime receipt must record wheel SHA256 hashes.' + } + foreach ($hash in $receipt.wheel_sha256.PSObject.Properties) { + if ($hash.Name -notlike '*.whl' -or $hash.Value -cnotmatch '^[a-f0-9]{64}$') { + throw 'The runtime receipt contains an invalid wheel SHA256 entry.' + } + } -if ($version -match '(dev|alpha|beta|rc)') { - Write-Host "FAIL: $name version '$version' is a pre-release ($($Matches[1]))" -ForegroundColor Red - Write-Error "$name version is pre-release. Official releases require stable versions." - exit 1 + $release = Invoke-RestMethod -Uri "https://pypi.org/pypi/mssql-python-rs/$version/json" -Method Get + if ($release.info.version -cne $version -or + $release.info.name -notmatch '^mssql[-_]python[-_]rs$' -or + @($release.urls | Where-Object { $_.packagetype -eq 'bdist_wheel' -and -not $_.yanked }).Count -eq 0) { + throw "mssql-python-rs==$version has no matching public PyPI wheel release." + } } -Write-Host "OK: $name version '$version'" -ForegroundColor Green +Write-Host "OK: mssql-python-rs==$version is eligible for Official $(if ($BuildOnly) { 'build' } else { 'release' })." diff --git a/OneBranchPipelines/scripts/validate-rs-wheel-dependency.ps1 b/OneBranchPipelines/scripts/validate-rs-wheel-dependency.ps1 index 1b15dd99e..2a42acad9 100644 --- a/OneBranchPipelines/scripts/validate-rs-wheel-dependency.ps1 +++ b/OneBranchPipelines/scripts/validate-rs-wheel-dependency.ps1 @@ -3,7 +3,9 @@ param( [string]$WheelsDir, [Parameter(Mandatory = $true)] - [string]$VersionFile + [string]$VersionFile, + + [string]$ReceiptFile ) $ErrorActionPreference = 'Stop' @@ -12,6 +14,12 @@ $expectedVersion = (Get-Content $VersionFile -Raw).Trim() if (-not $expectedVersion) { throw "Version file is empty: $VersionFile" } +if ($ReceiptFile) { + $receipt = Get-Content -LiteralPath $ReceiptFile -Raw | ConvertFrom-Json + if ($receipt.distribution_version -cne $expectedVersion) { + throw 'The wheel dependency version and runtime receipt disagree.' + } +} $wheels = @(Get-ChildItem $WheelsDir -Filter 'mssql_python-*.whl' -File) if ($wheels.Count -eq 0) { diff --git a/OneBranchPipelines/stages/build-linux-single-stage.yml b/OneBranchPipelines/stages/build-linux-single-stage.yml index e50123769..3a383d447 100644 --- a/OneBranchPipelines/stages/build-linux-single-stage.yml +++ b/OneBranchPipelines/stages/build-linux-single-stage.yml @@ -74,6 +74,15 @@ stages: steps: - checkout: self fetchDepth: 0 + + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: PowerShell@2 + displayName: 'Reject test overrides and prereleases before Official build' + inputs: + pwsh: true + targetType: filePath + filePath: OneBranchPipelines/scripts/validate-release-versions.ps1 + arguments: '-BuildOnly' # Docker is pre-installed now # Verify it's working and start the daemon @@ -127,7 +136,7 @@ stages: exit 1 fi - docker run -d --name build-$(LINUX_TAG)-$(ARCH) \ + docker run -e MSSQL_PYTHON_RS_VERSION -d --name build-$(LINUX_TAG)-$(ARCH) \ --platform $(DOCKER_PLATFORM) \ -v $(Build.SourcesDirectory):/workspace \ -w /workspace \ @@ -314,6 +323,7 @@ stages: $PY -m pip install -q --require-hashes -r /workspace/eng/requirements-test-linux.txt; echo "Installing: $WHEEL"; $PY -m pip install -q "$WHEEL"; + $PY /workspace/eng/scripts/mssql_python_rs.py --verify; # Step 6: Verify package imports correctly echo "Verifying package installation..."; @@ -330,7 +340,9 @@ stages: mkdir -p $TEST_DIR/OneBranchPipelines/stages; cp /workspace/eng/scripts/run-mssql-odbc-tests.sh /workspace/eng/scripts/verify_mssql_odbc_provider.py $TEST_DIR/eng/scripts/; cp /workspace/eng/pipelines/mssql-odbc-daily-validation-pipeline.yml $TEST_DIR/eng/pipelines/; - cp /workspace/eng/versions/mssql-python-rs-nuget.version $TEST_DIR/eng/versions/; + cp -r /workspace/eng/pipelines/. $TEST_DIR/eng/pipelines/; + cp -r /workspace/OneBranchPipelines/. $TEST_DIR/OneBranchPipelines/; + cp /workspace/eng/versions/mssql-python-rs.version $TEST_DIR/eng/versions/; cp /workspace/OneBranchPipelines/stages/build-linux-single-stage.yml $TEST_DIR/OneBranchPipelines/stages/; cp /workspace/pytest.ini $TEST_DIR/ || echo "WARNING: No pytest.ini"; @@ -395,6 +407,7 @@ stages: $PY -m pip install -q --require-hashes -r /workspace/eng/requirements-test-linux.txt; echo "Installing: $WHEEL"; $PY -m pip install -q "$WHEEL"; + $PY /workspace/eng/scripts/mssql_python_rs.py --verify; # Step 6: Verify package imports correctly echo "Verifying package installation..."; @@ -411,7 +424,9 @@ stages: mkdir -p $TEST_DIR/OneBranchPipelines/stages; cp /workspace/eng/scripts/run-mssql-odbc-tests.sh /workspace/eng/scripts/verify_mssql_odbc_provider.py $TEST_DIR/eng/scripts/; cp /workspace/eng/pipelines/mssql-odbc-daily-validation-pipeline.yml $TEST_DIR/eng/pipelines/; - cp /workspace/eng/versions/mssql-python-rs-nuget.version $TEST_DIR/eng/versions/; + cp -r /workspace/eng/pipelines/. $TEST_DIR/eng/pipelines/; + cp -r /workspace/OneBranchPipelines/. $TEST_DIR/OneBranchPipelines/; + cp /workspace/eng/versions/mssql-python-rs.version $TEST_DIR/eng/versions/; cp /workspace/OneBranchPipelines/stages/build-linux-single-stage.yml $TEST_DIR/OneBranchPipelines/stages/; cp /workspace/pytest.ini $TEST_DIR/ || echo "WARNING: No pytest.ini"; diff --git a/OneBranchPipelines/stages/build-macos-single-stage.yml b/OneBranchPipelines/stages/build-macos-single-stage.yml index 65909d55b..901d93de3 100644 --- a/OneBranchPipelines/stages/build-macos-single-stage.yml +++ b/OneBranchPipelines/stages/build-macos-single-stage.yml @@ -77,6 +77,15 @@ stages: # fetchDepth: 0 = full git history (needed for version tagging) - checkout: self fetchDepth: 0 + + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: PowerShell@2 + displayName: 'Reject test overrides and prereleases before Official build' + inputs: + pwsh: true + targetType: filePath + filePath: OneBranchPipelines/scripts/validate-release-versions.ps1 + arguments: '-BuildOnly' # ========================= # PYTHON INSTALLATION @@ -217,7 +226,7 @@ stages: # Tests use localhost:1433 connection (SA user with password from variable) # -v = verbose output (show test names and results) - script: | - python -m pytest -v + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v displayName: 'Run pytests' env: # Connection string uses localhost (SQL Server container exposed on port 1433) diff --git a/OneBranchPipelines/stages/build-windows-single-stage.yml b/OneBranchPipelines/stages/build-windows-single-stage.yml index b1c8cdbb3..ec6ae6d37 100644 --- a/OneBranchPipelines/stages/build-windows-single-stage.yml +++ b/OneBranchPipelines/stages/build-windows-single-stage.yml @@ -86,6 +86,14 @@ stages: steps: - checkout: self fetchDepth: 0 + + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: PowerShell@2 + displayName: 'Reject test overrides and prereleases before Official build' + inputs: + targetType: filePath + filePath: OneBranchPipelines/scripts/validate-release-versions.ps1 + arguments: '-BuildOnly' # Python 3.14 Installation: Download from NuGet (not yet in UsePythonVersion@0 task) # Microsoft hasn't added Python 3.14 to the standard Python registry yet @@ -362,6 +370,8 @@ stages: if ("$(targetArch)" -eq "arm64") { Write-Host "Skipping pytests on Windows ARM64" } else { + python eng/scripts/mssql_python_rs.py --verify + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } python -m pytest -v } displayName: 'Run pytests' diff --git a/OneBranchPipelines/steps/conda-publish-step.yml b/OneBranchPipelines/steps/conda-publish-step.yml index 9c406726a..d1050776b 100644 --- a/OneBranchPipelines/steps/conda-publish-step.yml +++ b/OneBranchPipelines/steps/conda-publish-step.yml @@ -49,9 +49,6 @@ parameters: - name: releaseVersions type: string default: '' - - name: rsTransportVersion - type: string - default: '' # Comma-separated subdirs a complete release MUST contain (full PyPI parity minus # musllinux, which has no conda subdir). win-arm64 IS included -- the build enforces # its --dry-run solve + PE-arch assert, so a missing/broken win-arm64 blocks the @@ -153,7 +150,6 @@ steps: CONDA_LABEL: '${{ parameters.condaLabel }}' MSSQL_PYTHON_VERSION: '${{ parameters.mssqlPythonVersion }}' RELEASE_VERSIONS: '${{ parameters.releaseVersions }}' - RS_TRANSPORT_VERSION: '${{ parameters.rsTransportVersion }}' REQUIRED_SUBDIRS: '${{ parameters.requiredSubdirs }}' ALLOWED_SUBDIRS: '${{ parameters.allowedSubdirs }}' PYTHON_VERSIONS: '${{ parameters.pythonVersions }}' diff --git a/OneBranchPipelines/steps/conda-release-step.yml b/OneBranchPipelines/steps/conda-release-step.yml index 3e3cce161..61e402523 100644 --- a/OneBranchPipelines/steps/conda-release-step.yml +++ b/OneBranchPipelines/steps/conda-release-step.yml @@ -74,9 +74,6 @@ parameters: - name: releaseVersions type: string default: '' - - name: rsTransportVersion - type: string - default: '' # The audit dependency hash targets CPython 3.12 / Windows x64, like the publisher. - name: pythonVersion type: string @@ -160,7 +157,6 @@ steps: PYTHON_VERSIONS: '${{ parameters.pythonVersions }}' MSSQL_PYTHON_VERSION: '${{ parameters.mssqlPythonVersion }}' RELEASE_VERSIONS: '${{ parameters.releaseVersions }}' - RS_TRANSPORT_VERSION: '${{ parameters.rsTransportVersion }}' inputs: targetType: 'inline' workingDirectory: '$(Build.SourcesDirectory)' diff --git a/OneBranchPipelines/stress-test-pipeline.yml b/OneBranchPipelines/stress-test-pipeline.yml index 548f52186..d7a5dc7d3 100644 --- a/OneBranchPipelines/stress-test-pipeline.yml +++ b/OneBranchPipelines/stress-test-pipeline.yml @@ -27,7 +27,15 @@ resources: source: 'Build-Release-Package-Pipeline' # Name of the build pipeline trigger: none # Don't auto-trigger, we run on schedule or manually +parameters: + - name: rsVersionOverride + displayName: 'Exact mssql-python-rs test version (empty uses committed PyPI release)' + type: string + default: '' + variables: + - name: MSSQL_PYTHON_RS_VERSION + value: ${{ parameters.rsVersionOverride }} # Common variables - template: /OneBranchPipelines/variables/common-variables.yml@self - name: pythonVersion @@ -62,17 +70,28 @@ jobs: pip install psutil pytest-timeout displayName: 'Install dependencies' - # Download pre-built wheel from build-whl-pipeline instead of building from source + # Select a matching NonOfficial build resource when testing a runtime override. - task: DownloadPipelineArtifact@2 inputs: buildType: 'specific' project: '$(System.TeamProject)' - definition: 2162 # build-whl-pipeline definition ID - buildVersionToDownload: 'latest' - branchName: 'refs/heads/main' - artifactName: 'mssql-python-wheels-dist' + definition: '$(resources.pipeline.buildPipeline.pipelineID)' + buildVersionToDownload: 'specific' + pipelineId: '$(resources.pipeline.buildPipeline.runID)' + artifactName: 'drop_Consolidate_ConsolidateArtifacts' targetPath: '$(Build.SourcesDirectory)\wheels' - displayName: 'Download pre-built wheel from build-whl-pipeline' + displayName: 'Download selected build (choose matching override artifacts for dev tests)' + + - script: | + python eng/scripts/mssql_python_rs.py --write-version-file "$(Pipeline.Workspace)\effective-runtime.version" + displayName: 'Resolve effective stress-test runtime' + + - task: PowerShell@2 + displayName: 'Require test wheel metadata and receipt to match effective runtime' + inputs: + targetType: filePath + filePath: OneBranchPipelines/scripts/validate-rs-wheel-dependency.ps1 + arguments: '-WheelsDir "$(Build.SourcesDirectory)\wheels\dist" -VersionFile "$(Pipeline.Workspace)\effective-runtime.version" -ReceiptFile "$(Build.SourcesDirectory)\wheels\dist\transport.json"' - script: | python eng/scripts/download_mssql_python_rs_wheels.py --output-dir "$(Pipeline.Workspace)\mssql-python-rs-wheels" @@ -82,7 +101,9 @@ jobs: - script: | echo "Installing pre-built wheel..." dir "$(Build.SourcesDirectory)\wheels" - pip install --find-links=$(Build.SourcesDirectory)\wheels --find-links=$(Pipeline.Workspace)\mssql-python-rs-wheels mssql-python --force-reinstall + python -c "import subprocess,sys; from eng.scripts.mssql_python_rs import resolve_runtime; subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--no-index', '--no-deps', '--find-links', r'$(Build.SourcesDirectory)\wheels\dist', '--find-links', r'$(Pipeline.Workspace)\mssql-python-rs-wheels', 'mssql-python', 'mssql-python-rs=='+resolve_runtime().version, '--force-reinstall'])" + if errorlevel 1 exit /b 1 + python eng/scripts/mssql_python_rs.py --verify displayName: 'Install pre-built mssql-python wheel' # Rename local source folder to avoid import conflicts with installed wheel @@ -232,17 +253,29 @@ jobs: pip install psutil pytest-timeout displayName: 'Install Python dependencies' - # Download pre-built wheel from build-whl-pipeline instead of building from source + # Select a matching NonOfficial build resource when testing a runtime override. - task: DownloadPipelineArtifact@2 inputs: buildType: 'specific' project: '$(System.TeamProject)' - definition: 2162 # build-whl-pipeline definition ID - buildVersionToDownload: 'latest' - branchName: 'refs/heads/main' - artifactName: 'mssql-python-wheels-dist' + definition: '$(resources.pipeline.buildPipeline.pipelineID)' + buildVersionToDownload: 'specific' + pipelineId: '$(resources.pipeline.buildPipeline.runID)' + artifactName: 'drop_Consolidate_ConsolidateArtifacts' targetPath: '$(Build.SourcesDirectory)/wheels' - displayName: 'Download pre-built wheel from build-whl-pipeline' + displayName: 'Download selected build (choose matching override artifacts for dev tests)' + + - script: | + python eng/scripts/mssql_python_rs.py --write-version-file "$(Pipeline.Workspace)/effective-runtime.version" + displayName: 'Resolve effective stress-test runtime' + + - task: PowerShell@2 + displayName: 'Require test wheel metadata and receipt to match effective runtime' + inputs: + pwsh: true + targetType: filePath + filePath: OneBranchPipelines/scripts/validate-rs-wheel-dependency.ps1 + arguments: '-WheelsDir "$(Build.SourcesDirectory)/wheels/dist" -VersionFile "$(Pipeline.Workspace)/effective-runtime.version" -ReceiptFile "$(Build.SourcesDirectory)/wheels/dist/transport.json"' - script: | python eng/scripts/download_mssql_python_rs_wheels.py --output-dir "$(Pipeline.Workspace)/mssql-python-rs-wheels" @@ -250,9 +283,11 @@ jobs: # Install the pre-built wheel - script: | + set -e echo "Installing pre-built wheel..." ls -la "$(Build.SourcesDirectory)/wheels" - pip install --find-links=$(Build.SourcesDirectory)/wheels --find-links=$(Pipeline.Workspace)/mssql-python-rs-wheels mssql-python --force-reinstall + python -c "import subprocess,sys; from eng.scripts.mssql_python_rs import resolve_runtime; subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--no-index', '--no-deps', '--find-links', '$(Build.SourcesDirectory)/wheels/dist', '--find-links', '$(Pipeline.Workspace)/mssql-python-rs-wheels', 'mssql-python', 'mssql-python-rs=='+resolve_runtime().version, '--force-reinstall'])" + python eng/scripts/mssql_python_rs.py --verify displayName: 'Install pre-built mssql-python wheel' # Rename local source folder to avoid import conflicts with installed wheel diff --git a/conda/README.md b/conda/README.md index efbd5fb92..3dc64a15a 100644 --- a/conda/README.md +++ b/conda/README.md @@ -45,7 +45,7 @@ receives a bounded frame-structure pass because its streaming API alone accepts truncated frame endings. ZIP envelopes must use stored or deflate compression; sparse TAR extensions are rejected before sparse-map processing. Wheel metadata reads share the ZIP and metadata budgets. These are package-reader limits, not a -claim that NuGet transport downloads or total process memory have the same bounds. +claim that runtime wheel downloads or total process memory have the same bounds. The GitHub audit job installs its complete Linux x64 / CPython 3.11 test-tool closure from `requirements-audit.txt` with `--require-hashes --only-binary=:all:`. @@ -90,10 +90,13 @@ hashes with the final Conda archive hash after Linux relocation. Use `build --rs-wheel-dir /rs-wheels --rs-version-file /dist/mssql-python-rs.version` for RS inputs. -The version file asserts the selected producer's source contract, not a new release-version -setting. The distinct NuGet transport pin remains in `mssql-python-rs-nuget.version`; -the downloader records its content hash and individual wheel hashes in `transport.json`. -Conda consumers use staged producer bytes, not a fresh download using their checkout's pin. +The version file asserts the selected producer's runtime selection against the binding's +exact dependency, not a new release-version setting. The producer's `transport.json` +records `distribution_version`, `index_url`, `version_override` and individual +`wheel_sha256` values. Released runtimes use PyPI; an explicitly selected `.devN` +version uses the public `mssql-rs_Public` Python index. Conda consumers validate this +receipt and use staged producer bytes, not a fresh download using their checkout's +pin or `MSSQL_PYTHON_RS_VERSION`. No NuGet package or transport pin is required. Published historical bindings that declare no RS dependency are accepted only when their payload and RECORD own the core initializer and native extension. They produce an explicit @@ -269,13 +272,14 @@ acquisition; configuration readback alone is not evidence that a run holds the l The release pipeline resolves AUTO from the **recorded upstream wheel commit**, not the release checkout or a latest-version lookup. It cross-checks binding setup/runtime versions and reads ODBC and, when explicitly required by that source, RS distribution -versions. RS's NuGet transport pin is separate. Missing or ambiguous source declarations +versions. Source analysis reads the clean `X.Y.Z` RS pin without executing the +runtime-selection helper or consulting the consumer's environment override. +Missing or ambiguous source declarations fail rather than select a historical profile. A supplied binding version is only an assertion against this resolved version. Both readiness and the credentialed job use `validate --release-versions` to read the -verified `RELEASE_VERSIONS` JSON from the environment; `RS_TRANSPORT_VERSION` supplies -the separate transport assertion. This avoids passing JSON quoting or empty arguments +verified `RELEASE_VERSIONS` JSON from the environment. This avoids passing JSON quoting or empty arguments through Windows PowerShell. Explicit JSON may also follow `--release-versions` in source-only controls. With this flag, absent versions fail closed. Without it, `validate` remains the metadata/matrix-only inspection command and makes no source-input claim. @@ -283,7 +287,9 @@ The source-bound gate checks each installed component's METADATA and RECORD owne the binding's exact dependency pins, and the selected RS filename/WHEEL tags against `rs-transport.json`. Raw wheel/transport hashes remain input evidence; relocated installed native files are checked by the native audits, not falsely compared with raw -wheel hashes. No numeric RS build ID is inferred from a transport-version suffix. +wheel hashes. The receipt must name the verified RS distribution version and its +release/development index; a receipt cannot override the verified source version. +No numeric RS build ID is inferred from a version suffix. Release and native auditing also check all installed core paths, not only RECORD-filtered members. This detects unowned or cross-owned additions; it cannot recover file origin after an overwrite of an already owned path, which is why the raw pre-extraction gate @@ -313,7 +319,7 @@ The `probe-driver` module command is a convenience diagnostic in the calling interpreter, not a substitute for this neutral-directory installed-package check. The files remaining under `conda/` are the conventional recipe entrypoints, -metadata, documentation and line-ending configuration. Shared NuGet transport, -feed-resolution and output-safety utilities remain under `eng/scripts` because +metadata, documentation and line-ending configuration. Shared Python-index download, +runtime-selection and output-safety utilities remain under `eng/scripts` because non-Conda native build/development workflows also use them; Conda reuses those implementations rather than keeping private copies. diff --git a/eng/conda_tools/build.py b/eng/conda_tools/build.py index f68392cb0..1dd18b63d 100644 --- a/eng/conda_tools/build.py +++ b/eng/conda_tools/build.py @@ -13,7 +13,7 @@ from eng.scripts.download_mssql_python_rs_wheels import file_sha256 -from . import archive, contracts, environment, verify +from . import archive, contracts, environment, inputs, verify from .formats import elf, macho, pe _KNOWN_SUBDIRS = ("win-64", "win-arm64", "osx-64", "osx-arm64", "linux-64", "linux-aarch64") @@ -167,19 +167,7 @@ def gather_wheels( receipt_path = Path(rs_dir, "transport.json") if receipt_path.exists(): receipt = json.loads(receipt_path.read_text(encoding="utf-8")) - if not isinstance(receipt, dict) or not isinstance( - receipt.get("wheel_sha256"), dict - ): - raise ValueError("RS transport receipt must contain a wheel_sha256 mapping") - if receipt.get("distribution_version") != rs_ver: - raise ValueError("RS transport receipt disagrees with the binding dependency") - if rs_version_file is not None: - pin_file = Path(rs_version_file).with_name("mssql-python-rs-nuget.version") - transport = pin_file.read_text(encoding="ascii").strip() - if not transport or receipt.get("transport_version") != transport: - raise ValueError( - "RS transport receipt disagrees with the producer transport pin" - ) + inputs.validate_rs_transport(receipt, {"mssql-python-rs": rs_ver}) for wheel in rs: if receipt["wheel_sha256"].get(Path(wheel).name) != file_sha256(Path(wheel)): raise ValueError( diff --git a/eng/conda_tools/inputs.py b/eng/conda_tools/inputs.py index 5cc43380b..efe326361 100644 --- a/eng/conda_tools/inputs.py +++ b/eng/conda_tools/inputs.py @@ -14,7 +14,7 @@ from urllib.error import URLError from urllib.request import urlopen -from eng.scripts.download_mssql_python_rs_wheels import DEFAULT_FEED, PACKAGE_ID, file_sha256 +from eng.scripts.download_mssql_python_rs_wheels import DEFAULT_INDEX, DEV_INDEX, file_sha256 from . import archive, contracts @@ -175,9 +175,7 @@ def fetch_wheels( violations = contracts.validate_core_layout(owned, python_tag, subdir) else: if source_versions.get("mssql-python-rs") != rs_version: - raise ValueError( - "Published binding RS requirement differs from the selected pin." - ) + raise ValueError("Published binding RS requirement differs from the selected pin.") versions["mssql-python-rs"] = rs_version hashes.update( _download_published( @@ -375,7 +373,7 @@ def read_release_versions(read_source: Callable[[str], str]) -> dict[str, str]: if _source_requires_rs(setup_source): path = "eng/versions/mssql-python-rs.version" version = read_source(path).strip() - if not re.fullmatch(r"[0-9][A-Za-z0-9.!+_-]*", version): + if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version): raise ValueError(f"{path} must contain one RS distribution release version.") result["mssql-python-rs"] = version return result @@ -449,31 +447,32 @@ def parse_release_versions(raw: str) -> dict[str, str]: return versions -def validate_rs_transport( - receipt: object, versions: dict[str, str], transport_version: str -) -> None: - """Check the selected producer's receipt against its separately verified source pins.""" +def validate_rs_transport(receipt: object, versions: dict[str, str]) -> None: + """Check index evidence against the selected producer's verified distribution pin.""" version = versions.get("mssql-python-rs") if version is None: - if receipt is not None or transport_version: + if receipt is not None: raise ValueError("An embedded-core producer must not supply an RS transport receipt.") return if not isinstance(receipt, dict): raise ValueError("The RS producer requires its recorded rs-transport.json receipt.") expected = { "distribution_version": version, - "transport_version": transport_version, - "feed_url": DEFAULT_FEED, - "package_id": PACKAGE_ID, + "index_url": DEV_INDEX if ".dev" in version else DEFAULT_INDEX, } - if not transport_version or any(receipt.get(key) != value for key, value in expected.items()): + override = receipt.get("version_override") + if ( + any(receipt.get(key) != value for key, value in expected.items()) + or not isinstance(override, bool) + or (".dev" in version and not override) + ): raise ValueError( - "RS transport receipt differs from the verified producer's source pins/feed." + "RS transport receipt differs from the verified producer's distribution pin/index." ) wheels = receipt.get("wheel_sha256") - if not _is_sha256(receipt.get("package_sha256")) or not isinstance(wheels, dict) or not wheels: - raise ValueError("RS transport receipt requires its package and wheel SHA256 values.") + if not isinstance(wheels, dict) or not wheels: + raise ValueError("RS transport receipt requires its wheel SHA256 values.") for name, digest in wheels.items(): if ( not isinstance(name, str) diff --git a/eng/conda_tools/provenance.py b/eng/conda_tools/provenance.py index 126b500e9..3f95d2fa1 100644 --- a/eng/conda_tools/provenance.py +++ b/eng/conda_tools/provenance.py @@ -60,15 +60,7 @@ def read_source(path: str) -> str: raise ValueError(f"Missing or mismatched wheel producer source for {path} at {commit}.") return item["content"] - versions = read_release_versions(read_source) - result = {"versions": versions, "rsTransportVersion": ""} - if "mssql-python-rs" in versions: - path = "eng/versions/mssql-python-rs-nuget.version" - transport_version = read_source(path).strip() - if not re.fullmatch(r"[0-9][A-Za-z0-9.+_-]*", transport_version): - raise ValueError(f"{path} must contain one RS transport version.") - result["rsTransportVersion"] = transport_version - return result + return {"versions": read_release_versions(read_source)} def _verify_run(build: dict, run: dict, pipeline_id: int, run_id: int) -> dict: @@ -202,8 +194,6 @@ def execute() -> None: print(f"##vso[task.setvariable variable=mssqlPythonVersion;isOutput=true]{version}") versions = json.dumps(result["versions"], sort_keys=True) print(f"##vso[task.setvariable variable=releaseVersions;isOutput=true]{versions}") - transport = result["rsTransportVersion"] - print(f"##vso[task.setvariable variable=rsTransportVersion;isOutput=true]{transport}") if publish == "false": print("Validate-only: provenance verified; package readiness is gated separately.") diff --git a/eng/conda_tools/release.py b/eng/conda_tools/release.py index bf7ab52da..7981e3be8 100644 --- a/eng/conda_tools/release.py +++ b/eng/conda_tools/release.py @@ -400,9 +400,6 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: default=None, help="Exact component version JSON; without a value, read RELEASE_VERSIONS.", ) - parser.add_argument( - "--rs-transport-version", default=os.environ.get("RS_TRANSPORT_VERSION", "") - ) def execute(args: argparse.Namespace) -> int: @@ -424,7 +421,7 @@ def execute(args: argparse.Namespace) -> int: if receipt_path.exists() else None ) - inputs.validate_rs_transport(receipt, versions, args.rs_transport_version) + inputs.validate_rs_transport(receipt, versions) except archive.READ_ERRORS as exc: print(f"ERROR: {exc}", file=sys.stderr) return 1 diff --git a/eng/pipelines/mssql-odbc-daily-validation-pipeline.yml b/eng/pipelines/mssql-odbc-daily-validation-pipeline.yml index 6a6712a64..7702b078a 100644 --- a/eng/pipelines/mssql-odbc-daily-validation-pipeline.yml +++ b/eng/pipelines/mssql-odbc-daily-validation-pipeline.yml @@ -11,6 +11,15 @@ schedules: - main always: true +parameters: + - name: rsVersionOverride + displayName: 'Exact mssql-python-rs test version (empty uses committed PyPI release)' + type: string + default: '' + +variables: + MSSQL_PYTHON_RS_VERSION: ${{ parameters.rsVersionOverride }} + jobs: - job: PytestOnMssqlOdbc displayName: 'Complete mssql-python suite on mssql-odbc' @@ -32,7 +41,7 @@ jobs: displayName: 'Checkout latest GitHub main' - script: | - docker run -d --name $(testContainer) \ + docker run -e MSSQL_PYTHON_RS_VERSION -d --name $(testContainer) \ -v $(Build.SourcesDirectory):/workspace \ -w /workspace \ --network bridge \ @@ -128,6 +137,7 @@ jobs: -e PYTHONPATH=/workspace \ $(testContainer) bash -c ' source /opt/venv/bin/activate + python eng/scripts/mssql_python_rs.py --verify || exit 2 chmod +x eng/scripts/run-mssql-odbc-tests.sh eng/scripts/run-mssql-odbc-tests.sh ' || rc=$? diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index c19a3e8aa..3b94d1163 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -6,7 +6,14 @@ trigger: include: - main +parameters: + - name: rsVersionOverride + displayName: 'Exact mssql-python-rs test version (empty uses committed PyPI release)' + type: string + default: '' + variables: + MSSQL_PYTHON_RS_VERSION: ${{ parameters.rsVersionOverride }} # QEMU 10.2.3, qualified against the ARM64 GCC startup crash and shutdown tests. arm64QemuImage: 'tonistiigi/binfmt@sha256:400a4873b838d1b89194d982c45e5fb3cda4593fbfd7e08a02e76b03b21166f0' @@ -355,7 +362,7 @@ jobs: # Run tests for LocalDB - script: | - python -m pytest -v --junitxml=test-results-localdb.xml --cov=. --cov-report=xml:coverage-localdb.xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results-localdb.xml --cov=. --cov-report=xml:coverage-localdb.xml --capture=tee-sys --cache-clear displayName: 'Run tests with coverage on LocalDB' condition: and(succeeded(), eq(variables['sqlVersion'], 'LocalDB')) env: @@ -363,7 +370,7 @@ jobs: # Run tests for SQL Server 2022 - script: | - python -m pytest -v --junitxml=test-results-sql2022.xml --cov=. --cov-report=xml:coverage-sql2022.xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results-sql2022.xml --cov=. --cov-report=xml:coverage-sql2022.xml --capture=tee-sys --cache-clear displayName: 'Run tests with coverage on SQL Server 2022' condition: and(succeeded(), eq(variables['sqlVersion'], 'SQL2022')) env: @@ -371,7 +378,7 @@ jobs: # Run tests for SQL Server 2025 - script: | - python -m pytest -v --junitxml=test-results-sql2025.xml --cov=. --cov-report=xml:coverage-sql2025.xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results-sql2025.xml --cov=. --cov-report=xml:coverage-sql2025.xml --capture=tee-sys --cache-clear displayName: 'Run tests with coverage on SQL Server 2025' condition: and(succeeded(), eq(variables['sqlVersion'], 'SQL2025')) env: @@ -716,7 +723,7 @@ jobs: - script: | echo "Build successful, running tests now" - python -m pytest -v --junitxml=test-results.xml --cov=. --cov-report=xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results.xml --cov=. --cov-report=xml --capture=tee-sys --cache-clear displayName: 'Run pytest with coverage' env: DB_CONNECTION_STRING: 'Server=tcp:127.0.0.1,1433;Database=master;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' @@ -789,7 +796,7 @@ jobs: - script: | # Create a Docker container for testing - docker run -d --name test-container-$(distroName) \ + docker run -e MSSQL_PYTHON_RS_VERSION -d --name test-container-$(distroName) \ -v $(Build.SourcesDirectory):/workspace \ -w /workspace \ --network bridge \ @@ -951,7 +958,7 @@ jobs: source /opt/venv/bin/activate echo 'Build successful, running tests now on $(distroName) with Azure SQL' echo 'Using Azure SQL connection string' - python -m pytest -v --junitxml=test-results-$(distroName).xml --cov=. --cov-report=xml:coverage-$(distroName).xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results-$(distroName).xml --cov=. --cov-report=xml:coverage-$(distroName).xml --capture=tee-sys --cache-clear " else # Local SQL Server testing @@ -966,7 +973,7 @@ jobs: source /opt/venv/bin/activate echo 'Build successful, running tests now on $(distroName)' echo 'Using connection string: Server=$SQLSERVER_IP;Database=TestDB;Uid=SA;Pwd=***;TrustServerCertificate=yes' - python -m pytest -v --junitxml=test-results-$(distroName).xml --cov=. --cov-report=xml:coverage-$(distroName).xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results-$(distroName).xml --cov=. --cov-report=xml:coverage-$(distroName).xml --capture=tee-sys --cache-clear " fi displayName: 'Run pytest with coverage in $(distroName) container' @@ -1122,7 +1129,7 @@ jobs: - script: | # Create a Docker container for testing on ARM64 - docker run -d --name test-container-$(distroName)-$(archName) \ + docker run -e MSSQL_PYTHON_RS_VERSION -d --name test-container-$(distroName)-$(archName) \ --platform linux/arm64 \ -v $(Build.SourcesDirectory):/workspace \ -w /workspace \ @@ -1293,7 +1300,7 @@ jobs: echo 'Architecture:' \$(uname -m) echo 'Using connection string: Server=$SQLSERVER_IP;Database=TestDB;Uid=SA;Pwd=***;TrustServerCertificate=yes' python main.py - python -m pytest -v --junitxml=test-results-$(distroName)-$(archName).xml --cov=. --cov-report=xml:coverage-$(distroName)-$(archName).xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results-$(distroName)-$(archName).xml --cov=. --cov-report=xml:coverage-$(distroName)-$(archName).xml --capture=tee-sys --cache-clear " displayName: 'Run pytest with coverage in $(distroName) ARM64 container' env: @@ -1335,7 +1342,7 @@ jobs: steps: - script: | # Create a Docker container for testing - docker run -d --name test-container-rhel9 \ + docker run -e MSSQL_PYTHON_RS_VERSION -d --name test-container-rhel9 \ -v $(Build.SourcesDirectory):/workspace \ -w /workspace \ --network bridge \ @@ -1495,7 +1502,7 @@ jobs: echo 'Build successful, running tests now on RHEL 9' echo 'Using connection string: Server=$SQLSERVER_IP;Database=TestDB;Uid=SA;Pwd=***;TrustServerCertificate=yes' python main.py - python -m pytest -v --junitxml=test-results-rhel9.xml --cov=. --cov-report=xml:coverage-rhel9.xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results-rhel9.xml --cov=. --cov-report=xml:coverage-rhel9.xml --capture=tee-sys --cache-clear " displayName: 'Run pytest with coverage in RHEL 9 container' env: @@ -1548,7 +1555,7 @@ jobs: - script: | # Create a Docker container for testing on ARM64 - docker run -d --name test-container-rhel9-arm64 \ + docker run -e MSSQL_PYTHON_RS_VERSION -d --name test-container-rhel9-arm64 \ --platform linux/arm64 \ -v $(Build.SourcesDirectory):/workspace \ -w /workspace \ @@ -1712,7 +1719,7 @@ jobs: echo 'Build successful, running tests now on RHEL 9 ARM64' echo 'Architecture:' \$(uname -m) echo 'Using connection string: Server=$SQLSERVER_IP;Database=TestDB;Uid=SA;Pwd=***;TrustServerCertificate=yes' - python -m pytest -v --junitxml=test-results-rhel9-arm64.xml --cov=. --cov-report=xml:coverage-rhel9-arm64.xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results-rhel9-arm64.xml --cov=. --cov-report=xml:coverage-rhel9-arm64.xml --capture=tee-sys --cache-clear " displayName: 'Run pytest with coverage in RHEL 9 ARM64 container' env: @@ -1761,7 +1768,7 @@ jobs: - script: | # Create a Docker container for testing on x86_64 - docker run -d --name test-container-alpine \ + docker run -e MSSQL_PYTHON_RS_VERSION -d --name test-container-alpine \ --platform linux/amd64 \ -v $(Build.SourcesDirectory):/workspace \ -w /workspace \ @@ -1948,7 +1955,7 @@ jobs: fi # Run pytest - python -m pytest -v --junitxml=test-results-alpine.xml --cov=. --cov-report=xml:coverage-alpine.xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results-alpine.xml --cov=. --cov-report=xml:coverage-alpine.xml --capture=tee-sys --cache-clear " displayName: 'Run pytest with coverage in Alpine x86_64 container' env: @@ -2003,7 +2010,7 @@ jobs: # Create a Docker container for testing on ARM64 # TODO(AB#40901): Temporary pin to 3.22 due to msodbcsql ARM64 package arch mismatch # Revert to alpine:latest once ODBC team releases fixed ARM64 package - docker run -d --name test-container-alpine-arm64 \ + docker run -e MSSQL_PYTHON_RS_VERSION -d --name test-container-alpine-arm64 \ --platform linux/arm64 \ -v $(Build.SourcesDirectory):/workspace \ -w /workspace \ @@ -2190,7 +2197,7 @@ jobs: fi # Run pytest - python -m pytest -v --junitxml=test-results-alpine-arm64.xml --cov=. --cov-report=xml:coverage-alpine-arm64.xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results-alpine-arm64.xml --cov=. --cov-report=xml:coverage-alpine-arm64.xml --capture=tee-sys --cache-clear " displayName: 'Run pytest with coverage in Alpine ARM64 container' env: diff --git a/eng/pipelines/steps/install-mssql-py-core.yml b/eng/pipelines/steps/install-mssql-py-core.yml index a10cf0b46..d7277caf1 100644 --- a/eng/pipelines/steps/install-mssql-py-core.yml +++ b/eng/pipelines/steps/install-mssql-py-core.yml @@ -1,4 +1,4 @@ -# Step template: Install mssql-python-rs from the internal NuGet transport +# Install the committed PyPI release, or the explicit MSSQL_PYTHON_RS_VERSION test override. # # Usage: # # Windows (host) @@ -54,7 +54,7 @@ steps: # Inside a Docker container - ${{ if eq(parameters.platform, 'container') }}: - script: | - docker exec ${{ parameters.containerName }} bash -c " + docker exec -e MSSQL_PYTHON_RS_VERSION ${{ parameters.containerName }} bash -c " ${{ parameters.venvActivate }} chmod +x eng/scripts/install-mssql-py-core.sh ./eng/scripts/install-mssql-py-core.sh diff --git a/eng/scripts/download_mssql_python_rs_wheels.py b/eng/scripts/download_mssql_python_rs_wheels.py index f1e6bb498..212929ea6 100644 --- a/eng/scripts/download_mssql_python_rs_wheels.py +++ b/eng/scripts/download_mssql_python_rs_wheels.py @@ -1,36 +1,28 @@ #!/usr/bin/env python3 -"""Download and stage mssql-python-rs wheels from the internal NuGet transport.""" +"""Stage exact-version runtime wheels from PyPI or the development wheel index.""" from __future__ import annotations import argparse import hashlib import json +import re import shutil import urllib.request import zipfile -from xml.etree import ElementTree -from pathlib import Path, PurePosixPath +from email.parser import BytesParser +from html.parser import HTMLParser +from pathlib import Path +from urllib.parse import parse_qs, unquote, urldefrag, urljoin, urlsplit if __package__: from . import mssql_python_build_safety - from .resolve_nuget_feed import resolve + from .mssql_python_rs import DEFAULT_INDEX, DEV_INDEX, VERSION_FILE, resolve_runtime else: import mssql_python_build_safety - from resolve_nuget_feed import resolve + from mssql_python_rs import DEFAULT_INDEX, DEV_INDEX, VERSION_FILE, resolve_runtime -DEFAULT_FEED = ( - "https://pkgs.dev.azure.com/sqlclientdrivers/public/" - "_packaging/mssql-rs_Public/nuget/v3/index.json" -) -PACKAGE_ID = "mssql-python-rs-wheels" - - -def _read_version(version_file: Path) -> str: - version = version_file.read_text(encoding="ascii").strip() - if not version: - raise ValueError(f"Version file is empty: {version_file}") - return version +PACKAGE_ID = "mssql-python-rs" def file_sha256(path: Path) -> str: @@ -41,104 +33,105 @@ def file_sha256(path: Path) -> str: return digest.hexdigest() +def validate_wheel(path: Path, version: str) -> None: + if not path.name.startswith(f"mssql_python_rs-{version}-") or path.suffix != ".whl": + raise ValueError(f"Unexpected wheel name for {PACKAGE_ID} {version}: {path.name}") + with zipfile.ZipFile(path) as wheel: + metadata = [name for name in wheel.namelist() if name.endswith(".dist-info/METADATA")] + if len(metadata) != 1: + raise ValueError(f"{path.name} must contain exactly one wheel METADATA") + identity = BytesParser().parsebytes(wheel.read(metadata[0])) + name = re.sub(r"[-_.]+", "-", identity.get("Name", "")).lower() + if name != PACKAGE_ID or identity.get("Version") != version: + raise ValueError(f"{path.name} metadata does not match {PACKAGE_ID}=={version}") + + +class _WheelLinks(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.links: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag == "a": + attributes = dict(attrs) + href = attributes.get("href") + if href and "data-yanked" not in attributes: + self.links.append(href) + + def download_wheels( - feed_url: str, + index_url: str | None, distribution_version_file: Path, - transport_version_file: Path, output_dir: Path, + *, + version: str | None = None, ) -> list[Path]: - distribution_version = _read_version(distribution_version_file) - transport_version = _read_version(transport_version_file) - - output_dir = getattr(mssql_python_build_safety, "resolve_safe_output_directory")(output_dir) - - package_base = resolve(feed_url).rstrip("/") + "/" - normalized_version = transport_version.lower() - package_url = ( - f"{package_base}{PACKAGE_ID}/{normalized_version}/" - f"{PACKAGE_ID}.{normalized_version}.nupkg" + runtime = resolve_runtime(distribution_version_file, version=version) + index_url = index_url or runtime.index_url + package_url = index_url.rstrip("/") + f"/{PACKAGE_ID}/" + if urlsplit(package_url).scheme != "https": + raise ValueError("The runtime wheel index must use HTTPS") + output_dir = mssql_python_build_safety.resolve_safe_output_directory(output_dir) + with urllib.request.urlopen(package_url, timeout=120) as response: + page = response.read().decode("utf-8") + parser = _WheelLinks() + parser.feed(page) + candidates: dict[str, tuple[str, str]] = {} + pattern = re.compile( + rf"mssql_python_rs-{re.escape(runtime.version)}-cp310-abi3-[A-Za-z0-9_]+\.whl" ) - + for link in parser.links: + url, fragment = urldefrag(urljoin(package_url, link)) + filename = unquote(urlsplit(url).path.rsplit("/", 1)[-1]) + if not pattern.fullmatch(filename): + continue + if urlsplit(url).scheme != "https": + raise ValueError(f"Wheel download must use HTTPS: {filename}") + if filename in candidates: + raise ValueError(f"Duplicate wheel filename in index: {filename}") + hashes = parse_qs(fragment).get("sha256", []) + if len(hashes) != 1 or not re.fullmatch(r"[a-fA-F0-9]{64}", hashes[0]): + raise ValueError(f"Missing or invalid SHA-256 for {filename}") + candidates[filename] = (url, hashes[0].lower()) + if not candidates: + raise ValueError(f"No {PACKAGE_ID}=={runtime.version} wheels found in {package_url}") if output_dir.exists(): shutil.rmtree(output_dir) output_dir.mkdir(parents=True) - package_path = output_dir / f"{PACKAGE_ID}.{normalized_version}.nupkg" - with urllib.request.urlopen(package_url, timeout=120) as response: - with package_path.open("wb") as destination: - shutil.copyfileobj(response, destination) - - staged: list[Path] = [] - seen: set[str] = set() - expected_prefix = f"mssql_python_rs-{distribution_version}-" - with zipfile.ZipFile(package_path) as package: - nuspecs = [name for name in package.namelist() if name.endswith(".nuspec")] - if len(nuspecs) != 1: - raise ValueError("RS transport must contain exactly one Nuspec") - metadata = ElementTree.fromstring(package.read(nuspecs[0])).find("{*}metadata") - if ( - metadata is None - or metadata.findtext("{*}id") != PACKAGE_ID - or metadata.findtext("{*}version", "").lower() != normalized_version - ): - raise ValueError("RS transport Nuspec identity does not match the pinned package") - description = metadata.findtext("{*}description", "") - for entry in package.infolist(): - path = PurePosixPath(entry.filename) - if len(path.parts) != 2 or path.parts[0] != "wheels" or path.suffix != ".whl": - continue - if not path.name.startswith(expected_prefix) or "\\" in path.name: - raise ValueError( - "Unexpected wheel name for mssql-python-rs " - f"{distribution_version}: {path.name}" - ) - if path.name in seen: - raise ValueError(f"Duplicate wheel filename in NuGet package: {path.name}") - seen.add(path.name) - destination = output_dir / path.name - with package.open(entry) as source, destination.open("wb") as target: - shutil.copyfileobj(source, target) - staged.append(destination) - - if not staged: - raise ValueError(f"No mssql-python-rs {distribution_version} wheels found in {package_url}") + staged = [] + for filename, (url, expected_hash) in sorted(candidates.items()): + destination = output_dir / filename + with urllib.request.urlopen(url, timeout=120) as response: + with destination.open("wb") as target: + shutil.copyfileobj(response, target) + if file_sha256(destination) != expected_hash: + raise ValueError(f"SHA-256 mismatch for {filename}") + validate_wheel(destination, runtime.version) + staged.append(destination) receipt = { - "distribution_version": distribution_version, - "transport_version": transport_version, - "feed_url": feed_url, - "package_id": PACKAGE_ID, - "package_sha256": file_sha256(package_path), - "nuspec_description": description, - "wheel_sha256": {path.name: file_sha256(path) for path in sorted(staged)}, + "distribution_version": runtime.version, + "index_url": index_url, + "version_override": runtime.version_override, + "wheel_sha256": {path.name: file_sha256(path) for path in staged}, } (output_dir / "transport.json").write_text( json.dumps(receipt, indent=2) + "\n", encoding="utf-8", newline="\n" ) - package_path.unlink() - return sorted(staged) + return staged def main() -> None: - repo_root = Path(__file__).resolve().parents[2] parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--feed-url", default=DEFAULT_FEED) - parser.add_argument( - "--distribution-version-file", - type=Path, - default=repo_root / "eng" / "versions" / "mssql-python-rs.version", - ) - parser.add_argument( - "--transport-version-file", - type=Path, - default=repo_root / "eng" / "versions" / "mssql-python-rs-nuget.version", - ) + parser.add_argument("--index-url") + parser.add_argument("--version") + parser.add_argument("--distribution-version-file", type=Path, default=VERSION_FILE) parser.add_argument("--output-dir", type=Path, required=True) args = parser.parse_args() - wheels = download_wheels( - args.feed_url, + args.index_url, args.distribution_version_file, - args.transport_version_file, args.output_dir, + version=args.version, ) for wheel in wheels: print(wheel) diff --git a/eng/scripts/install-mssql-py-core.ps1 b/eng/scripts/install-mssql-py-core.ps1 index 91d22eff2..51681afa8 100644 --- a/eng/scripts/install-mssql-py-core.ps1 +++ b/eng/scripts/install-mssql-py-core.ps1 @@ -1,170 +1,19 @@ <# .SYNOPSIS - Installs mssql-python-rs from the pinned internal NuGet transport package. - -.PARAMETER FeedUrl - The NuGet v3 feed URL. This is a public feed — no authentication required. - -.PARAMETER OutputDir - Temporary directory for downloaded artifacts. Cleaned up after installation. - Defaults to $env:TEMP\mssql-python-rs-wheels. - + Installs the pinned released runtime, or MSSQL_PYTHON_RS_VERSION for feature tests. .PARAMETER TargetArch - Target CPU architecture ('x64' or 'arm64') for cross-compilation builds. - When set, selects the matching-architecture wheel. Cross-architecture wheels - are not installed into the host interpreter. + Validates but does not install a cross-architecture Windows wheel. #> - param( - [string]$FeedUrl = "https://pkgs.dev.azure.com/sqlclientdrivers/public/_packaging/mssql-rs_Public/nuget/v3/index.json", - [string]$OutputDir = "$env:TEMP\mssql-python-rs-wheels", + [string]$OutputDir = "", + [ValidateSet("", "x64", "arm64")] [string]$TargetArch = "" ) $ErrorActionPreference = 'Stop' -$ScriptDir = $PSScriptRoot -$RepoRoot = (Get-Item "$ScriptDir\..\..").FullName - -function Read-PackageVersion { - $distributionVersionFile = Join-Path $RepoRoot "eng\versions\mssql-python-rs.version" - $transportVersionFile = Join-Path $RepoRoot "eng\versions\mssql-python-rs-nuget.version" - foreach ($versionFile in @($distributionVersionFile, $transportVersionFile)) { - if (-not (Test-Path $versionFile -PathType Leaf)) { - throw "Version file not found: $versionFile" - } - } - $script:DistributionVersion = (Get-Content $distributionVersionFile -Raw).Trim() - $script:TransportVersion = (Get-Content $transportVersionFile -Raw).Trim() - if (-not $script:DistributionVersion -or -not $script:TransportVersion) { - throw "mssql-python-rs version files must not be empty" - } - Write-Host "Distribution version: $script:DistributionVersion" - Write-Host "NuGet transport version: $script:TransportVersion" -} - -function Get-PlatformInfo { - # Single python call to get version, platform, and HOST arch - $info = & python -c "import sys, platform; v = sys.version_info; print(f'cp{v.major}{v.minor} {platform.system().lower()} {platform.machine().lower()}')" - if ($LASTEXITCODE -ne 0) { throw "Failed to detect Python platform info" } - - $parts = $info -split ' ' - $script:PyVersion = $parts[0] - $script:Platform = $parts[1] - $hostArch = $parts[2] - - # Normalize an arch string (platform.machine() or -TargetArch) to a wheel tag. - function ConvertTo-ArchTag($a) { - switch -Regex ($a) { - 'amd64|x86_64|x64' { return 'x86_64' } - 'arm64|aarch64' { return 'aarch64' } - default { throw "Unsupported architecture: $a" } - } - } - - # Select the wheel by TARGET arch, not host arch. Empty -TargetArch uses host. - $hostArchTag = ConvertTo-ArchTag $hostArch - $archTag = if ($TargetArch) { ConvertTo-ArchTag $TargetArch } else { $hostArchTag } - $script:IsCrossArch = ($archTag -ne $hostArchTag) - - Write-Host "Python: $script:PyVersion | Platform: $script:Platform | Arch: $archTag (host: $hostArchTag, target: $(if ($TargetArch) { $TargetArch } else { '' }))" - - if ($script:Platform -ne 'windows') { - throw "The PowerShell installer supports Windows only; use install-mssql-py-core.sh on $script:Platform" - } - # aarch64 -> arm64 so a Windows arm64 target resolves win_arm64, not win_aarch64. - $script:WheelPlatform = "win_$($archTag -replace 'x86_64','amd64' -replace 'aarch64','arm64')" - - $script:WheelPattern = "mssql_python_rs-$script:DistributionVersion-cp310-abi3-$script:WheelPlatform.whl" - Write-Host "Wheel pattern: $script:WheelPattern" -} - -function Get-NupkgFromFeed { - param([string]$FeedUrl, [string]$OutputDir) - - $resolvedOutput = & python "$ScriptDir\mssql_python_build_safety.py" $OutputDir - if ($LASTEXITCODE -ne 0) { throw "Unsafe OutputDir: $OutputDir" } - if (Test-Path $resolvedOutput) { Remove-Item $resolvedOutput -Recurse -Force } - New-Item -ItemType Directory -Path $resolvedOutput -Force | Out-Null - $script:ResolvedOutputDir = $resolvedOutput - - Write-Host "Resolving feed: $FeedUrl" - # Fetch the NuGet v3 service index and extract the PackageBaseAddress URL. - # See resolve_nuget_feed.py for the JSON schema and detailed explanation. - $feedIndex = Invoke-RestMethod -Uri $FeedUrl - $packageBaseUrl = ($feedIndex.resources | Where-Object { $_.'@type' -like 'PackageBaseAddress*' } | Select-Object -First 1).'@id' - if (-not $packageBaseUrl) { throw "Could not resolve PackageBaseAddress from feed" } - $packageBaseUrl = $packageBaseUrl.TrimEnd('/') + '/' - - $versionLower = $script:TransportVersion.ToLowerInvariant() - $packageId = "mssql-python-rs-wheels" - $nupkgUrl = "${packageBaseUrl}${packageId}/${versionLower}/${packageId}.${versionLower}.nupkg" - $script:NupkgPath = Join-Path $resolvedOutput "${packageId}.${versionLower}.nupkg" - Write-Host "Downloading: $nupkgUrl" - Invoke-WebRequest -Uri $nupkgUrl -OutFile $script:NupkgPath - - $sizeMB = [math]::Round((Get-Item $script:NupkgPath).Length / 1MB, 2) - Write-Host "Downloaded: $script:NupkgPath ($sizeMB MB)" -} - -function Find-MatchingWheel { - param([string]$OutputDir) - - # nupkg is a ZIP — rename so Expand-Archive accepts it - $zipPath = $script:NupkgPath -replace '\.nupkg$', '.zip' - Rename-Item -Path $script:NupkgPath -NewName (Split-Path $zipPath -Leaf) - - $extractDir = Join-Path $script:ResolvedOutputDir "extracted" - Expand-Archive -Path $zipPath -DestinationPath $extractDir -Force - - $wheelsDir = Join-Path $extractDir "wheels" - if (-not (Test-Path $wheelsDir)) { - throw "No 'wheels' directory found in NuGet package" - } - - $script:MatchingWheel = Get-ChildItem $wheelsDir -Filter $script:WheelPattern | Select-Object -First 1 - if (-not $script:MatchingWheel) { - Write-Host "Available wheels:" - Get-ChildItem $wheelsDir -Filter *.whl | ForEach-Object { Write-Host " $_" } - throw "No wheel found matching: $script:WheelPattern" - } - - Write-Host "Found: $($script:MatchingWheel.Name)" -} - -function Install-AndVerify { - $coreDir = Join-Path $RepoRoot "mssql_py_core" - if (Test-Path $coreDir) { - Remove-Item $coreDir -Recurse -Force - Write-Host "Cleaned previous mssql_py_core/" - } - - # An arm64 wheel cannot be installed into the x64 build-host interpreter. - if ($script:IsCrossArch) { - Write-Host "Skipping mssql-python-rs installation (cross-arch build: target != host)" - return - } - - & python -m pip install --force-reinstall --no-deps $script:MatchingWheel.FullName - if ($LASTEXITCODE -ne 0) { throw "Failed to install mssql-python-rs" } - - & python -c "import importlib.metadata as m, mssql_py_core; assert m.version('mssql-python-rs') == '$script:DistributionVersion'; print('mssql-python-rs', m.version('mssql-python-rs'), 'loaded from', mssql_py_core.__file__)" - if ($LASTEXITCODE -ne 0) { throw "Failed to verify installed mssql-python-rs" } -} - -# --- main --- - -Write-Host "=== Install mssql-python-rs from NuGet transport ===" - -try { - Read-PackageVersion - Get-PlatformInfo - Get-NupkgFromFeed -FeedUrl $FeedUrl -OutputDir $OutputDir - Find-MatchingWheel -OutputDir $OutputDir - Install-AndVerify - Write-Host "=== mssql-python-rs installed successfully ===" -} -finally { - if ($script:ResolvedOutputDir) { - Remove-Item $script:ResolvedOutputDir -Recurse -Force -ErrorAction SilentlyContinue - } -} +$python = if ($env:PYTHON) { $env:PYTHON } else { 'python' } +$arguments = @("$PSScriptRoot\install_mssql_python_rs.py") +if ($OutputDir) { $arguments += @('--output-dir', $OutputDir) } +if ($TargetArch) { $arguments += @('--target-arch', $TargetArch) } +& $python @arguments +if ($LASTEXITCODE -ne 0) { throw 'Failed to install or validate mssql-python-rs' } diff --git a/eng/scripts/install-mssql-py-core.sh b/eng/scripts/install-mssql-py-core.sh index a27729701..21370d2fc 100644 --- a/eng/scripts/install-mssql-py-core.sh +++ b/eng/scripts/install-mssql-py-core.sh @@ -1,206 +1,7 @@ #!/usr/bin/env bash -# Installs mssql-python-rs from the pinned internal NuGet transport package. -# -# This script is used identically for: -# - Local development (dev runs it after build.sh) -# - PR validation pipelines -# - Official build pipelines and tests -# -# The Python distribution and NuGet transport versions are pinned separately. -# -# Usage: -# ./install-mssql-py-core.sh [--feed-url URL] - +# Install the release pin, or an explicit MSSQL_PYTHON_RS_VERSION feature-test override. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" PYTHON="${PYTHON:-$(command -v python || command -v python3)}" - -read_version() { - local distribution_file="$REPO_ROOT/eng/versions/mssql-python-rs.version" - local transport_file="$REPO_ROOT/eng/versions/mssql-python-rs-nuget.version" - for version_file in "$distribution_file" "$transport_file"; do - if [ ! -f "$version_file" ]; then - echo "ERROR: Version file not found: $version_file" - exit 1 - fi - done - DISTRIBUTION_VERSION=$(tr -d '[:space:]' < "$distribution_file") - TRANSPORT_VERSION=$(tr -d '[:space:]' < "$transport_file") - if [ -z "$DISTRIBUTION_VERSION" ] || [ -z "$TRANSPORT_VERSION" ]; then - echo "ERROR: mssql-python-rs version files must not be empty" - exit 1 - fi - echo "Distribution version: $DISTRIBUTION_VERSION" - echo "NuGet transport version: $TRANSPORT_VERSION" -} - -detect_platform() { - read -r PY_VERSION PLATFORM ARCH <<< "$("$PYTHON" -c " -import sys, platform -v = sys.version_info -print(f'cp{v.major}{v.minor} {platform.system().lower()} {platform.machine().lower()}')" - )" - - echo "Python: $PY_VERSION | Platform: $PLATFORM | Arch: $ARCH" - - case "$PLATFORM" in - linux) - case "$ARCH" in - x86_64|amd64) ARCH_TAG="x86_64" ;; - aarch64|arm64) ARCH_TAG="aarch64" ;; - *) echo "Unsupported Linux architecture: $ARCH"; exit 1 ;; - esac - - # Detect musl libc (Alpine) vs glibc. - # ldd --version exits 1 on musl, so capture output instead of piping. - local ldd_output - ldd_output=$(ldd --version 2>&1 || true) - if echo "$ldd_output" | grep -qi musl || [ -f /etc/alpine-release ]; then - WHEEL_PLATFORM="musllinux_1_2_${ARCH_TAG}" - else - local glibc_version major minor - glibc_version=$(printf '%s\n' "$ldd_output" | head -1 | grep -Eo '[0-9]+\.[0-9]+' | tail -1) - major=${glibc_version%%.*} - minor=${glibc_version#*.} - if [ -z "$glibc_version" ] || ! [ "$major" -eq "$major" ] 2>/dev/null || ! [ "$minor" -eq "$minor" ] 2>/dev/null; then - echo "ERROR: Could not determine glibc version from: $ldd_output" - exit 1 - fi - if [ "$major" -gt 2 ] || { [ "$major" -eq 2 ] && [ "$minor" -ge 34 ]; }; then - # glibc 2.34+ systems carry OpenSSL 3. - WHEEL_PLATFORM="manylinux_2_34_${ARCH_TAG}" - elif [ "$major" -eq 2 ] && [ "$minor" -ge 28 ]; then - # glibc 2.28-2.33 systems carry OpenSSL 1.1. - WHEEL_PLATFORM="manylinux_2_28_${ARCH_TAG}" - else - echo "ERROR: mssql-python-rs requires glibc 2.28 or newer; found $glibc_version" - exit 1 - fi - fi - ;; - darwin) - WHEEL_PLATFORM="macosx_15_0_universal2" - ;; - *) - echo "Unsupported platform: $PLATFORM" - exit 1 - ;; - esac - - WHEEL_PATTERN="mssql_python_rs-${DISTRIBUTION_VERSION}-cp310-abi3-${WHEEL_PLATFORM}.whl" - echo "Wheel pattern: $WHEEL_PATTERN" -} - -download_nupkg() { - local feed_url="$1" - local output_dir="$2" - - if ! output_dir=$("$PYTHON" "$SCRIPT_DIR/mssql_python_build_safety.py" "$output_dir"); then - exit 1 - fi - rm -rf "$output_dir" - mkdir -p "$output_dir" - RESOLVED_OUTPUT_DIR="$output_dir" - - echo "Resolving feed: $feed_url" - PACKAGE_BASE_URL=$("$PYTHON" "$SCRIPT_DIR/resolve_nuget_feed.py" "$feed_url") - if [ -z "$PACKAGE_BASE_URL" ]; then - echo "ERROR: Could not resolve PackageBaseAddress from feed" - exit 1 - fi - PACKAGE_BASE_URL="${PACKAGE_BASE_URL%/}/" - - local version_lower - version_lower=$(echo "$TRANSPORT_VERSION" | tr '[:upper:]' '[:lower:]') - local package_id="mssql-python-rs-wheels" - NUPKG_URL="${PACKAGE_BASE_URL}${package_id}/${version_lower}/${package_id}.${version_lower}.nupkg" - NUPKG_PATH="$output_dir/${package_id}.${version_lower}.nupkg" - local http_status - echo "Downloading: $NUPKG_URL" - if ! http_status=$(curl -sSL -o "$NUPKG_PATH" -w '%{http_code}' "$NUPKG_URL"); then - rm -f "$NUPKG_PATH" - echo "ERROR: Failed to download NuGet package: $package_id $TRANSPORT_VERSION" >&2 - exit 1 - fi - if [ "$http_status" != "200" ]; then - rm -f "$NUPKG_PATH" - echo "ERROR: Failed to download NuGet package: $package_id $TRANSPORT_VERSION (HTTP $http_status)" >&2 - exit 1 - fi - - local filesize - filesize=$(wc -c < "$NUPKG_PATH") - echo "Downloaded: $NUPKG_PATH ($filesize bytes)" - - if [ "$filesize" -eq 0 ]; then - echo "ERROR: Downloaded file is empty" - exit 1 - fi -} - -find_matching_wheel() { - local output_dir="$1" - local extract_dir="$output_dir/extracted" - - mkdir -p "$extract_dir" - if command -v unzip &>/dev/null; then - unzip -q "$NUPKG_PATH" -d "$extract_dir" - else - "$PYTHON" -c "import zipfile, sys; zipfile.ZipFile(sys.argv[1]).extractall(sys.argv[2])" "$NUPKG_PATH" "$extract_dir" - fi - - local wheels_dir="$extract_dir/wheels" - if [ ! -d "$wheels_dir" ]; then - echo "ERROR: No 'wheels' directory found in NuGet package" - ls -la "$extract_dir" - exit 1 - fi - - MATCHING_WHEEL=$(find "$wheels_dir" -name "$WHEEL_PATTERN" -print -quit) - if [ -z "$MATCHING_WHEEL" ]; then - echo "Available wheels:" - ls "$wheels_dir"/*.whl 2>/dev/null || echo " (none)" - echo "ERROR: No wheel found matching: $WHEEL_PATTERN" - exit 1 - fi - - echo "Found: $(basename "$MATCHING_WHEEL")" -} - -install_and_verify() { - local core_dir="$REPO_ROOT/mssql_py_core" - - if [ -d "$core_dir" ]; then - rm -rf "$core_dir" - echo "Cleaned previous mssql_py_core/" - fi - - "$PYTHON" -m pip install --force-reinstall --no-deps "$MATCHING_WHEEL" - "$PYTHON" -c "import importlib.metadata as m, mssql_py_core; assert m.version('mssql-python-rs') == '$DISTRIBUTION_VERSION'; print('mssql-python-rs', m.version('mssql-python-rs'), 'loaded from', mssql_py_core.__file__)" -} - -# --- main --- - -FEED_URL="${FEED_URL:-https://pkgs.dev.azure.com/sqlclientdrivers/public/_packaging/mssql-rs_Public/nuget/v3/index.json}" -OUTPUT_DIR="${TMPDIR:-/tmp}/mssql-python-rs-wheels" - -while [[ $# -gt 0 ]]; do - case "$1" in - --feed-url) FEED_URL="$2"; shift 2 ;; - *) echo "Unknown argument: $1"; exit 1 ;; - esac -done - -echo "=== Install mssql-python-rs from NuGet transport ===" - -RESOLVED_OUTPUT_DIR="" -trap 'if [ -n "$RESOLVED_OUTPUT_DIR" ]; then rm -rf "$RESOLVED_OUTPUT_DIR"; fi' EXIT - -read_version -detect_platform -download_nupkg "$FEED_URL" "$OUTPUT_DIR" -find_matching_wheel "$RESOLVED_OUTPUT_DIR" -install_and_verify -echo "=== mssql-python-rs installed successfully ===" +exec "$PYTHON" "$SCRIPT_DIR/install_mssql_python_rs.py" "$@" diff --git a/eng/scripts/install_mssql_python_rs.py b/eng/scripts/install_mssql_python_rs.py new file mode 100644 index 000000000..8f4e4f2eb --- /dev/null +++ b/eng/scripts/install_mssql_python_rs.py @@ -0,0 +1,113 @@ +"""Install the selected runtime, or download a Windows cross-architecture wheel.""" + +from __future__ import annotations + +import argparse +import os +import platform +import subprocess +import sys +import tempfile +from pathlib import Path + +if __package__: + from .download_mssql_python_rs_wheels import validate_wheel + from .mssql_python_build_safety import resolve_safe_output_directory + from .mssql_python_rs import resolve_runtime, verify_installed +else: + from download_mssql_python_rs_wheels import validate_wheel + from mssql_python_build_safety import resolve_safe_output_directory + from mssql_python_rs import resolve_runtime, verify_installed + + +def install_runtime(target_arch: str = "", output_dir: Path | None = None) -> None: + runtime = resolve_runtime() + host = platform.machine().lower() + aliases = {"amd64": "x64", "x86_64": "x64", "arm64": "arm64", "aarch64": "arm64"} + host_arch = aliases.get(host, host) + cross_arch = bool(target_arch and target_arch != host_arch) + if target_arch and target_arch not in ("x64", "arm64"): + raise ValueError(f"Unsupported target architecture: {target_arch}") + if cross_arch and sys.platform != "win32": + raise ValueError("--target-arch cross-compilation is supported only on Windows") + parent = None + if output_dir is not None: + parent = resolve_safe_output_directory(output_dir) + parent.mkdir(parents=True, exist_ok=True) + transport_settings = {"PIP_CERT", "PIP_CLIENT_CERT", "PIP_PROXY", "PIP_TIMEOUT", "PIP_RETRIES"} + env = { + key: value + for key, value in os.environ.items() + if not key.upper().startswith("PIP_") or key.upper() in transport_settings + } + env["PIP_CONFIG_FILE"] = os.devnull + print( + f"Selected mssql-python-rs=={runtime.version} from {runtime.index_url} " + f"(override={runtime.version_override})", + flush=True, + ) + with tempfile.TemporaryDirectory(prefix="mssql-python-rs-", dir=parent) as directory: + command = [ + sys.executable, + "-m", + "pip", + "--disable-pip-version-check", + "--no-input", + "download", + "--only-binary=:all:", + "--no-deps", + "--index-url", + runtime.index_url, + "--dest", + directory, + ] + if cross_arch: + command.extend( + [ + "--platform", + "win_arm64" if target_arch == "arm64" else "win_amd64", + "--implementation", + "cp", + "--python-version", + "311", + "--abi", + "abi3", + ] + ) + command.append(f"mssql-python-rs=={runtime.version}") + subprocess.run(command, env=env, check=True) + wheels = list(Path(directory).glob("*.whl")) + if len(wheels) != 1: + raise ValueError(f"Expected exactly one runtime wheel, found {len(wheels)}") + validate_wheel(wheels[0], runtime.version) + if cross_arch: + print(f"Validated {wheels[0].name}; not installed into the {host_arch} host") + return + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "--disable-pip-version-check", + "--no-input", + "install", + "--force-reinstall", + "--no-deps", + str(wheels[0]), + ], + env=env, + check=True, + ) + verify_installed(runtime) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--target-arch", choices=("x64", "arm64"), default="") + parser.add_argument("--output-dir", type=Path) + args = parser.parse_args() + install_runtime(args.target_arch, args.output_dir) + + +if __name__ == "__main__": + main() diff --git a/eng/scripts/mssql_python_rs.py b/eng/scripts/mssql_python_rs.py new file mode 100644 index 000000000..0e1608f37 --- /dev/null +++ b/eng/scripts/mssql_python_rs.py @@ -0,0 +1,80 @@ +"""Resolve the native runtime once for installation and Python package metadata.""" + +from __future__ import annotations + +import argparse +import importlib +import importlib.metadata +import json +import os +import re +from dataclasses import asdict, dataclass +from pathlib import Path + +DEFAULT_INDEX = "https://pypi.org/simple/" +DEV_INDEX = ( + "https://pkgs.dev.azure.com/sqlclientdrivers/public/" "_packaging/mssql-rs_Public/pypi/simple/" +) +VERSION_ENV = "MSSQL_PYTHON_RS_VERSION" +VERSION_FILE = Path(__file__).resolve().parents[2] / "eng/versions/mssql-python-rs.version" +_RELEASE = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+") +_VERSION = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+(?:\.dev[0-9]+)?") + + +@dataclass(frozen=True) +class Runtime: + version: str + index_url: str + version_override: bool + + +def resolve_runtime(version_file: Path | None = None, *, version: str | None = None) -> Runtime: + selected = version if version is not None else os.environ.get(VERSION_ENV, "") + overridden = bool(selected.strip()) + if overridden: + selected = selected.strip() + if not _VERSION.fullmatch(selected): + raise ValueError( + f"Invalid {VERSION_ENV}: {selected!r}; use an exact X.Y.Z or X.Y.Z.devN version" + ) + else: + source = version_file if version_file is not None else VERSION_FILE + selected = source.read_text(encoding="ascii").strip() + if not _RELEASE.fullmatch(selected): + raise ValueError(f"{source} must contain an exact released X.Y.Z version") + return Runtime( + selected, + DEV_INDEX if ".dev" in selected else DEFAULT_INDEX, + overridden, + ) + + +def verify_installed(runtime: Runtime) -> None: + installed = importlib.metadata.version("mssql-python-rs") + if installed != runtime.version: + raise ValueError( + f"Installed mssql-python-rs=={installed}, expected {runtime.version}; " + "a later dependency installation may have replaced the selected runtime" + ) + core = importlib.import_module("mssql_py_core") + print(f"Verified mssql-python-rs=={installed} loaded from {core.__file__}") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version-file", type=Path) + parser.add_argument("--version") + parser.add_argument("--write-version-file", type=Path) + parser.add_argument("--verify", action="store_true") + args = parser.parse_args() + runtime = resolve_runtime(args.version_file, version=args.version) + if args.write_version_file: + args.write_version_file.write_text(runtime.version + "\n", encoding="ascii") + if args.verify: + verify_installed(runtime) + else: + print(json.dumps(asdict(runtime))) + + +if __name__ == "__main__": + main() diff --git a/eng/versions/mssql-python-rs-nuget.version b/eng/versions/mssql-python-rs-nuget.version deleted file mode 100644 index 9325c3ccd..000000000 --- a/eng/versions/mssql-python-rs-nuget.version +++ /dev/null @@ -1 +0,0 @@ -0.3.0 \ No newline at end of file diff --git a/setup.py b/setup.py index e2f68457e..f748be834 100644 --- a/setup.py +++ b/setup.py @@ -7,15 +7,14 @@ from setuptools.dist import Distribution from wheel.bdist_wheel import bdist_wheel +from eng.scripts.mssql_python_rs import resolve_runtime + PROJECT_ROOT = Path(__file__).resolve().parent def _read_mssql_python_rs_version() -> str: version_file = PROJECT_ROOT / "eng" / "versions" / "mssql-python-rs.version" - version = version_file.read_text(encoding="utf-8").strip() - if not version: - raise SystemExit(f"Version file is empty: {version_file}") - return version + return resolve_runtime(version_file).version def _read_odbc_version() -> str: diff --git a/tests/test_030_pe_machine_assert.py b/tests/test_030_pe_machine_assert.py index 9f2b77852..855578238 100644 --- a/tests/test_030_pe_machine_assert.py +++ b/tests/test_030_pe_machine_assert.py @@ -124,6 +124,9 @@ def test_binding_wheel_excludes_core_and_requires_source_rs_version(tmp_path): shutil.copy2(_ROOT / "setup.py", tmp_path / "setup.py") sources = { "PyPI_Description.md": "Packaging fixture", + "eng/scripts/mssql_python_rs.py": (_ROOT / "eng/scripts/mssql_python_rs.py").read_text( + encoding="utf-8" + ), "mssql_python/__init__.py": "", "mssql_python_odbc/__init__.py": '__version__ = "18.6.2.1"\n', "mssql_py_core/__init__.py": "from .mssql_py_core import *\n", diff --git a/tests/test_034_conda_verify_cwd.py b/tests/test_034_conda_verify_cwd.py index befe385c4..3f6d318df 100644 --- a/tests/test_034_conda_verify_cwd.py +++ b/tests/test_034_conda_verify_cwd.py @@ -15,6 +15,7 @@ """ import importlib.machinery +import json import os import subprocess import sys @@ -718,6 +719,48 @@ def _rs_inputs(tmp_path, platform="win_amd64", python_tags=("cp313",)): return code, odbc, links, rs +@pytest.mark.parametrize( + "problem", ["valid", "distribution", "source", "index", "hash", "missing-hash", "override"] +) +def test_gather_rs_receipt_uses_producer_version_and_bytes(tmp_path, monkeypatch, capsys, problem): + code, odbc, links, rs = _rs_inputs(tmp_path) + selected = next(rs.glob("*.whl")) + assertion = tmp_path / "mssql-python-rs.version" + assertion.write_text("0.2.0\n" if problem == "source" else "0.1.0\n") + receipt = { + "distribution_version": "0.2.0" if problem == "distribution" else "0.1.0", + "index_url": build.inputs.DEV_INDEX if problem == "index" else build.inputs.DEFAULT_INDEX, + "version_override": "false" if problem == "override" else False, + "wheel_sha256": { + selected.name: "a" * 64 if problem == "hash" else build.file_sha256(selected) + }, + } + if problem == "missing-hash": + receipt["wheel_sha256"] = {} + (rs / "transport.json").write_text(json.dumps(receipt)) + monkeypatch.setenv("MSSQL_PYTHON_RS_VERSION", "9.9.9.dev123") + arguments = ( + str(code), + "*.whl", + str(odbc), + "*.whl", + str(links), + str(rs), + str(assertion), + "win-64", + "3.13", + ) + if problem == "valid": + assert build.gather_wheels(*arguments) == ("1.2.3", "18.6.2", "0.1.0") + assert (links / selected.name).read_bytes() == selected.read_bytes() + else: + with pytest.raises(SystemExit) as error: + build.gather_wheels(*arguments) + assert error.value.code == 1 + assert "RS" in capsys.readouterr().err + assert not list(links.iterdir()) + + @pytest.mark.parametrize("with_rs", [False, True]) @pytest.mark.parametrize("recorded", [False, True]) @pytest.mark.parametrize( diff --git a/tests/test_036_conda_provenance.py b/tests/test_036_conda_provenance.py index 445b13cbc..8d57fb64f 100644 --- a/tests/test_036_conda_provenance.py +++ b/tests/test_036_conda_provenance.py @@ -362,7 +362,7 @@ def download(argv, *, check): "missing-core", "missing-private-driver", "missing-receipt", - "wrong-transport", + "wrong-index", "missing-selected-wheel", "wrong-selected-wheel", "wrong-wheel-tags", @@ -407,12 +407,12 @@ def test_source_bound_release_components(tmp_path, monkeypatch, capsys, problem, selected = f"mssql_python_rs-0.2.0-{rs_tag}-win_amd64.whl" receipt = { "distribution_version": "0.2.0", - "transport_version": "0.2.0-dev.transport", - "feed_url": inputs.DEFAULT_FEED, - "package_id": inputs.PACKAGE_ID, - "package_sha256": "a" * 64, + "index_url": inputs.DEFAULT_INDEX, + "version_override": False, "wheel_sha256": {selected: "b" * 64}, } + if problem == "wrong-index": + receipt["index_url"] = inputs.DEV_INDEX if with_rs: files[rs_prefix + "conda-wheel-source.txt"] = (selected + "\n").encode() if "-alias-" in problem: @@ -517,14 +517,6 @@ def test_source_bound_release_components(tmp_path, monkeypatch, capsys, problem, monkeypatch.setenv( "RELEASE_VERSIONS", json.dumps(versions) if problem != "missing-version-env" else "" ) - monkeypatch.setenv( - "RS_TRANSPORT_VERSION", - ( - "wrong" - if problem == "wrong-transport" - else "0.2.0-dev.transport" if "mssql-python-rs" in versions else "" - ), - ) result = cli.main( [ "validate", @@ -1162,7 +1154,24 @@ def read_source(path): ] -@pytest.mark.parametrize("version", ["", "0.2.0\n0.3.0", "$(version)", "../0.2.0", "0.2.0;bad"]) +def test_helper_based_setup_source_uses_release_pin_not_consumer_override( + release_sources, monkeypatch +): + _source_dependencies(release_sources, '[f"mssql-python-rs=={_read_mssql_python_rs_version()}"]') + release_sources["setup.py"] = ( + "from eng.scripts.mssql_python_rs import resolve_runtime\n" + "def _read_mssql_python_rs_version():\n" + " return resolve_runtime(version_file).version\n" + release_sources["setup.py"] + ) + release_sources["eng/versions/mssql-python-rs.version"] = "0.2.0\n" + monkeypatch.setenv("MSSQL_PYTHON_RS_VERSION", "0.3.0.dev123") + assert inputs.read_release_versions(release_sources.__getitem__)["mssql-python-rs"] == "0.2.0" + + +@pytest.mark.parametrize( + "version", + ["", "0.2.0\n0.3.0", "$(version)", "../0.2.0", "0.2.0;bad", "0.2.0.dev123", "0.2.0+local"], +) def test_rs_distribution_pin_must_be_present_and_unambiguous(release_sources, version): _source_dependencies(release_sources, '[f"mssql-python-rs=={_read_mssql_python_rs_version()}"]') release_sources["eng/versions/mssql-python-rs.version"] = version @@ -1217,10 +1226,6 @@ def test_recorded_rs_source_is_read_only_at_verified_wheel_commit(release_source record.pop("content") if problem != "missing": chain[key] = record - transport_key, transport = _source_record( - "eng/versions/mssql-python-rs-nuget.version", "0.2.0-dev.transport" - ) - chain[transport_key] = transport reads = [] def get_json(path): @@ -1232,39 +1237,13 @@ def get_json(path): if problem == "valid": result = _verify(chain, get_json=get_json) assert result["versions"]["mssql-python-rs"] == "0.2.0" - assert result["rsTransportVersion"] == "0.2.0-dev.transport" - assert reads[-2:] == [key, transport_key] + assert "rsTransportVersion" not in result + assert reads[-1] == key else: with pytest.raises((ValueError, KeyError, HTTPError)): _verify(chain, get_json=get_json) assert reads[-1] == key - assert len(reads) == (9 if problem == "valid" else 8) - - -@pytest.mark.parametrize("problem", ["missing", "wrong-commit", "invalid", "forbidden"]) -def test_rs_transport_source_cannot_fall_back_to_distribution_version( - release_sources, chain, problem -): - _source_dependencies(release_sources, '[f"mssql-python-rs=={_read_mssql_python_rs_version()}"]') - chain.update([_source_record("setup.py", release_sources["setup.py"])]) - chain.update([_source_record("eng/versions/mssql-python-rs.version", "0.2.0")]) - key, record = _source_record( - "eng/versions/mssql-python-rs-nuget.version", "0.2.0-dev.transport" - ) - if problem == "wrong-commit": - record["commitId"] = "c" * 40 - elif problem == "invalid": - record["content"] = "0.2.0\n0.3.0" - if problem != "missing": - chain[key] = record - - def get_json(path): - if path == key and problem == "forbidden": - raise HTTPError("https://dev.azure.com", 403, "Forbidden", {}, None) - return chain[path] - - with pytest.raises((ValueError, KeyError, HTTPError)): - _verify(chain, get_json=get_json) + assert len(reads) == 8 @pytest.mark.parametrize( @@ -1273,10 +1252,10 @@ def get_json(path): "valid", "absent", "distribution", - "transport", - "feed", - "package", - "package-hash", + "index", + "override-absent", + "override-nonbool", + "legacy-transport", "empty-wheels", "wheel-hash", "wheel-version", @@ -1286,25 +1265,29 @@ def get_json(path): def test_rs_transport_receipt_is_source_bound_without_inferred_producer_id(problem): receipt = { "distribution_version": "0.2.0", - "transport_version": "0.2.0-dev.transport", - "feed_url": ( - "https://pkgs.dev.azure.com/sqlclientdrivers/public/" - "_packaging/mssql-rs_Public/nuget/v3/index.json" - ), - "package_id": "mssql-python-rs-wheels", - "package_sha256": "a" * 64, + "index_url": inputs.DEFAULT_INDEX, + "version_override": False, "wheel_sha256": {"mssql_python_rs-0.2.0-cp313-cp313-win_amd64.whl": "b" * 64}, - "nuspec_description": "InternalBuild; no independently verified numeric producer ID", } fields = { "distribution": "distribution_version", - "transport": "transport_version", - "feed": "feed_url", - "package": "package_id", - "package-hash": "package_sha256", + "index": "index_url", } if problem in fields: receipt[fields[problem]] = "wrong" + elif problem == "override-absent": + receipt.pop("version_override") + elif problem == "override-nonbool": + receipt["version_override"] = "false" + elif problem == "legacy-transport": + receipt.pop("index_url") + receipt.pop("version_override") + receipt.update( + transport_version="0.2.0-dev.transport", + feed_url="https://example.test/nuget/v3/index.json", + package_id="mssql-python-rs-wheels", + package_sha256="a" * 64, + ) elif problem == "absent": receipt = None elif problem == "empty-wheels": @@ -1318,7 +1301,7 @@ def test_rs_transport_receipt_is_source_bound_without_inferred_producer_id(probl else: name = "../" + name receipt["wheel_sha256"] = {name: digest} - arguments = (receipt, {"mssql-python-rs": "0.2.0"}, "0.2.0-dev.transport") + arguments = (receipt, {"mssql-python-rs": "0.2.0"}) if problem == "valid": inputs.validate_rs_transport(*arguments) else: @@ -1327,11 +1310,38 @@ def test_rs_transport_receipt_is_source_bound_without_inferred_producer_id(probl def test_verified_legacy_transport_absence_is_not_a_new_producer_fallback(): - inputs.validate_rs_transport(None, {"mssql-python": "1.15.0"}, "") + inputs.validate_rs_transport(None, {"mssql-python": "1.15.0"}) with pytest.raises(ValueError, match="embedded-core"): - inputs.validate_rs_transport({}, {"mssql-python": "1.15.0"}, "") + inputs.validate_rs_transport({}, {"mssql-python": "1.15.0"}) with pytest.raises(ValueError, match="recorded"): - inputs.validate_rs_transport(None, {"mssql-python-rs": "0.2.0"}, "") + inputs.validate_rs_transport(None, {"mssql-python-rs": "0.2.0"}) + + +@pytest.mark.parametrize( + ("version", "index", "override", "valid"), + [ + ("0.2.0", inputs.DEFAULT_INDEX, False, True), + ("0.2.0", inputs.DEFAULT_INDEX, True, True), + ("0.2.0", inputs.DEV_INDEX, False, False), + ("0.2.0.dev123", inputs.DEV_INDEX, True, True), + ("0.2.0.dev123", inputs.DEFAULT_INDEX, True, False), + ("0.2.0.dev123", inputs.DEV_INDEX, False, False), + ], +) +def test_rs_index_receipt_cannot_switch_release_and_development_sources( + version, index, override, valid +): + receipt = { + "distribution_version": version, + "index_url": index, + "version_override": override, + "wheel_sha256": {f"mssql_python_rs-{version}-cp310-abi3-win_amd64.whl": "a" * 64}, + } + if valid: + inputs.validate_rs_transport(receipt, {"mssql-python-rs": version}) + else: + with pytest.raises(ValueError, match="distribution pin/index"): + inputs.validate_rs_transport(receipt, {"mssql-python-rs": version}) @pytest.mark.parametrize( diff --git a/tests/test_037_rs_wheel_download.py b/tests/test_037_rs_wheel_download.py index c03ded8ed..4fe5c20f7 100644 --- a/tests/test_037_rs_wheel_download.py +++ b/tests/test_037_rs_wheel_download.py @@ -8,169 +8,132 @@ import pytest -SCRIPTS_DIR = Path(__file__).parents[1] / "eng" / "scripts" -MODULE_PATH = SCRIPTS_DIR / "download_mssql_python_rs_wheels.py" - -if not MODULE_PATH.is_file() or not (Path(__file__).parents[1] / "OneBranchPipelines").is_dir(): +ROOT = Path(__file__).resolve().parents[1] +if ( + not (ROOT / "eng/scripts/download_mssql_python_rs_wheels.py").is_file() + or not (ROOT / "OneBranchPipelines").is_dir() +): pytest.skip( - "rs wheel download contracts require a complete source checkout", - allow_module_level=True, + "Runtime staging contracts require a complete source checkout", allow_module_level=True ) - from eng.scripts import download_mssql_python_rs_wheels as downloader +from eng.scripts.mssql_python_rs import DEFAULT_INDEX, DEV_INDEX, VERSION_ENV + + +@pytest.fixture(autouse=True) +def clear_override(monkeypatch): + monkeypatch.delenv(VERSION_ENV, raising=False) -def _package(entries, transport_version="0.1.0-dev.1"): - package = io.BytesIO() - with zipfile.ZipFile(package, "w") as archive: +def _wheel(version="0.3.0", metadata_version=None): + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as archive: archive.writestr( - "mssql-python-rs-wheels.nuspec", - "mssql-python-rs-wheels" - f"{transport_version}" - "RS transport fixture", + f"mssql_python_rs-{version}.dist-info/METADATA", + f"Name: mssql-python-rs\nVersion: {metadata_version or version}\n", ) - for name in entries: - archive.writestr(name, name.encode("ascii")) - package.seek(0) - return package.getvalue() + return stream.getvalue() -def _mock_download(monkeypatch, module, content, package_base="https://example.test/flat/"): +def _index(monkeypatch, *, version="0.3.0", content=None, duplicate=False, hash_value=None): + content = _wheel(version) if content is None else content + name = f"mssql_python_rs-{version}-cp310-abi3-win_amd64.whl" + digest = hash_value or hashlib.sha256(content).hexdigest() + anchor = f'{name}' + page = (anchor * (2 if duplicate else 1)).encode() requested = [] - monkeypatch.setattr(module, "resolve", lambda _url: package_base) def urlopen(url, timeout): - requested.append((url, timeout)) - return io.BytesIO(content) + requested.append(url) + return io.BytesIO(page if url.endswith("/mssql-python-rs/") else content) - monkeypatch.setattr(module.urllib.request, "urlopen", urlopen) - return requested + monkeypatch.setattr(downloader.urllib.request, "urlopen", urlopen) + return name, content, requested -def test_downloads_only_matching_wheels_and_clears_stale_output(tmp_path, monkeypatch): - module = downloader - version_file = tmp_path / "version" - version_file.write_text("0.1.0\n", encoding="ascii") - transport_file = tmp_path / "transport-version" - transport_file.write_text("0.1.0-dev.20260914.174990\n", encoding="ascii") - output = tmp_path / "wheels" - output.mkdir() - (output / "stale.whl").write_bytes(b"stale") - content = _package( - [ - "wheels/mssql_python_rs-0.1.0-cp310-abi3-win_amd64.whl", - "symbols/debug.pdb", - ], - "0.1.0-dev.20260914.174990", - ) - requested = _mock_download(monkeypatch, module, content, "https://example.test/flat") +def _pin(tmp_path): + path = tmp_path / "version" + path.write_text("0.3.0\n", encoding="ascii") + return path - wheels = module.download_wheels( - "https://example.test/index.json", version_file, transport_file, output - ) - assert [wheel.name for wheel in wheels] == ["mssql_python_rs-0.1.0-cp310-abi3-win_amd64.whl"] - assert sorted(path.name for path in output.iterdir()) == [wheels[0].name, "transport.json"] - receipt = json.loads((output / "transport.json").read_text()) - assert receipt == { - "distribution_version": "0.1.0", - "transport_version": "0.1.0-dev.20260914.174990", - "feed_url": "https://example.test/index.json", - "package_id": "mssql-python-rs-wheels", - "package_sha256": hashlib.sha256(content).hexdigest(), - "nuspec_description": "RS transport fixture", - "wheel_sha256": {wheels[0].name: hashlib.sha256(wheels[0].read_bytes()).hexdigest()}, +@pytest.mark.parametrize("override", [None, "0.4.0.dev20261028187000"]) +def test_stages_selected_wheels_and_receipt(tmp_path, monkeypatch, override): + pin = _pin(tmp_path) + if override: + monkeypatch.setenv(VERSION_ENV, override) + version = override or "0.3.0" + name, content, requested = _index(monkeypatch, version=version) + output = tmp_path / "out" + output.mkdir() + (output / "stale.whl").touch() + + wheels = downloader.download_wheels(None, pin, output) + + index = DEV_INDEX if override else DEFAULT_INDEX + assert wheels == [output / name] + assert sorted(path.name for path in output.iterdir()) == [name, "transport.json"] + assert json.loads((output / "transport.json").read_text()) == { + "distribution_version": version, + "index_url": index, + "version_override": bool(override), + "wheel_sha256": {name: hashlib.sha256(content).hexdigest()}, } assert requested == [ - ( - "https://example.test/flat/mssql-python-rs-wheels/0.1.0-dev.20260914.174990/" - "mssql-python-rs-wheels.0.1.0-dev.20260914.174990.nupkg", - 120, - ) + index + "mssql-python-rs/", + f"https://files.example.test/{name}", ] -def test_rejects_wheel_with_unexpected_version(tmp_path, monkeypatch): - module = downloader - version_file = tmp_path / "version" - version_file.write_text("0.1.0", encoding="ascii") - transport_file = tmp_path / "transport-version" - transport_file.write_text("0.1.0-dev.1", encoding="ascii") - content = _package(["wheels/mssql_python_rs-0.1.1-cp310-abi3-win_amd64.whl"]) - _mock_download(monkeypatch, module, content) - - with pytest.raises(ValueError, match="Unexpected wheel name"): - module.download_wheels( - "https://example.test/index.json", - version_file, - transport_file, - tmp_path / "out", - ) +def test_explicit_version_selects_dev_index(tmp_path, monkeypatch): + _index(monkeypatch, version="0.4.0.dev1") + wheels = downloader.download_wheels( + None, _pin(tmp_path), tmp_path / "out", version="0.4.0.dev1" + ) + assert wheels[0].name.startswith("mssql_python_rs-0.4.0.dev1-") -def test_rejects_duplicate_wheel_filename(tmp_path, monkeypatch): - module = downloader - version_file = tmp_path / "version" - version_file.write_text("0.1.0", encoding="ascii") - transport_file = tmp_path / "transport-version" - transport_file.write_text("0.1.0-dev.1", encoding="ascii") - wheel = "wheels/mssql_python_rs-0.1.0-cp310-abi3-win_amd64.whl" - with pytest.warns(UserWarning, match="Duplicate name"): - content = _package([wheel, wheel]) - _mock_download(monkeypatch, module, content) - - with pytest.raises(ValueError, match="Duplicate wheel filename"): - module.download_wheels( - "https://example.test/index.json", - version_file, - transport_file, - tmp_path / "out", - ) +@pytest.mark.parametrize( + ("options", "error"), + [ + ({"duplicate": True}, "Duplicate wheel filename"), + ({"hash_value": "0" * 64}, "SHA-256 mismatch"), + ({"hash_value": "bad"}, "Missing or invalid SHA-256"), + ({"content": _wheel(metadata_version="0.4.0")}, "metadata does not match"), + ({"version": "0.4.0"}, "No mssql-python-rs==0.3.0 wheels"), + ], +) +def test_rejects_bad_or_missing_wheels(tmp_path, monkeypatch, options, error): + _index(monkeypatch, **options) + with pytest.raises(ValueError, match=error): + downloader.download_wheels(None, _pin(tmp_path), tmp_path / "out") + assert not (tmp_path / "out" / "transport.json").exists() -def test_rejects_filesystem_root_as_output(tmp_path): - module = downloader - version_file = tmp_path / "version" - version_file.write_text("0.1.0", encoding="ascii") - transport_file = tmp_path / "transport-version" - transport_file.write_text("0.1.0-dev.1", encoding="ascii") +def test_missing_requested_dev_build_does_not_fall_back(tmp_path, monkeypatch): + _index(monkeypatch) + monkeypatch.setenv(VERSION_ENV, "0.4.0.dev1") + with pytest.raises(ValueError, match="No mssql-python-rs==0.4.0.dev1"): + downloader.download_wheels(None, _pin(tmp_path), tmp_path / "out") + +def test_rejects_filesystem_root_as_output(tmp_path): with pytest.raises(ValueError, match="filesystem root"): - module.download_wheels( - "https://example.test/index.json", - version_file, - transport_file, - Path(tmp_path.anchor), - ) + downloader.download_wheels(None, _pin(tmp_path), Path(tmp_path.anchor)) def test_rejects_working_directory_and_ancestor_as_output(tmp_path, monkeypatch): - module = downloader - version_file = tmp_path / "version" - version_file.write_text("0.1.0", encoding="ascii") - transport_file = tmp_path / "transport-version" - transport_file.write_text("0.1.0-dev.1", encoding="ascii") - working_directory = tmp_path / "checkout" / "subdirectory" - working_directory.mkdir(parents=True) - monkeypatch.chdir(working_directory) - - for unsafe in (working_directory, working_directory.parent): + pin = _pin(tmp_path) + cwd = tmp_path / "checkout" / "subdirectory" + cwd.mkdir(parents=True) + monkeypatch.chdir(cwd) + for unsafe in (cwd, cwd.parent): with pytest.raises(ValueError, match="current working directory|ancestors"): - module.download_wheels( - "https://example.test/index.json", - version_file, - transport_file, - unsafe, - ) - - -def test_rejects_symlink_output_without_deleting_target(tmp_path, monkeypatch): - module = downloader - version_file = tmp_path / "version" - version_file.write_text("0.1.0", encoding="ascii") - transport_file = tmp_path / "transport-version" - transport_file.write_text("0.1.0-dev.1", encoding="ascii") + downloader.download_wheels(None, pin, unsafe) + + +def test_rejects_symlink_output_without_deleting_target(tmp_path): target = tmp_path / "unrelated" target.mkdir() marker = target / "keep.txt" @@ -184,35 +147,21 @@ def test_rejects_symlink_output_without_deleting_target(tmp_path, monkeypatch): ) else: output.symlink_to(target, target_is_directory=True) - - content = _package(["wheels/mssql_python_rs-0.1.0-cp310-abi3-win_amd64.whl"]) - _mock_download(monkeypatch, module, content) - with pytest.raises(ValueError, match="symbolic link|junction"): - module.download_wheels( - "https://example.test/index.json", version_file, transport_file, output - ) - + downloader.download_wheels(None, _pin(tmp_path), output) assert marker.read_text(encoding="ascii") == "keep" -def test_stress_jobs_resolve_dependency_from_pinned_transport(): - pipeline = ( - Path(__file__).parents[1] / "OneBranchPipelines" / "stress-test-pipeline.yml" - ).read_text(encoding="utf-8") +def test_rejects_insecure_index(tmp_path): + with pytest.raises(ValueError, match="HTTPS"): + downloader.download_wheels("http://example.test/simple/", _pin(tmp_path), tmp_path / "out") - assert pipeline.count("download_mssql_python_rs_wheels.py --output-dir") == 2 - assert pipeline.count("--find-links=$(Pipeline.Workspace)\\mssql-python-rs-wheels") == 1 - assert pipeline.count("--find-links=$(Pipeline.Workspace)/mssql-python-rs-wheels") == 1 - -@pytest.mark.parametrize( - "script_name", - ["install-mssql-py-core.ps1", "install-mssql-py-core.sh"], -) -def test_local_installers_select_stable_abi_wheel(script_name): - script = (SCRIPTS_DIR / script_name).read_text(encoding="utf-8") - - assert "cp310-abi3" in script - assert "PyVersion-$script:PyVersion" not in script - assert "PY_VERSION}-${PY_VERSION" not in script +def test_ignores_yanked_wheels(tmp_path, monkeypatch): + name = "mssql_python_rs-0.3.0-cp310-abi3-win_amd64.whl" + page = f'wheel' + monkeypatch.setattr( + downloader.urllib.request, "urlopen", lambda *args, **kwargs: io.BytesIO(page.encode()) + ) + with pytest.raises(ValueError, match="No mssql-python-rs"): + downloader.download_wheels(None, _pin(tmp_path), tmp_path / "out") diff --git a/tests/test_038_mssql_odbc_daily_validation.py b/tests/test_038_mssql_odbc_daily_validation.py index 11c76c710..7a63fcc40 100644 --- a/tests/test_038_mssql_odbc_daily_validation.py +++ b/tests/test_038_mssql_odbc_daily_validation.py @@ -19,7 +19,7 @@ @unittest.skipUnless(sys.platform.startswith("linux"), "runner requires Linux GNU timeout and bash") class RunnerTests(unittest.TestCase): def run_runner(self, behavior: str, total_budget: str = "60s", provider: str = "mssql-odbc"): - with tempfile.TemporaryDirectory() as directory: + with tempfile.TemporaryDirectory(dir=ROOT, prefix=".odbc-contract-") as directory: work = Path(directory) (work / "tests").mkdir() (work / "tests" / "test_sample.py").write_text("pass\n", encoding="utf-8") @@ -120,11 +120,14 @@ def test_release_wheel_fixture_includes_runner_contract_files(self): "eng/scripts/run-mssql-odbc-tests.sh", "eng/scripts/verify_mssql_odbc_provider.py", "eng/pipelines/mssql-odbc-daily-validation-pipeline.yml", - "eng/versions/mssql-python-rs-nuget.version", + "eng/versions/mssql-python-rs.version", "OneBranchPipelines/stages/build-linux-single-stage.yml", ): self.assertEqual(pipeline.count(f"/workspace/{path}"), 2) self.assertNotRegex(pipeline, r"\bcp\s+-r\s+/workspace/eng(?:\s|$)") + self.assertNotIn("mssql-python-rs-nuget.version", pipeline) + self.assertEqual(pipeline.count("cp -r /workspace/eng/pipelines/."), 2) + self.assertEqual(pipeline.count("cp -r /workspace/OneBranchPipelines/."), 2) def test_runner_uses_busybox_compatible_timeout_options(self): runner = RUNNER.read_text(encoding="utf-8") @@ -133,7 +136,7 @@ def test_runner_uses_busybox_compatible_timeout_options(self): self.assertNotIn("--kill-after", runner) def test_optimized_preflight_rejects_wrong_provider(self): - with tempfile.TemporaryDirectory() as directory: + with tempfile.TemporaryDirectory(dir=ROOT, prefix=".odbc-contract-") as directory: fake_module = Path(directory) / "mssql_python.py" fake_module.write_text( "def get_native_provider_info():\n" @@ -162,12 +165,20 @@ def test_pipeline_authenticates_advisory_status_and_keeps_publish_strict(self): self.assertIn('exit "$cleanup_rc"', pipeline) self.assertIn("grep -q 'MSSQL_ODBC_PREFLIGHT_OK'", pipeline) - def test_stable_rs_transport_is_pinned(self): - version = (ROOT / "eng" / "versions" / "mssql-python-rs-nuget.version").read_text( - encoding="ascii" - ) + def test_default_rs_runtime_is_an_exact_pypi_release(self): + version_file = ROOT / "eng" / "versions" / "mssql-python-rs.version" + version = version_file.read_text(encoding="ascii") + + self.assertRegex(version.strip(), r"^[0-9]+\.[0-9]+\.[0-9]+$") + self.assertFalse(version_file.with_name("mssql-python-rs-nuget.version").exists()) + + def test_daily_pipeline_carries_explicit_override_into_test_container(self): + pipeline = PIPELINE.read_text(encoding="utf-8") - self.assertEqual(version.strip(), "0.3.0") + self.assertIn("name: rsVersionOverride", pipeline) + self.assertIn("MSSQL_PYTHON_RS_VERSION: ${{ parameters.rsVersionOverride }}", pipeline) + self.assertIn("docker run -e MSSQL_PYTHON_RS_VERSION ", pipeline) + self.assertNotIn("$(MSSQL_PYTHON_RS_VERSION)", pipeline) if __name__ == "__main__": diff --git a/tests/test_039_rs_runtime_pipelines.py b/tests/test_039_rs_runtime_pipelines.py new file mode 100644 index 000000000..1e7867ccd --- /dev/null +++ b/tests/test_039_rs_runtime_pipelines.py @@ -0,0 +1,281 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Source-only contracts for runtime selection and the pre-ESRP release gates.""" + +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import uuid +import zipfile + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +PIPELINES = ROOT / "OneBranchPipelines" +if not (PIPELINES / "scripts/validate-release-versions.ps1").is_file(): + pytest.skip("Pipeline contracts require a complete source checkout", allow_module_level=True) + +VERSION_ENV = "MSSQL_PYTHON_RS_VERSION" +ENTRY_POINTS = ( + "eng/pipelines/pr-validation-pipeline.yml", + "eng/pipelines/mssql-odbc-daily-validation-pipeline.yml", + "OneBranchPipelines/build-release-package-pipeline.yml", + "OneBranchPipelines/stress-test-pipeline.yml", +) + + +@pytest.mark.parametrize("filename", ENTRY_POINTS) +def test_pipeline_override_is_explicit_and_defaults_to_empty(filename): + source = (ROOT / filename).read_text(encoding="utf-8") + assert re.search(r"name: rsVersionOverride[\s\S]*?default: ''", source) + assert VERSION_ENV in source + assert "${{ parameters.rsVersionOverride }}" in source + assert "$(rsVersionOverride)" not in source + assert "$(MSSQL_PYTHON_RS_VERSION)" not in source + + +def test_modified_pipeline_yaml_parses(): + yaml = pytest.importorskip("yaml") + for folder in (ROOT / "eng/pipelines", PIPELINES): + for path in folder.rglob("*.yml"): + assert isinstance(yaml.safe_load(path.read_text(encoding="utf-8")), dict), path + + +def test_conda_pipeline_no_longer_passes_removed_transport_provenance(): + for path in ( + PIPELINES / "conda-release-pipeline.yml", + PIPELINES / "steps/conda-publish-step.yml", + PIPELINES / "steps/conda-release-step.yml", + ): + source = path.read_text(encoding="utf-8") + for obsolete in ("rsTransportVersion", "RS_TRANSPORT_VERSION", "--rs-transport-version"): + assert obsolete not in source, path + assert "releaseVersions" in source + assert "mssqlPythonVersion" in source + + +def test_every_runtime_container_inherits_version_without_shell_interpolation(): + for filename in ENTRY_POINTS[:2] + ("OneBranchPipelines/stages/build-linux-single-stage.yml",): + source = (ROOT / filename).read_text(encoding="utf-8") + runs = [ + line + for line in source.splitlines() + if not line.lstrip().startswith("#") + and "docker run " in line + and ("--name test-container" in line or "--name build-" in line) + ] + if "daily" in filename: + runs = [line for line in source.splitlines() if "docker run " in line] + assert runs, filename + assert all("-e MSSQL_PYTHON_RS_VERSION " in line for line in runs) + assert not re.search(r"-e MSSQL_PYTHON_RS_VERSION=", source) + installer = (ROOT / "eng/pipelines/steps/install-mssql-py-core.yml").read_text(encoding="utf-8") + assert "docker exec -e MSSQL_PYTHON_RS_VERSION " in installer + + +def test_consolidation_stages_effective_receipt_version_not_source_pin(): + source = (PIPELINES / "jobs/consolidate-artifacts-job.yml").read_text(encoding="utf-8") + assert "receipt['distribution_version']" in source + assert "dist/transport.json" in source + assert "filePath: OneBranchPipelines/scripts/validate-rs-wheel-dependency.ps1" in source + assert "mssql-python-rs-nuget.version" not in source + assert 'cp "$(Build.SourcesDirectory)/eng/versions/mssql-python-rs.version"' not in source + linux = (PIPELINES / "stages/build-linux-single-stage.yml").read_text(encoding="utf-8") + assert "mssql-python-rs-nuget.version" not in linux + + +def test_stress_uses_selected_build_and_checks_effective_runtime_metadata(): + source = (PIPELINES / "stress-test-pipeline.yml").read_text(encoding="utf-8") + assert source.count("pipelineId: '$(resources.pipeline.buildPipeline.runID)'") == 2 + assert ( + source.count("filePath: OneBranchPipelines/scripts/validate-rs-wheel-dependency.ps1") == 2 + ) + assert source.count("'--no-index', '--no-deps'") == 2 + assert source.count("'mssql-python-rs=='+resolve_runtime().version") == 2 + + +def test_runtime_is_verified_after_dependency_setup_before_testing(): + pr = (ROOT / ENTRY_POINTS[0]).read_text(encoding="utf-8") + test_commands = [ + line + for line in pr.splitlines() + if not line.lstrip().startswith("#") and "python -m pytest " in line + ] + assert test_commands + assert all("mssql_python_rs.py --verify && python -m pytest " in line for line in test_commands) + daily = (ROOT / ENTRY_POINTS[1]).read_text(encoding="utf-8") + assert "python eng/scripts/mssql_python_rs.py --verify || exit 2" in daily + linux = (PIPELINES / "stages/build-linux-single-stage.yml").read_text(encoding="utf-8") + assert ( + linux.count( + '$PY -m pip install -q "$WHEEL";\n $PY /workspace/eng/scripts/mssql_python_rs.py --verify;' + ) + == 2 + ) + for platform in ("windows", "macos"): + source = (PIPELINES / f"stages/build-{platform}-single-stage.yml").read_text( + encoding="utf-8" + ) + verify = source.index("python eng/scripts/mssql_python_rs.py --verify") + assert verify < source.index("python -m pytest -v") + stress = (PIPELINES / "stress-test-pipeline.yml").read_text(encoding="utf-8") + assert stress.count("python eng/scripts/mssql_python_rs.py --verify") == 2 + + +@pytest.mark.parametrize("platform", ("windows", "macos", "linux")) +def test_official_build_guards_run_before_runtime_installation(platform): + source = (PIPELINES / f"stages/build-{platform}-single-stage.yml").read_text(encoding="utf-8") + guard = source.index("validate-release-versions.ps1") + assert source.index("if eq(parameters.oneBranchType, 'Official')") < guard + assert "arguments: '-BuildOnly'" in source + assert guard < source.index("install-mssql-py-core") + + +@pytest.mark.parametrize("pipeline", ("official", "dummy")) +def test_release_validates_selected_artifact_before_esrp(pipeline): + source = (PIPELINES / f"{pipeline}-release-pipeline.yml").read_text(encoding="utf-8") + guard = source.index( + "filePath: '$(Build.SourcesDirectory)\\OneBranchPipelines\\scripts\\validate-release-versions.ps1'" + ) + metadata = source.index("validate-rs-wheel-dependency.ps1") + assert guard < metadata + assert source[guard:metadata].count(r"artifacts\dist\transport.json") == 1 + assert source.index("dependsOn: PrepAndValidate", metadata) > metadata + + +@pytest.fixture +def workspace(): + path = ROOT / f".pipeline-contract-{uuid.uuid4().hex}" + path.mkdir() + try: + yield path + finally: + shutil.rmtree(path) + + +def _receipt(version="0.3.0"): + return { + "distribution_version": version, + "index_url": "https://pypi.org/simple/", + "version_override": False, + "wheel_sha256": {"mssql_python_rs-0.3.0-py3-none-any.whl": "a" * 64}, + } + + +def _guard(workspace, *, version="0.3.0", receipt=None, override="", build_only=False, public=True): + pwsh = shutil.which("pwsh") + if not pwsh: + pytest.skip("PowerShell Core is not available") + (workspace / "runtime.version").write_text(version) + if receipt is not None: + (workspace / "transport.json").write_text(json.dumps(receipt)) + env = dict(os.environ) + env.update( + { + VERSION_ENV: override, + "GUARD": str(PIPELINES / "scripts/validate-release-versions.ps1"), + "VERSION_FILE": str(workspace / "runtime.version"), + "PUBLIC_VERSION": version if public else "9.9.9", + "BUILD_ONLY": str(build_only).lower(), + } + ) + command = """ + $ErrorActionPreference = 'Stop' + function Invoke-RestMethod { + param($Uri, $Method) + if ($Uri -cne "https://pypi.org/pypi/mssql-python-rs/$env:PUBLIC_VERSION/json") { + throw 'No exact release on public PyPI' + } + return @{ + info = @{ version = $env:PUBLIC_VERSION; name = 'mssql-python-rs' } + urls = @(@{ packagetype = 'bdist_wheel'; yanked = $false }) + } + } + & $env:GUARD -VersionFile $env:VERSION_FILE -BuildOnly:($env:BUILD_ONLY -eq 'true') + """ + return subprocess.run( + [pwsh, "-NoProfile", "-NonInteractive", "-Command", command], + capture_output=True, + text=True, + env=env, + check=False, + ) + + +@pytest.mark.parametrize("override", ("0.3.0", "0.3.1.dev123")) +def test_official_build_rejects_all_explicit_overrides(workspace, override): + result = _guard(workspace, override=override, build_only=True) + assert result.returncode != 0 + assert "forbid" in result.stderr + + +@pytest.mark.parametrize("version", ("0.3.1.dev123", "0.3.1rc1", "", ">=0.3.0")) +def test_official_build_rejects_non_release_pins(workspace, version): + assert _guard(workspace, version=version, build_only=True).returncode != 0 + + +def test_official_build_accepts_committed_release_without_network_or_receipt(workspace): + result = _guard(workspace, build_only=True, public=False) + assert result.returncode == 0, result.stderr + + +def test_release_requires_receipt_and_exact_public_pypi_release(workspace): + assert _guard(workspace).returncode != 0 + result = _guard(workspace, receipt=_receipt(), public=False) + assert result.returncode != 0 + result = _guard(workspace, receipt=_receipt()) + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize( + "field,value", + ( + ("version_override", True), + ("version_override", "false"), + ("distribution_version", "0.3.1"), + ("index_url", "https://pkgs.dev.azure.com/example/simple/"), + ("wheel_sha256", {}), + ("wheel_sha256", {"test.whl": "bad"}), + ), +) +def test_release_rejects_override_or_inconsistent_receipt(workspace, field, value): + receipt = _receipt() + receipt[field] = value + assert _guard(workspace, receipt=receipt).returncode != 0 + + +@pytest.mark.parametrize("dependency", ("0.3.0", "0.3.1.dev123")) +def test_wheel_metadata_must_match_effective_staged_version(workspace, dependency): + pwsh = shutil.which("pwsh") + if not pwsh: + pytest.skip("PowerShell Core is not available") + (workspace / "runtime.version").write_text("0.3.1.dev123") + (workspace / "transport.json").write_text(json.dumps(_receipt("0.3.1.dev123"))) + with zipfile.ZipFile(workspace / "mssql_python-1.0.0-py3-none-any.whl", "w") as archive: + archive.writestr( + "mssql_python-1.0.0.dist-info/METADATA", + f"Name: mssql-python\nVersion: 1.0.0\nRequires-Dist: mssql-python-rs=={dependency}\n", + ) + result = subprocess.run( + [ + pwsh, + "-NoProfile", + "-NonInteractive", + "-File", + str(PIPELINES / "scripts/validate-rs-wheel-dependency.ps1"), + "-WheelsDir", + str(workspace), + "-VersionFile", + str(workspace / "runtime.version"), + "-ReceiptFile", + str(workspace / "transport.json"), + ], + capture_output=True, + text=True, + check=False, + ) + assert (result.returncode == 0) == (dependency == "0.3.1.dev123"), result.stderr diff --git a/tests/test_040_rs_runtime_selection.py b/tests/test_040_rs_runtime_selection.py new file mode 100644 index 000000000..0ef356642 --- /dev/null +++ b/tests/test_040_rs_runtime_selection.py @@ -0,0 +1,181 @@ +import json +import runpy +import subprocess +import sys +import zipfile +from pathlib import Path +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if not (ROOT / "eng/scripts/mssql_python_rs.py").is_file() or not (ROOT / "setup.py").is_file(): + pytest.skip( + "Runtime selection contracts require a complete source checkout", allow_module_level=True + ) + +from eng.scripts import install_mssql_python_rs as installer +from eng.scripts import mssql_python_rs as runtime + + +@pytest.fixture(autouse=True) +def clear_override(monkeypatch): + monkeypatch.delenv(runtime.VERSION_ENV, raising=False) + + +def _pin(tmp_path, value="0.3.0"): + path = tmp_path / "runtime.version" + path.write_text(value, encoding="ascii") + return path + + +@pytest.mark.parametrize("override", [None, "", "0.4.0", "0.4.0.dev20261028187000"]) +def test_resolves_exact_release_or_override(tmp_path, monkeypatch, override): + if override is not None: + monkeypatch.setenv(runtime.VERSION_ENV, override) + selected = runtime.resolve_runtime(_pin(tmp_path)) + assert selected.version == (override or "0.3.0") + assert selected.version_override == bool(override) + assert selected.index_url == ( + runtime.DEV_INDEX if override and ".dev" in override else runtime.DEFAULT_INDEX + ) + + +@pytest.mark.parametrize( + "version", + ["latest", ">=0.3.0", "0.3", "0.3.0;echo bad", "0.3.0rc1", "0.3.0.dev", "0.3.0+local"], +) +def test_rejects_non_exact_or_unsupported_override(tmp_path, monkeypatch, version): + monkeypatch.setenv(runtime.VERSION_ENV, version) + with pytest.raises(ValueError, match="Invalid MSSQL_PYTHON_RS_VERSION"): + runtime.resolve_runtime(_pin(tmp_path)) + + +@pytest.mark.parametrize("version", ["", "0.4.0.dev1", "0.4.0rc1", ">=0.3.0"]) +def test_committed_pin_must_be_a_release(tmp_path, version): + with pytest.raises(ValueError, match="exact released"): + runtime.resolve_runtime(_pin(tmp_path, version)) + + +@pytest.mark.parametrize("override", [None, "0.4.0.dev20261028187000"]) +def test_setup_metadata_uses_same_effective_runtime(monkeypatch, override): + if override: + monkeypatch.setenv(runtime.VERSION_ENV, override) + captured = {} + monkeypatch.chdir(ROOT) + monkeypatch.setattr("setuptools.setup", lambda **kwargs: captured.update(kwargs)) + runpy.run_path(str(ROOT / "setup.py")) + selected = runtime.resolve_runtime() + assert f"mssql-python-rs=={selected.version}" in captured["install_requires"] + + +@pytest.mark.parametrize("override", [None, "0.4.0.dev20261028187000"]) +def test_generated_package_metadata_matches_selection(tmp_path, monkeypatch, override): + if override: + monkeypatch.setenv(runtime.VERSION_ENV, override) + subprocess.run( + [sys.executable, "setup.py", "egg_info", "--egg-base", str(tmp_path)], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + metadata = tmp_path / "mssql_python.egg-info" + requirements = (metadata / "requires.txt").read_text(encoding="utf-8").splitlines() + assert f"mssql-python-rs=={runtime.resolve_runtime().version}" in requirements + sources = (metadata / "SOURCES.txt").read_text(encoding="utf-8") + assert "eng/scripts/mssql_python_rs.py" in sources + assert "eng/versions/mssql-python-rs.version" in sources + + +def test_verify_rejects_runtime_replaced_by_later_pip_install(monkeypatch): + monkeypatch.setattr(runtime.importlib.metadata, "version", lambda _name: "0.3.0") + with pytest.raises(ValueError, match="expected 0.4.0.dev1"): + runtime.verify_installed(runtime.Runtime("0.4.0.dev1", runtime.DEV_INDEX, True)) + + +def test_verify_imports_native_module(monkeypatch): + monkeypatch.setattr(runtime.importlib.metadata, "version", lambda _name: "0.3.0") + imports = [] + + def import_module(name): + imports.append(name) + return SimpleNamespace(__file__="test-native-module") + + monkeypatch.setattr(runtime.importlib, "import_module", import_module) + runtime.verify_installed(runtime.Runtime("0.3.0", runtime.DEFAULT_INDEX, False)) + assert imports == ["mssql_py_core"] + + +@pytest.mark.parametrize("cross_arch", [False, True]) +def test_installer_selects_index_and_preserves_cross_arch_behavior( + tmp_path, monkeypatch, cross_arch +): + monkeypatch.setenv(runtime.VERSION_ENV, "0.4.0.dev1") + monkeypatch.setenv("PIP_EXTRA_INDEX_URL", "https://wrong.example/simple/") + monkeypatch.setenv("PIP_CERT", "corporate-ca.pem") + monkeypatch.setattr(installer.platform, "machine", lambda: "AMD64") + monkeypatch.setattr(installer.sys, "platform", "win32") + commands = [] + verified = [] + + def run(command, *, env, check): + assert check is True + assert "PIP_EXTRA_INDEX_URL" not in env + assert env["PIP_CERT"] == "corporate-ca.pem" + commands.append(command) + if "download" in command: + target = Path(command[command.index("--dest") + 1]) + name = "mssql_python_rs-0.4.0.dev1-cp310-abi3-win_arm64.whl" + with zipfile.ZipFile(target / name, "w") as wheel: + wheel.writestr( + "mssql_python_rs-0.4.0.dev1.dist-info/METADATA", + "Name: mssql-python-rs\nVersion: 0.4.0.dev1\n", + ) + + monkeypatch.setattr(installer.subprocess, "run", run) + monkeypatch.setattr(installer, "verify_installed", verified.append) + installer.install_runtime("arm64" if cross_arch else "", tmp_path / "downloads") + download = commands[0] + assert download[download.index("--index-url") + 1] == runtime.DEV_INDEX + assert download[-1] == "mssql-python-rs==0.4.0.dev1" + if cross_arch: + assert download[download.index("--platform") + 1] == "win_arm64" + assert len(commands) == 1 + assert verified == [] + else: + assert len(commands) == 2 + assert "--no-deps" in commands[1] + assert verified[0].version == "0.4.0.dev1" + assert not list((tmp_path / "downloads").iterdir()) + + +def test_installer_does_not_fall_back_after_download_failure(monkeypatch): + commands = [] + + def run(command, **kwargs): + commands.append(command) + raise subprocess.CalledProcessError(1, command) + + monkeypatch.setattr(installer.subprocess, "run", run) + with pytest.raises(subprocess.CalledProcessError): + installer.install_runtime() + assert len(commands) == 1 + + +def test_resolver_cli_writes_effective_artifact_pin(tmp_path, monkeypatch): + monkeypatch.setenv(runtime.VERSION_ENV, "0.4.0.dev1") + destination = tmp_path / "mssql-python-rs.version" + result = subprocess.run( + [ + sys.executable, + str(ROOT / "eng/scripts/mssql_python_rs.py"), + "--write-version-file", + str(destination), + ], + capture_output=True, + text=True, + check=True, + ) + assert destination.read_text(encoding="ascii") == "0.4.0.dev1\n" + assert json.loads(result.stdout)["version_override"] is True From a9f162db130e4e02c3e75389fd07358102896144 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Tue, 27 Oct 2026 14:45:48 +0530 Subject: [PATCH 2/4] Fix runtime release guards and stress dependencies Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- OneBranchPipelines/stress-test-pipeline.yml | 30 +++- conda/README.md | 2 + eng/conda_tools/release.py | 2 + eng/pipelines/pr-validation-pipeline.yml | 6 + requirements.txt | 1 + tests/test_030_pe_machine_assert.py | 3 +- tests/test_036_conda_provenance.py | 10 +- tests/test_037_rs_wheel_download.py | 6 +- tests/test_039_rs_runtime_pipelines.py | 158 +++++++++++++++++++- tests/test_040_rs_runtime_selection.py | 7 +- 10 files changed, 215 insertions(+), 10 deletions(-) diff --git a/OneBranchPipelines/stress-test-pipeline.yml b/OneBranchPipelines/stress-test-pipeline.yml index d7a5dc7d3..8f27f8f97 100644 --- a/OneBranchPipelines/stress-test-pipeline.yml +++ b/OneBranchPipelines/stress-test-pipeline.yml @@ -82,6 +82,17 @@ jobs: targetPath: '$(Build.SourcesDirectory)\wheels' displayName: 'Download selected build (choose matching override artifacts for dev tests)' + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: '$(resources.pipeline.buildPipeline.pipelineID)' + buildVersionToDownload: 'specific' + pipelineId: '$(resources.pipeline.buildPipeline.runID)' + artifactName: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + targetPath: '$(Pipeline.Workspace)\odbc-wheels' + displayName: 'Download ODBC wheels from the same selected build' + - script: | python eng/scripts/mssql_python_rs.py --write-version-file "$(Pipeline.Workspace)\effective-runtime.version" displayName: 'Resolve effective stress-test runtime' @@ -97,11 +108,11 @@ jobs: python eng/scripts/download_mssql_python_rs_wheels.py --output-dir "$(Pipeline.Workspace)\mssql-python-rs-wheels" displayName: 'Download pinned mssql-python-rs wheels' - # Install the pre-built wheel + # Read ODBC pins from the selected build, not the stress checkout's release versions. - script: | echo "Installing pre-built wheel..." dir "$(Build.SourcesDirectory)\wheels" - python -c "import subprocess,sys; from eng.scripts.mssql_python_rs import resolve_runtime; subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--no-index', '--no-deps', '--find-links', r'$(Build.SourcesDirectory)\wheels\dist', '--find-links', r'$(Pipeline.Workspace)\mssql-python-rs-wheels', 'mssql-python', 'mssql-python-rs=='+resolve_runtime().version, '--force-reinstall'])" + python -c "import subprocess,sys; from pathlib import Path; from eng.conda_tools.archive import read_wheel_metadata; from eng.conda_tools.contracts import exact_dependency_pin; from eng.scripts.mssql_python_rs import resolve_runtime; odbc_version, = {exact_dependency_pin(read_wheel_metadata(w)['requires_dist'], 'mssql-python-odbc') for w in Path(r'$(Build.SourcesDirectory)\wheels\dist').glob('mssql_python-*.whl')}; subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--no-index', '--no-deps', '--find-links', r'$(Build.SourcesDirectory)\wheels\dist', '--find-links', r'$(Pipeline.Workspace)\odbc-wheels\dist', '--find-links', r'$(Pipeline.Workspace)\mssql-python-rs-wheels', 'mssql-python', 'mssql-python-odbc=='+odbc_version, 'mssql-python-rs=='+resolve_runtime().version, '--force-reinstall'])" if errorlevel 1 exit /b 1 python eng/scripts/mssql_python_rs.py --verify displayName: 'Install pre-built mssql-python wheel' @@ -265,6 +276,17 @@ jobs: targetPath: '$(Build.SourcesDirectory)/wheels' displayName: 'Download selected build (choose matching override artifacts for dev tests)' + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: '$(resources.pipeline.buildPipeline.pipelineID)' + buildVersionToDownload: 'specific' + pipelineId: '$(resources.pipeline.buildPipeline.runID)' + artifactName: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + targetPath: '$(Pipeline.Workspace)/odbc-wheels' + displayName: 'Download ODBC wheels from the same selected build' + - script: | python eng/scripts/mssql_python_rs.py --write-version-file "$(Pipeline.Workspace)/effective-runtime.version" displayName: 'Resolve effective stress-test runtime' @@ -281,12 +303,12 @@ jobs: python eng/scripts/download_mssql_python_rs_wheels.py --output-dir "$(Pipeline.Workspace)/mssql-python-rs-wheels" displayName: 'Download pinned mssql-python-rs wheels' - # Install the pre-built wheel + # Read ODBC pins from the selected build, not the stress checkout's release versions. - script: | set -e echo "Installing pre-built wheel..." ls -la "$(Build.SourcesDirectory)/wheels" - python -c "import subprocess,sys; from eng.scripts.mssql_python_rs import resolve_runtime; subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--no-index', '--no-deps', '--find-links', '$(Build.SourcesDirectory)/wheels/dist', '--find-links', '$(Pipeline.Workspace)/mssql-python-rs-wheels', 'mssql-python', 'mssql-python-rs=='+resolve_runtime().version, '--force-reinstall'])" + python -c "import subprocess,sys; from pathlib import Path; from eng.conda_tools.archive import read_wheel_metadata; from eng.conda_tools.contracts import exact_dependency_pin; from eng.scripts.mssql_python_rs import resolve_runtime; odbc_version, = {exact_dependency_pin(read_wheel_metadata(w)['requires_dist'], 'mssql-python-odbc') for w in Path('$(Build.SourcesDirectory)/wheels/dist').glob('mssql_python-*.whl')}; subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--no-index', '--no-deps', '--find-links', '$(Build.SourcesDirectory)/wheels/dist', '--find-links', '$(Pipeline.Workspace)/odbc-wheels/dist', '--find-links', '$(Pipeline.Workspace)/mssql-python-rs-wheels', 'mssql-python', 'mssql-python-odbc=='+odbc_version, 'mssql-python-rs=='+resolve_runtime().version, '--force-reinstall'])" python eng/scripts/mssql_python_rs.py --verify displayName: 'Install pre-built mssql-python wheel' diff --git a/conda/README.md b/conda/README.md index 3dc64a15a..97749c16a 100644 --- a/conda/README.md +++ b/conda/README.md @@ -97,6 +97,8 @@ records `distribution_version`, `index_url`, `version_override` and individual version uses the public `mssql-rs_Public` Python index. Conda consumers validate this receipt and use staged producer bytes, not a fresh download using their checkout's pin or `MSSQL_PYTHON_RS_VERSION`. No NuGet package or transport pin is required. +Release validation rejects any receipt with `version_override=true`, including +an override equal to the committed stable pin. Such artifacts remain test-only. Published historical bindings that declare no RS dependency are accepted only when their payload and RECORD own the core initializer and native extension. They produce an explicit diff --git a/eng/conda_tools/release.py b/eng/conda_tools/release.py index 7981e3be8..cf9796b53 100644 --- a/eng/conda_tools/release.py +++ b/eng/conda_tools/release.py @@ -422,6 +422,8 @@ def execute(args: argparse.Namespace) -> int: else None ) inputs.validate_rs_transport(receipt, versions) + if isinstance(receipt, dict) and receipt.get("version_override"): + raise ValueError("RS version overrides are not eligible for Conda release.") except archive.READ_ERRORS as exc: print(f"ERROR: {exc}", file=sys.stderr) return 1 diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 3b94d1163..6004f9422 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -2463,6 +2463,12 @@ jobs: vmImage: 'ubuntu-latest' steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.12' + addToPath: true + displayName: 'Use Python 3.12 for coverage build and tests' + - script: | # Install build dependencies sudo apt-get update diff --git a/requirements.txt b/requirements.txt index 2b18cd919..77640bae2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ azure-identity # Build dependencies pybind11 setuptools +wheel # Code formatting and linting black diff --git a/tests/test_030_pe_machine_assert.py b/tests/test_030_pe_machine_assert.py index 855578238..cb80c8ab0 100644 --- a/tests/test_030_pe_machine_assert.py +++ b/tests/test_030_pe_machine_assert.py @@ -118,7 +118,8 @@ def test_zstd_backend_is_available_for_conda_audit_tests(): ) -def test_binding_wheel_excludes_core_and_requires_source_rs_version(tmp_path): +def test_binding_wheel_excludes_core_and_requires_source_rs_version(tmp_path, monkeypatch): + monkeypatch.delenv("MSSQL_PYTHON_RS_VERSION", raising=False) pytest.importorskip("setuptools", reason="Wheel archive regression requires setuptools") pytest.importorskip("wheel", reason="Wheel archive regression requires the wheel build backend") shutil.copy2(_ROOT / "setup.py", tmp_path / "setup.py") diff --git a/tests/test_036_conda_provenance.py b/tests/test_036_conda_provenance.py index 8d57fb64f..e2dc27075 100644 --- a/tests/test_036_conda_provenance.py +++ b/tests/test_036_conda_provenance.py @@ -363,6 +363,7 @@ def download(argv, *, check): "missing-private-driver", "missing-receipt", "wrong-index", + "stable-rs-override", "missing-selected-wheel", "wrong-selected-wheel", "wrong-wheel-tags", @@ -413,6 +414,8 @@ def test_source_bound_release_components(tmp_path, monkeypatch, capsys, problem, } if problem == "wrong-index": receipt["index_url"] = inputs.DEV_INDEX + elif problem == "stable-rs-override": + receipt["version_override"] = True if with_rs: files[rs_prefix + "conda-wheel-source.txt"] = (selected + "\n").encode() if "-alias-" in problem: @@ -533,9 +536,14 @@ def test_source_bound_release_components(tmp_path, monkeypatch, capsys, problem, "--release-versions", ] ) + output = capsys.readouterr() assert result == ( 0 if problem in {"legacy", "rs", "rs-extra-owned", "rs-pyc-owned"} else 1 - ), capsys.readouterr() + ), output + if problem == "stable-rs-override": + assert "RS version overrides are not eligible for Conda release." in output.err + assert "COMPONENT_INPUT_OK" not in output.out + assert "release-ready" not in output.out @pytest.mark.parametrize( diff --git a/tests/test_037_rs_wheel_download.py b/tests/test_037_rs_wheel_download.py index 4fe5c20f7..e7ff6920d 100644 --- a/tests/test_037_rs_wheel_download.py +++ b/tests/test_037_rs_wheel_download.py @@ -154,7 +154,11 @@ def test_rejects_symlink_output_without_deleting_target(tmp_path): def test_rejects_insecure_index(tmp_path): with pytest.raises(ValueError, match="HTTPS"): - downloader.download_wheels("http://example.test/simple/", _pin(tmp_path), tmp_path / "out") + downloader.download_wheels( + "http://example.test/simple/", # DevSkim: ignore DS137138 - HTTPS rejection fixture + _pin(tmp_path), + tmp_path / "out", + ) def test_ignores_yanked_wheels(tmp_path, monkeypatch): diff --git a/tests/test_039_rs_runtime_pipelines.py b/tests/test_039_rs_runtime_pipelines.py index 1e7867ccd..f23660a20 100644 --- a/tests/test_039_rs_runtime_pipelines.py +++ b/tests/test_039_rs_runtime_pipelines.py @@ -9,6 +9,7 @@ import re import shutil import subprocess +from types import SimpleNamespace import uuid import zipfile @@ -90,7 +91,7 @@ def test_consolidation_stages_effective_receipt_version_not_source_pin(): def test_stress_uses_selected_build_and_checks_effective_runtime_metadata(): source = (PIPELINES / "stress-test-pipeline.yml").read_text(encoding="utf-8") - assert source.count("pipelineId: '$(resources.pipeline.buildPipeline.runID)'") == 2 + assert source.count("pipelineId: '$(resources.pipeline.buildPipeline.runID)'") == 4 assert ( source.count("filePath: OneBranchPipelines/scripts/validate-rs-wheel-dependency.ps1") == 2 ) @@ -98,6 +99,161 @@ def test_stress_uses_selected_build_and_checks_effective_runtime_metadata(): assert source.count("'mssql-python-rs=='+resolve_runtime().version") == 2 +@pytest.fixture(params=("StressTestWindows", "StressTestLinux")) +def stress_job(request): + yaml = pytest.importorskip("yaml") + pipeline = yaml.safe_load((PIPELINES / "stress-test-pipeline.yml").read_text(encoding="utf-8")) + return next(job for job in pipeline["jobs"] if job["job"] == request.param) + + +def _stress_wheel(folder, name, version, requirements=()): + folder.mkdir(parents=True, exist_ok=True) + stem = f"{name.replace('-', '_')}-{version}" + path = folder / f"{stem}-py3-none-any.whl" + with zipfile.ZipFile(path, "w") as archive: + archive.writestr( + f"{stem}.dist-info/METADATA", + f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n" + + "".join(f"Requires-Dist: {requirement}\n" for requirement in requirements), + ) + archive.writestr( + f"{stem}.dist-info/WHEEL", + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + ) + archive.writestr(f"{stem}.dist-info/RECORD", "") + return path + + +def _stress_install_command(stress_job, workspace, monkeypatch): + from eng.scripts import mssql_python_rs + + calls = [] + monkeypatch.setattr(subprocess, "check_call", calls.append) + monkeypatch.setattr( + mssql_python_rs, "resolve_runtime", lambda: SimpleNamespace(version="0.3.1.dev123") + ) + step = next( + step + for step in stress_job["steps"] + if step.get("displayName") == "Install pre-built mssql-python wheel" + ) + script = step["script"] + command = next(line.strip() for line in script.splitlines() if 'python -c "' in line) + code = command.removeprefix('python -c "').removesuffix('"') + source_path = ( + str(workspace) if stress_job["job"] == "StressTestWindows" else workspace.as_posix() + ) + code = code.replace("$(Build.SourcesDirectory)", source_path).replace( + "$(Pipeline.Workspace)", source_path + ) + if os.name != "nt": + code = code.replace("\\", "/") + exec(compile(code, "", "exec"), {}) + assert len(calls) == 1 + assert script.index("python -c ") < script.index("mssql_python_rs.py --verify") + return calls[0] + + +def test_stress_downloads_odbc_from_same_selected_build(stress_job): + downloads = { + step["inputs"]["artifactName"]: step["inputs"] + for step in stress_job["steps"] + if step.get("task") == "DownloadPipelineArtifact@2" + } + binding = downloads["drop_Consolidate_ConsolidateArtifacts"] + odbc = downloads["drop_ConsolidateOdbc_ConsolidateArtifacts"] + for key in ("buildType", "project", "definition", "buildVersionToDownload", "pipelineId"): + assert binding[key] == odbc[key] + assert odbc["buildType"] == odbc["buildVersionToDownload"] == "specific" + assert odbc["pipelineId"] == "$(resources.pipeline.buildPipeline.runID)" + assert odbc["targetPath"].replace("\\", "/") == "$(Pipeline.Workspace)/odbc-wheels" + + +@pytest.mark.parametrize("odbc_available", (True, False)) +def test_stress_installs_metadata_pinned_odbc_offline( + stress_job, workspace, monkeypatch, odbc_available +): + _stress_wheel( + workspace / "wheels/dist", + "mssql-python", + "9.8.7", + ("mssql-python-odbc==8.7.6", "mssql-python-rs==0.3.1.dev123"), + ) + odbc_dir = workspace / "odbc-wheels/dist" + _stress_wheel(odbc_dir, "mssql-python-odbc", "99.0.0") + if odbc_available: + _stress_wheel(odbc_dir, "mssql-python-odbc", "8.7.6") + _stress_wheel(workspace / "mssql-python-rs-wheels", "mssql-python-rs", "0.3.1.dev123") + command = _stress_install_command(stress_job, workspace, monkeypatch) + assert "mssql-python-odbc==8.7.6" in command + assert "mssql-python-rs==0.3.1.dev123" in command + assert "--no-index" in command and "--no-deps" in command + links = [command[index + 1] for index, value in enumerate(command) if value == "--find-links"] + assert str(odbc_dir) in [str(Path(link)) for link in links] + result = subprocess.run( + [*command, "--dry-run", "--ignore-installed"], + capture_output=True, + text=True, + check=False, + ) + assert (result.returncode == 0) == odbc_available, result.stdout + result.stderr + if odbc_available: + assert "mssql-python-odbc-8.7.6" in result.stdout + else: + assert "mssql-python-odbc==8.7.6" in result.stderr + + +@pytest.mark.parametrize( + "requirements", + ( + (), + ("mssql-python-odbc>=8.7.6",), + ("mssql-python-odbc==8.7.*",), + ("mssql-python-odbc==8.7.6; sys_platform == 'win32'",), + ("mssql-python-odbc==8.7.6", "mssql-python-odbc==8.7.5"), + ), +) +def test_stress_rejects_missing_or_nonexact_odbc_pin( + stress_job, workspace, monkeypatch, requirements +): + _stress_wheel(workspace / "wheels/dist", "mssql-python", "9.8.7", requirements) + with pytest.raises((ValueError, TypeError)): + _stress_install_command(stress_job, workspace, monkeypatch) + + +def test_stress_rejects_inconsistent_binding_odbc_pins(stress_job, workspace, monkeypatch): + for version, pin in (("9.8.7", "8.7.6"), ("9.8.8", "8.7.5")): + _stress_wheel( + workspace / "wheels/dist", "mssql-python", version, (f"mssql-python-odbc=={pin}",) + ) + with pytest.raises(ValueError): + _stress_install_command(stress_job, workspace, monkeypatch) + + +def test_coverage_selects_python_before_dependencies_native_build_and_runtime(): + yaml = pytest.importorskip("yaml") + pipeline = yaml.safe_load((ROOT / ENTRY_POINTS[0]).read_text(encoding="utf-8")) + job = next(job for job in pipeline["jobs"] if job["job"] == "CodeCoverageReport") + steps = job["steps"] + selectors = [ + index for index, step in enumerate(steps) if step.get("task") == "UsePythonVersion@0" + ] + assert len(selectors) == 1 + selected = selectors[0] + assert steps[selected]["inputs"]["versionSpec"] == "3.12" + assert steps[selected]["inputs"]["addToPath"] is True + for required in ("pip install", "./build.sh codecov", "./generate_codecov.sh"): + assert any(required in step.get("script", "") for step in steps), required + assert any(step.get("template") == "steps/install-mssql-py-core.yml" for step in steps) + for index, step in enumerate(steps): + uses_python = re.search( + r"\b(?:python\d*|pip\d*)\b|(?:build|generate_codecov)\.sh", + step.get("script", ""), + ) + if uses_python or step.get("template") == "steps/install-mssql-py-core.yml": + assert selected < index, step + + def test_runtime_is_verified_after_dependency_setup_before_testing(): pr = (ROOT / ENTRY_POINTS[0]).read_text(encoding="utf-8") test_commands = [ diff --git a/tests/test_040_rs_runtime_selection.py b/tests/test_040_rs_runtime_selection.py index 0ef356642..10ec0630a 100644 --- a/tests/test_040_rs_runtime_selection.py +++ b/tests/test_040_rs_runtime_selection.py @@ -112,7 +112,10 @@ def test_installer_selects_index_and_preserves_cross_arch_behavior( tmp_path, monkeypatch, cross_arch ): monkeypatch.setenv(runtime.VERSION_ENV, "0.4.0.dev1") - monkeypatch.setenv("PIP_EXTRA_INDEX_URL", "https://wrong.example/simple/") + monkeypatch.setenv( + "PIP_EXTRA_INDEX_URL", # DevSkim: ignore DS205001 - test removal from child environment + "https://wrong.example/simple/", + ) monkeypatch.setenv("PIP_CERT", "corporate-ca.pem") monkeypatch.setattr(installer.platform, "machine", lambda: "AMD64") monkeypatch.setattr(installer.sys, "platform", "win32") @@ -121,7 +124,7 @@ def test_installer_selects_index_and_preserves_cross_arch_behavior( def run(command, *, env, check): assert check is True - assert "PIP_EXTRA_INDEX_URL" not in env + assert "PIP_EXTRA_INDEX_URL" not in env # DevSkim: ignore DS205001 - assert removal assert env["PIP_CERT"] == "corporate-ca.pem" commands.append(command) if "download" in command: From 8a65995c33dad64067a35128dc11423f7f8c2cbc Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Tue, 27 Oct 2026 15:21:24 +0530 Subject: [PATCH 3/4] Fix stress import isolation and pipeline test prerequisites Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- OneBranchPipelines/stress-test-pipeline.yml | 32 ++--- eng/requirements-build-linux.in | 1 + eng/requirements-build-linux.txt | 75 ++++++++++++ eng/requirements-build-macos.txt | 79 +++++++++++- eng/requirements-build-windows.in | 1 + eng/requirements-build-windows.txt | 75 ++++++++++++ eng/requirements-test-linux.txt | 79 ++++++++++++ requirements.txt | 1 + tests/test_039_rs_runtime_pipelines.py | 127 +++++++++++++++++++- 9 files changed, 450 insertions(+), 20 deletions(-) diff --git a/OneBranchPipelines/stress-test-pipeline.yml b/OneBranchPipelines/stress-test-pipeline.yml index 8f27f8f97..168e86319 100644 --- a/OneBranchPipelines/stress-test-pipeline.yml +++ b/OneBranchPipelines/stress-test-pipeline.yml @@ -117,15 +117,17 @@ jobs: python eng/scripts/mssql_python_rs.py --verify displayName: 'Install pre-built mssql-python wheel' - # Rename local source folder to avoid import conflicts with installed wheel + # Hide checkout packages before loading the selected build's installed wheels. - powershell: | - Write-Host "Renaming local mssql_python folder to avoid import conflicts..." - if (Test-Path "$(Build.SourcesDirectory)\mssql_python") { - Rename-Item -Path "$(Build.SourcesDirectory)\mssql_python" -NewName "mssql_python_src" -Force - Write-Host "Renamed mssql_python to mssql_python_src" + $ErrorActionPreference = 'Stop' + foreach ($package in @('mssql_python', 'mssql_python_odbc')) { + $source = Join-Path $env:BUILD_SOURCESDIRECTORY $package + if (Test-Path -LiteralPath $source) { + Rename-Item -LiteralPath $source -NewName "${package}_src" -ErrorAction Stop + } } - # Verify import from installed wheel - python -c "import mssql_python; print('Imported from:', mssql_python.__file__)" + python -c "import os; from pathlib import Path; import mssql_python_odbc as odbc; source=Path(os.environ['BUILD_SOURCESDIRECTORY']).resolve(); p=Path(odbc.__file__).resolve(); assert not p.is_relative_to(source), 'mssql_python_odbc resolved inside checkout: '+str(p); print('mssql_python_odbc imported from:', p); import mssql_python; p=Path(mssql_python.__file__).resolve(); assert not p.is_relative_to(source), 'mssql_python resolved inside checkout: '+str(p); print('mssql_python imported from:', p)" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } displayName: 'Ensure tests use installed wheel' # PYTHON-1ES-MMS2022 has LocalDB pre-installed (no full SQL Server instance) @@ -312,15 +314,15 @@ jobs: python eng/scripts/mssql_python_rs.py --verify displayName: 'Install pre-built mssql-python wheel' - # Rename local source folder to avoid import conflicts with installed wheel + # Hide checkout packages before loading the selected build's installed wheels. - script: | - echo "Renaming local mssql_python folder to avoid import conflicts..." - if [ -d "$(Build.SourcesDirectory)/mssql_python" ]; then - mv "$(Build.SourcesDirectory)/mssql_python" "$(Build.SourcesDirectory)/mssql_python_src" - echo "Renamed mssql_python to mssql_python_src" - fi - # Verify import from installed wheel - python -c "import mssql_python; print('Imported from:', mssql_python.__file__)" + set -e + for package in mssql_python mssql_python_odbc; do + if [ -d "$(Build.SourcesDirectory)/$package" ]; then + mv "$(Build.SourcesDirectory)/$package" "$(Build.SourcesDirectory)/${package}_src" + fi + done + python -c "import os; from pathlib import Path; import mssql_python_odbc as odbc; source=Path(os.environ['BUILD_SOURCESDIRECTORY']).resolve(); p=Path(odbc.__file__).resolve(); assert not p.is_relative_to(source), 'mssql_python_odbc resolved inside checkout: '+str(p); print('mssql_python_odbc imported from:', p); import mssql_python; p=Path(mssql_python.__file__).resolve(); assert not p.is_relative_to(source), 'mssql_python resolved inside checkout: '+str(p); print('mssql_python imported from:', p)" displayName: 'Ensure tests use installed wheel' # Docker is pre-installed by linux-1es-pt-prerequisites-v2 artifact diff --git a/eng/requirements-build-linux.in b/eng/requirements-build-linux.in index e153979ba..cd3a97adb 100644 --- a/eng/requirements-build-linux.in +++ b/eng/requirements-build-linux.in @@ -2,5 +2,6 @@ pip pybind11 pytest +PyYAML setuptools wheel diff --git a/eng/requirements-build-linux.txt b/eng/requirements-build-linux.txt index ce528b83e..bb38e7f71 100644 --- a/eng/requirements-build-linux.txt +++ b/eng/requirements-build-linux.txt @@ -36,6 +36,81 @@ pytest==9.1.1 \ --hash=sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313 \ --hash=sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c # via -r eng/requirements-build-linux.in +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via -r eng/requirements-build-linux.in setuptools==84.0.0 \ --hash=sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670 \ --hash=sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73 diff --git a/eng/requirements-build-macos.txt b/eng/requirements-build-macos.txt index 939ff74d1..642c60e96 100644 --- a/eng/requirements-build-macos.txt +++ b/eng/requirements-build-macos.txt @@ -1307,6 +1307,81 @@ pytokens==0.4.1 \ --hash=sha256:ee44d0f85b803321710f9239f335aafe16553b39106384cef8e6de40cb4ef2f6 \ --hash=sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324 # via black +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via -r eng/../requirements.txt requests==2.34.2 \ --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed @@ -1404,7 +1479,9 @@ urllib3==2.7.0 \ wheel==0.48.0 \ --hash=sha256:3217dcc807155e45db462d7ef2431f5ddda0d7273b700d05a67b271ceb1287ab \ --hash=sha256:94800765601e9171bf5d58d066e640662842bcedcbab982b2c90787a2c987322 - # via -r eng/requirements-build-macos.in + # via + # -r eng/../requirements.txt + # -r eng/requirements-build-macos.in zstandard==0.25.0 \ --hash=sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64 \ --hash=sha256:01582723b3ccd6939ab7b3a78622c573799d5d8737b534b86d0e06ac18dbde4a \ diff --git a/eng/requirements-build-windows.in b/eng/requirements-build-windows.in index 3687afb64..e0bfbc0bf 100644 --- a/eng/requirements-build-windows.in +++ b/eng/requirements-build-windows.in @@ -5,6 +5,7 @@ pybind11 pyodbc pytest pytest-asyncio +PyYAML setuptools wheel zstandard diff --git a/eng/requirements-build-windows.txt b/eng/requirements-build-windows.txt index 395e7ffc5..f8962f0d5 100644 --- a/eng/requirements-build-windows.txt +++ b/eng/requirements-build-windows.txt @@ -135,6 +135,81 @@ pytest-asyncio==1.4.0 \ --hash=sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1 \ --hash=sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42 # via -r eng/requirements-build-windows.in +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via -r eng/requirements-build-windows.in setuptools==84.0.0 \ --hash=sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670 \ --hash=sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73 diff --git a/eng/requirements-test-linux.txt b/eng/requirements-test-linux.txt index 5f7a6019d..d7a427168 100644 --- a/eng/requirements-test-linux.txt +++ b/eng/requirements-test-linux.txt @@ -1298,6 +1298,84 @@ pytokens==0.4.1 \ --hash=sha256:ee44d0f85b803321710f9239f335aafe16553b39106384cef8e6de40cb4ef2f6 \ --hash=sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324 # via black +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via + # -c eng/requirements-build-linux.txt + # -r eng/../requirements.txt + # -r eng/requirements-build-linux.in requests==2.34.2 \ --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed @@ -1402,6 +1480,7 @@ wheel==0.48.0 \ --hash=sha256:94800765601e9171bf5d58d066e640662842bcedcbab982b2c90787a2c987322 # via # -c eng/requirements-build-linux.txt + # -r eng/../requirements.txt # -r eng/requirements-build-linux.in zstandard==0.25.0 \ --hash=sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64 \ diff --git a/requirements.txt b/requirements.txt index 77640bae2..a7a69563b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ pytest pytest-cov pytest-asyncio +PyYAML zstandard coverage unittest-xml-reporting diff --git a/tests/test_039_rs_runtime_pipelines.py b/tests/test_039_rs_runtime_pipelines.py index f23660a20..bc30c87ab 100644 --- a/tests/test_039_rs_runtime_pipelines.py +++ b/tests/test_039_rs_runtime_pipelines.py @@ -9,6 +9,7 @@ import re import shutil import subprocess +import sys from types import SimpleNamespace import uuid import zipfile @@ -17,9 +18,17 @@ ROOT = Path(__file__).resolve().parents[1] PIPELINES = ROOT / "OneBranchPipelines" -if not (PIPELINES / "scripts/validate-release-versions.ps1").is_file(): +SOURCE_PREREQUISITES = ( + "OneBranchPipelines/scripts/validate-release-versions.ps1", + "eng/scripts/mssql_python_rs.py", + "eng/conda_tools/archive.py", + "eng/conda_tools/contracts.py", +) +if not all((ROOT / relative).is_file() for relative in SOURCE_PREREQUISITES): pytest.skip("Pipeline contracts require a complete source checkout", allow_module_level=True) +import yaml + VERSION_ENV = "MSSQL_PYTHON_RS_VERSION" ENTRY_POINTS = ( "eng/pipelines/pr-validation-pipeline.yml", @@ -40,7 +49,6 @@ def test_pipeline_override_is_explicit_and_defaults_to_empty(filename): def test_modified_pipeline_yaml_parses(): - yaml = pytest.importorskip("yaml") for folder in (ROOT / "eng/pipelines", PIPELINES): for path in folder.rglob("*.yml"): assert isinstance(yaml.safe_load(path.read_text(encoding="utf-8")), dict), path @@ -101,11 +109,123 @@ def test_stress_uses_selected_build_and_checks_effective_runtime_metadata(): @pytest.fixture(params=("StressTestWindows", "StressTestLinux")) def stress_job(request): - yaml = pytest.importorskip("yaml") pipeline = yaml.safe_load((PIPELINES / "stress-test-pipeline.yml").read_text(encoding="utf-8")) return next(job for job in pipeline["jobs"] if job["job"] == request.param) +@pytest.mark.parametrize( + "missing", + ( + "eng/scripts/mssql_python_rs.py", + "eng/conda_tools/archive.py", + "eng/conda_tools/contracts.py", + ), +) +def test_pipeline_contracts_skip_incomplete_wheel_fixture(tmp_path, missing): + shutil.copytree(PIPELINES, tmp_path / "OneBranchPipelines") + for relative in SOURCE_PREREQUISITES[1:]: + if relative != missing: + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + copied_test = tmp_path / "tests" / Path(__file__).name + copied_test.parent.mkdir() + shutil.copy2(__file__, copied_test) + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "--noconftest", + "--collect-only", + "-q", + "-rs", + str(copied_test), + ], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 5, result.stdout + result.stderr + assert "complete source checkout" in result.stdout + assert "SKIPPED [1]" in result.stdout + + +def test_pipeline_contracts_require_yaml_in_complete_checkout(): + command = ( + "import sys, pytest; sys.modules['yaml'] = None; " + "raise SystemExit(pytest.main(" + + repr(["--noconftest", "--collect-only", "-q", str(Path(__file__).resolve())]) + + "))" + ) + result = subprocess.run( + [sys.executable, "-c", command], + cwd=ROOT, + env=dict(os.environ, PYTEST_DISABLE_PLUGIN_AUTOLOAD="1"), + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 2, result.stdout + result.stderr + assert "ModuleNotFoundError" in result.stdout + assert "yaml" in result.stdout + + +@pytest.mark.parametrize("shadow", (None, "mssql_python_odbc", "mssql_python")) +def test_stress_checks_import_provenance_before_binding_load(stress_job, tmp_path, shadow): + checkout = tmp_path / "checkout" + checkout.mkdir() + installed = tmp_path / "checkout-site-packages" + for name in ("mssql_python_odbc", "mssql_python"): + package = installed / name + package.mkdir(parents=True) + (package / "__init__.py").write_text( + "print('BINDING_IMPORTED')\n" if name == "mssql_python" else "", + encoding="utf-8", + ) + if shadow: + package = checkout / shadow + package.mkdir() + (package / "__init__.py").write_text( + "print('BINDING_IMPORTED')\n" if shadow == "mssql_python" else "", + encoding="utf-8", + ) + step = next( + step + for step in stress_job["steps"] + if step.get("displayName") == "Ensure tests use installed wheel" + ) + script = step.get("script", step.get("powershell")) + command = next(line.strip() for line in script.splitlines() if 'python -c "' in line) + code = command.removeprefix('python -c "').removesuffix('"') + result = subprocess.run( + [sys.executable, "-c", code], + cwd=checkout, + env=dict( + os.environ, + PYTHONPATH=str(installed), + BUILD_SOURCESDIRECTORY=str(checkout), + ), + capture_output=True, + text=True, + timeout=30, + ) + if shadow: + assert result.returncode != 0, result.stdout + assert f"{shadow} resolved inside checkout:" in result.stderr + if shadow == "mssql_python_odbc": + assert "BINDING_IMPORTED" not in result.stdout + else: + assert result.returncode == 0, result.stderr + assert str(installed) in result.stdout + assert "BINDING_IMPORTED" in result.stdout + renaming = script[: script.index('python -c "')] + assert "mssql_python_odbc" in renaming + assert "mssql_python" in renaming + assert "set -e" in script if "script" in step else "$ErrorActionPreference" in script + + def _stress_wheel(folder, name, version, requirements=()): folder.mkdir(parents=True, exist_ok=True) stem = f"{name.replace('-', '_')}-{version}" @@ -231,7 +351,6 @@ def test_stress_rejects_inconsistent_binding_odbc_pins(stress_job, workspace, mo def test_coverage_selects_python_before_dependencies_native_build_and_runtime(): - yaml = pytest.importorskip("yaml") pipeline = yaml.safe_load((ROOT / ENTRY_POINTS[0]).read_text(encoding="utf-8")) job = next(job for job in pipeline["jobs"] if job["job"] == "CodeCoverageReport") steps = job["steps"] From 0efc34c29b1321324f90c414b95c916374f2a0ad Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Tue, 27 Oct 2026 17:11:57 +0530 Subject: [PATCH 4/4] Use published Rust nightlies for PR validation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CONTRIBUTING.md | 30 +- eng/pipelines/pr-validation-pipeline.yml | 54 +++- eng/scripts/resolve_pr_rs_runtime.py | 205 +++++++++++++ tests/test_039_rs_runtime_pipelines.py | 58 ++++ tests/test_041_rs_nightly_selection.py | 373 +++++++++++++++++++++++ 5 files changed, 713 insertions(+), 7 deletions(-) create mode 100644 eng/scripts/resolve_pr_rs_runtime.py create mode 100644 tests/test_041_rs_nightly_selection.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 37602e9b7..797332e9a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,7 +17,8 @@ or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any addi ### Selecting the Rust Runtime -Normal builds and tests install the exact released `mssql-python-rs` version in +Local builds and official/release pipelines install the exact released +`mssql-python-rs` version in [`eng/versions/mssql-python-rs.version`](eng/versions/mssql-python-rs.version) from public PyPI. The runtime is a separate dependency, not embedded in the `mssql-python` wheel. No NuGet transport package is needed. @@ -44,10 +45,29 @@ index; clean releases come from public PyPI. A missing requested wheel fails; the installer never falls back to another version. Cross-architecture Windows builds validate the target wheel without installing it into the host Python. -Azure build/test pipelines expose the same selection as `rsVersionOverride`. -Leave it empty for the committed release pin. Record the exact dev version in -the feature's PR so another developer can reproduce the tests. Before running -tests after other package installations, verify the runtime has not changed: +The GitHub PR-validation checks run in Azure DevOps using +`eng/pipelines/pr-validation-pipeline.yml`. This validation pipeline (including +its main-branch CI runs) defaults to the latest published nightly from the Azure +Python feed, not the stable PyPI pin. It resolves the version once before the +matrix and passes that exact version to every build/test job. The +`rs-runtime-selection` artifact records the version, source index and nightly +wheel hashes. + +Nightlies use `X.Y.Z.devYYYYMMDD`; per-merge development builds append a build ID +and are not selected automatically. Selection requires all nine platform wheels +for the newest nightly, with valid SHA-256 links and no yanked files. Missing or +incomplete publication fails validation rather than falling back to an older +nightly or a stable release. Feed contents establish wheel publication, not +the overall status or source commit of the producer's Azure build. + +Azure build/test pipelines expose the explicit selection as `rsVersionOverride`. +For PR validation, leave it empty for the nightly, or provide an exact version +to reproduce a run or test newer per-merge Rust changes. Official/release builds +retain the committed stable PyPI pin and reject overrides. Other test pipelines +retain their existing stable default unless explicitly overridden. Record the +exact dev version in the feature's PR so another developer can reproduce the +tests. Before running tests after other package installations, verify the runtime +has not changed: ```console python eng/scripts/mssql_python_rs.py --verify diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 6004f9422..d89684674 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -8,17 +8,41 @@ trigger: parameters: - name: rsVersionOverride - displayName: 'Exact mssql-python-rs test version (empty uses committed PyPI release)' + displayName: 'Exact mssql-python-rs test version (empty uses latest published nightly)' type: string default: '' variables: - MSSQL_PYTHON_RS_VERSION: ${{ parameters.rsVersionOverride }} # QEMU 10.2.3, qualified against the ARM64 GCC startup crash and shutdown tests. arm64QemuImage: 'tonistiigi/binfmt@sha256:400a4873b838d1b89194d982c45e5fb3cda4593fbfd7e08a02e76b03b21166f0' jobs: +- job: SelectRsRuntime + displayName: 'Select one published Rust nightly for the validation run' + pool: + vmImage: 'ubuntu-latest' + steps: + - checkout: self + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.12' + addToPath: true + - script: | + python eng/scripts/resolve_pr_rs_runtime.py --output-dir "$(Build.ArtifactStagingDirectory)/rs-runtime" + name: select + displayName: 'Resolve nightly or exact override from the Python package index' + env: + MSSQL_PYTHON_RS_VERSION: ${{ parameters.rsVersionOverride }} + - task: PublishPipelineArtifact@1 + displayName: 'Record exact runtime selection and wheel hashes' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)/rs-runtime' + artifact: 'rs-runtime-selection' + - job: CodeQLAnalysis + dependsOn: SelectRsRuntime + variables: + MSSQL_PYTHON_RS_VERSION: $[ dependencies.SelectRsRuntime.outputs['select.version'] ] displayName: 'CodeQL Security Analysis' pool: vmImage: 'ubuntu-latest' @@ -57,6 +81,9 @@ jobs: displayName: 'Finalize CodeQL' - job: pytestonwindows + dependsOn: SelectRsRuntime + variables: + MSSQL_PYTHON_RS_VERSION: $[ dependencies.SelectRsRuntime.outputs['select.version'] ] displayName: 'Windows x64' timeoutInMinutes: 160 pool: @@ -568,12 +595,14 @@ jobs: # displayName: 'Publish code coverage results' - job: PytestOnMacOS + dependsOn: SelectRsRuntime displayName: 'macOS x86_64' timeoutInMinutes: 90 pool: vmImage: 'macos-latest' variables: + MSSQL_PYTHON_RS_VERSION: $[ dependencies.SelectRsRuntime.outputs['select.version'] ] # Apply to setup, benchmark consumers and final cleanup without changing # the Docker client's global context. DOCKER_CONTEXT: colima @@ -749,6 +778,9 @@ jobs: DB_PASSWORD: $(DB_PASSWORD) - job: PytestOnLinux + dependsOn: SelectRsRuntime + variables: + MSSQL_PYTHON_RS_VERSION: $[ dependencies.SelectRsRuntime.outputs['select.version'] ] displayName: 'Linux x86_64' timeoutInMinutes: 160 pool: @@ -1092,6 +1124,9 @@ jobs: testRunTitle: 'Publish pytest results on $(distroName)' - job: PytestOnLinux_ARM64 + dependsOn: SelectRsRuntime + variables: + MSSQL_PYTHON_RS_VERSION: $[ dependencies.SelectRsRuntime.outputs['select.version'] ] displayName: 'Linux ARM64' pool: vmImage: 'ubuntu-latest' @@ -1335,6 +1370,9 @@ jobs: testRunTitle: 'Publish pytest results on $(distroName) ARM64' - job: PytestOnLinux_RHEL9 + dependsOn: SelectRsRuntime + variables: + MSSQL_PYTHON_RS_VERSION: $[ dependencies.SelectRsRuntime.outputs['select.version'] ] displayName: 'Linux RedHat x86_64' pool: vmImage: 'ubuntu-latest' @@ -1537,6 +1575,9 @@ jobs: testRunTitle: 'Publish pytest results on RHEL 9' - job: PytestOnLinux_RHEL9_ARM64 + dependsOn: SelectRsRuntime + variables: + MSSQL_PYTHON_RS_VERSION: $[ dependencies.SelectRsRuntime.outputs['select.version'] ] displayName: 'Linux RedHat ARM64' pool: vmImage: 'ubuntu-latest' @@ -1754,6 +1795,9 @@ jobs: testRunTitle: 'Publish pytest results on RHEL 9 ARM64' - job: PytestOnLinux_Alpine + dependsOn: SelectRsRuntime + variables: + MSSQL_PYTHON_RS_VERSION: $[ dependencies.SelectRsRuntime.outputs['select.version'] ] displayName: 'Linux Alpine x86_64' pool: vmImage: 'ubuntu-latest' @@ -1990,6 +2034,9 @@ jobs: testRunTitle: 'Publish pytest results on Alpine x86_64' - job: PytestOnLinux_Alpine_ARM64 + dependsOn: SelectRsRuntime + variables: + MSSQL_PYTHON_RS_VERSION: $[ dependencies.SelectRsRuntime.outputs['select.version'] ] displayName: 'Linux Alpine ARM64' pool: vmImage: 'ubuntu-latest' @@ -2458,6 +2505,9 @@ jobs: # testRunTitle: 'Publish pytest results on Ubuntu AzureSQL' - job: CodeCoverageReport + dependsOn: SelectRsRuntime + variables: + MSSQL_PYTHON_RS_VERSION: $[ dependencies.SelectRsRuntime.outputs['select.version'] ] displayName: 'Full Code Coverage Report in Ubuntu x86_64' pool: vmImage: 'ubuntu-latest' diff --git a/eng/scripts/resolve_pr_rs_runtime.py b/eng/scripts/resolve_pr_rs_runtime.py new file mode 100644 index 000000000..a91a4e516 --- /dev/null +++ b/eng/scripts/resolve_pr_rs_runtime.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Select an exact override or the newest feed-validated nightly runtime for PR tests.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import urllib.request +from dataclasses import asdict, dataclass +from datetime import datetime +from itertools import chain +from pathlib import Path +from urllib.parse import parse_qs, unquote, urldefrag, urljoin, urlsplit + +if __package__: + from .download_mssql_python_rs_wheels import PACKAGE_ID, _WheelLinks + from .mssql_python_build_safety import resolve_safe_output_directory + from .mssql_python_rs import DEV_INDEX, VERSION_ENV, resolve_runtime +else: + from download_mssql_python_rs_wheels import PACKAGE_ID, _WheelLinks + from mssql_python_build_safety import resolve_safe_output_directory + from mssql_python_rs import DEV_INDEX, VERSION_ENV, resolve_runtime + +EXPECTED_PLATFORMS = frozenset( + { + "win_amd64", + "win_arm64", + "manylinux_2_28_x86_64", + "manylinux_2_28_aarch64", + "manylinux_2_34_x86_64", + "manylinux_2_34_aarch64", + "musllinux_1_2_x86_64", + "musllinux_1_2_aarch64", + "macosx_15_0_universal2", + } +) +_NIGHTLY = re.compile(r"([0-9]+)\.([0-9]+)\.([0-9]+)\.dev([0-9]{8})") +_WHEEL = re.compile(r"mssql_python_rs-([^-]+?)(?:-(.*))?\.whl") +_NO_NIGHTLY = ( + "No scheduled nightly (X.Y.Z.devYYYYMMDD) wheels found; " + f"set {VERSION_ENV} to an exact published version to override selection" +) + + +@dataclass(frozen=True) +class RuntimeSelection: + """Overrides are unprobed; nightly evidence consists only of public feed links.""" + + version: str + index_url: str + channel: str + version_override: bool + wheel_sha256: dict[str, str] + wheel_urls: dict[str, str] + + +class _NightlyLinks(_WheelLinks): + def __init__(self) -> None: + super().__init__() + self.yanked: list[str] = [] + self.anchor_open = False + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag == "a": + attributes = dict(attrs) + if self.anchor_open or len(attributes) != len(attrs) or not attributes.get("href"): + raise ValueError("Malformed wheel index anchor") + self.anchor_open = True + if "data-yanked" in attributes: + self.yanked.append(attributes["href"]) + super().handle_starttag(tag, attrs) + + def handle_endtag(self, tag: str) -> None: + if tag == "a": + if not self.anchor_open: + raise ValueError("Malformed wheel index anchor") + self.anchor_open = False + + def handle_data(self, data: str) -> None: + if data.lstrip().startswith("<"): + raise ValueError("Malformed wheel index HTML") + + def close(self) -> None: + if self.anchor_open or self.rawdata.strip(): + raise ValueError("Malformed wheel index HTML") + super().close() + + +def select_nightly(page: str, index_url: str = DEV_INDEX) -> RuntimeSelection: + """Validate link evidence, not publisher build status or commit provenance.""" + package_url = index_url.rstrip("/") + f"/{PACKAGE_ID}/" + if urlsplit(package_url).scheme != "https" or not urlsplit(package_url).hostname: + raise ValueError("The runtime wheel index must use HTTPS") + parser = _NightlyLinks() + parser.feed(page) + parser.close() + candidates = [] + for link, yanked in chain( + ((link, False) for link in parser.links), + ((link, True) for link in parser.yanked), + ): + url, fragment = urldefrag(urljoin(package_url, link)) + parts = urlsplit(url) + if not parts.path or parts.path == "/": + raise ValueError("Invalid wheel URL; HTTPS wheel file required") + filename = unquote(parts.path.rsplit("/", 1)[-1]) + wheel = _WHEEL.fullmatch(filename) + if not wheel: + continue + version, tags = wheel.groups() + nightly = _NIGHTLY.fullmatch(version) + if not nightly: + continue + major, minor, patch, stamp = nightly.groups() + try: + date = datetime.strptime(stamp, "%Y%m%d").date() + except ValueError as exc: + raise ValueError(f"Invalid nightly calendar date in {filename}") from exc + if date.strftime("%Y%m%d") != stamp: + raise ValueError(f"Invalid nightly calendar date in {filename}") + key = (date, int(major), int(minor), int(patch)) + candidates.append((key, version, tags or "", filename, url, fragment, yanked)) + if not candidates: + raise ValueError(_NO_NIGHTLY) + newest = max(candidate[0] for candidate in candidates) + selected = [candidate for candidate in candidates if candidate[0] == newest] + versions = {candidate[1] for candidate in selected} + if len(versions) != 1: + raise ValueError(f"Ambiguous nightly version spellings: {sorted(versions)}") + version = versions.pop() + hashes = {} + urls = {} + platforms = set() + for _, _, tags, filename, url, fragment, yanked in selected: + if filename in urls: + raise ValueError(f"Duplicate wheel filename in index: {filename}") + if yanked: + raise ValueError(f"Newest nightly {version} has a yanked wheel: {filename}") + parts = urlsplit(url) + if parts.scheme != "https" or not parts.hostname or parts.username or parts.password: + raise ValueError(f"Wheel download must use HTTPS without credentials: {filename}") + platform = tags.removeprefix("cp310-abi3-") + if not tags.startswith("cp310-abi3-") or platform not in EXPECTED_PLATFORMS: + raise ValueError(f"Unexpected newest nightly wheel tags: {filename}") + digest = parse_qs(fragment, keep_blank_values=True).get("sha256", []) + if len(digest) != 1 or not re.fullmatch(r"[a-fA-F0-9]{64}", digest[0]): + raise ValueError(f"Missing or invalid SHA-256 for {filename}") + hashes[filename] = digest[0].lower() + urls[filename] = url + platforms.add(platform) + missing = EXPECTED_PLATFORMS - platforms + if missing: + raise ValueError( + f"Newest nightly {version} is incomplete; missing platforms: {', '.join(sorted(missing))}" + ) + return RuntimeSelection(version, index_url, "nightly", False, hashes, urls) + + +def resolve_pr_runtime(*, version: str | None = None) -> RuntimeSelection: + selected = version if version is not None else os.environ.get(VERSION_ENV, "") + if selected.strip(): + runtime = resolve_runtime(version=selected) + return RuntimeSelection(runtime.version, runtime.index_url, "override", True, {}, {}) + package_url = DEV_INDEX.rstrip("/") + f"/{PACKAGE_ID}/" + with urllib.request.urlopen(package_url, timeout=120) as response: + page = response.read().decode("utf-8") + return select_nightly(page) + + +def write_selection(selection: RuntimeSelection, output_dir: Path) -> Path: + output = resolve_safe_output_directory(output_dir) + # Validate again before publishing a version into an Azure logging command. + runtime = resolve_runtime(version=selection.version) + if runtime.version != selection.version or not selection.version: + raise ValueError("Selection version must be an exact validated version") + output.mkdir(parents=True, exist_ok=True) + for filename in ("runtime-selection.json", "mssql-python-rs.version"): + destination = output / filename + if destination.is_symlink() or destination.exists() and not destination.is_file(): + raise ValueError(f"Unsafe selection artifact destination: {destination}") + (output / "runtime-selection.json").write_text( + json.dumps(asdict(selection), indent=2) + "\n", encoding="utf-8", newline="\n" + ) + (output / "mssql-python-rs.version").write_text( + selection.version + "\n", encoding="ascii", newline="\n" + ) + return output + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version") + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + selection = resolve_pr_runtime(version=args.version) + write_selection(selection, args.output_dir) + label = "feed-validated nightly" if selection.channel == "nightly" else "exact override" + print(f"Selected {label}: {selection.version} from {selection.index_url}") + print(f"##vso[task.setvariable variable=version;isOutput=true]{selection.version}") + + +if __name__ == "__main__": + main() diff --git a/tests/test_039_rs_runtime_pipelines.py b/tests/test_039_rs_runtime_pipelines.py index bc30c87ab..fa58bcd10 100644 --- a/tests/test_039_rs_runtime_pipelines.py +++ b/tests/test_039_rs_runtime_pipelines.py @@ -21,6 +21,7 @@ SOURCE_PREREQUISITES = ( "OneBranchPipelines/scripts/validate-release-versions.ps1", "eng/scripts/mssql_python_rs.py", + "eng/scripts/resolve_pr_rs_runtime.py", "eng/conda_tools/archive.py", "eng/conda_tools/contracts.py", ) @@ -48,6 +49,62 @@ def test_pipeline_override_is_explicit_and_defaults_to_empty(filename): assert "$(MSSQL_PYTHON_RS_VERSION)" not in source +def test_pr_validation_resolves_nightly_once_and_shares_it_across_all_jobs(): + pipeline = yaml.safe_load((ROOT / ENTRY_POINTS[0]).read_text(encoding="utf-8")) + assert VERSION_ENV not in pipeline["variables"] + jobs = pipeline["jobs"] + selectors = [job for job in jobs if job["job"] == "SelectRsRuntime"] + assert len(selectors) == 1 + selector = selectors[0] + selection_steps = [ + step + for job in jobs + for step in job["steps"] + if "resolve_pr_rs_runtime.py" in step.get("script", "") + ] + assert len(selection_steps) == 1 + selection = selection_steps[0] + assert selection in selector["steps"] + assert selection["name"] == "select" + assert selection["env"][VERSION_ENV] == "${{ parameters.rsVersionOverride }}" + assert "--output-dir" in selection["script"] + artifacts = [ + step["inputs"] + for step in selector["steps"] + if step.get("task") == "PublishPipelineArtifact@1" + ] + assert artifacts == [ + { + "targetPath": "$(Build.ArtifactStagingDirectory)/rs-runtime", + "artifact": "rs-runtime-selection", + } + ] + consumers = [job for job in jobs if job is not selector] + assert len(consumers) == 10 + for job in consumers: + assert job["dependsOn"] == "SelectRsRuntime", job["job"] + assert job["variables"][VERSION_ENV] == ( + "$[ dependencies.SelectRsRuntime.outputs['select.version'] ]" + ), job["job"] + assert "condition" not in job, "A failed selection must block all consumers" + + +def test_official_entry_points_keep_stable_defaults_without_nightly_resolution(): + for path in ( + PIPELINES / "build-release-package-pipeline.yml", + PIPELINES / "official-release-pipeline.yml", + PIPELINES / "conda-release-pipeline.yml", + ): + source = path.read_text(encoding="utf-8") + assert "resolve_pr_rs_runtime.py" not in source, path + assert "SelectRsRuntime" not in source, path + pipeline = yaml.safe_load( + (PIPELINES / "build-release-package-pipeline.yml").read_text(encoding="utf-8") + ) + override = next(p for p in pipeline["parameters"] if p["name"] == "rsVersionOverride") + assert override["default"] == "" + + def test_modified_pipeline_yaml_parses(): for folder in (ROOT / "eng/pipelines", PIPELINES): for path in folder.rglob("*.yml"): @@ -117,6 +174,7 @@ def stress_job(request): "missing", ( "eng/scripts/mssql_python_rs.py", + "eng/scripts/resolve_pr_rs_runtime.py", "eng/conda_tools/archive.py", "eng/conda_tools/contracts.py", ), diff --git a/tests/test_041_rs_nightly_selection.py b/tests/test_041_rs_nightly_selection.py new file mode 100644 index 000000000..d834b8b74 --- /dev/null +++ b/tests/test_041_rs_nightly_selection.py @@ -0,0 +1,373 @@ +import io +import json +import subprocess +import sys +import urllib.error +from dataclasses import replace +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if not all( + (ROOT / "eng" / "scripts" / name).is_file() + for name in ( + "resolve_pr_rs_runtime.py", + "download_mssql_python_rs_wheels.py", + "mssql_python_rs.py", + "mssql_python_build_safety.py", + ) +): + pytest.skip("Nightly selection requires a complete source checkout", allow_module_level=True) + +from eng.scripts import resolve_pr_rs_runtime as resolver +from eng.scripts.mssql_python_rs import DEFAULT_INDEX, DEV_INDEX, VERSION_ENV + +VERSION = "0.4.0.dev20261026" +DIGEST = "abcdef01" * 8 +PLATFORMS = ( + "win_amd64", + "win_arm64", + "manylinux_2_28_x86_64", + "manylinux_2_28_aarch64", + "manylinux_2_34_x86_64", + "manylinux_2_34_aarch64", + "musllinux_1_2_x86_64", + "musllinux_1_2_aarch64", + "macosx_15_0_universal2", +) + + +@pytest.fixture(autouse=True) +def clear_override(monkeypatch): + monkeypatch.delenv(VERSION_ENV, raising=False) + + +def _anchor(version=VERSION, platform="win_amd64", *, hash_value=DIGEST, yanked=False): + name = f"mssql_python_rs-{version}-cp310-abi3-{platform}.whl" + attribute = ' data-yanked="broken"' if yanked else "" + fragment = f"#sha256={hash_value}" if hash_value is not None else "" + return f'{name}' + + +def _page(version=VERSION, platforms=PLATFORMS, **kwargs): + return ( + "" + + "".join(_anchor(version, p, **kwargs) for p in platforms) + + "" + ) + + +def _serve(monkeypatch, page): + requests = [] + + def urlopen(url, timeout): + requests.append((url, timeout)) + return io.BytesIO(page.encode("utf-8")) + + monkeypatch.setattr(resolver.urllib.request, "urlopen", urlopen) + return requests + + +def test_selects_all_nine_platforms_with_hash_and_url_evidence(monkeypatch): + requests = _serve(monkeypatch, _page()) + selection = resolver.resolve_pr_runtime() + assert selection.version == VERSION + assert selection.index_url == DEV_INDEX + assert selection.channel == "nightly" + assert selection.version_override is False + assert resolver.EXPECTED_PLATFORMS == set(PLATFORMS) + assert len(selection.wheel_sha256) == len(selection.wheel_urls) == 9 + for name, digest in selection.wheel_sha256.items(): + assert digest == DIGEST + assert selection.wheel_urls[name] == f"https://files.example.test/{name}" + assert requests == [(DEV_INDEX + "mssql-python-rs/", 120)] + + +@pytest.mark.parametrize( + ("versions", "expected"), + [ + (["99.99.99.dev20261025", VERSION], VERSION), + (["0.9.9.dev20261026", "0.10.0.dev20261026"], "0.10.0.dev20261026"), + (["9.0.0.dev20261026", "10.0.0.dev20261026"], "10.0.0.dev20261026"), + (["0.0.9.dev20261026", "0.0.10.dev20261026"], "0.0.10.dev20261026"), + ([VERSION, "99.0.0.dev20261026186010", "999.0.0"], VERSION), + (["0.1.0.dev20240229", "0.1.0.dev20240301"], "0.1.0.dev20240301"), + ], +) +def test_date_first_then_numeric_base_ordering(versions, expected): + assert ( + resolver.select_nightly("".join(_page(version) for version in versions)).version == expected + ) + + +@pytest.mark.parametrize("version", ["1.0.0.dev20260229", "1.0.0.dev20261301", "1.0.0.dev20261000"]) +def test_rejects_invalid_calendar_dates(version): + with pytest.raises(ValueError, match="Invalid nightly calendar date"): + resolver.select_nightly(_page(version)) + + +@pytest.mark.parametrize( + "page", + [ + "", + "Please log in", + _page("0.3.0"), + _page("0.4.0.dev20261026186010"), + _page("0.4.0.dev1"), + _page("0.4.0.dev2026102"), + ], +) +def test_no_nightly_fails_with_exact_override_remedy(page): + with pytest.raises(ValueError, match="No scheduled nightly.*MSSQL_PYTHON_RS_VERSION"): + resolver.select_nightly(page) + + +@pytest.mark.parametrize( + "page", + ['wheel", _anchor()[:-4], ""], +) +def test_rejects_malformed_html(page): + with pytest.raises(ValueError, match="Malformed wheel index"): + resolver.select_nightly(page) + + +def test_rejects_duplicate_href_attributes(): + page = _page().replace('href="', 'href="ignored" href="', 1) + with pytest.raises(ValueError, match="Malformed wheel index anchor"): + resolver.select_nightly(page) + + +def test_http_error_is_visible_and_not_retried(monkeypatch): + requests = [] + + def urlopen(url, timeout): + requests.append(url) + raise urllib.error.HTTPError(url, 503, "Unavailable", {}, None) + + monkeypatch.setattr(resolver.urllib.request, "urlopen", urlopen) + with pytest.raises(urllib.error.HTTPError, match="503"): + resolver.resolve_pr_runtime() + assert requests == [DEV_INDEX + "mssql-python-rs/"] + + +@pytest.mark.parametrize("yanked", [False, True]) +def test_newest_partial_or_yanked_never_falls_back_to_complete_older(yanked): + page = _page("99.0.0.dev20261025") + _page(platforms=PLATFORMS[:1], yanked=yanked) + with pytest.raises(ValueError, match="Newest nightly.*(incomplete|yanked)"): + resolver.select_nightly(page) + + +def test_fully_yanked_newest_never_falls_back(): + with pytest.raises(ValueError, match="Newest nightly.*yanked"): + resolver.select_nightly(_page("0.3.0.dev20261025") + _page(yanked=True)) + + +def test_partial_yank_is_not_treated_as_complete(): + page = _page(platforms=PLATFORMS[1:]) + _anchor(yanked=True) + with pytest.raises(ValueError, match="yanked"): + resolver.select_nightly(page) + + +@pytest.mark.parametrize( + "hash_value", [None, "", "bad", "g" * 64, DIGEST + "&sha256=" + DIGEST, DIGEST + "&sha256="] +) +def test_rejects_missing_invalid_or_duplicate_hashes(hash_value): + page = _page(platforms=PLATFORMS[1:]) + _anchor(hash_value=hash_value) + with pytest.raises(ValueError, match="Missing or invalid SHA-256"): + resolver.select_nightly(page) + + +def test_uppercase_hash_is_normalized(): + assert set(resolver.select_nightly(_page(hash_value=DIGEST.upper())).wheel_sha256.values()) == { + DIGEST + } + + +def test_duplicate_wheel_links_fail(): + with pytest.raises(ValueError, match="Duplicate wheel filename"): + resolver.select_nightly(_page() + _anchor()) + + +@pytest.mark.parametrize( + "replacement", + [ + "http://files.example.test/", + "//", + "https://:443/", + "https://user:password@files.example.test/", + ], +) +def test_rejects_invalid_wheel_urls(replacement): + page = _page().replace("https://files.example.test/", replacement) + with pytest.raises(ValueError, match="HTTPS"): + resolver.select_nightly(page) + + +@pytest.mark.parametrize("index", ["http://example.test/simple/", "https:///simple/"]) +def test_rejects_insecure_or_missing_index_host(index): + with pytest.raises(ValueError, match="HTTPS"): + resolver.select_nightly(_page(), index) + + +def test_relative_wheel_urls_resolve_against_package_index(): + selection = resolver.select_nightly(_page().replace("https://files.example.test/", "")) + assert all( + url.startswith(DEV_INDEX + "mssql-python-rs/") for url in selection.wheel_urls.values() + ) + + +@pytest.mark.parametrize( + "tags", ["cp311-abi3-win_amd64", "cp310-abi3-unexpected", "cp310-cp310-win_amd64"] +) +def test_rejects_unexpected_newest_wheel_tags(tags): + page = _page().replace("cp310-abi3-win_amd64", tags) + with pytest.raises(ValueError, match="Unexpected newest nightly wheel tags"): + resolver.select_nightly(page) + + +def test_malformed_newest_wheel_name_does_not_fall_back(): + page = _page("0.3.0.dev20261025") + _anchor().replace("-cp310-abi3-win_amd64", "") + with pytest.raises(ValueError, match="Unexpected newest nightly wheel tags"): + resolver.select_nightly(page) + + +def test_rejects_ambiguous_numeric_version_spellings(): + with pytest.raises(ValueError, match="Ambiguous nightly"): + resolver.select_nightly(_page() + _page("00.4.0.dev20261026")) + + +@pytest.mark.parametrize("source", ["environment", "argument"]) +@pytest.mark.parametrize("version", ["0.3.0", "0.4.0.dev20261026186010", " 0.4.0.dev1 "]) +def test_exact_overrides_do_not_access_network(monkeypatch, source, version): + def unexpected_network(*args, **kwargs): + pytest.fail("Exact override must not access the network") + + monkeypatch.setattr(resolver.urllib.request, "urlopen", unexpected_network) + if source == "environment": + monkeypatch.setenv(VERSION_ENV, version) + selection = resolver.resolve_pr_runtime( + **({"version": version} if source == "argument" else {}) + ) + assert selection.version == version.strip() + assert selection.index_url == (DEV_INDEX if ".dev" in version else DEFAULT_INDEX) + assert selection.channel == "override" + assert selection.version_override + assert selection.wheel_sha256 == selection.wheel_urls == {} + + +def test_explicit_version_takes_precedence_over_environment(monkeypatch): + monkeypatch.setenv(VERSION_ENV, "invalid") + assert resolver.resolve_pr_runtime(version="0.3.0").version == "0.3.0" + + +@pytest.mark.parametrize( + "version", ["latest", ">=0.3.0", "0.3.0\n##vso[bad]", "0.3.0;echo bad", "0.3.0rc1"] +) +def test_invalid_override_fails_before_network(monkeypatch, version): + def unexpected_network(*args, **kwargs): + pytest.fail("Invalid override must not access the network") + + monkeypatch.setattr(resolver.urllib.request, "urlopen", unexpected_network) + monkeypatch.setenv(VERSION_ENV, version) + with pytest.raises(ValueError, match="Invalid MSSQL_PYTHON_RS_VERSION"): + resolver.resolve_pr_runtime() + + +@pytest.mark.parametrize("value", ["", " "]) +def test_empty_environment_selects_feed_not_committed_release(monkeypatch, value): + monkeypatch.setenv(VERSION_ENV, value) + _serve(monkeypatch, _page()) + assert resolver.resolve_pr_runtime().version == VERSION + + +def test_explicit_empty_version_selects_nightly_despite_environment(monkeypatch): + monkeypatch.setenv(VERSION_ENV, "0.3.0") + _serve(monkeypatch, _page()) + assert resolver.resolve_pr_runtime(version="").version == VERSION + + +def test_writes_reproducible_selection_files(tmp_path): + selection = resolver.select_nightly(_page()) + output = resolver.write_selection(selection, tmp_path / "selection") + assert (output / "mssql-python-rs.version").read_bytes() == (VERSION + "\n").encode("ascii") + receipt = json.loads((output / "runtime-selection.json").read_text(encoding="utf-8")) + assert receipt == { + "version": VERSION, + "index_url": DEV_INDEX, + "channel": "nightly", + "version_override": False, + "wheel_sha256": selection.wheel_sha256, + "wheel_urls": selection.wheel_urls, + } + assert sorted(path.name for path in output.iterdir()) == [ + "mssql-python-rs.version", + "runtime-selection.json", + ] + + +def test_output_directory_uses_shared_safety_helper(monkeypatch): + monkeypatch.chdir(ROOT) + with pytest.raises(ValueError, match="current working directory"): + resolver.write_selection(resolver.select_nightly(_page()), ROOT) + + +def test_output_artifact_directory_is_rejected_without_partial_write(tmp_path): + output = tmp_path / "selection" + (output / "mssql-python-rs.version").mkdir(parents=True) + with pytest.raises(ValueError, match="Unsafe selection artifact"): + resolver.write_selection(resolver.select_nightly(_page()), output) + assert not (output / "runtime-selection.json").exists() + + +@pytest.mark.parametrize("version", ["bad", "0.3.0\n##vso[bad]", "0.3.0\n"]) +def test_artifact_writer_rejects_unvalidated_version(tmp_path, version): + selection = replace(resolver.select_nightly(_page()), version=version) + with pytest.raises(ValueError): + resolver.write_selection(selection, tmp_path / "selection") + assert not (tmp_path / "selection").exists() + + +def test_nightly_cli_logs_feed_validation_and_output_variable(monkeypatch, tmp_path, capsys): + _serve(monkeypatch, _page()) + monkeypatch.setattr( + sys, "argv", ["resolve_pr_rs_runtime.py", "--output-dir", str(tmp_path / "out")] + ) + resolver.main() + stdout = capsys.readouterr().out + assert "Selected feed-validated nightly:" in stdout + assert "successful build" not in stdout + assert f"##vso[task.setvariable variable=version;isOutput=true]{VERSION}\n" in stdout + assert (tmp_path / "out" / "mssql-python-rs.version").read_text() == VERSION + "\n" + + +def test_failed_selection_emits_no_output_variable_or_files(monkeypatch, tmp_path, capsys): + _serve(monkeypatch, _page("0.4.0.dev20261026186010")) + output = tmp_path / "out" + monkeypatch.setattr(sys, "argv", ["resolve_pr_rs_runtime.py", "--output-dir", str(output)]) + with pytest.raises(ValueError, match="No scheduled nightly"): + resolver.main() + assert "##vso[" not in capsys.readouterr().out + assert not output.exists() + + +def test_standalone_cli_supports_override_and_writes_files(tmp_path, monkeypatch): + monkeypatch.setenv(VERSION_ENV, "0.4.0.dev20261026186010") + output = tmp_path / "out" + result = subprocess.run( + [ + sys.executable, + str(ROOT / "eng" / "scripts" / "resolve_pr_rs_runtime.py"), + "--output-dir", + str(output), + ], + capture_output=True, + text=True, + check=True, + ) + assert "Selected exact override:" in result.stdout + assert ( + "##vso[task.setvariable variable=version;isOutput=true]0.4.0.dev20261026186010" + in result.stdout + ) + assert json.loads((output / "runtime-selection.json").read_text())["channel"] == "override"