diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 83ff7dd67..797332e9a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,6 +15,70 @@ or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any addi ## Before Contributing +### Selecting the Rust Runtime + +Local builds and official/release pipelines install the exact released +`mssql-python-rs` version in +[`eng/versions/mssql-python-rs.version`](eng/versions/mssql-python-rs.version) +from public PyPI. The runtime is a separate dependency, not embedded in the +`mssql-python` wheel. No NuGet transport package is needed. + +To test a Python feature against merged but unreleased Rust changes, select the +exact version from a successful `mssql-rs` dev or nightly publishing run: + +```powershell +# Windows, with the development environment activated +$env:MSSQL_PYTHON_RS_VERSION = "0.4.0.dev20261028187000" # Replace with a published build +.\eng\scripts\install-mssql-py-core.ps1 +``` + +```bash +# Linux or macOS, with the development environment activated +export MSSQL_PYTHON_RS_VERSION=0.4.0.dev20261028187000 # Replace with a published build +bash eng/scripts/install-mssql-py-core.sh +``` + +Keep the environment variable set while building/installing `mssql-python` and +running tests. Its generated dependency metadata then uses the same selected +version. Development versions come from the anonymous Azure Artifacts PyPI +index; clean releases come from public PyPI. A missing requested wheel fails; +the installer never falls back to another version. Cross-architecture Windows +builds validate the target wheel without installing it into the host Python. + +The GitHub PR-validation checks run in Azure DevOps using +`eng/pipelines/pr-validation-pipeline.yml`. This validation pipeline (including +its main-branch CI runs) defaults to the latest published nightly from the Azure +Python feed, not the stable PyPI pin. It resolves the version once before the +matrix and passes that exact version to every build/test job. The +`rs-runtime-selection` artifact records the version, source index and nightly +wheel hashes. + +Nightlies use `X.Y.Z.devYYYYMMDD`; per-merge development builds append a build ID +and are not selected automatically. Selection requires all nine platform wheels +for the newest nightly, with valid SHA-256 links and no yanked files. Missing or +incomplete publication fails validation rather than falling back to an older +nightly or a stable release. Feed contents establish wheel publication, not +the overall status or source commit of the producer's Azure build. + +Azure build/test pipelines expose the explicit selection as `rsVersionOverride`. +For PR validation, leave it empty for the nightly, or provide an exact version +to reproduce a run or test newer per-merge Rust changes. Official/release builds +retain the committed stable PyPI pin and reject overrides. Other test pipelines +retain their existing stable default unless explicitly overridden. Record the +exact dev version in the feature's PR so another developer can reproduce the +tests. Before running tests after other package installations, verify the runtime +has not changed: + +```console +python eng/scripts/mssql_python_rs.py --verify +``` + +Feature-test artifacts are not eligible for public release, even if an override +selects a stable version. Once the required Rust runtime is released on PyPI, +update the committed pin and validate with no override before releasing Python. +Unset `MSSQL_PYTHON_RS_VERSION` to return to the committed release pin and rerun +the installer. + ### Install Local Formatting Hooks From the repository root, with your Python 3.10+ development virtual environment diff --git a/MANIFEST.in b/MANIFEST.in index 8a680962e..43edbd917 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ -include eng/versions/mssql-python-rs.version \ No newline at end of file +include eng/versions/mssql-python-rs.version +include eng/scripts/mssql_python_rs.py \ No newline at end of file diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index bf0904658..68a579214 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -51,6 +51,11 @@ schedules: # PIPELINE PARAMETERS # ========================= parameters: + - name: rsVersionOverride + displayName: 'Exact mssql-python-rs test version (NonOfficial only)' + type: string + default: '' + # OneBranch build type determines compliance level # - Official: Production builds with full SDL compliance, all security scanning enabled # - NonOfficial: Development/test builds with reduced security scanning @@ -182,6 +187,8 @@ parameters: # PIPELINE VARIABLES # ========================= variables: + - name: MSSQL_PYTHON_RS_VERSION + value: ${{ parameters.rsVersionOverride }} # Determine effective build type: scheduled builds are Official, manual/PR builds use parameter # Build.Reason values: Schedule, Manual, IndividualCI, PullRequest, BatchedCI - name: effectiveOneBranchType diff --git a/OneBranchPipelines/conda-release-pipeline.yml b/OneBranchPipelines/conda-release-pipeline.yml index a2611e809..2b1da3947 100644 --- a/OneBranchPipelines/conda-release-pipeline.yml +++ b/OneBranchPipelines/conda-release-pipeline.yml @@ -165,7 +165,6 @@ extends: parameters: mssqlPythonVersion: '$(Provenance.mssqlPythonVersion)' releaseVersions: '$(Provenance.releaseVersions)' - rsTransportVersion: '$(Provenance.rsTransportVersion)' # Dry run (publishToConda=false): enumerate every package and print the exact # `anaconda upload` plan WITHOUT uploading, so a validate-only run shows precisely @@ -250,7 +249,6 @@ extends: ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' mssqlPythonVersion: $[ dependencies.ValidateConda.outputs['Provenance.mssqlPythonVersion'] ] releaseVersions: $[ dependencies.ValidateConda.outputs['Provenance.releaseVersions'] ] - rsTransportVersion: $[ dependencies.ValidateConda.outputs['Provenance.rsTransportVersion'] ] steps: - checkout: self fetchDepth: 1 @@ -260,4 +258,3 @@ extends: condaLabel: '${{ parameters.condaLabel }}' mssqlPythonVersion: '$(mssqlPythonVersion)' releaseVersions: '$(releaseVersions)' - rsTransportVersion: '$(rsTransportVersion)' diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index a7808aed7..a464d4bf3 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -202,14 +202,14 @@ extends: inputs: targetType: 'filePath' filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' - arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version"' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version" -ReceiptFile "$(Build.SourcesDirectory)\artifacts\dist\transport.json"' - task: PowerShell@2 displayName: '[TEST] Validate mssql-python-rs wheel dependency' inputs: targetType: 'filePath' filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-rs-wheel-dependency.ps1' - arguments: '-WheelsDir "$(Build.SourcesDirectory)\artifacts\dist" -VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version"' + arguments: '-WheelsDir "$(Build.SourcesDirectory)\artifacts\dist" -VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version" -ReceiptFile "$(Build.SourcesDirectory)\artifacts\dist\transport.json"' # Guard: if the released mssql-python wheel declares an # mssql-python-odbc dependency, it MUST be the exact pinned version. diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index d2b7b9ff1..19d7f0cb1 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -125,16 +125,22 @@ jobs: displayName: 'Consolidate Windows symbols (optional)' continueOnError: true - # Stage the actual RS payload using THIS producer source's immutable pins. - # Downstream Conda jobs consume this artifact, never their checkout's NuGet pin. + # Carry the effective runtime selection with the artifact, including test overrides. - bash: | set -e python3 "$(Build.SourcesDirectory)/eng/scripts/download_mssql_python_rs_wheels.py" \ --output-dir "$(ob_outputDirectory)/rs-wheels" - cp "$(Build.SourcesDirectory)/eng/versions/mssql-python-rs.version" "$(ob_outputDirectory)/dist/" - cp "$(Build.SourcesDirectory)/eng/versions/mssql-python-rs-nuget.version" "$(ob_outputDirectory)/dist/" + python3 -c "import json,pathlib; root=pathlib.Path('$(ob_outputDirectory)'); receipt=json.loads((root/'rs-wheels/transport.json').read_text()); (root/'dist/mssql-python-rs.version').write_text(receipt['distribution_version']+'\n'); (root/'dist/transport.json').write_bytes((root/'rs-wheels/transport.json').read_bytes())" echo "mssql-python-rs version: $(cat "$(ob_outputDirectory)/dist/mssql-python-rs.version")" - displayName: 'Stage source-pinned mssql-python-rs wheels and transport receipt' + displayName: 'Stage effective mssql-python-rs wheels, version and receipt' + + - task: PowerShell@2 + displayName: 'Validate every wheel uses the effective runtime version' + inputs: + pwsh: true + targetType: filePath + filePath: OneBranchPipelines/scripts/validate-rs-wheel-dependency.ps1 + arguments: '-WheelsDir "$(ob_outputDirectory)/dist" -VersionFile "$(ob_outputDirectory)/dist/mssql-python-rs.version" -ReceiptFile "$(ob_outputDirectory)/dist/transport.json"' # Verify consolidation - bash: | diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index f84240d23..262cc0944 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -202,14 +202,14 @@ extends: inputs: targetType: 'filePath' filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' - arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version"' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version" -ReceiptFile "$(Build.SourcesDirectory)\artifacts\dist\transport.json"' - task: PowerShell@2 displayName: 'Validate mssql-python-rs wheel dependency' inputs: targetType: 'filePath' filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-rs-wheel-dependency.ps1' - arguments: '-WheelsDir "$(Build.SourcesDirectory)\artifacts\dist" -VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version"' + arguments: '-WheelsDir "$(Build.SourcesDirectory)\artifacts\dist" -VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version" -ReceiptFile "$(Build.SourcesDirectory)\artifacts\dist\transport.json"' # Guard: if the released mssql-python wheel declares an # mssql-python-odbc dependency, it MUST be the exact pinned version. diff --git a/OneBranchPipelines/scripts/validate-release-versions.ps1 b/OneBranchPipelines/scripts/validate-release-versions.ps1 index ec77286dd..84d470776 100644 --- a/OneBranchPipelines/scripts/validate-release-versions.ps1 +++ b/OneBranchPipelines/scripts/validate-release-versions.ps1 @@ -3,47 +3,62 @@ <# .SYNOPSIS - Validates that a version file contains a stable (non-prerelease) version. - + Gates Official builds and releases on an exact public PyPI runtime release. .DESCRIPTION - Reads a single .version file and rejects any version containing - dev, alpha, beta, or rc tags. Intended to gate official releases. - -.PARAMETER VersionFile - Path to a .version file. Defaults to eng/versions/mssql-python-rs.version - relative to the repository root. - -.EXAMPLE - # Run from repo root (uses default path): - .\eng\scripts\validate-release-versions.ps1 - - # Explicit path: - .\OneBranchPipelines\scripts\validate-release-versions.ps1 -VersionFile C:\work\mssql-python\eng\versions\mssql-python-rs.version + BuildOnly rejects overrides and non-release source pins without network access. + Release validation additionally requires the selected artifact's transport + receipt and verifies the exact version exists on public PyPI before ESRP. #> - param( - [string]$VersionFile + [string]$VersionFile, + [string]$ReceiptFile, + [switch]$BuildOnly ) $ErrorActionPreference = 'Stop' if (-not $VersionFile) { - $repoRoot = (Resolve-Path "$PSScriptRoot\..\..").Path - $VersionFile = Join-Path $repoRoot 'eng\versions\mssql-python-rs.version' + $VersionFile = Join-Path $PSScriptRoot '../../eng/versions/mssql-python-rs.version' +} + +if (-not [string]::IsNullOrWhiteSpace($env:MSSQL_PYTHON_RS_VERSION)) { + throw 'Official builds and releases forbid MSSQL_PYTHON_RS_VERSION overrides, including stable versions.' } -if (-not (Test-Path $VersionFile)) { - Write-Error "Version file not found: $VersionFile" - exit 1 +$version = (Get-Content -LiteralPath $VersionFile -Raw).Trim() +if ($version -cnotmatch '^[0-9]+\.[0-9]+\.[0-9]+$') { + throw "Official builds and releases require an exact stable X.Y.Z runtime version, got '$version'." } -$version = (Get-Content $VersionFile -Raw).Trim() -$name = [System.IO.Path]::GetFileNameWithoutExtension($VersionFile) +if (-not $BuildOnly) { + if (-not $ReceiptFile) { + $ReceiptFile = Join-Path (Split-Path -Parent $VersionFile) 'transport.json' + } + $receipt = Get-Content -LiteralPath $ReceiptFile -Raw | ConvertFrom-Json + if ($receipt.version_override -isnot [bool] -or $receipt.version_override) { + throw 'Official releases require a receipt with version_override=false.' + } + if ($receipt.distribution_version -cne $version) { + throw 'The staged runtime version and transport receipt disagree.' + } + if ($receipt.index_url -cnotmatch '^https://pypi\.org/simple/?$') { + throw 'Official releases require the public PyPI runtime source.' + } + if (-not $receipt.wheel_sha256 -or @($receipt.wheel_sha256.PSObject.Properties).Count -eq 0) { + throw 'The runtime receipt must record wheel SHA256 hashes.' + } + foreach ($hash in $receipt.wheel_sha256.PSObject.Properties) { + if ($hash.Name -notlike '*.whl' -or $hash.Value -cnotmatch '^[a-f0-9]{64}$') { + throw 'The runtime receipt contains an invalid wheel SHA256 entry.' + } + } -if ($version -match '(dev|alpha|beta|rc)') { - Write-Host "FAIL: $name version '$version' is a pre-release ($($Matches[1]))" -ForegroundColor Red - Write-Error "$name version is pre-release. Official releases require stable versions." - exit 1 + $release = Invoke-RestMethod -Uri "https://pypi.org/pypi/mssql-python-rs/$version/json" -Method Get + if ($release.info.version -cne $version -or + $release.info.name -notmatch '^mssql[-_]python[-_]rs$' -or + @($release.urls | Where-Object { $_.packagetype -eq 'bdist_wheel' -and -not $_.yanked }).Count -eq 0) { + throw "mssql-python-rs==$version has no matching public PyPI wheel release." + } } -Write-Host "OK: $name version '$version'" -ForegroundColor Green +Write-Host "OK: mssql-python-rs==$version is eligible for Official $(if ($BuildOnly) { 'build' } else { 'release' })." diff --git a/OneBranchPipelines/scripts/validate-rs-wheel-dependency.ps1 b/OneBranchPipelines/scripts/validate-rs-wheel-dependency.ps1 index 1b15dd99e..2a42acad9 100644 --- a/OneBranchPipelines/scripts/validate-rs-wheel-dependency.ps1 +++ b/OneBranchPipelines/scripts/validate-rs-wheel-dependency.ps1 @@ -3,7 +3,9 @@ param( [string]$WheelsDir, [Parameter(Mandatory = $true)] - [string]$VersionFile + [string]$VersionFile, + + [string]$ReceiptFile ) $ErrorActionPreference = 'Stop' @@ -12,6 +14,12 @@ $expectedVersion = (Get-Content $VersionFile -Raw).Trim() if (-not $expectedVersion) { throw "Version file is empty: $VersionFile" } +if ($ReceiptFile) { + $receipt = Get-Content -LiteralPath $ReceiptFile -Raw | ConvertFrom-Json + if ($receipt.distribution_version -cne $expectedVersion) { + throw 'The wheel dependency version and runtime receipt disagree.' + } +} $wheels = @(Get-ChildItem $WheelsDir -Filter 'mssql_python-*.whl' -File) if ($wheels.Count -eq 0) { diff --git a/OneBranchPipelines/stages/build-linux-single-stage.yml b/OneBranchPipelines/stages/build-linux-single-stage.yml index e50123769..3a383d447 100644 --- a/OneBranchPipelines/stages/build-linux-single-stage.yml +++ b/OneBranchPipelines/stages/build-linux-single-stage.yml @@ -74,6 +74,15 @@ stages: steps: - checkout: self fetchDepth: 0 + + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: PowerShell@2 + displayName: 'Reject test overrides and prereleases before Official build' + inputs: + pwsh: true + targetType: filePath + filePath: OneBranchPipelines/scripts/validate-release-versions.ps1 + arguments: '-BuildOnly' # Docker is pre-installed now # Verify it's working and start the daemon @@ -127,7 +136,7 @@ stages: exit 1 fi - docker run -d --name build-$(LINUX_TAG)-$(ARCH) \ + docker run -e MSSQL_PYTHON_RS_VERSION -d --name build-$(LINUX_TAG)-$(ARCH) \ --platform $(DOCKER_PLATFORM) \ -v $(Build.SourcesDirectory):/workspace \ -w /workspace \ @@ -314,6 +323,7 @@ stages: $PY -m pip install -q --require-hashes -r /workspace/eng/requirements-test-linux.txt; echo "Installing: $WHEEL"; $PY -m pip install -q "$WHEEL"; + $PY /workspace/eng/scripts/mssql_python_rs.py --verify; # Step 6: Verify package imports correctly echo "Verifying package installation..."; @@ -330,7 +340,9 @@ stages: mkdir -p $TEST_DIR/OneBranchPipelines/stages; cp /workspace/eng/scripts/run-mssql-odbc-tests.sh /workspace/eng/scripts/verify_mssql_odbc_provider.py $TEST_DIR/eng/scripts/; cp /workspace/eng/pipelines/mssql-odbc-daily-validation-pipeline.yml $TEST_DIR/eng/pipelines/; - cp /workspace/eng/versions/mssql-python-rs-nuget.version $TEST_DIR/eng/versions/; + cp -r /workspace/eng/pipelines/. $TEST_DIR/eng/pipelines/; + cp -r /workspace/OneBranchPipelines/. $TEST_DIR/OneBranchPipelines/; + cp /workspace/eng/versions/mssql-python-rs.version $TEST_DIR/eng/versions/; cp /workspace/OneBranchPipelines/stages/build-linux-single-stage.yml $TEST_DIR/OneBranchPipelines/stages/; cp /workspace/pytest.ini $TEST_DIR/ || echo "WARNING: No pytest.ini"; @@ -395,6 +407,7 @@ stages: $PY -m pip install -q --require-hashes -r /workspace/eng/requirements-test-linux.txt; echo "Installing: $WHEEL"; $PY -m pip install -q "$WHEEL"; + $PY /workspace/eng/scripts/mssql_python_rs.py --verify; # Step 6: Verify package imports correctly echo "Verifying package installation..."; @@ -411,7 +424,9 @@ stages: mkdir -p $TEST_DIR/OneBranchPipelines/stages; cp /workspace/eng/scripts/run-mssql-odbc-tests.sh /workspace/eng/scripts/verify_mssql_odbc_provider.py $TEST_DIR/eng/scripts/; cp /workspace/eng/pipelines/mssql-odbc-daily-validation-pipeline.yml $TEST_DIR/eng/pipelines/; - cp /workspace/eng/versions/mssql-python-rs-nuget.version $TEST_DIR/eng/versions/; + cp -r /workspace/eng/pipelines/. $TEST_DIR/eng/pipelines/; + cp -r /workspace/OneBranchPipelines/. $TEST_DIR/OneBranchPipelines/; + cp /workspace/eng/versions/mssql-python-rs.version $TEST_DIR/eng/versions/; cp /workspace/OneBranchPipelines/stages/build-linux-single-stage.yml $TEST_DIR/OneBranchPipelines/stages/; cp /workspace/pytest.ini $TEST_DIR/ || echo "WARNING: No pytest.ini"; diff --git a/OneBranchPipelines/stages/build-macos-single-stage.yml b/OneBranchPipelines/stages/build-macos-single-stage.yml index 65909d55b..901d93de3 100644 --- a/OneBranchPipelines/stages/build-macos-single-stage.yml +++ b/OneBranchPipelines/stages/build-macos-single-stage.yml @@ -77,6 +77,15 @@ stages: # fetchDepth: 0 = full git history (needed for version tagging) - checkout: self fetchDepth: 0 + + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: PowerShell@2 + displayName: 'Reject test overrides and prereleases before Official build' + inputs: + pwsh: true + targetType: filePath + filePath: OneBranchPipelines/scripts/validate-release-versions.ps1 + arguments: '-BuildOnly' # ========================= # PYTHON INSTALLATION @@ -217,7 +226,7 @@ stages: # Tests use localhost:1433 connection (SA user with password from variable) # -v = verbose output (show test names and results) - script: | - python -m pytest -v + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v displayName: 'Run pytests' env: # Connection string uses localhost (SQL Server container exposed on port 1433) diff --git a/OneBranchPipelines/stages/build-windows-single-stage.yml b/OneBranchPipelines/stages/build-windows-single-stage.yml index b1c8cdbb3..ec6ae6d37 100644 --- a/OneBranchPipelines/stages/build-windows-single-stage.yml +++ b/OneBranchPipelines/stages/build-windows-single-stage.yml @@ -86,6 +86,14 @@ stages: steps: - checkout: self fetchDepth: 0 + + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: PowerShell@2 + displayName: 'Reject test overrides and prereleases before Official build' + inputs: + targetType: filePath + filePath: OneBranchPipelines/scripts/validate-release-versions.ps1 + arguments: '-BuildOnly' # Python 3.14 Installation: Download from NuGet (not yet in UsePythonVersion@0 task) # Microsoft hasn't added Python 3.14 to the standard Python registry yet @@ -362,6 +370,8 @@ stages: if ("$(targetArch)" -eq "arm64") { Write-Host "Skipping pytests on Windows ARM64" } else { + python eng/scripts/mssql_python_rs.py --verify + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } python -m pytest -v } displayName: 'Run pytests' diff --git a/OneBranchPipelines/steps/conda-publish-step.yml b/OneBranchPipelines/steps/conda-publish-step.yml index 9c406726a..d1050776b 100644 --- a/OneBranchPipelines/steps/conda-publish-step.yml +++ b/OneBranchPipelines/steps/conda-publish-step.yml @@ -49,9 +49,6 @@ parameters: - name: releaseVersions type: string default: '' - - name: rsTransportVersion - type: string - default: '' # Comma-separated subdirs a complete release MUST contain (full PyPI parity minus # musllinux, which has no conda subdir). win-arm64 IS included -- the build enforces # its --dry-run solve + PE-arch assert, so a missing/broken win-arm64 blocks the @@ -153,7 +150,6 @@ steps: CONDA_LABEL: '${{ parameters.condaLabel }}' MSSQL_PYTHON_VERSION: '${{ parameters.mssqlPythonVersion }}' RELEASE_VERSIONS: '${{ parameters.releaseVersions }}' - RS_TRANSPORT_VERSION: '${{ parameters.rsTransportVersion }}' REQUIRED_SUBDIRS: '${{ parameters.requiredSubdirs }}' ALLOWED_SUBDIRS: '${{ parameters.allowedSubdirs }}' PYTHON_VERSIONS: '${{ parameters.pythonVersions }}' diff --git a/OneBranchPipelines/steps/conda-release-step.yml b/OneBranchPipelines/steps/conda-release-step.yml index 3e3cce161..61e402523 100644 --- a/OneBranchPipelines/steps/conda-release-step.yml +++ b/OneBranchPipelines/steps/conda-release-step.yml @@ -74,9 +74,6 @@ parameters: - name: releaseVersions type: string default: '' - - name: rsTransportVersion - type: string - default: '' # The audit dependency hash targets CPython 3.12 / Windows x64, like the publisher. - name: pythonVersion type: string @@ -160,7 +157,6 @@ steps: PYTHON_VERSIONS: '${{ parameters.pythonVersions }}' MSSQL_PYTHON_VERSION: '${{ parameters.mssqlPythonVersion }}' RELEASE_VERSIONS: '${{ parameters.releaseVersions }}' - RS_TRANSPORT_VERSION: '${{ parameters.rsTransportVersion }}' inputs: targetType: 'inline' workingDirectory: '$(Build.SourcesDirectory)' diff --git a/OneBranchPipelines/stress-test-pipeline.yml b/OneBranchPipelines/stress-test-pipeline.yml index 548f52186..168e86319 100644 --- a/OneBranchPipelines/stress-test-pipeline.yml +++ b/OneBranchPipelines/stress-test-pipeline.yml @@ -27,7 +27,15 @@ resources: source: 'Build-Release-Package-Pipeline' # Name of the build pipeline trigger: none # Don't auto-trigger, we run on schedule or manually +parameters: + - name: rsVersionOverride + displayName: 'Exact mssql-python-rs test version (empty uses committed PyPI release)' + type: string + default: '' + variables: + - name: MSSQL_PYTHON_RS_VERSION + value: ${{ parameters.rsVersionOverride }} # Common variables - template: /OneBranchPipelines/variables/common-variables.yml@self - name: pythonVersion @@ -62,38 +70,64 @@ jobs: pip install psutil pytest-timeout displayName: 'Install dependencies' - # Download pre-built wheel from build-whl-pipeline instead of building from source + # Select a matching NonOfficial build resource when testing a runtime override. - task: DownloadPipelineArtifact@2 inputs: buildType: 'specific' project: '$(System.TeamProject)' - definition: 2162 # build-whl-pipeline definition ID - buildVersionToDownload: 'latest' - branchName: 'refs/heads/main' - artifactName: 'mssql-python-wheels-dist' + definition: '$(resources.pipeline.buildPipeline.pipelineID)' + buildVersionToDownload: 'specific' + pipelineId: '$(resources.pipeline.buildPipeline.runID)' + artifactName: 'drop_Consolidate_ConsolidateArtifacts' targetPath: '$(Build.SourcesDirectory)\wheels' - displayName: 'Download pre-built wheel from build-whl-pipeline' + displayName: 'Download selected build (choose matching override artifacts for dev tests)' + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: '$(resources.pipeline.buildPipeline.pipelineID)' + buildVersionToDownload: 'specific' + pipelineId: '$(resources.pipeline.buildPipeline.runID)' + artifactName: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + targetPath: '$(Pipeline.Workspace)\odbc-wheels' + displayName: 'Download ODBC wheels from the same selected build' + + - script: | + python eng/scripts/mssql_python_rs.py --write-version-file "$(Pipeline.Workspace)\effective-runtime.version" + displayName: 'Resolve effective stress-test runtime' + + - task: PowerShell@2 + displayName: 'Require test wheel metadata and receipt to match effective runtime' + inputs: + targetType: filePath + filePath: OneBranchPipelines/scripts/validate-rs-wheel-dependency.ps1 + arguments: '-WheelsDir "$(Build.SourcesDirectory)\wheels\dist" -VersionFile "$(Pipeline.Workspace)\effective-runtime.version" -ReceiptFile "$(Build.SourcesDirectory)\wheels\dist\transport.json"' - script: | python eng/scripts/download_mssql_python_rs_wheels.py --output-dir "$(Pipeline.Workspace)\mssql-python-rs-wheels" displayName: 'Download pinned mssql-python-rs wheels' - # Install the pre-built wheel + # Read ODBC pins from the selected build, not the stress checkout's release versions. - script: | echo "Installing pre-built wheel..." dir "$(Build.SourcesDirectory)\wheels" - pip install --find-links=$(Build.SourcesDirectory)\wheels --find-links=$(Pipeline.Workspace)\mssql-python-rs-wheels mssql-python --force-reinstall + python -c "import subprocess,sys; from pathlib import Path; from eng.conda_tools.archive import read_wheel_metadata; from eng.conda_tools.contracts import exact_dependency_pin; from eng.scripts.mssql_python_rs import resolve_runtime; odbc_version, = {exact_dependency_pin(read_wheel_metadata(w)['requires_dist'], 'mssql-python-odbc') for w in Path(r'$(Build.SourcesDirectory)\wheels\dist').glob('mssql_python-*.whl')}; subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--no-index', '--no-deps', '--find-links', r'$(Build.SourcesDirectory)\wheels\dist', '--find-links', r'$(Pipeline.Workspace)\odbc-wheels\dist', '--find-links', r'$(Pipeline.Workspace)\mssql-python-rs-wheels', 'mssql-python', 'mssql-python-odbc=='+odbc_version, 'mssql-python-rs=='+resolve_runtime().version, '--force-reinstall'])" + if errorlevel 1 exit /b 1 + python eng/scripts/mssql_python_rs.py --verify displayName: 'Install pre-built mssql-python wheel' - # Rename local source folder to avoid import conflicts with installed wheel + # Hide checkout packages before loading the selected build's installed wheels. - powershell: | - Write-Host "Renaming local mssql_python folder to avoid import conflicts..." - if (Test-Path "$(Build.SourcesDirectory)\mssql_python") { - Rename-Item -Path "$(Build.SourcesDirectory)\mssql_python" -NewName "mssql_python_src" -Force - Write-Host "Renamed mssql_python to mssql_python_src" + $ErrorActionPreference = 'Stop' + foreach ($package in @('mssql_python', 'mssql_python_odbc')) { + $source = Join-Path $env:BUILD_SOURCESDIRECTORY $package + if (Test-Path -LiteralPath $source) { + Rename-Item -LiteralPath $source -NewName "${package}_src" -ErrorAction Stop + } } - # Verify import from installed wheel - python -c "import mssql_python; print('Imported from:', mssql_python.__file__)" + python -c "import os; from pathlib import Path; import mssql_python_odbc as odbc; source=Path(os.environ['BUILD_SOURCESDIRECTORY']).resolve(); p=Path(odbc.__file__).resolve(); assert not p.is_relative_to(source), 'mssql_python_odbc resolved inside checkout: '+str(p); print('mssql_python_odbc imported from:', p); import mssql_python; p=Path(mssql_python.__file__).resolve(); assert not p.is_relative_to(source), 'mssql_python resolved inside checkout: '+str(p); print('mssql_python imported from:', p)" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } displayName: 'Ensure tests use installed wheel' # PYTHON-1ES-MMS2022 has LocalDB pre-installed (no full SQL Server instance) @@ -232,38 +266,63 @@ jobs: pip install psutil pytest-timeout displayName: 'Install Python dependencies' - # Download pre-built wheel from build-whl-pipeline instead of building from source + # Select a matching NonOfficial build resource when testing a runtime override. - task: DownloadPipelineArtifact@2 inputs: buildType: 'specific' project: '$(System.TeamProject)' - definition: 2162 # build-whl-pipeline definition ID - buildVersionToDownload: 'latest' - branchName: 'refs/heads/main' - artifactName: 'mssql-python-wheels-dist' + definition: '$(resources.pipeline.buildPipeline.pipelineID)' + buildVersionToDownload: 'specific' + pipelineId: '$(resources.pipeline.buildPipeline.runID)' + artifactName: 'drop_Consolidate_ConsolidateArtifacts' targetPath: '$(Build.SourcesDirectory)/wheels' - displayName: 'Download pre-built wheel from build-whl-pipeline' + displayName: 'Download selected build (choose matching override artifacts for dev tests)' + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: '$(resources.pipeline.buildPipeline.pipelineID)' + buildVersionToDownload: 'specific' + pipelineId: '$(resources.pipeline.buildPipeline.runID)' + artifactName: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + targetPath: '$(Pipeline.Workspace)/odbc-wheels' + displayName: 'Download ODBC wheels from the same selected build' + + - script: | + python eng/scripts/mssql_python_rs.py --write-version-file "$(Pipeline.Workspace)/effective-runtime.version" + displayName: 'Resolve effective stress-test runtime' + + - task: PowerShell@2 + displayName: 'Require test wheel metadata and receipt to match effective runtime' + inputs: + pwsh: true + targetType: filePath + filePath: OneBranchPipelines/scripts/validate-rs-wheel-dependency.ps1 + arguments: '-WheelsDir "$(Build.SourcesDirectory)/wheels/dist" -VersionFile "$(Pipeline.Workspace)/effective-runtime.version" -ReceiptFile "$(Build.SourcesDirectory)/wheels/dist/transport.json"' - script: | python eng/scripts/download_mssql_python_rs_wheels.py --output-dir "$(Pipeline.Workspace)/mssql-python-rs-wheels" displayName: 'Download pinned mssql-python-rs wheels' - # Install the pre-built wheel + # Read ODBC pins from the selected build, not the stress checkout's release versions. - script: | + set -e echo "Installing pre-built wheel..." ls -la "$(Build.SourcesDirectory)/wheels" - pip install --find-links=$(Build.SourcesDirectory)/wheels --find-links=$(Pipeline.Workspace)/mssql-python-rs-wheels mssql-python --force-reinstall + python -c "import subprocess,sys; from pathlib import Path; from eng.conda_tools.archive import read_wheel_metadata; from eng.conda_tools.contracts import exact_dependency_pin; from eng.scripts.mssql_python_rs import resolve_runtime; odbc_version, = {exact_dependency_pin(read_wheel_metadata(w)['requires_dist'], 'mssql-python-odbc') for w in Path('$(Build.SourcesDirectory)/wheels/dist').glob('mssql_python-*.whl')}; subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--no-index', '--no-deps', '--find-links', '$(Build.SourcesDirectory)/wheels/dist', '--find-links', '$(Pipeline.Workspace)/odbc-wheels/dist', '--find-links', '$(Pipeline.Workspace)/mssql-python-rs-wheels', 'mssql-python', 'mssql-python-odbc=='+odbc_version, 'mssql-python-rs=='+resolve_runtime().version, '--force-reinstall'])" + python eng/scripts/mssql_python_rs.py --verify displayName: 'Install pre-built mssql-python wheel' - # Rename local source folder to avoid import conflicts with installed wheel + # Hide checkout packages before loading the selected build's installed wheels. - script: | - echo "Renaming local mssql_python folder to avoid import conflicts..." - if [ -d "$(Build.SourcesDirectory)/mssql_python" ]; then - mv "$(Build.SourcesDirectory)/mssql_python" "$(Build.SourcesDirectory)/mssql_python_src" - echo "Renamed mssql_python to mssql_python_src" - fi - # Verify import from installed wheel - python -c "import mssql_python; print('Imported from:', mssql_python.__file__)" + set -e + for package in mssql_python mssql_python_odbc; do + if [ -d "$(Build.SourcesDirectory)/$package" ]; then + mv "$(Build.SourcesDirectory)/$package" "$(Build.SourcesDirectory)/${package}_src" + fi + done + python -c "import os; from pathlib import Path; import mssql_python_odbc as odbc; source=Path(os.environ['BUILD_SOURCESDIRECTORY']).resolve(); p=Path(odbc.__file__).resolve(); assert not p.is_relative_to(source), 'mssql_python_odbc resolved inside checkout: '+str(p); print('mssql_python_odbc imported from:', p); import mssql_python; p=Path(mssql_python.__file__).resolve(); assert not p.is_relative_to(source), 'mssql_python resolved inside checkout: '+str(p); print('mssql_python imported from:', p)" displayName: 'Ensure tests use installed wheel' # Docker is pre-installed by linux-1es-pt-prerequisites-v2 artifact diff --git a/conda/README.md b/conda/README.md index efbd5fb92..97749c16a 100644 --- a/conda/README.md +++ b/conda/README.md @@ -45,7 +45,7 @@ receives a bounded frame-structure pass because its streaming API alone accepts truncated frame endings. ZIP envelopes must use stored or deflate compression; sparse TAR extensions are rejected before sparse-map processing. Wheel metadata reads share the ZIP and metadata budgets. These are package-reader limits, not a -claim that NuGet transport downloads or total process memory have the same bounds. +claim that runtime wheel downloads or total process memory have the same bounds. The GitHub audit job installs its complete Linux x64 / CPython 3.11 test-tool closure from `requirements-audit.txt` with `--require-hashes --only-binary=:all:`. @@ -90,10 +90,15 @@ hashes with the final Conda archive hash after Linux relocation. Use `build --rs-wheel-dir /rs-wheels --rs-version-file /dist/mssql-python-rs.version` for RS inputs. -The version file asserts the selected producer's source contract, not a new release-version -setting. The distinct NuGet transport pin remains in `mssql-python-rs-nuget.version`; -the downloader records its content hash and individual wheel hashes in `transport.json`. -Conda consumers use staged producer bytes, not a fresh download using their checkout's pin. +The version file asserts the selected producer's runtime selection against the binding's +exact dependency, not a new release-version setting. The producer's `transport.json` +records `distribution_version`, `index_url`, `version_override` and individual +`wheel_sha256` values. Released runtimes use PyPI; an explicitly selected `.devN` +version uses the public `mssql-rs_Public` Python index. Conda consumers validate this +receipt and use staged producer bytes, not a fresh download using their checkout's +pin or `MSSQL_PYTHON_RS_VERSION`. No NuGet package or transport pin is required. +Release validation rejects any receipt with `version_override=true`, including +an override equal to the committed stable pin. Such artifacts remain test-only. Published historical bindings that declare no RS dependency are accepted only when their payload and RECORD own the core initializer and native extension. They produce an explicit @@ -269,13 +274,14 @@ acquisition; configuration readback alone is not evidence that a run holds the l The release pipeline resolves AUTO from the **recorded upstream wheel commit**, not the release checkout or a latest-version lookup. It cross-checks binding setup/runtime versions and reads ODBC and, when explicitly required by that source, RS distribution -versions. RS's NuGet transport pin is separate. Missing or ambiguous source declarations +versions. Source analysis reads the clean `X.Y.Z` RS pin without executing the +runtime-selection helper or consulting the consumer's environment override. +Missing or ambiguous source declarations fail rather than select a historical profile. A supplied binding version is only an assertion against this resolved version. Both readiness and the credentialed job use `validate --release-versions` to read the -verified `RELEASE_VERSIONS` JSON from the environment; `RS_TRANSPORT_VERSION` supplies -the separate transport assertion. This avoids passing JSON quoting or empty arguments +verified `RELEASE_VERSIONS` JSON from the environment. This avoids passing JSON quoting or empty arguments through Windows PowerShell. Explicit JSON may also follow `--release-versions` in source-only controls. With this flag, absent versions fail closed. Without it, `validate` remains the metadata/matrix-only inspection command and makes no source-input claim. @@ -283,7 +289,9 @@ The source-bound gate checks each installed component's METADATA and RECORD owne the binding's exact dependency pins, and the selected RS filename/WHEEL tags against `rs-transport.json`. Raw wheel/transport hashes remain input evidence; relocated installed native files are checked by the native audits, not falsely compared with raw -wheel hashes. No numeric RS build ID is inferred from a transport-version suffix. +wheel hashes. The receipt must name the verified RS distribution version and its +release/development index; a receipt cannot override the verified source version. +No numeric RS build ID is inferred from a version suffix. Release and native auditing also check all installed core paths, not only RECORD-filtered members. This detects unowned or cross-owned additions; it cannot recover file origin after an overwrite of an already owned path, which is why the raw pre-extraction gate @@ -313,7 +321,7 @@ The `probe-driver` module command is a convenience diagnostic in the calling interpreter, not a substitute for this neutral-directory installed-package check. The files remaining under `conda/` are the conventional recipe entrypoints, -metadata, documentation and line-ending configuration. Shared NuGet transport, -feed-resolution and output-safety utilities remain under `eng/scripts` because +metadata, documentation and line-ending configuration. Shared Python-index download, +runtime-selection and output-safety utilities remain under `eng/scripts` because non-Conda native build/development workflows also use them; Conda reuses those implementations rather than keeping private copies. diff --git a/eng/conda_tools/build.py b/eng/conda_tools/build.py index f68392cb0..1dd18b63d 100644 --- a/eng/conda_tools/build.py +++ b/eng/conda_tools/build.py @@ -13,7 +13,7 @@ from eng.scripts.download_mssql_python_rs_wheels import file_sha256 -from . import archive, contracts, environment, verify +from . import archive, contracts, environment, inputs, verify from .formats import elf, macho, pe _KNOWN_SUBDIRS = ("win-64", "win-arm64", "osx-64", "osx-arm64", "linux-64", "linux-aarch64") @@ -167,19 +167,7 @@ def gather_wheels( receipt_path = Path(rs_dir, "transport.json") if receipt_path.exists(): receipt = json.loads(receipt_path.read_text(encoding="utf-8")) - if not isinstance(receipt, dict) or not isinstance( - receipt.get("wheel_sha256"), dict - ): - raise ValueError("RS transport receipt must contain a wheel_sha256 mapping") - if receipt.get("distribution_version") != rs_ver: - raise ValueError("RS transport receipt disagrees with the binding dependency") - if rs_version_file is not None: - pin_file = Path(rs_version_file).with_name("mssql-python-rs-nuget.version") - transport = pin_file.read_text(encoding="ascii").strip() - if not transport or receipt.get("transport_version") != transport: - raise ValueError( - "RS transport receipt disagrees with the producer transport pin" - ) + inputs.validate_rs_transport(receipt, {"mssql-python-rs": rs_ver}) for wheel in rs: if receipt["wheel_sha256"].get(Path(wheel).name) != file_sha256(Path(wheel)): raise ValueError( diff --git a/eng/conda_tools/inputs.py b/eng/conda_tools/inputs.py index 5cc43380b..efe326361 100644 --- a/eng/conda_tools/inputs.py +++ b/eng/conda_tools/inputs.py @@ -14,7 +14,7 @@ from urllib.error import URLError from urllib.request import urlopen -from eng.scripts.download_mssql_python_rs_wheels import DEFAULT_FEED, PACKAGE_ID, file_sha256 +from eng.scripts.download_mssql_python_rs_wheels import DEFAULT_INDEX, DEV_INDEX, file_sha256 from . import archive, contracts @@ -175,9 +175,7 @@ def fetch_wheels( violations = contracts.validate_core_layout(owned, python_tag, subdir) else: if source_versions.get("mssql-python-rs") != rs_version: - raise ValueError( - "Published binding RS requirement differs from the selected pin." - ) + raise ValueError("Published binding RS requirement differs from the selected pin.") versions["mssql-python-rs"] = rs_version hashes.update( _download_published( @@ -375,7 +373,7 @@ def read_release_versions(read_source: Callable[[str], str]) -> dict[str, str]: if _source_requires_rs(setup_source): path = "eng/versions/mssql-python-rs.version" version = read_source(path).strip() - if not re.fullmatch(r"[0-9][A-Za-z0-9.!+_-]*", version): + if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version): raise ValueError(f"{path} must contain one RS distribution release version.") result["mssql-python-rs"] = version return result @@ -449,31 +447,32 @@ def parse_release_versions(raw: str) -> dict[str, str]: return versions -def validate_rs_transport( - receipt: object, versions: dict[str, str], transport_version: str -) -> None: - """Check the selected producer's receipt against its separately verified source pins.""" +def validate_rs_transport(receipt: object, versions: dict[str, str]) -> None: + """Check index evidence against the selected producer's verified distribution pin.""" version = versions.get("mssql-python-rs") if version is None: - if receipt is not None or transport_version: + if receipt is not None: raise ValueError("An embedded-core producer must not supply an RS transport receipt.") return if not isinstance(receipt, dict): raise ValueError("The RS producer requires its recorded rs-transport.json receipt.") expected = { "distribution_version": version, - "transport_version": transport_version, - "feed_url": DEFAULT_FEED, - "package_id": PACKAGE_ID, + "index_url": DEV_INDEX if ".dev" in version else DEFAULT_INDEX, } - if not transport_version or any(receipt.get(key) != value for key, value in expected.items()): + override = receipt.get("version_override") + if ( + any(receipt.get(key) != value for key, value in expected.items()) + or not isinstance(override, bool) + or (".dev" in version and not override) + ): raise ValueError( - "RS transport receipt differs from the verified producer's source pins/feed." + "RS transport receipt differs from the verified producer's distribution pin/index." ) wheels = receipt.get("wheel_sha256") - if not _is_sha256(receipt.get("package_sha256")) or not isinstance(wheels, dict) or not wheels: - raise ValueError("RS transport receipt requires its package and wheel SHA256 values.") + if not isinstance(wheels, dict) or not wheels: + raise ValueError("RS transport receipt requires its wheel SHA256 values.") for name, digest in wheels.items(): if ( not isinstance(name, str) diff --git a/eng/conda_tools/provenance.py b/eng/conda_tools/provenance.py index 126b500e9..3f95d2fa1 100644 --- a/eng/conda_tools/provenance.py +++ b/eng/conda_tools/provenance.py @@ -60,15 +60,7 @@ def read_source(path: str) -> str: raise ValueError(f"Missing or mismatched wheel producer source for {path} at {commit}.") return item["content"] - versions = read_release_versions(read_source) - result = {"versions": versions, "rsTransportVersion": ""} - if "mssql-python-rs" in versions: - path = "eng/versions/mssql-python-rs-nuget.version" - transport_version = read_source(path).strip() - if not re.fullmatch(r"[0-9][A-Za-z0-9.+_-]*", transport_version): - raise ValueError(f"{path} must contain one RS transport version.") - result["rsTransportVersion"] = transport_version - return result + return {"versions": read_release_versions(read_source)} def _verify_run(build: dict, run: dict, pipeline_id: int, run_id: int) -> dict: @@ -202,8 +194,6 @@ def execute() -> None: print(f"##vso[task.setvariable variable=mssqlPythonVersion;isOutput=true]{version}") versions = json.dumps(result["versions"], sort_keys=True) print(f"##vso[task.setvariable variable=releaseVersions;isOutput=true]{versions}") - transport = result["rsTransportVersion"] - print(f"##vso[task.setvariable variable=rsTransportVersion;isOutput=true]{transport}") if publish == "false": print("Validate-only: provenance verified; package readiness is gated separately.") diff --git a/eng/conda_tools/release.py b/eng/conda_tools/release.py index bf7ab52da..cf9796b53 100644 --- a/eng/conda_tools/release.py +++ b/eng/conda_tools/release.py @@ -400,9 +400,6 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: default=None, help="Exact component version JSON; without a value, read RELEASE_VERSIONS.", ) - parser.add_argument( - "--rs-transport-version", default=os.environ.get("RS_TRANSPORT_VERSION", "") - ) def execute(args: argparse.Namespace) -> int: @@ -424,7 +421,9 @@ def execute(args: argparse.Namespace) -> int: if receipt_path.exists() else None ) - inputs.validate_rs_transport(receipt, versions, args.rs_transport_version) + inputs.validate_rs_transport(receipt, versions) + if isinstance(receipt, dict) and receipt.get("version_override"): + raise ValueError("RS version overrides are not eligible for Conda release.") except archive.READ_ERRORS as exc: print(f"ERROR: {exc}", file=sys.stderr) return 1 diff --git a/eng/pipelines/mssql-odbc-daily-validation-pipeline.yml b/eng/pipelines/mssql-odbc-daily-validation-pipeline.yml index 6a6712a64..7702b078a 100644 --- a/eng/pipelines/mssql-odbc-daily-validation-pipeline.yml +++ b/eng/pipelines/mssql-odbc-daily-validation-pipeline.yml @@ -11,6 +11,15 @@ schedules: - main always: true +parameters: + - name: rsVersionOverride + displayName: 'Exact mssql-python-rs test version (empty uses committed PyPI release)' + type: string + default: '' + +variables: + MSSQL_PYTHON_RS_VERSION: ${{ parameters.rsVersionOverride }} + jobs: - job: PytestOnMssqlOdbc displayName: 'Complete mssql-python suite on mssql-odbc' @@ -32,7 +41,7 @@ jobs: displayName: 'Checkout latest GitHub main' - script: | - docker run -d --name $(testContainer) \ + docker run -e MSSQL_PYTHON_RS_VERSION -d --name $(testContainer) \ -v $(Build.SourcesDirectory):/workspace \ -w /workspace \ --network bridge \ @@ -128,6 +137,7 @@ jobs: -e PYTHONPATH=/workspace \ $(testContainer) bash -c ' source /opt/venv/bin/activate + python eng/scripts/mssql_python_rs.py --verify || exit 2 chmod +x eng/scripts/run-mssql-odbc-tests.sh eng/scripts/run-mssql-odbc-tests.sh ' || rc=$? diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index c19a3e8aa..d89684674 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -6,12 +6,43 @@ trigger: include: - main +parameters: + - name: rsVersionOverride + displayName: 'Exact mssql-python-rs test version (empty uses latest published nightly)' + type: string + default: '' + variables: # QEMU 10.2.3, qualified against the ARM64 GCC startup crash and shutdown tests. arm64QemuImage: 'tonistiigi/binfmt@sha256:400a4873b838d1b89194d982c45e5fb3cda4593fbfd7e08a02e76b03b21166f0' jobs: +- job: SelectRsRuntime + displayName: 'Select one published Rust nightly for the validation run' + pool: + vmImage: 'ubuntu-latest' + steps: + - checkout: self + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.12' + addToPath: true + - script: | + python eng/scripts/resolve_pr_rs_runtime.py --output-dir "$(Build.ArtifactStagingDirectory)/rs-runtime" + name: select + displayName: 'Resolve nightly or exact override from the Python package index' + env: + MSSQL_PYTHON_RS_VERSION: ${{ parameters.rsVersionOverride }} + - task: PublishPipelineArtifact@1 + displayName: 'Record exact runtime selection and wheel hashes' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)/rs-runtime' + artifact: 'rs-runtime-selection' + - job: CodeQLAnalysis + dependsOn: SelectRsRuntime + variables: + MSSQL_PYTHON_RS_VERSION: $[ dependencies.SelectRsRuntime.outputs['select.version'] ] displayName: 'CodeQL Security Analysis' pool: vmImage: 'ubuntu-latest' @@ -50,6 +81,9 @@ jobs: displayName: 'Finalize CodeQL' - job: pytestonwindows + dependsOn: SelectRsRuntime + variables: + MSSQL_PYTHON_RS_VERSION: $[ dependencies.SelectRsRuntime.outputs['select.version'] ] displayName: 'Windows x64' timeoutInMinutes: 160 pool: @@ -355,7 +389,7 @@ jobs: # Run tests for LocalDB - script: | - python -m pytest -v --junitxml=test-results-localdb.xml --cov=. --cov-report=xml:coverage-localdb.xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results-localdb.xml --cov=. --cov-report=xml:coverage-localdb.xml --capture=tee-sys --cache-clear displayName: 'Run tests with coverage on LocalDB' condition: and(succeeded(), eq(variables['sqlVersion'], 'LocalDB')) env: @@ -363,7 +397,7 @@ jobs: # Run tests for SQL Server 2022 - script: | - python -m pytest -v --junitxml=test-results-sql2022.xml --cov=. --cov-report=xml:coverage-sql2022.xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results-sql2022.xml --cov=. --cov-report=xml:coverage-sql2022.xml --capture=tee-sys --cache-clear displayName: 'Run tests with coverage on SQL Server 2022' condition: and(succeeded(), eq(variables['sqlVersion'], 'SQL2022')) env: @@ -371,7 +405,7 @@ jobs: # Run tests for SQL Server 2025 - script: | - python -m pytest -v --junitxml=test-results-sql2025.xml --cov=. --cov-report=xml:coverage-sql2025.xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results-sql2025.xml --cov=. --cov-report=xml:coverage-sql2025.xml --capture=tee-sys --cache-clear displayName: 'Run tests with coverage on SQL Server 2025' condition: and(succeeded(), eq(variables['sqlVersion'], 'SQL2025')) env: @@ -561,12 +595,14 @@ jobs: # displayName: 'Publish code coverage results' - job: PytestOnMacOS + dependsOn: SelectRsRuntime displayName: 'macOS x86_64' timeoutInMinutes: 90 pool: vmImage: 'macos-latest' variables: + MSSQL_PYTHON_RS_VERSION: $[ dependencies.SelectRsRuntime.outputs['select.version'] ] # Apply to setup, benchmark consumers and final cleanup without changing # the Docker client's global context. DOCKER_CONTEXT: colima @@ -716,7 +752,7 @@ jobs: - script: | echo "Build successful, running tests now" - python -m pytest -v --junitxml=test-results.xml --cov=. --cov-report=xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results.xml --cov=. --cov-report=xml --capture=tee-sys --cache-clear displayName: 'Run pytest with coverage' env: DB_CONNECTION_STRING: 'Server=tcp:127.0.0.1,1433;Database=master;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' @@ -742,6 +778,9 @@ jobs: DB_PASSWORD: $(DB_PASSWORD) - job: PytestOnLinux + dependsOn: SelectRsRuntime + variables: + MSSQL_PYTHON_RS_VERSION: $[ dependencies.SelectRsRuntime.outputs['select.version'] ] displayName: 'Linux x86_64' timeoutInMinutes: 160 pool: @@ -789,7 +828,7 @@ jobs: - script: | # Create a Docker container for testing - docker run -d --name test-container-$(distroName) \ + docker run -e MSSQL_PYTHON_RS_VERSION -d --name test-container-$(distroName) \ -v $(Build.SourcesDirectory):/workspace \ -w /workspace \ --network bridge \ @@ -951,7 +990,7 @@ jobs: source /opt/venv/bin/activate echo 'Build successful, running tests now on $(distroName) with Azure SQL' echo 'Using Azure SQL connection string' - python -m pytest -v --junitxml=test-results-$(distroName).xml --cov=. --cov-report=xml:coverage-$(distroName).xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results-$(distroName).xml --cov=. --cov-report=xml:coverage-$(distroName).xml --capture=tee-sys --cache-clear " else # Local SQL Server testing @@ -966,7 +1005,7 @@ jobs: source /opt/venv/bin/activate echo 'Build successful, running tests now on $(distroName)' echo 'Using connection string: Server=$SQLSERVER_IP;Database=TestDB;Uid=SA;Pwd=***;TrustServerCertificate=yes' - python -m pytest -v --junitxml=test-results-$(distroName).xml --cov=. --cov-report=xml:coverage-$(distroName).xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results-$(distroName).xml --cov=. --cov-report=xml:coverage-$(distroName).xml --capture=tee-sys --cache-clear " fi displayName: 'Run pytest with coverage in $(distroName) container' @@ -1085,6 +1124,9 @@ jobs: testRunTitle: 'Publish pytest results on $(distroName)' - job: PytestOnLinux_ARM64 + dependsOn: SelectRsRuntime + variables: + MSSQL_PYTHON_RS_VERSION: $[ dependencies.SelectRsRuntime.outputs['select.version'] ] displayName: 'Linux ARM64' pool: vmImage: 'ubuntu-latest' @@ -1122,7 +1164,7 @@ jobs: - script: | # Create a Docker container for testing on ARM64 - docker run -d --name test-container-$(distroName)-$(archName) \ + docker run -e MSSQL_PYTHON_RS_VERSION -d --name test-container-$(distroName)-$(archName) \ --platform linux/arm64 \ -v $(Build.SourcesDirectory):/workspace \ -w /workspace \ @@ -1293,7 +1335,7 @@ jobs: echo 'Architecture:' \$(uname -m) echo 'Using connection string: Server=$SQLSERVER_IP;Database=TestDB;Uid=SA;Pwd=***;TrustServerCertificate=yes' python main.py - python -m pytest -v --junitxml=test-results-$(distroName)-$(archName).xml --cov=. --cov-report=xml:coverage-$(distroName)-$(archName).xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results-$(distroName)-$(archName).xml --cov=. --cov-report=xml:coverage-$(distroName)-$(archName).xml --capture=tee-sys --cache-clear " displayName: 'Run pytest with coverage in $(distroName) ARM64 container' env: @@ -1328,6 +1370,9 @@ jobs: testRunTitle: 'Publish pytest results on $(distroName) ARM64' - job: PytestOnLinux_RHEL9 + dependsOn: SelectRsRuntime + variables: + MSSQL_PYTHON_RS_VERSION: $[ dependencies.SelectRsRuntime.outputs['select.version'] ] displayName: 'Linux RedHat x86_64' pool: vmImage: 'ubuntu-latest' @@ -1335,7 +1380,7 @@ jobs: steps: - script: | # Create a Docker container for testing - docker run -d --name test-container-rhel9 \ + docker run -e MSSQL_PYTHON_RS_VERSION -d --name test-container-rhel9 \ -v $(Build.SourcesDirectory):/workspace \ -w /workspace \ --network bridge \ @@ -1495,7 +1540,7 @@ jobs: echo 'Build successful, running tests now on RHEL 9' echo 'Using connection string: Server=$SQLSERVER_IP;Database=TestDB;Uid=SA;Pwd=***;TrustServerCertificate=yes' python main.py - python -m pytest -v --junitxml=test-results-rhel9.xml --cov=. --cov-report=xml:coverage-rhel9.xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results-rhel9.xml --cov=. --cov-report=xml:coverage-rhel9.xml --capture=tee-sys --cache-clear " displayName: 'Run pytest with coverage in RHEL 9 container' env: @@ -1530,6 +1575,9 @@ jobs: testRunTitle: 'Publish pytest results on RHEL 9' - job: PytestOnLinux_RHEL9_ARM64 + dependsOn: SelectRsRuntime + variables: + MSSQL_PYTHON_RS_VERSION: $[ dependencies.SelectRsRuntime.outputs['select.version'] ] displayName: 'Linux RedHat ARM64' pool: vmImage: 'ubuntu-latest' @@ -1548,7 +1596,7 @@ jobs: - script: | # Create a Docker container for testing on ARM64 - docker run -d --name test-container-rhel9-arm64 \ + docker run -e MSSQL_PYTHON_RS_VERSION -d --name test-container-rhel9-arm64 \ --platform linux/arm64 \ -v $(Build.SourcesDirectory):/workspace \ -w /workspace \ @@ -1712,7 +1760,7 @@ jobs: echo 'Build successful, running tests now on RHEL 9 ARM64' echo 'Architecture:' \$(uname -m) echo 'Using connection string: Server=$SQLSERVER_IP;Database=TestDB;Uid=SA;Pwd=***;TrustServerCertificate=yes' - python -m pytest -v --junitxml=test-results-rhel9-arm64.xml --cov=. --cov-report=xml:coverage-rhel9-arm64.xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results-rhel9-arm64.xml --cov=. --cov-report=xml:coverage-rhel9-arm64.xml --capture=tee-sys --cache-clear " displayName: 'Run pytest with coverage in RHEL 9 ARM64 container' env: @@ -1747,6 +1795,9 @@ jobs: testRunTitle: 'Publish pytest results on RHEL 9 ARM64' - job: PytestOnLinux_Alpine + dependsOn: SelectRsRuntime + variables: + MSSQL_PYTHON_RS_VERSION: $[ dependencies.SelectRsRuntime.outputs['select.version'] ] displayName: 'Linux Alpine x86_64' pool: vmImage: 'ubuntu-latest' @@ -1761,7 +1812,7 @@ jobs: - script: | # Create a Docker container for testing on x86_64 - docker run -d --name test-container-alpine \ + docker run -e MSSQL_PYTHON_RS_VERSION -d --name test-container-alpine \ --platform linux/amd64 \ -v $(Build.SourcesDirectory):/workspace \ -w /workspace \ @@ -1948,7 +1999,7 @@ jobs: fi # Run pytest - python -m pytest -v --junitxml=test-results-alpine.xml --cov=. --cov-report=xml:coverage-alpine.xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results-alpine.xml --cov=. --cov-report=xml:coverage-alpine.xml --capture=tee-sys --cache-clear " displayName: 'Run pytest with coverage in Alpine x86_64 container' env: @@ -1983,6 +2034,9 @@ jobs: testRunTitle: 'Publish pytest results on Alpine x86_64' - job: PytestOnLinux_Alpine_ARM64 + dependsOn: SelectRsRuntime + variables: + MSSQL_PYTHON_RS_VERSION: $[ dependencies.SelectRsRuntime.outputs['select.version'] ] displayName: 'Linux Alpine ARM64' pool: vmImage: 'ubuntu-latest' @@ -2003,7 +2057,7 @@ jobs: # Create a Docker container for testing on ARM64 # TODO(AB#40901): Temporary pin to 3.22 due to msodbcsql ARM64 package arch mismatch # Revert to alpine:latest once ODBC team releases fixed ARM64 package - docker run -d --name test-container-alpine-arm64 \ + docker run -e MSSQL_PYTHON_RS_VERSION -d --name test-container-alpine-arm64 \ --platform linux/arm64 \ -v $(Build.SourcesDirectory):/workspace \ -w /workspace \ @@ -2190,7 +2244,7 @@ jobs: fi # Run pytest - python -m pytest -v --junitxml=test-results-alpine-arm64.xml --cov=. --cov-report=xml:coverage-alpine-arm64.xml --capture=tee-sys --cache-clear + python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v --junitxml=test-results-alpine-arm64.xml --cov=. --cov-report=xml:coverage-alpine-arm64.xml --capture=tee-sys --cache-clear " displayName: 'Run pytest with coverage in Alpine ARM64 container' env: @@ -2451,11 +2505,20 @@ jobs: # testRunTitle: 'Publish pytest results on Ubuntu AzureSQL' - job: CodeCoverageReport + dependsOn: SelectRsRuntime + variables: + MSSQL_PYTHON_RS_VERSION: $[ dependencies.SelectRsRuntime.outputs['select.version'] ] displayName: 'Full Code Coverage Report in Ubuntu x86_64' pool: vmImage: 'ubuntu-latest' steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.12' + addToPath: true + displayName: 'Use Python 3.12 for coverage build and tests' + - script: | # Install build dependencies sudo apt-get update diff --git a/eng/pipelines/steps/install-mssql-py-core.yml b/eng/pipelines/steps/install-mssql-py-core.yml index a10cf0b46..d7277caf1 100644 --- a/eng/pipelines/steps/install-mssql-py-core.yml +++ b/eng/pipelines/steps/install-mssql-py-core.yml @@ -1,4 +1,4 @@ -# Step template: Install mssql-python-rs from the internal NuGet transport +# Install the committed PyPI release, or the explicit MSSQL_PYTHON_RS_VERSION test override. # # Usage: # # Windows (host) @@ -54,7 +54,7 @@ steps: # Inside a Docker container - ${{ if eq(parameters.platform, 'container') }}: - script: | - docker exec ${{ parameters.containerName }} bash -c " + docker exec -e MSSQL_PYTHON_RS_VERSION ${{ parameters.containerName }} bash -c " ${{ parameters.venvActivate }} chmod +x eng/scripts/install-mssql-py-core.sh ./eng/scripts/install-mssql-py-core.sh diff --git a/eng/requirements-build-linux.in b/eng/requirements-build-linux.in index e153979ba..cd3a97adb 100644 --- a/eng/requirements-build-linux.in +++ b/eng/requirements-build-linux.in @@ -2,5 +2,6 @@ pip pybind11 pytest +PyYAML setuptools wheel diff --git a/eng/requirements-build-linux.txt b/eng/requirements-build-linux.txt index ce528b83e..bb38e7f71 100644 --- a/eng/requirements-build-linux.txt +++ b/eng/requirements-build-linux.txt @@ -36,6 +36,81 @@ pytest==9.1.1 \ --hash=sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313 \ --hash=sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c # via -r eng/requirements-build-linux.in +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via -r eng/requirements-build-linux.in setuptools==84.0.0 \ --hash=sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670 \ --hash=sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73 diff --git a/eng/requirements-build-macos.txt b/eng/requirements-build-macos.txt index 939ff74d1..642c60e96 100644 --- a/eng/requirements-build-macos.txt +++ b/eng/requirements-build-macos.txt @@ -1307,6 +1307,81 @@ pytokens==0.4.1 \ --hash=sha256:ee44d0f85b803321710f9239f335aafe16553b39106384cef8e6de40cb4ef2f6 \ --hash=sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324 # via black +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via -r eng/../requirements.txt requests==2.34.2 \ --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed @@ -1404,7 +1479,9 @@ urllib3==2.7.0 \ wheel==0.48.0 \ --hash=sha256:3217dcc807155e45db462d7ef2431f5ddda0d7273b700d05a67b271ceb1287ab \ --hash=sha256:94800765601e9171bf5d58d066e640662842bcedcbab982b2c90787a2c987322 - # via -r eng/requirements-build-macos.in + # via + # -r eng/../requirements.txt + # -r eng/requirements-build-macos.in zstandard==0.25.0 \ --hash=sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64 \ --hash=sha256:01582723b3ccd6939ab7b3a78622c573799d5d8737b534b86d0e06ac18dbde4a \ diff --git a/eng/requirements-build-windows.in b/eng/requirements-build-windows.in index 3687afb64..e0bfbc0bf 100644 --- a/eng/requirements-build-windows.in +++ b/eng/requirements-build-windows.in @@ -5,6 +5,7 @@ pybind11 pyodbc pytest pytest-asyncio +PyYAML setuptools wheel zstandard diff --git a/eng/requirements-build-windows.txt b/eng/requirements-build-windows.txt index 395e7ffc5..f8962f0d5 100644 --- a/eng/requirements-build-windows.txt +++ b/eng/requirements-build-windows.txt @@ -135,6 +135,81 @@ pytest-asyncio==1.4.0 \ --hash=sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1 \ --hash=sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42 # via -r eng/requirements-build-windows.in +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via -r eng/requirements-build-windows.in setuptools==84.0.0 \ --hash=sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670 \ --hash=sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73 diff --git a/eng/requirements-test-linux.txt b/eng/requirements-test-linux.txt index 5f7a6019d..d7a427168 100644 --- a/eng/requirements-test-linux.txt +++ b/eng/requirements-test-linux.txt @@ -1298,6 +1298,84 @@ pytokens==0.4.1 \ --hash=sha256:ee44d0f85b803321710f9239f335aafe16553b39106384cef8e6de40cb4ef2f6 \ --hash=sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324 # via black +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via + # -c eng/requirements-build-linux.txt + # -r eng/../requirements.txt + # -r eng/requirements-build-linux.in requests==2.34.2 \ --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed @@ -1402,6 +1480,7 @@ wheel==0.48.0 \ --hash=sha256:94800765601e9171bf5d58d066e640662842bcedcbab982b2c90787a2c987322 # via # -c eng/requirements-build-linux.txt + # -r eng/../requirements.txt # -r eng/requirements-build-linux.in zstandard==0.25.0 \ --hash=sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64 \ diff --git a/eng/scripts/download_mssql_python_rs_wheels.py b/eng/scripts/download_mssql_python_rs_wheels.py index f1e6bb498..212929ea6 100644 --- a/eng/scripts/download_mssql_python_rs_wheels.py +++ b/eng/scripts/download_mssql_python_rs_wheels.py @@ -1,36 +1,28 @@ #!/usr/bin/env python3 -"""Download and stage mssql-python-rs wheels from the internal NuGet transport.""" +"""Stage exact-version runtime wheels from PyPI or the development wheel index.""" from __future__ import annotations import argparse import hashlib import json +import re import shutil import urllib.request import zipfile -from xml.etree import ElementTree -from pathlib import Path, PurePosixPath +from email.parser import BytesParser +from html.parser import HTMLParser +from pathlib import Path +from urllib.parse import parse_qs, unquote, urldefrag, urljoin, urlsplit if __package__: from . import mssql_python_build_safety - from .resolve_nuget_feed import resolve + from .mssql_python_rs import DEFAULT_INDEX, DEV_INDEX, VERSION_FILE, resolve_runtime else: import mssql_python_build_safety - from resolve_nuget_feed import resolve + from mssql_python_rs import DEFAULT_INDEX, DEV_INDEX, VERSION_FILE, resolve_runtime -DEFAULT_FEED = ( - "https://pkgs.dev.azure.com/sqlclientdrivers/public/" - "_packaging/mssql-rs_Public/nuget/v3/index.json" -) -PACKAGE_ID = "mssql-python-rs-wheels" - - -def _read_version(version_file: Path) -> str: - version = version_file.read_text(encoding="ascii").strip() - if not version: - raise ValueError(f"Version file is empty: {version_file}") - return version +PACKAGE_ID = "mssql-python-rs" def file_sha256(path: Path) -> str: @@ -41,104 +33,105 @@ def file_sha256(path: Path) -> str: return digest.hexdigest() +def validate_wheel(path: Path, version: str) -> None: + if not path.name.startswith(f"mssql_python_rs-{version}-") or path.suffix != ".whl": + raise ValueError(f"Unexpected wheel name for {PACKAGE_ID} {version}: {path.name}") + with zipfile.ZipFile(path) as wheel: + metadata = [name for name in wheel.namelist() if name.endswith(".dist-info/METADATA")] + if len(metadata) != 1: + raise ValueError(f"{path.name} must contain exactly one wheel METADATA") + identity = BytesParser().parsebytes(wheel.read(metadata[0])) + name = re.sub(r"[-_.]+", "-", identity.get("Name", "")).lower() + if name != PACKAGE_ID or identity.get("Version") != version: + raise ValueError(f"{path.name} metadata does not match {PACKAGE_ID}=={version}") + + +class _WheelLinks(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.links: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag == "a": + attributes = dict(attrs) + href = attributes.get("href") + if href and "data-yanked" not in attributes: + self.links.append(href) + + def download_wheels( - feed_url: str, + index_url: str | None, distribution_version_file: Path, - transport_version_file: Path, output_dir: Path, + *, + version: str | None = None, ) -> list[Path]: - distribution_version = _read_version(distribution_version_file) - transport_version = _read_version(transport_version_file) - - output_dir = getattr(mssql_python_build_safety, "resolve_safe_output_directory")(output_dir) - - package_base = resolve(feed_url).rstrip("/") + "/" - normalized_version = transport_version.lower() - package_url = ( - f"{package_base}{PACKAGE_ID}/{normalized_version}/" - f"{PACKAGE_ID}.{normalized_version}.nupkg" + runtime = resolve_runtime(distribution_version_file, version=version) + index_url = index_url or runtime.index_url + package_url = index_url.rstrip("/") + f"/{PACKAGE_ID}/" + if urlsplit(package_url).scheme != "https": + raise ValueError("The runtime wheel index must use HTTPS") + output_dir = mssql_python_build_safety.resolve_safe_output_directory(output_dir) + with urllib.request.urlopen(package_url, timeout=120) as response: + page = response.read().decode("utf-8") + parser = _WheelLinks() + parser.feed(page) + candidates: dict[str, tuple[str, str]] = {} + pattern = re.compile( + rf"mssql_python_rs-{re.escape(runtime.version)}-cp310-abi3-[A-Za-z0-9_]+\.whl" ) - + for link in parser.links: + url, fragment = urldefrag(urljoin(package_url, link)) + filename = unquote(urlsplit(url).path.rsplit("/", 1)[-1]) + if not pattern.fullmatch(filename): + continue + if urlsplit(url).scheme != "https": + raise ValueError(f"Wheel download must use HTTPS: {filename}") + if filename in candidates: + raise ValueError(f"Duplicate wheel filename in index: {filename}") + hashes = parse_qs(fragment).get("sha256", []) + if len(hashes) != 1 or not re.fullmatch(r"[a-fA-F0-9]{64}", hashes[0]): + raise ValueError(f"Missing or invalid SHA-256 for {filename}") + candidates[filename] = (url, hashes[0].lower()) + if not candidates: + raise ValueError(f"No {PACKAGE_ID}=={runtime.version} wheels found in {package_url}") if output_dir.exists(): shutil.rmtree(output_dir) output_dir.mkdir(parents=True) - package_path = output_dir / f"{PACKAGE_ID}.{normalized_version}.nupkg" - with urllib.request.urlopen(package_url, timeout=120) as response: - with package_path.open("wb") as destination: - shutil.copyfileobj(response, destination) - - staged: list[Path] = [] - seen: set[str] = set() - expected_prefix = f"mssql_python_rs-{distribution_version}-" - with zipfile.ZipFile(package_path) as package: - nuspecs = [name for name in package.namelist() if name.endswith(".nuspec")] - if len(nuspecs) != 1: - raise ValueError("RS transport must contain exactly one Nuspec") - metadata = ElementTree.fromstring(package.read(nuspecs[0])).find("{*}metadata") - if ( - metadata is None - or metadata.findtext("{*}id") != PACKAGE_ID - or metadata.findtext("{*}version", "").lower() != normalized_version - ): - raise ValueError("RS transport Nuspec identity does not match the pinned package") - description = metadata.findtext("{*}description", "") - for entry in package.infolist(): - path = PurePosixPath(entry.filename) - if len(path.parts) != 2 or path.parts[0] != "wheels" or path.suffix != ".whl": - continue - if not path.name.startswith(expected_prefix) or "\\" in path.name: - raise ValueError( - "Unexpected wheel name for mssql-python-rs " - f"{distribution_version}: {path.name}" - ) - if path.name in seen: - raise ValueError(f"Duplicate wheel filename in NuGet package: {path.name}") - seen.add(path.name) - destination = output_dir / path.name - with package.open(entry) as source, destination.open("wb") as target: - shutil.copyfileobj(source, target) - staged.append(destination) - - if not staged: - raise ValueError(f"No mssql-python-rs {distribution_version} wheels found in {package_url}") + staged = [] + for filename, (url, expected_hash) in sorted(candidates.items()): + destination = output_dir / filename + with urllib.request.urlopen(url, timeout=120) as response: + with destination.open("wb") as target: + shutil.copyfileobj(response, target) + if file_sha256(destination) != expected_hash: + raise ValueError(f"SHA-256 mismatch for {filename}") + validate_wheel(destination, runtime.version) + staged.append(destination) receipt = { - "distribution_version": distribution_version, - "transport_version": transport_version, - "feed_url": feed_url, - "package_id": PACKAGE_ID, - "package_sha256": file_sha256(package_path), - "nuspec_description": description, - "wheel_sha256": {path.name: file_sha256(path) for path in sorted(staged)}, + "distribution_version": runtime.version, + "index_url": index_url, + "version_override": runtime.version_override, + "wheel_sha256": {path.name: file_sha256(path) for path in staged}, } (output_dir / "transport.json").write_text( json.dumps(receipt, indent=2) + "\n", encoding="utf-8", newline="\n" ) - package_path.unlink() - return sorted(staged) + return staged def main() -> None: - repo_root = Path(__file__).resolve().parents[2] parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--feed-url", default=DEFAULT_FEED) - parser.add_argument( - "--distribution-version-file", - type=Path, - default=repo_root / "eng" / "versions" / "mssql-python-rs.version", - ) - parser.add_argument( - "--transport-version-file", - type=Path, - default=repo_root / "eng" / "versions" / "mssql-python-rs-nuget.version", - ) + parser.add_argument("--index-url") + parser.add_argument("--version") + parser.add_argument("--distribution-version-file", type=Path, default=VERSION_FILE) parser.add_argument("--output-dir", type=Path, required=True) args = parser.parse_args() - wheels = download_wheels( - args.feed_url, + args.index_url, args.distribution_version_file, - args.transport_version_file, args.output_dir, + version=args.version, ) for wheel in wheels: print(wheel) diff --git a/eng/scripts/install-mssql-py-core.ps1 b/eng/scripts/install-mssql-py-core.ps1 index 91d22eff2..51681afa8 100644 --- a/eng/scripts/install-mssql-py-core.ps1 +++ b/eng/scripts/install-mssql-py-core.ps1 @@ -1,170 +1,19 @@ <# .SYNOPSIS - Installs mssql-python-rs from the pinned internal NuGet transport package. - -.PARAMETER FeedUrl - The NuGet v3 feed URL. This is a public feed — no authentication required. - -.PARAMETER OutputDir - Temporary directory for downloaded artifacts. Cleaned up after installation. - Defaults to $env:TEMP\mssql-python-rs-wheels. - + Installs the pinned released runtime, or MSSQL_PYTHON_RS_VERSION for feature tests. .PARAMETER TargetArch - Target CPU architecture ('x64' or 'arm64') for cross-compilation builds. - When set, selects the matching-architecture wheel. Cross-architecture wheels - are not installed into the host interpreter. + Validates but does not install a cross-architecture Windows wheel. #> - param( - [string]$FeedUrl = "https://pkgs.dev.azure.com/sqlclientdrivers/public/_packaging/mssql-rs_Public/nuget/v3/index.json", - [string]$OutputDir = "$env:TEMP\mssql-python-rs-wheels", + [string]$OutputDir = "", + [ValidateSet("", "x64", "arm64")] [string]$TargetArch = "" ) $ErrorActionPreference = 'Stop' -$ScriptDir = $PSScriptRoot -$RepoRoot = (Get-Item "$ScriptDir\..\..").FullName - -function Read-PackageVersion { - $distributionVersionFile = Join-Path $RepoRoot "eng\versions\mssql-python-rs.version" - $transportVersionFile = Join-Path $RepoRoot "eng\versions\mssql-python-rs-nuget.version" - foreach ($versionFile in @($distributionVersionFile, $transportVersionFile)) { - if (-not (Test-Path $versionFile -PathType Leaf)) { - throw "Version file not found: $versionFile" - } - } - $script:DistributionVersion = (Get-Content $distributionVersionFile -Raw).Trim() - $script:TransportVersion = (Get-Content $transportVersionFile -Raw).Trim() - if (-not $script:DistributionVersion -or -not $script:TransportVersion) { - throw "mssql-python-rs version files must not be empty" - } - Write-Host "Distribution version: $script:DistributionVersion" - Write-Host "NuGet transport version: $script:TransportVersion" -} - -function Get-PlatformInfo { - # Single python call to get version, platform, and HOST arch - $info = & python -c "import sys, platform; v = sys.version_info; print(f'cp{v.major}{v.minor} {platform.system().lower()} {platform.machine().lower()}')" - if ($LASTEXITCODE -ne 0) { throw "Failed to detect Python platform info" } - - $parts = $info -split ' ' - $script:PyVersion = $parts[0] - $script:Platform = $parts[1] - $hostArch = $parts[2] - - # Normalize an arch string (platform.machine() or -TargetArch) to a wheel tag. - function ConvertTo-ArchTag($a) { - switch -Regex ($a) { - 'amd64|x86_64|x64' { return 'x86_64' } - 'arm64|aarch64' { return 'aarch64' } - default { throw "Unsupported architecture: $a" } - } - } - - # Select the wheel by TARGET arch, not host arch. Empty -TargetArch uses host. - $hostArchTag = ConvertTo-ArchTag $hostArch - $archTag = if ($TargetArch) { ConvertTo-ArchTag $TargetArch } else { $hostArchTag } - $script:IsCrossArch = ($archTag -ne $hostArchTag) - - Write-Host "Python: $script:PyVersion | Platform: $script:Platform | Arch: $archTag (host: $hostArchTag, target: $(if ($TargetArch) { $TargetArch } else { '' }))" - - if ($script:Platform -ne 'windows') { - throw "The PowerShell installer supports Windows only; use install-mssql-py-core.sh on $script:Platform" - } - # aarch64 -> arm64 so a Windows arm64 target resolves win_arm64, not win_aarch64. - $script:WheelPlatform = "win_$($archTag -replace 'x86_64','amd64' -replace 'aarch64','arm64')" - - $script:WheelPattern = "mssql_python_rs-$script:DistributionVersion-cp310-abi3-$script:WheelPlatform.whl" - Write-Host "Wheel pattern: $script:WheelPattern" -} - -function Get-NupkgFromFeed { - param([string]$FeedUrl, [string]$OutputDir) - - $resolvedOutput = & python "$ScriptDir\mssql_python_build_safety.py" $OutputDir - if ($LASTEXITCODE -ne 0) { throw "Unsafe OutputDir: $OutputDir" } - if (Test-Path $resolvedOutput) { Remove-Item $resolvedOutput -Recurse -Force } - New-Item -ItemType Directory -Path $resolvedOutput -Force | Out-Null - $script:ResolvedOutputDir = $resolvedOutput - - Write-Host "Resolving feed: $FeedUrl" - # Fetch the NuGet v3 service index and extract the PackageBaseAddress URL. - # See resolve_nuget_feed.py for the JSON schema and detailed explanation. - $feedIndex = Invoke-RestMethod -Uri $FeedUrl - $packageBaseUrl = ($feedIndex.resources | Where-Object { $_.'@type' -like 'PackageBaseAddress*' } | Select-Object -First 1).'@id' - if (-not $packageBaseUrl) { throw "Could not resolve PackageBaseAddress from feed" } - $packageBaseUrl = $packageBaseUrl.TrimEnd('/') + '/' - - $versionLower = $script:TransportVersion.ToLowerInvariant() - $packageId = "mssql-python-rs-wheels" - $nupkgUrl = "${packageBaseUrl}${packageId}/${versionLower}/${packageId}.${versionLower}.nupkg" - $script:NupkgPath = Join-Path $resolvedOutput "${packageId}.${versionLower}.nupkg" - Write-Host "Downloading: $nupkgUrl" - Invoke-WebRequest -Uri $nupkgUrl -OutFile $script:NupkgPath - - $sizeMB = [math]::Round((Get-Item $script:NupkgPath).Length / 1MB, 2) - Write-Host "Downloaded: $script:NupkgPath ($sizeMB MB)" -} - -function Find-MatchingWheel { - param([string]$OutputDir) - - # nupkg is a ZIP — rename so Expand-Archive accepts it - $zipPath = $script:NupkgPath -replace '\.nupkg$', '.zip' - Rename-Item -Path $script:NupkgPath -NewName (Split-Path $zipPath -Leaf) - - $extractDir = Join-Path $script:ResolvedOutputDir "extracted" - Expand-Archive -Path $zipPath -DestinationPath $extractDir -Force - - $wheelsDir = Join-Path $extractDir "wheels" - if (-not (Test-Path $wheelsDir)) { - throw "No 'wheels' directory found in NuGet package" - } - - $script:MatchingWheel = Get-ChildItem $wheelsDir -Filter $script:WheelPattern | Select-Object -First 1 - if (-not $script:MatchingWheel) { - Write-Host "Available wheels:" - Get-ChildItem $wheelsDir -Filter *.whl | ForEach-Object { Write-Host " $_" } - throw "No wheel found matching: $script:WheelPattern" - } - - Write-Host "Found: $($script:MatchingWheel.Name)" -} - -function Install-AndVerify { - $coreDir = Join-Path $RepoRoot "mssql_py_core" - if (Test-Path $coreDir) { - Remove-Item $coreDir -Recurse -Force - Write-Host "Cleaned previous mssql_py_core/" - } - - # An arm64 wheel cannot be installed into the x64 build-host interpreter. - if ($script:IsCrossArch) { - Write-Host "Skipping mssql-python-rs installation (cross-arch build: target != host)" - return - } - - & python -m pip install --force-reinstall --no-deps $script:MatchingWheel.FullName - if ($LASTEXITCODE -ne 0) { throw "Failed to install mssql-python-rs" } - - & python -c "import importlib.metadata as m, mssql_py_core; assert m.version('mssql-python-rs') == '$script:DistributionVersion'; print('mssql-python-rs', m.version('mssql-python-rs'), 'loaded from', mssql_py_core.__file__)" - if ($LASTEXITCODE -ne 0) { throw "Failed to verify installed mssql-python-rs" } -} - -# --- main --- - -Write-Host "=== Install mssql-python-rs from NuGet transport ===" - -try { - Read-PackageVersion - Get-PlatformInfo - Get-NupkgFromFeed -FeedUrl $FeedUrl -OutputDir $OutputDir - Find-MatchingWheel -OutputDir $OutputDir - Install-AndVerify - Write-Host "=== mssql-python-rs installed successfully ===" -} -finally { - if ($script:ResolvedOutputDir) { - Remove-Item $script:ResolvedOutputDir -Recurse -Force -ErrorAction SilentlyContinue - } -} +$python = if ($env:PYTHON) { $env:PYTHON } else { 'python' } +$arguments = @("$PSScriptRoot\install_mssql_python_rs.py") +if ($OutputDir) { $arguments += @('--output-dir', $OutputDir) } +if ($TargetArch) { $arguments += @('--target-arch', $TargetArch) } +& $python @arguments +if ($LASTEXITCODE -ne 0) { throw 'Failed to install or validate mssql-python-rs' } diff --git a/eng/scripts/install-mssql-py-core.sh b/eng/scripts/install-mssql-py-core.sh index a27729701..21370d2fc 100644 --- a/eng/scripts/install-mssql-py-core.sh +++ b/eng/scripts/install-mssql-py-core.sh @@ -1,206 +1,7 @@ #!/usr/bin/env bash -# Installs mssql-python-rs from the pinned internal NuGet transport package. -# -# This script is used identically for: -# - Local development (dev runs it after build.sh) -# - PR validation pipelines -# - Official build pipelines and tests -# -# The Python distribution and NuGet transport versions are pinned separately. -# -# Usage: -# ./install-mssql-py-core.sh [--feed-url URL] - +# Install the release pin, or an explicit MSSQL_PYTHON_RS_VERSION feature-test override. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" PYTHON="${PYTHON:-$(command -v python || command -v python3)}" - -read_version() { - local distribution_file="$REPO_ROOT/eng/versions/mssql-python-rs.version" - local transport_file="$REPO_ROOT/eng/versions/mssql-python-rs-nuget.version" - for version_file in "$distribution_file" "$transport_file"; do - if [ ! -f "$version_file" ]; then - echo "ERROR: Version file not found: $version_file" - exit 1 - fi - done - DISTRIBUTION_VERSION=$(tr -d '[:space:]' < "$distribution_file") - TRANSPORT_VERSION=$(tr -d '[:space:]' < "$transport_file") - if [ -z "$DISTRIBUTION_VERSION" ] || [ -z "$TRANSPORT_VERSION" ]; then - echo "ERROR: mssql-python-rs version files must not be empty" - exit 1 - fi - echo "Distribution version: $DISTRIBUTION_VERSION" - echo "NuGet transport version: $TRANSPORT_VERSION" -} - -detect_platform() { - read -r PY_VERSION PLATFORM ARCH <<< "$("$PYTHON" -c " -import sys, platform -v = sys.version_info -print(f'cp{v.major}{v.minor} {platform.system().lower()} {platform.machine().lower()}')" - )" - - echo "Python: $PY_VERSION | Platform: $PLATFORM | Arch: $ARCH" - - case "$PLATFORM" in - linux) - case "$ARCH" in - x86_64|amd64) ARCH_TAG="x86_64" ;; - aarch64|arm64) ARCH_TAG="aarch64" ;; - *) echo "Unsupported Linux architecture: $ARCH"; exit 1 ;; - esac - - # Detect musl libc (Alpine) vs glibc. - # ldd --version exits 1 on musl, so capture output instead of piping. - local ldd_output - ldd_output=$(ldd --version 2>&1 || true) - if echo "$ldd_output" | grep -qi musl || [ -f /etc/alpine-release ]; then - WHEEL_PLATFORM="musllinux_1_2_${ARCH_TAG}" - else - local glibc_version major minor - glibc_version=$(printf '%s\n' "$ldd_output" | head -1 | grep -Eo '[0-9]+\.[0-9]+' | tail -1) - major=${glibc_version%%.*} - minor=${glibc_version#*.} - if [ -z "$glibc_version" ] || ! [ "$major" -eq "$major" ] 2>/dev/null || ! [ "$minor" -eq "$minor" ] 2>/dev/null; then - echo "ERROR: Could not determine glibc version from: $ldd_output" - exit 1 - fi - if [ "$major" -gt 2 ] || { [ "$major" -eq 2 ] && [ "$minor" -ge 34 ]; }; then - # glibc 2.34+ systems carry OpenSSL 3. - WHEEL_PLATFORM="manylinux_2_34_${ARCH_TAG}" - elif [ "$major" -eq 2 ] && [ "$minor" -ge 28 ]; then - # glibc 2.28-2.33 systems carry OpenSSL 1.1. - WHEEL_PLATFORM="manylinux_2_28_${ARCH_TAG}" - else - echo "ERROR: mssql-python-rs requires glibc 2.28 or newer; found $glibc_version" - exit 1 - fi - fi - ;; - darwin) - WHEEL_PLATFORM="macosx_15_0_universal2" - ;; - *) - echo "Unsupported platform: $PLATFORM" - exit 1 - ;; - esac - - WHEEL_PATTERN="mssql_python_rs-${DISTRIBUTION_VERSION}-cp310-abi3-${WHEEL_PLATFORM}.whl" - echo "Wheel pattern: $WHEEL_PATTERN" -} - -download_nupkg() { - local feed_url="$1" - local output_dir="$2" - - if ! output_dir=$("$PYTHON" "$SCRIPT_DIR/mssql_python_build_safety.py" "$output_dir"); then - exit 1 - fi - rm -rf "$output_dir" - mkdir -p "$output_dir" - RESOLVED_OUTPUT_DIR="$output_dir" - - echo "Resolving feed: $feed_url" - PACKAGE_BASE_URL=$("$PYTHON" "$SCRIPT_DIR/resolve_nuget_feed.py" "$feed_url") - if [ -z "$PACKAGE_BASE_URL" ]; then - echo "ERROR: Could not resolve PackageBaseAddress from feed" - exit 1 - fi - PACKAGE_BASE_URL="${PACKAGE_BASE_URL%/}/" - - local version_lower - version_lower=$(echo "$TRANSPORT_VERSION" | tr '[:upper:]' '[:lower:]') - local package_id="mssql-python-rs-wheels" - NUPKG_URL="${PACKAGE_BASE_URL}${package_id}/${version_lower}/${package_id}.${version_lower}.nupkg" - NUPKG_PATH="$output_dir/${package_id}.${version_lower}.nupkg" - local http_status - echo "Downloading: $NUPKG_URL" - if ! http_status=$(curl -sSL -o "$NUPKG_PATH" -w '%{http_code}' "$NUPKG_URL"); then - rm -f "$NUPKG_PATH" - echo "ERROR: Failed to download NuGet package: $package_id $TRANSPORT_VERSION" >&2 - exit 1 - fi - if [ "$http_status" != "200" ]; then - rm -f "$NUPKG_PATH" - echo "ERROR: Failed to download NuGet package: $package_id $TRANSPORT_VERSION (HTTP $http_status)" >&2 - exit 1 - fi - - local filesize - filesize=$(wc -c < "$NUPKG_PATH") - echo "Downloaded: $NUPKG_PATH ($filesize bytes)" - - if [ "$filesize" -eq 0 ]; then - echo "ERROR: Downloaded file is empty" - exit 1 - fi -} - -find_matching_wheel() { - local output_dir="$1" - local extract_dir="$output_dir/extracted" - - mkdir -p "$extract_dir" - if command -v unzip &>/dev/null; then - unzip -q "$NUPKG_PATH" -d "$extract_dir" - else - "$PYTHON" -c "import zipfile, sys; zipfile.ZipFile(sys.argv[1]).extractall(sys.argv[2])" "$NUPKG_PATH" "$extract_dir" - fi - - local wheels_dir="$extract_dir/wheels" - if [ ! -d "$wheels_dir" ]; then - echo "ERROR: No 'wheels' directory found in NuGet package" - ls -la "$extract_dir" - exit 1 - fi - - MATCHING_WHEEL=$(find "$wheels_dir" -name "$WHEEL_PATTERN" -print -quit) - if [ -z "$MATCHING_WHEEL" ]; then - echo "Available wheels:" - ls "$wheels_dir"/*.whl 2>/dev/null || echo " (none)" - echo "ERROR: No wheel found matching: $WHEEL_PATTERN" - exit 1 - fi - - echo "Found: $(basename "$MATCHING_WHEEL")" -} - -install_and_verify() { - local core_dir="$REPO_ROOT/mssql_py_core" - - if [ -d "$core_dir" ]; then - rm -rf "$core_dir" - echo "Cleaned previous mssql_py_core/" - fi - - "$PYTHON" -m pip install --force-reinstall --no-deps "$MATCHING_WHEEL" - "$PYTHON" -c "import importlib.metadata as m, mssql_py_core; assert m.version('mssql-python-rs') == '$DISTRIBUTION_VERSION'; print('mssql-python-rs', m.version('mssql-python-rs'), 'loaded from', mssql_py_core.__file__)" -} - -# --- main --- - -FEED_URL="${FEED_URL:-https://pkgs.dev.azure.com/sqlclientdrivers/public/_packaging/mssql-rs_Public/nuget/v3/index.json}" -OUTPUT_DIR="${TMPDIR:-/tmp}/mssql-python-rs-wheels" - -while [[ $# -gt 0 ]]; do - case "$1" in - --feed-url) FEED_URL="$2"; shift 2 ;; - *) echo "Unknown argument: $1"; exit 1 ;; - esac -done - -echo "=== Install mssql-python-rs from NuGet transport ===" - -RESOLVED_OUTPUT_DIR="" -trap 'if [ -n "$RESOLVED_OUTPUT_DIR" ]; then rm -rf "$RESOLVED_OUTPUT_DIR"; fi' EXIT - -read_version -detect_platform -download_nupkg "$FEED_URL" "$OUTPUT_DIR" -find_matching_wheel "$RESOLVED_OUTPUT_DIR" -install_and_verify -echo "=== mssql-python-rs installed successfully ===" +exec "$PYTHON" "$SCRIPT_DIR/install_mssql_python_rs.py" "$@" diff --git a/eng/scripts/install_mssql_python_rs.py b/eng/scripts/install_mssql_python_rs.py new file mode 100644 index 000000000..8f4e4f2eb --- /dev/null +++ b/eng/scripts/install_mssql_python_rs.py @@ -0,0 +1,113 @@ +"""Install the selected runtime, or download a Windows cross-architecture wheel.""" + +from __future__ import annotations + +import argparse +import os +import platform +import subprocess +import sys +import tempfile +from pathlib import Path + +if __package__: + from .download_mssql_python_rs_wheels import validate_wheel + from .mssql_python_build_safety import resolve_safe_output_directory + from .mssql_python_rs import resolve_runtime, verify_installed +else: + from download_mssql_python_rs_wheels import validate_wheel + from mssql_python_build_safety import resolve_safe_output_directory + from mssql_python_rs import resolve_runtime, verify_installed + + +def install_runtime(target_arch: str = "", output_dir: Path | None = None) -> None: + runtime = resolve_runtime() + host = platform.machine().lower() + aliases = {"amd64": "x64", "x86_64": "x64", "arm64": "arm64", "aarch64": "arm64"} + host_arch = aliases.get(host, host) + cross_arch = bool(target_arch and target_arch != host_arch) + if target_arch and target_arch not in ("x64", "arm64"): + raise ValueError(f"Unsupported target architecture: {target_arch}") + if cross_arch and sys.platform != "win32": + raise ValueError("--target-arch cross-compilation is supported only on Windows") + parent = None + if output_dir is not None: + parent = resolve_safe_output_directory(output_dir) + parent.mkdir(parents=True, exist_ok=True) + transport_settings = {"PIP_CERT", "PIP_CLIENT_CERT", "PIP_PROXY", "PIP_TIMEOUT", "PIP_RETRIES"} + env = { + key: value + for key, value in os.environ.items() + if not key.upper().startswith("PIP_") or key.upper() in transport_settings + } + env["PIP_CONFIG_FILE"] = os.devnull + print( + f"Selected mssql-python-rs=={runtime.version} from {runtime.index_url} " + f"(override={runtime.version_override})", + flush=True, + ) + with tempfile.TemporaryDirectory(prefix="mssql-python-rs-", dir=parent) as directory: + command = [ + sys.executable, + "-m", + "pip", + "--disable-pip-version-check", + "--no-input", + "download", + "--only-binary=:all:", + "--no-deps", + "--index-url", + runtime.index_url, + "--dest", + directory, + ] + if cross_arch: + command.extend( + [ + "--platform", + "win_arm64" if target_arch == "arm64" else "win_amd64", + "--implementation", + "cp", + "--python-version", + "311", + "--abi", + "abi3", + ] + ) + command.append(f"mssql-python-rs=={runtime.version}") + subprocess.run(command, env=env, check=True) + wheels = list(Path(directory).glob("*.whl")) + if len(wheels) != 1: + raise ValueError(f"Expected exactly one runtime wheel, found {len(wheels)}") + validate_wheel(wheels[0], runtime.version) + if cross_arch: + print(f"Validated {wheels[0].name}; not installed into the {host_arch} host") + return + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "--disable-pip-version-check", + "--no-input", + "install", + "--force-reinstall", + "--no-deps", + str(wheels[0]), + ], + env=env, + check=True, + ) + verify_installed(runtime) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--target-arch", choices=("x64", "arm64"), default="") + parser.add_argument("--output-dir", type=Path) + args = parser.parse_args() + install_runtime(args.target_arch, args.output_dir) + + +if __name__ == "__main__": + main() diff --git a/eng/scripts/mssql_python_rs.py b/eng/scripts/mssql_python_rs.py new file mode 100644 index 000000000..0e1608f37 --- /dev/null +++ b/eng/scripts/mssql_python_rs.py @@ -0,0 +1,80 @@ +"""Resolve the native runtime once for installation and Python package metadata.""" + +from __future__ import annotations + +import argparse +import importlib +import importlib.metadata +import json +import os +import re +from dataclasses import asdict, dataclass +from pathlib import Path + +DEFAULT_INDEX = "https://pypi.org/simple/" +DEV_INDEX = ( + "https://pkgs.dev.azure.com/sqlclientdrivers/public/" "_packaging/mssql-rs_Public/pypi/simple/" +) +VERSION_ENV = "MSSQL_PYTHON_RS_VERSION" +VERSION_FILE = Path(__file__).resolve().parents[2] / "eng/versions/mssql-python-rs.version" +_RELEASE = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+") +_VERSION = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+(?:\.dev[0-9]+)?") + + +@dataclass(frozen=True) +class Runtime: + version: str + index_url: str + version_override: bool + + +def resolve_runtime(version_file: Path | None = None, *, version: str | None = None) -> Runtime: + selected = version if version is not None else os.environ.get(VERSION_ENV, "") + overridden = bool(selected.strip()) + if overridden: + selected = selected.strip() + if not _VERSION.fullmatch(selected): + raise ValueError( + f"Invalid {VERSION_ENV}: {selected!r}; use an exact X.Y.Z or X.Y.Z.devN version" + ) + else: + source = version_file if version_file is not None else VERSION_FILE + selected = source.read_text(encoding="ascii").strip() + if not _RELEASE.fullmatch(selected): + raise ValueError(f"{source} must contain an exact released X.Y.Z version") + return Runtime( + selected, + DEV_INDEX if ".dev" in selected else DEFAULT_INDEX, + overridden, + ) + + +def verify_installed(runtime: Runtime) -> None: + installed = importlib.metadata.version("mssql-python-rs") + if installed != runtime.version: + raise ValueError( + f"Installed mssql-python-rs=={installed}, expected {runtime.version}; " + "a later dependency installation may have replaced the selected runtime" + ) + core = importlib.import_module("mssql_py_core") + print(f"Verified mssql-python-rs=={installed} loaded from {core.__file__}") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version-file", type=Path) + parser.add_argument("--version") + parser.add_argument("--write-version-file", type=Path) + parser.add_argument("--verify", action="store_true") + args = parser.parse_args() + runtime = resolve_runtime(args.version_file, version=args.version) + if args.write_version_file: + args.write_version_file.write_text(runtime.version + "\n", encoding="ascii") + if args.verify: + verify_installed(runtime) + else: + print(json.dumps(asdict(runtime))) + + +if __name__ == "__main__": + main() diff --git a/eng/scripts/resolve_pr_rs_runtime.py b/eng/scripts/resolve_pr_rs_runtime.py new file mode 100644 index 000000000..a91a4e516 --- /dev/null +++ b/eng/scripts/resolve_pr_rs_runtime.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Select an exact override or the newest feed-validated nightly runtime for PR tests.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import urllib.request +from dataclasses import asdict, dataclass +from datetime import datetime +from itertools import chain +from pathlib import Path +from urllib.parse import parse_qs, unquote, urldefrag, urljoin, urlsplit + +if __package__: + from .download_mssql_python_rs_wheels import PACKAGE_ID, _WheelLinks + from .mssql_python_build_safety import resolve_safe_output_directory + from .mssql_python_rs import DEV_INDEX, VERSION_ENV, resolve_runtime +else: + from download_mssql_python_rs_wheels import PACKAGE_ID, _WheelLinks + from mssql_python_build_safety import resolve_safe_output_directory + from mssql_python_rs import DEV_INDEX, VERSION_ENV, resolve_runtime + +EXPECTED_PLATFORMS = frozenset( + { + "win_amd64", + "win_arm64", + "manylinux_2_28_x86_64", + "manylinux_2_28_aarch64", + "manylinux_2_34_x86_64", + "manylinux_2_34_aarch64", + "musllinux_1_2_x86_64", + "musllinux_1_2_aarch64", + "macosx_15_0_universal2", + } +) +_NIGHTLY = re.compile(r"([0-9]+)\.([0-9]+)\.([0-9]+)\.dev([0-9]{8})") +_WHEEL = re.compile(r"mssql_python_rs-([^-]+?)(?:-(.*))?\.whl") +_NO_NIGHTLY = ( + "No scheduled nightly (X.Y.Z.devYYYYMMDD) wheels found; " + f"set {VERSION_ENV} to an exact published version to override selection" +) + + +@dataclass(frozen=True) +class RuntimeSelection: + """Overrides are unprobed; nightly evidence consists only of public feed links.""" + + version: str + index_url: str + channel: str + version_override: bool + wheel_sha256: dict[str, str] + wheel_urls: dict[str, str] + + +class _NightlyLinks(_WheelLinks): + def __init__(self) -> None: + super().__init__() + self.yanked: list[str] = [] + self.anchor_open = False + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag == "a": + attributes = dict(attrs) + if self.anchor_open or len(attributes) != len(attrs) or not attributes.get("href"): + raise ValueError("Malformed wheel index anchor") + self.anchor_open = True + if "data-yanked" in attributes: + self.yanked.append(attributes["href"]) + super().handle_starttag(tag, attrs) + + def handle_endtag(self, tag: str) -> None: + if tag == "a": + if not self.anchor_open: + raise ValueError("Malformed wheel index anchor") + self.anchor_open = False + + def handle_data(self, data: str) -> None: + if data.lstrip().startswith("<"): + raise ValueError("Malformed wheel index HTML") + + def close(self) -> None: + if self.anchor_open or self.rawdata.strip(): + raise ValueError("Malformed wheel index HTML") + super().close() + + +def select_nightly(page: str, index_url: str = DEV_INDEX) -> RuntimeSelection: + """Validate link evidence, not publisher build status or commit provenance.""" + package_url = index_url.rstrip("/") + f"/{PACKAGE_ID}/" + if urlsplit(package_url).scheme != "https" or not urlsplit(package_url).hostname: + raise ValueError("The runtime wheel index must use HTTPS") + parser = _NightlyLinks() + parser.feed(page) + parser.close() + candidates = [] + for link, yanked in chain( + ((link, False) for link in parser.links), + ((link, True) for link in parser.yanked), + ): + url, fragment = urldefrag(urljoin(package_url, link)) + parts = urlsplit(url) + if not parts.path or parts.path == "/": + raise ValueError("Invalid wheel URL; HTTPS wheel file required") + filename = unquote(parts.path.rsplit("/", 1)[-1]) + wheel = _WHEEL.fullmatch(filename) + if not wheel: + continue + version, tags = wheel.groups() + nightly = _NIGHTLY.fullmatch(version) + if not nightly: + continue + major, minor, patch, stamp = nightly.groups() + try: + date = datetime.strptime(stamp, "%Y%m%d").date() + except ValueError as exc: + raise ValueError(f"Invalid nightly calendar date in {filename}") from exc + if date.strftime("%Y%m%d") != stamp: + raise ValueError(f"Invalid nightly calendar date in {filename}") + key = (date, int(major), int(minor), int(patch)) + candidates.append((key, version, tags or "", filename, url, fragment, yanked)) + if not candidates: + raise ValueError(_NO_NIGHTLY) + newest = max(candidate[0] for candidate in candidates) + selected = [candidate for candidate in candidates if candidate[0] == newest] + versions = {candidate[1] for candidate in selected} + if len(versions) != 1: + raise ValueError(f"Ambiguous nightly version spellings: {sorted(versions)}") + version = versions.pop() + hashes = {} + urls = {} + platforms = set() + for _, _, tags, filename, url, fragment, yanked in selected: + if filename in urls: + raise ValueError(f"Duplicate wheel filename in index: {filename}") + if yanked: + raise ValueError(f"Newest nightly {version} has a yanked wheel: {filename}") + parts = urlsplit(url) + if parts.scheme != "https" or not parts.hostname or parts.username or parts.password: + raise ValueError(f"Wheel download must use HTTPS without credentials: {filename}") + platform = tags.removeprefix("cp310-abi3-") + if not tags.startswith("cp310-abi3-") or platform not in EXPECTED_PLATFORMS: + raise ValueError(f"Unexpected newest nightly wheel tags: {filename}") + digest = parse_qs(fragment, keep_blank_values=True).get("sha256", []) + if len(digest) != 1 or not re.fullmatch(r"[a-fA-F0-9]{64}", digest[0]): + raise ValueError(f"Missing or invalid SHA-256 for {filename}") + hashes[filename] = digest[0].lower() + urls[filename] = url + platforms.add(platform) + missing = EXPECTED_PLATFORMS - platforms + if missing: + raise ValueError( + f"Newest nightly {version} is incomplete; missing platforms: {', '.join(sorted(missing))}" + ) + return RuntimeSelection(version, index_url, "nightly", False, hashes, urls) + + +def resolve_pr_runtime(*, version: str | None = None) -> RuntimeSelection: + selected = version if version is not None else os.environ.get(VERSION_ENV, "") + if selected.strip(): + runtime = resolve_runtime(version=selected) + return RuntimeSelection(runtime.version, runtime.index_url, "override", True, {}, {}) + package_url = DEV_INDEX.rstrip("/") + f"/{PACKAGE_ID}/" + with urllib.request.urlopen(package_url, timeout=120) as response: + page = response.read().decode("utf-8") + return select_nightly(page) + + +def write_selection(selection: RuntimeSelection, output_dir: Path) -> Path: + output = resolve_safe_output_directory(output_dir) + # Validate again before publishing a version into an Azure logging command. + runtime = resolve_runtime(version=selection.version) + if runtime.version != selection.version or not selection.version: + raise ValueError("Selection version must be an exact validated version") + output.mkdir(parents=True, exist_ok=True) + for filename in ("runtime-selection.json", "mssql-python-rs.version"): + destination = output / filename + if destination.is_symlink() or destination.exists() and not destination.is_file(): + raise ValueError(f"Unsafe selection artifact destination: {destination}") + (output / "runtime-selection.json").write_text( + json.dumps(asdict(selection), indent=2) + "\n", encoding="utf-8", newline="\n" + ) + (output / "mssql-python-rs.version").write_text( + selection.version + "\n", encoding="ascii", newline="\n" + ) + return output + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version") + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + selection = resolve_pr_runtime(version=args.version) + write_selection(selection, args.output_dir) + label = "feed-validated nightly" if selection.channel == "nightly" else "exact override" + print(f"Selected {label}: {selection.version} from {selection.index_url}") + print(f"##vso[task.setvariable variable=version;isOutput=true]{selection.version}") + + +if __name__ == "__main__": + main() diff --git a/eng/versions/mssql-python-rs-nuget.version b/eng/versions/mssql-python-rs-nuget.version deleted file mode 100644 index 9325c3ccd..000000000 --- a/eng/versions/mssql-python-rs-nuget.version +++ /dev/null @@ -1 +0,0 @@ -0.3.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 2b18cd919..a7a69563b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ pytest pytest-cov pytest-asyncio +PyYAML zstandard coverage unittest-xml-reporting @@ -16,6 +17,7 @@ azure-identity # Build dependencies pybind11 setuptools +wheel # Code formatting and linting black diff --git a/setup.py b/setup.py index e2f68457e..f748be834 100644 --- a/setup.py +++ b/setup.py @@ -7,15 +7,14 @@ from setuptools.dist import Distribution from wheel.bdist_wheel import bdist_wheel +from eng.scripts.mssql_python_rs import resolve_runtime + PROJECT_ROOT = Path(__file__).resolve().parent def _read_mssql_python_rs_version() -> str: version_file = PROJECT_ROOT / "eng" / "versions" / "mssql-python-rs.version" - version = version_file.read_text(encoding="utf-8").strip() - if not version: - raise SystemExit(f"Version file is empty: {version_file}") - return version + return resolve_runtime(version_file).version def _read_odbc_version() -> str: diff --git a/tests/test_030_pe_machine_assert.py b/tests/test_030_pe_machine_assert.py index 9f2b77852..cb80c8ab0 100644 --- a/tests/test_030_pe_machine_assert.py +++ b/tests/test_030_pe_machine_assert.py @@ -118,12 +118,16 @@ def test_zstd_backend_is_available_for_conda_audit_tests(): ) -def test_binding_wheel_excludes_core_and_requires_source_rs_version(tmp_path): +def test_binding_wheel_excludes_core_and_requires_source_rs_version(tmp_path, monkeypatch): + monkeypatch.delenv("MSSQL_PYTHON_RS_VERSION", raising=False) pytest.importorskip("setuptools", reason="Wheel archive regression requires setuptools") pytest.importorskip("wheel", reason="Wheel archive regression requires the wheel build backend") shutil.copy2(_ROOT / "setup.py", tmp_path / "setup.py") sources = { "PyPI_Description.md": "Packaging fixture", + "eng/scripts/mssql_python_rs.py": (_ROOT / "eng/scripts/mssql_python_rs.py").read_text( + encoding="utf-8" + ), "mssql_python/__init__.py": "", "mssql_python_odbc/__init__.py": '__version__ = "18.6.2.1"\n', "mssql_py_core/__init__.py": "from .mssql_py_core import *\n", diff --git a/tests/test_034_conda_verify_cwd.py b/tests/test_034_conda_verify_cwd.py index befe385c4..3f6d318df 100644 --- a/tests/test_034_conda_verify_cwd.py +++ b/tests/test_034_conda_verify_cwd.py @@ -15,6 +15,7 @@ """ import importlib.machinery +import json import os import subprocess import sys @@ -718,6 +719,48 @@ def _rs_inputs(tmp_path, platform="win_amd64", python_tags=("cp313",)): return code, odbc, links, rs +@pytest.mark.parametrize( + "problem", ["valid", "distribution", "source", "index", "hash", "missing-hash", "override"] +) +def test_gather_rs_receipt_uses_producer_version_and_bytes(tmp_path, monkeypatch, capsys, problem): + code, odbc, links, rs = _rs_inputs(tmp_path) + selected = next(rs.glob("*.whl")) + assertion = tmp_path / "mssql-python-rs.version" + assertion.write_text("0.2.0\n" if problem == "source" else "0.1.0\n") + receipt = { + "distribution_version": "0.2.0" if problem == "distribution" else "0.1.0", + "index_url": build.inputs.DEV_INDEX if problem == "index" else build.inputs.DEFAULT_INDEX, + "version_override": "false" if problem == "override" else False, + "wheel_sha256": { + selected.name: "a" * 64 if problem == "hash" else build.file_sha256(selected) + }, + } + if problem == "missing-hash": + receipt["wheel_sha256"] = {} + (rs / "transport.json").write_text(json.dumps(receipt)) + monkeypatch.setenv("MSSQL_PYTHON_RS_VERSION", "9.9.9.dev123") + arguments = ( + str(code), + "*.whl", + str(odbc), + "*.whl", + str(links), + str(rs), + str(assertion), + "win-64", + "3.13", + ) + if problem == "valid": + assert build.gather_wheels(*arguments) == ("1.2.3", "18.6.2", "0.1.0") + assert (links / selected.name).read_bytes() == selected.read_bytes() + else: + with pytest.raises(SystemExit) as error: + build.gather_wheels(*arguments) + assert error.value.code == 1 + assert "RS" in capsys.readouterr().err + assert not list(links.iterdir()) + + @pytest.mark.parametrize("with_rs", [False, True]) @pytest.mark.parametrize("recorded", [False, True]) @pytest.mark.parametrize( diff --git a/tests/test_036_conda_provenance.py b/tests/test_036_conda_provenance.py index 445b13cbc..e2dc27075 100644 --- a/tests/test_036_conda_provenance.py +++ b/tests/test_036_conda_provenance.py @@ -362,7 +362,8 @@ def download(argv, *, check): "missing-core", "missing-private-driver", "missing-receipt", - "wrong-transport", + "wrong-index", + "stable-rs-override", "missing-selected-wheel", "wrong-selected-wheel", "wrong-wheel-tags", @@ -407,12 +408,14 @@ def test_source_bound_release_components(tmp_path, monkeypatch, capsys, problem, selected = f"mssql_python_rs-0.2.0-{rs_tag}-win_amd64.whl" receipt = { "distribution_version": "0.2.0", - "transport_version": "0.2.0-dev.transport", - "feed_url": inputs.DEFAULT_FEED, - "package_id": inputs.PACKAGE_ID, - "package_sha256": "a" * 64, + "index_url": inputs.DEFAULT_INDEX, + "version_override": False, "wheel_sha256": {selected: "b" * 64}, } + if problem == "wrong-index": + receipt["index_url"] = inputs.DEV_INDEX + elif problem == "stable-rs-override": + receipt["version_override"] = True if with_rs: files[rs_prefix + "conda-wheel-source.txt"] = (selected + "\n").encode() if "-alias-" in problem: @@ -517,14 +520,6 @@ def test_source_bound_release_components(tmp_path, monkeypatch, capsys, problem, monkeypatch.setenv( "RELEASE_VERSIONS", json.dumps(versions) if problem != "missing-version-env" else "" ) - monkeypatch.setenv( - "RS_TRANSPORT_VERSION", - ( - "wrong" - if problem == "wrong-transport" - else "0.2.0-dev.transport" if "mssql-python-rs" in versions else "" - ), - ) result = cli.main( [ "validate", @@ -541,9 +536,14 @@ def test_source_bound_release_components(tmp_path, monkeypatch, capsys, problem, "--release-versions", ] ) + output = capsys.readouterr() assert result == ( 0 if problem in {"legacy", "rs", "rs-extra-owned", "rs-pyc-owned"} else 1 - ), capsys.readouterr() + ), output + if problem == "stable-rs-override": + assert "RS version overrides are not eligible for Conda release." in output.err + assert "COMPONENT_INPUT_OK" not in output.out + assert "release-ready" not in output.out @pytest.mark.parametrize( @@ -1162,7 +1162,24 @@ def read_source(path): ] -@pytest.mark.parametrize("version", ["", "0.2.0\n0.3.0", "$(version)", "../0.2.0", "0.2.0;bad"]) +def test_helper_based_setup_source_uses_release_pin_not_consumer_override( + release_sources, monkeypatch +): + _source_dependencies(release_sources, '[f"mssql-python-rs=={_read_mssql_python_rs_version()}"]') + release_sources["setup.py"] = ( + "from eng.scripts.mssql_python_rs import resolve_runtime\n" + "def _read_mssql_python_rs_version():\n" + " return resolve_runtime(version_file).version\n" + release_sources["setup.py"] + ) + release_sources["eng/versions/mssql-python-rs.version"] = "0.2.0\n" + monkeypatch.setenv("MSSQL_PYTHON_RS_VERSION", "0.3.0.dev123") + assert inputs.read_release_versions(release_sources.__getitem__)["mssql-python-rs"] == "0.2.0" + + +@pytest.mark.parametrize( + "version", + ["", "0.2.0\n0.3.0", "$(version)", "../0.2.0", "0.2.0;bad", "0.2.0.dev123", "0.2.0+local"], +) def test_rs_distribution_pin_must_be_present_and_unambiguous(release_sources, version): _source_dependencies(release_sources, '[f"mssql-python-rs=={_read_mssql_python_rs_version()}"]') release_sources["eng/versions/mssql-python-rs.version"] = version @@ -1217,10 +1234,6 @@ def test_recorded_rs_source_is_read_only_at_verified_wheel_commit(release_source record.pop("content") if problem != "missing": chain[key] = record - transport_key, transport = _source_record( - "eng/versions/mssql-python-rs-nuget.version", "0.2.0-dev.transport" - ) - chain[transport_key] = transport reads = [] def get_json(path): @@ -1232,39 +1245,13 @@ def get_json(path): if problem == "valid": result = _verify(chain, get_json=get_json) assert result["versions"]["mssql-python-rs"] == "0.2.0" - assert result["rsTransportVersion"] == "0.2.0-dev.transport" - assert reads[-2:] == [key, transport_key] + assert "rsTransportVersion" not in result + assert reads[-1] == key else: with pytest.raises((ValueError, KeyError, HTTPError)): _verify(chain, get_json=get_json) assert reads[-1] == key - assert len(reads) == (9 if problem == "valid" else 8) - - -@pytest.mark.parametrize("problem", ["missing", "wrong-commit", "invalid", "forbidden"]) -def test_rs_transport_source_cannot_fall_back_to_distribution_version( - release_sources, chain, problem -): - _source_dependencies(release_sources, '[f"mssql-python-rs=={_read_mssql_python_rs_version()}"]') - chain.update([_source_record("setup.py", release_sources["setup.py"])]) - chain.update([_source_record("eng/versions/mssql-python-rs.version", "0.2.0")]) - key, record = _source_record( - "eng/versions/mssql-python-rs-nuget.version", "0.2.0-dev.transport" - ) - if problem == "wrong-commit": - record["commitId"] = "c" * 40 - elif problem == "invalid": - record["content"] = "0.2.0\n0.3.0" - if problem != "missing": - chain[key] = record - - def get_json(path): - if path == key and problem == "forbidden": - raise HTTPError("https://dev.azure.com", 403, "Forbidden", {}, None) - return chain[path] - - with pytest.raises((ValueError, KeyError, HTTPError)): - _verify(chain, get_json=get_json) + assert len(reads) == 8 @pytest.mark.parametrize( @@ -1273,10 +1260,10 @@ def get_json(path): "valid", "absent", "distribution", - "transport", - "feed", - "package", - "package-hash", + "index", + "override-absent", + "override-nonbool", + "legacy-transport", "empty-wheels", "wheel-hash", "wheel-version", @@ -1286,25 +1273,29 @@ def get_json(path): def test_rs_transport_receipt_is_source_bound_without_inferred_producer_id(problem): receipt = { "distribution_version": "0.2.0", - "transport_version": "0.2.0-dev.transport", - "feed_url": ( - "https://pkgs.dev.azure.com/sqlclientdrivers/public/" - "_packaging/mssql-rs_Public/nuget/v3/index.json" - ), - "package_id": "mssql-python-rs-wheels", - "package_sha256": "a" * 64, + "index_url": inputs.DEFAULT_INDEX, + "version_override": False, "wheel_sha256": {"mssql_python_rs-0.2.0-cp313-cp313-win_amd64.whl": "b" * 64}, - "nuspec_description": "InternalBuild; no independently verified numeric producer ID", } fields = { "distribution": "distribution_version", - "transport": "transport_version", - "feed": "feed_url", - "package": "package_id", - "package-hash": "package_sha256", + "index": "index_url", } if problem in fields: receipt[fields[problem]] = "wrong" + elif problem == "override-absent": + receipt.pop("version_override") + elif problem == "override-nonbool": + receipt["version_override"] = "false" + elif problem == "legacy-transport": + receipt.pop("index_url") + receipt.pop("version_override") + receipt.update( + transport_version="0.2.0-dev.transport", + feed_url="https://example.test/nuget/v3/index.json", + package_id="mssql-python-rs-wheels", + package_sha256="a" * 64, + ) elif problem == "absent": receipt = None elif problem == "empty-wheels": @@ -1318,7 +1309,7 @@ def test_rs_transport_receipt_is_source_bound_without_inferred_producer_id(probl else: name = "../" + name receipt["wheel_sha256"] = {name: digest} - arguments = (receipt, {"mssql-python-rs": "0.2.0"}, "0.2.0-dev.transport") + arguments = (receipt, {"mssql-python-rs": "0.2.0"}) if problem == "valid": inputs.validate_rs_transport(*arguments) else: @@ -1327,11 +1318,38 @@ def test_rs_transport_receipt_is_source_bound_without_inferred_producer_id(probl def test_verified_legacy_transport_absence_is_not_a_new_producer_fallback(): - inputs.validate_rs_transport(None, {"mssql-python": "1.15.0"}, "") + inputs.validate_rs_transport(None, {"mssql-python": "1.15.0"}) with pytest.raises(ValueError, match="embedded-core"): - inputs.validate_rs_transport({}, {"mssql-python": "1.15.0"}, "") + inputs.validate_rs_transport({}, {"mssql-python": "1.15.0"}) with pytest.raises(ValueError, match="recorded"): - inputs.validate_rs_transport(None, {"mssql-python-rs": "0.2.0"}, "") + inputs.validate_rs_transport(None, {"mssql-python-rs": "0.2.0"}) + + +@pytest.mark.parametrize( + ("version", "index", "override", "valid"), + [ + ("0.2.0", inputs.DEFAULT_INDEX, False, True), + ("0.2.0", inputs.DEFAULT_INDEX, True, True), + ("0.2.0", inputs.DEV_INDEX, False, False), + ("0.2.0.dev123", inputs.DEV_INDEX, True, True), + ("0.2.0.dev123", inputs.DEFAULT_INDEX, True, False), + ("0.2.0.dev123", inputs.DEV_INDEX, False, False), + ], +) +def test_rs_index_receipt_cannot_switch_release_and_development_sources( + version, index, override, valid +): + receipt = { + "distribution_version": version, + "index_url": index, + "version_override": override, + "wheel_sha256": {f"mssql_python_rs-{version}-cp310-abi3-win_amd64.whl": "a" * 64}, + } + if valid: + inputs.validate_rs_transport(receipt, {"mssql-python-rs": version}) + else: + with pytest.raises(ValueError, match="distribution pin/index"): + inputs.validate_rs_transport(receipt, {"mssql-python-rs": version}) @pytest.mark.parametrize( diff --git a/tests/test_037_rs_wheel_download.py b/tests/test_037_rs_wheel_download.py index c03ded8ed..e7ff6920d 100644 --- a/tests/test_037_rs_wheel_download.py +++ b/tests/test_037_rs_wheel_download.py @@ -8,169 +8,132 @@ import pytest -SCRIPTS_DIR = Path(__file__).parents[1] / "eng" / "scripts" -MODULE_PATH = SCRIPTS_DIR / "download_mssql_python_rs_wheels.py" - -if not MODULE_PATH.is_file() or not (Path(__file__).parents[1] / "OneBranchPipelines").is_dir(): +ROOT = Path(__file__).resolve().parents[1] +if ( + not (ROOT / "eng/scripts/download_mssql_python_rs_wheels.py").is_file() + or not (ROOT / "OneBranchPipelines").is_dir() +): pytest.skip( - "rs wheel download contracts require a complete source checkout", - allow_module_level=True, + "Runtime staging contracts require a complete source checkout", allow_module_level=True ) - from eng.scripts import download_mssql_python_rs_wheels as downloader +from eng.scripts.mssql_python_rs import DEFAULT_INDEX, DEV_INDEX, VERSION_ENV + +@pytest.fixture(autouse=True) +def clear_override(monkeypatch): + monkeypatch.delenv(VERSION_ENV, raising=False) -def _package(entries, transport_version="0.1.0-dev.1"): - package = io.BytesIO() - with zipfile.ZipFile(package, "w") as archive: + +def _wheel(version="0.3.0", metadata_version=None): + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as archive: archive.writestr( - "mssql-python-rs-wheels.nuspec", - "mssql-python-rs-wheels" - f"{transport_version}" - "RS transport fixture", + f"mssql_python_rs-{version}.dist-info/METADATA", + f"Name: mssql-python-rs\nVersion: {metadata_version or version}\n", ) - for name in entries: - archive.writestr(name, name.encode("ascii")) - package.seek(0) - return package.getvalue() + return stream.getvalue() -def _mock_download(monkeypatch, module, content, package_base="https://example.test/flat/"): +def _index(monkeypatch, *, version="0.3.0", content=None, duplicate=False, hash_value=None): + content = _wheel(version) if content is None else content + name = f"mssql_python_rs-{version}-cp310-abi3-win_amd64.whl" + digest = hash_value or hashlib.sha256(content).hexdigest() + anchor = f'{name}' + page = (anchor * (2 if duplicate else 1)).encode() requested = [] - monkeypatch.setattr(module, "resolve", lambda _url: package_base) def urlopen(url, timeout): - requested.append((url, timeout)) - return io.BytesIO(content) + requested.append(url) + return io.BytesIO(page if url.endswith("/mssql-python-rs/") else content) - monkeypatch.setattr(module.urllib.request, "urlopen", urlopen) - return requested + monkeypatch.setattr(downloader.urllib.request, "urlopen", urlopen) + return name, content, requested -def test_downloads_only_matching_wheels_and_clears_stale_output(tmp_path, monkeypatch): - module = downloader - version_file = tmp_path / "version" - version_file.write_text("0.1.0\n", encoding="ascii") - transport_file = tmp_path / "transport-version" - transport_file.write_text("0.1.0-dev.20260914.174990\n", encoding="ascii") - output = tmp_path / "wheels" - output.mkdir() - (output / "stale.whl").write_bytes(b"stale") - content = _package( - [ - "wheels/mssql_python_rs-0.1.0-cp310-abi3-win_amd64.whl", - "symbols/debug.pdb", - ], - "0.1.0-dev.20260914.174990", - ) - requested = _mock_download(monkeypatch, module, content, "https://example.test/flat") +def _pin(tmp_path): + path = tmp_path / "version" + path.write_text("0.3.0\n", encoding="ascii") + return path - wheels = module.download_wheels( - "https://example.test/index.json", version_file, transport_file, output - ) - assert [wheel.name for wheel in wheels] == ["mssql_python_rs-0.1.0-cp310-abi3-win_amd64.whl"] - assert sorted(path.name for path in output.iterdir()) == [wheels[0].name, "transport.json"] - receipt = json.loads((output / "transport.json").read_text()) - assert receipt == { - "distribution_version": "0.1.0", - "transport_version": "0.1.0-dev.20260914.174990", - "feed_url": "https://example.test/index.json", - "package_id": "mssql-python-rs-wheels", - "package_sha256": hashlib.sha256(content).hexdigest(), - "nuspec_description": "RS transport fixture", - "wheel_sha256": {wheels[0].name: hashlib.sha256(wheels[0].read_bytes()).hexdigest()}, +@pytest.mark.parametrize("override", [None, "0.4.0.dev20261028187000"]) +def test_stages_selected_wheels_and_receipt(tmp_path, monkeypatch, override): + pin = _pin(tmp_path) + if override: + monkeypatch.setenv(VERSION_ENV, override) + version = override or "0.3.0" + name, content, requested = _index(monkeypatch, version=version) + output = tmp_path / "out" + output.mkdir() + (output / "stale.whl").touch() + + wheels = downloader.download_wheels(None, pin, output) + + index = DEV_INDEX if override else DEFAULT_INDEX + assert wheels == [output / name] + assert sorted(path.name for path in output.iterdir()) == [name, "transport.json"] + assert json.loads((output / "transport.json").read_text()) == { + "distribution_version": version, + "index_url": index, + "version_override": bool(override), + "wheel_sha256": {name: hashlib.sha256(content).hexdigest()}, } assert requested == [ - ( - "https://example.test/flat/mssql-python-rs-wheels/0.1.0-dev.20260914.174990/" - "mssql-python-rs-wheels.0.1.0-dev.20260914.174990.nupkg", - 120, - ) + index + "mssql-python-rs/", + f"https://files.example.test/{name}", ] -def test_rejects_wheel_with_unexpected_version(tmp_path, monkeypatch): - module = downloader - version_file = tmp_path / "version" - version_file.write_text("0.1.0", encoding="ascii") - transport_file = tmp_path / "transport-version" - transport_file.write_text("0.1.0-dev.1", encoding="ascii") - content = _package(["wheels/mssql_python_rs-0.1.1-cp310-abi3-win_amd64.whl"]) - _mock_download(monkeypatch, module, content) - - with pytest.raises(ValueError, match="Unexpected wheel name"): - module.download_wheels( - "https://example.test/index.json", - version_file, - transport_file, - tmp_path / "out", - ) +def test_explicit_version_selects_dev_index(tmp_path, monkeypatch): + _index(monkeypatch, version="0.4.0.dev1") + wheels = downloader.download_wheels( + None, _pin(tmp_path), tmp_path / "out", version="0.4.0.dev1" + ) + assert wheels[0].name.startswith("mssql_python_rs-0.4.0.dev1-") -def test_rejects_duplicate_wheel_filename(tmp_path, monkeypatch): - module = downloader - version_file = tmp_path / "version" - version_file.write_text("0.1.0", encoding="ascii") - transport_file = tmp_path / "transport-version" - transport_file.write_text("0.1.0-dev.1", encoding="ascii") - wheel = "wheels/mssql_python_rs-0.1.0-cp310-abi3-win_amd64.whl" - with pytest.warns(UserWarning, match="Duplicate name"): - content = _package([wheel, wheel]) - _mock_download(monkeypatch, module, content) - - with pytest.raises(ValueError, match="Duplicate wheel filename"): - module.download_wheels( - "https://example.test/index.json", - version_file, - transport_file, - tmp_path / "out", - ) +@pytest.mark.parametrize( + ("options", "error"), + [ + ({"duplicate": True}, "Duplicate wheel filename"), + ({"hash_value": "0" * 64}, "SHA-256 mismatch"), + ({"hash_value": "bad"}, "Missing or invalid SHA-256"), + ({"content": _wheel(metadata_version="0.4.0")}, "metadata does not match"), + ({"version": "0.4.0"}, "No mssql-python-rs==0.3.0 wheels"), + ], +) +def test_rejects_bad_or_missing_wheels(tmp_path, monkeypatch, options, error): + _index(monkeypatch, **options) + with pytest.raises(ValueError, match=error): + downloader.download_wheels(None, _pin(tmp_path), tmp_path / "out") + assert not (tmp_path / "out" / "transport.json").exists() -def test_rejects_filesystem_root_as_output(tmp_path): - module = downloader - version_file = tmp_path / "version" - version_file.write_text("0.1.0", encoding="ascii") - transport_file = tmp_path / "transport-version" - transport_file.write_text("0.1.0-dev.1", encoding="ascii") +def test_missing_requested_dev_build_does_not_fall_back(tmp_path, monkeypatch): + _index(monkeypatch) + monkeypatch.setenv(VERSION_ENV, "0.4.0.dev1") + with pytest.raises(ValueError, match="No mssql-python-rs==0.4.0.dev1"): + downloader.download_wheels(None, _pin(tmp_path), tmp_path / "out") + +def test_rejects_filesystem_root_as_output(tmp_path): with pytest.raises(ValueError, match="filesystem root"): - module.download_wheels( - "https://example.test/index.json", - version_file, - transport_file, - Path(tmp_path.anchor), - ) + downloader.download_wheels(None, _pin(tmp_path), Path(tmp_path.anchor)) def test_rejects_working_directory_and_ancestor_as_output(tmp_path, monkeypatch): - module = downloader - version_file = tmp_path / "version" - version_file.write_text("0.1.0", encoding="ascii") - transport_file = tmp_path / "transport-version" - transport_file.write_text("0.1.0-dev.1", encoding="ascii") - working_directory = tmp_path / "checkout" / "subdirectory" - working_directory.mkdir(parents=True) - monkeypatch.chdir(working_directory) - - for unsafe in (working_directory, working_directory.parent): + pin = _pin(tmp_path) + cwd = tmp_path / "checkout" / "subdirectory" + cwd.mkdir(parents=True) + monkeypatch.chdir(cwd) + for unsafe in (cwd, cwd.parent): with pytest.raises(ValueError, match="current working directory|ancestors"): - module.download_wheels( - "https://example.test/index.json", - version_file, - transport_file, - unsafe, - ) - - -def test_rejects_symlink_output_without_deleting_target(tmp_path, monkeypatch): - module = downloader - version_file = tmp_path / "version" - version_file.write_text("0.1.0", encoding="ascii") - transport_file = tmp_path / "transport-version" - transport_file.write_text("0.1.0-dev.1", encoding="ascii") + downloader.download_wheels(None, pin, unsafe) + + +def test_rejects_symlink_output_without_deleting_target(tmp_path): target = tmp_path / "unrelated" target.mkdir() marker = target / "keep.txt" @@ -184,35 +147,25 @@ def test_rejects_symlink_output_without_deleting_target(tmp_path, monkeypatch): ) else: output.symlink_to(target, target_is_directory=True) - - content = _package(["wheels/mssql_python_rs-0.1.0-cp310-abi3-win_amd64.whl"]) - _mock_download(monkeypatch, module, content) - with pytest.raises(ValueError, match="symbolic link|junction"): - module.download_wheels( - "https://example.test/index.json", version_file, transport_file, output - ) - + downloader.download_wheels(None, _pin(tmp_path), output) assert marker.read_text(encoding="ascii") == "keep" -def test_stress_jobs_resolve_dependency_from_pinned_transport(): - pipeline = ( - Path(__file__).parents[1] / "OneBranchPipelines" / "stress-test-pipeline.yml" - ).read_text(encoding="utf-8") - - assert pipeline.count("download_mssql_python_rs_wheels.py --output-dir") == 2 - assert pipeline.count("--find-links=$(Pipeline.Workspace)\\mssql-python-rs-wheels") == 1 - assert pipeline.count("--find-links=$(Pipeline.Workspace)/mssql-python-rs-wheels") == 1 - +def test_rejects_insecure_index(tmp_path): + with pytest.raises(ValueError, match="HTTPS"): + downloader.download_wheels( + "http://example.test/simple/", # DevSkim: ignore DS137138 - HTTPS rejection fixture + _pin(tmp_path), + tmp_path / "out", + ) -@pytest.mark.parametrize( - "script_name", - ["install-mssql-py-core.ps1", "install-mssql-py-core.sh"], -) -def test_local_installers_select_stable_abi_wheel(script_name): - script = (SCRIPTS_DIR / script_name).read_text(encoding="utf-8") - assert "cp310-abi3" in script - assert "PyVersion-$script:PyVersion" not in script - assert "PY_VERSION}-${PY_VERSION" not in script +def test_ignores_yanked_wheels(tmp_path, monkeypatch): + name = "mssql_python_rs-0.3.0-cp310-abi3-win_amd64.whl" + page = f'wheel' + monkeypatch.setattr( + downloader.urllib.request, "urlopen", lambda *args, **kwargs: io.BytesIO(page.encode()) + ) + with pytest.raises(ValueError, match="No mssql-python-rs"): + downloader.download_wheels(None, _pin(tmp_path), tmp_path / "out") diff --git a/tests/test_038_mssql_odbc_daily_validation.py b/tests/test_038_mssql_odbc_daily_validation.py index 11c76c710..7a63fcc40 100644 --- a/tests/test_038_mssql_odbc_daily_validation.py +++ b/tests/test_038_mssql_odbc_daily_validation.py @@ -19,7 +19,7 @@ @unittest.skipUnless(sys.platform.startswith("linux"), "runner requires Linux GNU timeout and bash") class RunnerTests(unittest.TestCase): def run_runner(self, behavior: str, total_budget: str = "60s", provider: str = "mssql-odbc"): - with tempfile.TemporaryDirectory() as directory: + with tempfile.TemporaryDirectory(dir=ROOT, prefix=".odbc-contract-") as directory: work = Path(directory) (work / "tests").mkdir() (work / "tests" / "test_sample.py").write_text("pass\n", encoding="utf-8") @@ -120,11 +120,14 @@ def test_release_wheel_fixture_includes_runner_contract_files(self): "eng/scripts/run-mssql-odbc-tests.sh", "eng/scripts/verify_mssql_odbc_provider.py", "eng/pipelines/mssql-odbc-daily-validation-pipeline.yml", - "eng/versions/mssql-python-rs-nuget.version", + "eng/versions/mssql-python-rs.version", "OneBranchPipelines/stages/build-linux-single-stage.yml", ): self.assertEqual(pipeline.count(f"/workspace/{path}"), 2) self.assertNotRegex(pipeline, r"\bcp\s+-r\s+/workspace/eng(?:\s|$)") + self.assertNotIn("mssql-python-rs-nuget.version", pipeline) + self.assertEqual(pipeline.count("cp -r /workspace/eng/pipelines/."), 2) + self.assertEqual(pipeline.count("cp -r /workspace/OneBranchPipelines/."), 2) def test_runner_uses_busybox_compatible_timeout_options(self): runner = RUNNER.read_text(encoding="utf-8") @@ -133,7 +136,7 @@ def test_runner_uses_busybox_compatible_timeout_options(self): self.assertNotIn("--kill-after", runner) def test_optimized_preflight_rejects_wrong_provider(self): - with tempfile.TemporaryDirectory() as directory: + with tempfile.TemporaryDirectory(dir=ROOT, prefix=".odbc-contract-") as directory: fake_module = Path(directory) / "mssql_python.py" fake_module.write_text( "def get_native_provider_info():\n" @@ -162,12 +165,20 @@ def test_pipeline_authenticates_advisory_status_and_keeps_publish_strict(self): self.assertIn('exit "$cleanup_rc"', pipeline) self.assertIn("grep -q 'MSSQL_ODBC_PREFLIGHT_OK'", pipeline) - def test_stable_rs_transport_is_pinned(self): - version = (ROOT / "eng" / "versions" / "mssql-python-rs-nuget.version").read_text( - encoding="ascii" - ) + def test_default_rs_runtime_is_an_exact_pypi_release(self): + version_file = ROOT / "eng" / "versions" / "mssql-python-rs.version" + version = version_file.read_text(encoding="ascii") + + self.assertRegex(version.strip(), r"^[0-9]+\.[0-9]+\.[0-9]+$") + self.assertFalse(version_file.with_name("mssql-python-rs-nuget.version").exists()) + + def test_daily_pipeline_carries_explicit_override_into_test_container(self): + pipeline = PIPELINE.read_text(encoding="utf-8") - self.assertEqual(version.strip(), "0.3.0") + self.assertIn("name: rsVersionOverride", pipeline) + self.assertIn("MSSQL_PYTHON_RS_VERSION: ${{ parameters.rsVersionOverride }}", pipeline) + self.assertIn("docker run -e MSSQL_PYTHON_RS_VERSION ", pipeline) + self.assertNotIn("$(MSSQL_PYTHON_RS_VERSION)", pipeline) if __name__ == "__main__": diff --git a/tests/test_039_rs_runtime_pipelines.py b/tests/test_039_rs_runtime_pipelines.py new file mode 100644 index 000000000..fa58bcd10 --- /dev/null +++ b/tests/test_039_rs_runtime_pipelines.py @@ -0,0 +1,614 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Source-only contracts for runtime selection and the pre-ESRP release gates.""" + +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +from types import SimpleNamespace +import uuid +import zipfile + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +PIPELINES = ROOT / "OneBranchPipelines" +SOURCE_PREREQUISITES = ( + "OneBranchPipelines/scripts/validate-release-versions.ps1", + "eng/scripts/mssql_python_rs.py", + "eng/scripts/resolve_pr_rs_runtime.py", + "eng/conda_tools/archive.py", + "eng/conda_tools/contracts.py", +) +if not all((ROOT / relative).is_file() for relative in SOURCE_PREREQUISITES): + pytest.skip("Pipeline contracts require a complete source checkout", allow_module_level=True) + +import yaml + +VERSION_ENV = "MSSQL_PYTHON_RS_VERSION" +ENTRY_POINTS = ( + "eng/pipelines/pr-validation-pipeline.yml", + "eng/pipelines/mssql-odbc-daily-validation-pipeline.yml", + "OneBranchPipelines/build-release-package-pipeline.yml", + "OneBranchPipelines/stress-test-pipeline.yml", +) + + +@pytest.mark.parametrize("filename", ENTRY_POINTS) +def test_pipeline_override_is_explicit_and_defaults_to_empty(filename): + source = (ROOT / filename).read_text(encoding="utf-8") + assert re.search(r"name: rsVersionOverride[\s\S]*?default: ''", source) + assert VERSION_ENV in source + assert "${{ parameters.rsVersionOverride }}" in source + assert "$(rsVersionOverride)" not in source + assert "$(MSSQL_PYTHON_RS_VERSION)" not in source + + +def test_pr_validation_resolves_nightly_once_and_shares_it_across_all_jobs(): + pipeline = yaml.safe_load((ROOT / ENTRY_POINTS[0]).read_text(encoding="utf-8")) + assert VERSION_ENV not in pipeline["variables"] + jobs = pipeline["jobs"] + selectors = [job for job in jobs if job["job"] == "SelectRsRuntime"] + assert len(selectors) == 1 + selector = selectors[0] + selection_steps = [ + step + for job in jobs + for step in job["steps"] + if "resolve_pr_rs_runtime.py" in step.get("script", "") + ] + assert len(selection_steps) == 1 + selection = selection_steps[0] + assert selection in selector["steps"] + assert selection["name"] == "select" + assert selection["env"][VERSION_ENV] == "${{ parameters.rsVersionOverride }}" + assert "--output-dir" in selection["script"] + artifacts = [ + step["inputs"] + for step in selector["steps"] + if step.get("task") == "PublishPipelineArtifact@1" + ] + assert artifacts == [ + { + "targetPath": "$(Build.ArtifactStagingDirectory)/rs-runtime", + "artifact": "rs-runtime-selection", + } + ] + consumers = [job for job in jobs if job is not selector] + assert len(consumers) == 10 + for job in consumers: + assert job["dependsOn"] == "SelectRsRuntime", job["job"] + assert job["variables"][VERSION_ENV] == ( + "$[ dependencies.SelectRsRuntime.outputs['select.version'] ]" + ), job["job"] + assert "condition" not in job, "A failed selection must block all consumers" + + +def test_official_entry_points_keep_stable_defaults_without_nightly_resolution(): + for path in ( + PIPELINES / "build-release-package-pipeline.yml", + PIPELINES / "official-release-pipeline.yml", + PIPELINES / "conda-release-pipeline.yml", + ): + source = path.read_text(encoding="utf-8") + assert "resolve_pr_rs_runtime.py" not in source, path + assert "SelectRsRuntime" not in source, path + pipeline = yaml.safe_load( + (PIPELINES / "build-release-package-pipeline.yml").read_text(encoding="utf-8") + ) + override = next(p for p in pipeline["parameters"] if p["name"] == "rsVersionOverride") + assert override["default"] == "" + + +def test_modified_pipeline_yaml_parses(): + for folder in (ROOT / "eng/pipelines", PIPELINES): + for path in folder.rglob("*.yml"): + assert isinstance(yaml.safe_load(path.read_text(encoding="utf-8")), dict), path + + +def test_conda_pipeline_no_longer_passes_removed_transport_provenance(): + for path in ( + PIPELINES / "conda-release-pipeline.yml", + PIPELINES / "steps/conda-publish-step.yml", + PIPELINES / "steps/conda-release-step.yml", + ): + source = path.read_text(encoding="utf-8") + for obsolete in ("rsTransportVersion", "RS_TRANSPORT_VERSION", "--rs-transport-version"): + assert obsolete not in source, path + assert "releaseVersions" in source + assert "mssqlPythonVersion" in source + + +def test_every_runtime_container_inherits_version_without_shell_interpolation(): + for filename in ENTRY_POINTS[:2] + ("OneBranchPipelines/stages/build-linux-single-stage.yml",): + source = (ROOT / filename).read_text(encoding="utf-8") + runs = [ + line + for line in source.splitlines() + if not line.lstrip().startswith("#") + and "docker run " in line + and ("--name test-container" in line or "--name build-" in line) + ] + if "daily" in filename: + runs = [line for line in source.splitlines() if "docker run " in line] + assert runs, filename + assert all("-e MSSQL_PYTHON_RS_VERSION " in line for line in runs) + assert not re.search(r"-e MSSQL_PYTHON_RS_VERSION=", source) + installer = (ROOT / "eng/pipelines/steps/install-mssql-py-core.yml").read_text(encoding="utf-8") + assert "docker exec -e MSSQL_PYTHON_RS_VERSION " in installer + + +def test_consolidation_stages_effective_receipt_version_not_source_pin(): + source = (PIPELINES / "jobs/consolidate-artifacts-job.yml").read_text(encoding="utf-8") + assert "receipt['distribution_version']" in source + assert "dist/transport.json" in source + assert "filePath: OneBranchPipelines/scripts/validate-rs-wheel-dependency.ps1" in source + assert "mssql-python-rs-nuget.version" not in source + assert 'cp "$(Build.SourcesDirectory)/eng/versions/mssql-python-rs.version"' not in source + linux = (PIPELINES / "stages/build-linux-single-stage.yml").read_text(encoding="utf-8") + assert "mssql-python-rs-nuget.version" not in linux + + +def test_stress_uses_selected_build_and_checks_effective_runtime_metadata(): + source = (PIPELINES / "stress-test-pipeline.yml").read_text(encoding="utf-8") + assert source.count("pipelineId: '$(resources.pipeline.buildPipeline.runID)'") == 4 + assert ( + source.count("filePath: OneBranchPipelines/scripts/validate-rs-wheel-dependency.ps1") == 2 + ) + assert source.count("'--no-index', '--no-deps'") == 2 + assert source.count("'mssql-python-rs=='+resolve_runtime().version") == 2 + + +@pytest.fixture(params=("StressTestWindows", "StressTestLinux")) +def stress_job(request): + pipeline = yaml.safe_load((PIPELINES / "stress-test-pipeline.yml").read_text(encoding="utf-8")) + return next(job for job in pipeline["jobs"] if job["job"] == request.param) + + +@pytest.mark.parametrize( + "missing", + ( + "eng/scripts/mssql_python_rs.py", + "eng/scripts/resolve_pr_rs_runtime.py", + "eng/conda_tools/archive.py", + "eng/conda_tools/contracts.py", + ), +) +def test_pipeline_contracts_skip_incomplete_wheel_fixture(tmp_path, missing): + shutil.copytree(PIPELINES, tmp_path / "OneBranchPipelines") + for relative in SOURCE_PREREQUISITES[1:]: + if relative != missing: + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + copied_test = tmp_path / "tests" / Path(__file__).name + copied_test.parent.mkdir() + shutil.copy2(__file__, copied_test) + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "--noconftest", + "--collect-only", + "-q", + "-rs", + str(copied_test), + ], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 5, result.stdout + result.stderr + assert "complete source checkout" in result.stdout + assert "SKIPPED [1]" in result.stdout + + +def test_pipeline_contracts_require_yaml_in_complete_checkout(): + command = ( + "import sys, pytest; sys.modules['yaml'] = None; " + "raise SystemExit(pytest.main(" + + repr(["--noconftest", "--collect-only", "-q", str(Path(__file__).resolve())]) + + "))" + ) + result = subprocess.run( + [sys.executable, "-c", command], + cwd=ROOT, + env=dict(os.environ, PYTEST_DISABLE_PLUGIN_AUTOLOAD="1"), + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 2, result.stdout + result.stderr + assert "ModuleNotFoundError" in result.stdout + assert "yaml" in result.stdout + + +@pytest.mark.parametrize("shadow", (None, "mssql_python_odbc", "mssql_python")) +def test_stress_checks_import_provenance_before_binding_load(stress_job, tmp_path, shadow): + checkout = tmp_path / "checkout" + checkout.mkdir() + installed = tmp_path / "checkout-site-packages" + for name in ("mssql_python_odbc", "mssql_python"): + package = installed / name + package.mkdir(parents=True) + (package / "__init__.py").write_text( + "print('BINDING_IMPORTED')\n" if name == "mssql_python" else "", + encoding="utf-8", + ) + if shadow: + package = checkout / shadow + package.mkdir() + (package / "__init__.py").write_text( + "print('BINDING_IMPORTED')\n" if shadow == "mssql_python" else "", + encoding="utf-8", + ) + step = next( + step + for step in stress_job["steps"] + if step.get("displayName") == "Ensure tests use installed wheel" + ) + script = step.get("script", step.get("powershell")) + command = next(line.strip() for line in script.splitlines() if 'python -c "' in line) + code = command.removeprefix('python -c "').removesuffix('"') + result = subprocess.run( + [sys.executable, "-c", code], + cwd=checkout, + env=dict( + os.environ, + PYTHONPATH=str(installed), + BUILD_SOURCESDIRECTORY=str(checkout), + ), + capture_output=True, + text=True, + timeout=30, + ) + if shadow: + assert result.returncode != 0, result.stdout + assert f"{shadow} resolved inside checkout:" in result.stderr + if shadow == "mssql_python_odbc": + assert "BINDING_IMPORTED" not in result.stdout + else: + assert result.returncode == 0, result.stderr + assert str(installed) in result.stdout + assert "BINDING_IMPORTED" in result.stdout + renaming = script[: script.index('python -c "')] + assert "mssql_python_odbc" in renaming + assert "mssql_python" in renaming + assert "set -e" in script if "script" in step else "$ErrorActionPreference" in script + + +def _stress_wheel(folder, name, version, requirements=()): + folder.mkdir(parents=True, exist_ok=True) + stem = f"{name.replace('-', '_')}-{version}" + path = folder / f"{stem}-py3-none-any.whl" + with zipfile.ZipFile(path, "w") as archive: + archive.writestr( + f"{stem}.dist-info/METADATA", + f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n" + + "".join(f"Requires-Dist: {requirement}\n" for requirement in requirements), + ) + archive.writestr( + f"{stem}.dist-info/WHEEL", + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + ) + archive.writestr(f"{stem}.dist-info/RECORD", "") + return path + + +def _stress_install_command(stress_job, workspace, monkeypatch): + from eng.scripts import mssql_python_rs + + calls = [] + monkeypatch.setattr(subprocess, "check_call", calls.append) + monkeypatch.setattr( + mssql_python_rs, "resolve_runtime", lambda: SimpleNamespace(version="0.3.1.dev123") + ) + step = next( + step + for step in stress_job["steps"] + if step.get("displayName") == "Install pre-built mssql-python wheel" + ) + script = step["script"] + command = next(line.strip() for line in script.splitlines() if 'python -c "' in line) + code = command.removeprefix('python -c "').removesuffix('"') + source_path = ( + str(workspace) if stress_job["job"] == "StressTestWindows" else workspace.as_posix() + ) + code = code.replace("$(Build.SourcesDirectory)", source_path).replace( + "$(Pipeline.Workspace)", source_path + ) + if os.name != "nt": + code = code.replace("\\", "/") + exec(compile(code, "", "exec"), {}) + assert len(calls) == 1 + assert script.index("python -c ") < script.index("mssql_python_rs.py --verify") + return calls[0] + + +def test_stress_downloads_odbc_from_same_selected_build(stress_job): + downloads = { + step["inputs"]["artifactName"]: step["inputs"] + for step in stress_job["steps"] + if step.get("task") == "DownloadPipelineArtifact@2" + } + binding = downloads["drop_Consolidate_ConsolidateArtifacts"] + odbc = downloads["drop_ConsolidateOdbc_ConsolidateArtifacts"] + for key in ("buildType", "project", "definition", "buildVersionToDownload", "pipelineId"): + assert binding[key] == odbc[key] + assert odbc["buildType"] == odbc["buildVersionToDownload"] == "specific" + assert odbc["pipelineId"] == "$(resources.pipeline.buildPipeline.runID)" + assert odbc["targetPath"].replace("\\", "/") == "$(Pipeline.Workspace)/odbc-wheels" + + +@pytest.mark.parametrize("odbc_available", (True, False)) +def test_stress_installs_metadata_pinned_odbc_offline( + stress_job, workspace, monkeypatch, odbc_available +): + _stress_wheel( + workspace / "wheels/dist", + "mssql-python", + "9.8.7", + ("mssql-python-odbc==8.7.6", "mssql-python-rs==0.3.1.dev123"), + ) + odbc_dir = workspace / "odbc-wheels/dist" + _stress_wheel(odbc_dir, "mssql-python-odbc", "99.0.0") + if odbc_available: + _stress_wheel(odbc_dir, "mssql-python-odbc", "8.7.6") + _stress_wheel(workspace / "mssql-python-rs-wheels", "mssql-python-rs", "0.3.1.dev123") + command = _stress_install_command(stress_job, workspace, monkeypatch) + assert "mssql-python-odbc==8.7.6" in command + assert "mssql-python-rs==0.3.1.dev123" in command + assert "--no-index" in command and "--no-deps" in command + links = [command[index + 1] for index, value in enumerate(command) if value == "--find-links"] + assert str(odbc_dir) in [str(Path(link)) for link in links] + result = subprocess.run( + [*command, "--dry-run", "--ignore-installed"], + capture_output=True, + text=True, + check=False, + ) + assert (result.returncode == 0) == odbc_available, result.stdout + result.stderr + if odbc_available: + assert "mssql-python-odbc-8.7.6" in result.stdout + else: + assert "mssql-python-odbc==8.7.6" in result.stderr + + +@pytest.mark.parametrize( + "requirements", + ( + (), + ("mssql-python-odbc>=8.7.6",), + ("mssql-python-odbc==8.7.*",), + ("mssql-python-odbc==8.7.6; sys_platform == 'win32'",), + ("mssql-python-odbc==8.7.6", "mssql-python-odbc==8.7.5"), + ), +) +def test_stress_rejects_missing_or_nonexact_odbc_pin( + stress_job, workspace, monkeypatch, requirements +): + _stress_wheel(workspace / "wheels/dist", "mssql-python", "9.8.7", requirements) + with pytest.raises((ValueError, TypeError)): + _stress_install_command(stress_job, workspace, monkeypatch) + + +def test_stress_rejects_inconsistent_binding_odbc_pins(stress_job, workspace, monkeypatch): + for version, pin in (("9.8.7", "8.7.6"), ("9.8.8", "8.7.5")): + _stress_wheel( + workspace / "wheels/dist", "mssql-python", version, (f"mssql-python-odbc=={pin}",) + ) + with pytest.raises(ValueError): + _stress_install_command(stress_job, workspace, monkeypatch) + + +def test_coverage_selects_python_before_dependencies_native_build_and_runtime(): + pipeline = yaml.safe_load((ROOT / ENTRY_POINTS[0]).read_text(encoding="utf-8")) + job = next(job for job in pipeline["jobs"] if job["job"] == "CodeCoverageReport") + steps = job["steps"] + selectors = [ + index for index, step in enumerate(steps) if step.get("task") == "UsePythonVersion@0" + ] + assert len(selectors) == 1 + selected = selectors[0] + assert steps[selected]["inputs"]["versionSpec"] == "3.12" + assert steps[selected]["inputs"]["addToPath"] is True + for required in ("pip install", "./build.sh codecov", "./generate_codecov.sh"): + assert any(required in step.get("script", "") for step in steps), required + assert any(step.get("template") == "steps/install-mssql-py-core.yml" for step in steps) + for index, step in enumerate(steps): + uses_python = re.search( + r"\b(?:python\d*|pip\d*)\b|(?:build|generate_codecov)\.sh", + step.get("script", ""), + ) + if uses_python or step.get("template") == "steps/install-mssql-py-core.yml": + assert selected < index, step + + +def test_runtime_is_verified_after_dependency_setup_before_testing(): + pr = (ROOT / ENTRY_POINTS[0]).read_text(encoding="utf-8") + test_commands = [ + line + for line in pr.splitlines() + if not line.lstrip().startswith("#") and "python -m pytest " in line + ] + assert test_commands + assert all("mssql_python_rs.py --verify && python -m pytest " in line for line in test_commands) + daily = (ROOT / ENTRY_POINTS[1]).read_text(encoding="utf-8") + assert "python eng/scripts/mssql_python_rs.py --verify || exit 2" in daily + linux = (PIPELINES / "stages/build-linux-single-stage.yml").read_text(encoding="utf-8") + assert ( + linux.count( + '$PY -m pip install -q "$WHEEL";\n $PY /workspace/eng/scripts/mssql_python_rs.py --verify;' + ) + == 2 + ) + for platform in ("windows", "macos"): + source = (PIPELINES / f"stages/build-{platform}-single-stage.yml").read_text( + encoding="utf-8" + ) + verify = source.index("python eng/scripts/mssql_python_rs.py --verify") + assert verify < source.index("python -m pytest -v") + stress = (PIPELINES / "stress-test-pipeline.yml").read_text(encoding="utf-8") + assert stress.count("python eng/scripts/mssql_python_rs.py --verify") == 2 + + +@pytest.mark.parametrize("platform", ("windows", "macos", "linux")) +def test_official_build_guards_run_before_runtime_installation(platform): + source = (PIPELINES / f"stages/build-{platform}-single-stage.yml").read_text(encoding="utf-8") + guard = source.index("validate-release-versions.ps1") + assert source.index("if eq(parameters.oneBranchType, 'Official')") < guard + assert "arguments: '-BuildOnly'" in source + assert guard < source.index("install-mssql-py-core") + + +@pytest.mark.parametrize("pipeline", ("official", "dummy")) +def test_release_validates_selected_artifact_before_esrp(pipeline): + source = (PIPELINES / f"{pipeline}-release-pipeline.yml").read_text(encoding="utf-8") + guard = source.index( + "filePath: '$(Build.SourcesDirectory)\\OneBranchPipelines\\scripts\\validate-release-versions.ps1'" + ) + metadata = source.index("validate-rs-wheel-dependency.ps1") + assert guard < metadata + assert source[guard:metadata].count(r"artifacts\dist\transport.json") == 1 + assert source.index("dependsOn: PrepAndValidate", metadata) > metadata + + +@pytest.fixture +def workspace(): + path = ROOT / f".pipeline-contract-{uuid.uuid4().hex}" + path.mkdir() + try: + yield path + finally: + shutil.rmtree(path) + + +def _receipt(version="0.3.0"): + return { + "distribution_version": version, + "index_url": "https://pypi.org/simple/", + "version_override": False, + "wheel_sha256": {"mssql_python_rs-0.3.0-py3-none-any.whl": "a" * 64}, + } + + +def _guard(workspace, *, version="0.3.0", receipt=None, override="", build_only=False, public=True): + pwsh = shutil.which("pwsh") + if not pwsh: + pytest.skip("PowerShell Core is not available") + (workspace / "runtime.version").write_text(version) + if receipt is not None: + (workspace / "transport.json").write_text(json.dumps(receipt)) + env = dict(os.environ) + env.update( + { + VERSION_ENV: override, + "GUARD": str(PIPELINES / "scripts/validate-release-versions.ps1"), + "VERSION_FILE": str(workspace / "runtime.version"), + "PUBLIC_VERSION": version if public else "9.9.9", + "BUILD_ONLY": str(build_only).lower(), + } + ) + command = """ + $ErrorActionPreference = 'Stop' + function Invoke-RestMethod { + param($Uri, $Method) + if ($Uri -cne "https://pypi.org/pypi/mssql-python-rs/$env:PUBLIC_VERSION/json") { + throw 'No exact release on public PyPI' + } + return @{ + info = @{ version = $env:PUBLIC_VERSION; name = 'mssql-python-rs' } + urls = @(@{ packagetype = 'bdist_wheel'; yanked = $false }) + } + } + & $env:GUARD -VersionFile $env:VERSION_FILE -BuildOnly:($env:BUILD_ONLY -eq 'true') + """ + return subprocess.run( + [pwsh, "-NoProfile", "-NonInteractive", "-Command", command], + capture_output=True, + text=True, + env=env, + check=False, + ) + + +@pytest.mark.parametrize("override", ("0.3.0", "0.3.1.dev123")) +def test_official_build_rejects_all_explicit_overrides(workspace, override): + result = _guard(workspace, override=override, build_only=True) + assert result.returncode != 0 + assert "forbid" in result.stderr + + +@pytest.mark.parametrize("version", ("0.3.1.dev123", "0.3.1rc1", "", ">=0.3.0")) +def test_official_build_rejects_non_release_pins(workspace, version): + assert _guard(workspace, version=version, build_only=True).returncode != 0 + + +def test_official_build_accepts_committed_release_without_network_or_receipt(workspace): + result = _guard(workspace, build_only=True, public=False) + assert result.returncode == 0, result.stderr + + +def test_release_requires_receipt_and_exact_public_pypi_release(workspace): + assert _guard(workspace).returncode != 0 + result = _guard(workspace, receipt=_receipt(), public=False) + assert result.returncode != 0 + result = _guard(workspace, receipt=_receipt()) + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize( + "field,value", + ( + ("version_override", True), + ("version_override", "false"), + ("distribution_version", "0.3.1"), + ("index_url", "https://pkgs.dev.azure.com/example/simple/"), + ("wheel_sha256", {}), + ("wheel_sha256", {"test.whl": "bad"}), + ), +) +def test_release_rejects_override_or_inconsistent_receipt(workspace, field, value): + receipt = _receipt() + receipt[field] = value + assert _guard(workspace, receipt=receipt).returncode != 0 + + +@pytest.mark.parametrize("dependency", ("0.3.0", "0.3.1.dev123")) +def test_wheel_metadata_must_match_effective_staged_version(workspace, dependency): + pwsh = shutil.which("pwsh") + if not pwsh: + pytest.skip("PowerShell Core is not available") + (workspace / "runtime.version").write_text("0.3.1.dev123") + (workspace / "transport.json").write_text(json.dumps(_receipt("0.3.1.dev123"))) + with zipfile.ZipFile(workspace / "mssql_python-1.0.0-py3-none-any.whl", "w") as archive: + archive.writestr( + "mssql_python-1.0.0.dist-info/METADATA", + f"Name: mssql-python\nVersion: 1.0.0\nRequires-Dist: mssql-python-rs=={dependency}\n", + ) + result = subprocess.run( + [ + pwsh, + "-NoProfile", + "-NonInteractive", + "-File", + str(PIPELINES / "scripts/validate-rs-wheel-dependency.ps1"), + "-WheelsDir", + str(workspace), + "-VersionFile", + str(workspace / "runtime.version"), + "-ReceiptFile", + str(workspace / "transport.json"), + ], + capture_output=True, + text=True, + check=False, + ) + assert (result.returncode == 0) == (dependency == "0.3.1.dev123"), result.stderr diff --git a/tests/test_040_rs_runtime_selection.py b/tests/test_040_rs_runtime_selection.py new file mode 100644 index 000000000..10ec0630a --- /dev/null +++ b/tests/test_040_rs_runtime_selection.py @@ -0,0 +1,184 @@ +import json +import runpy +import subprocess +import sys +import zipfile +from pathlib import Path +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if not (ROOT / "eng/scripts/mssql_python_rs.py").is_file() or not (ROOT / "setup.py").is_file(): + pytest.skip( + "Runtime selection contracts require a complete source checkout", allow_module_level=True + ) + +from eng.scripts import install_mssql_python_rs as installer +from eng.scripts import mssql_python_rs as runtime + + +@pytest.fixture(autouse=True) +def clear_override(monkeypatch): + monkeypatch.delenv(runtime.VERSION_ENV, raising=False) + + +def _pin(tmp_path, value="0.3.0"): + path = tmp_path / "runtime.version" + path.write_text(value, encoding="ascii") + return path + + +@pytest.mark.parametrize("override", [None, "", "0.4.0", "0.4.0.dev20261028187000"]) +def test_resolves_exact_release_or_override(tmp_path, monkeypatch, override): + if override is not None: + monkeypatch.setenv(runtime.VERSION_ENV, override) + selected = runtime.resolve_runtime(_pin(tmp_path)) + assert selected.version == (override or "0.3.0") + assert selected.version_override == bool(override) + assert selected.index_url == ( + runtime.DEV_INDEX if override and ".dev" in override else runtime.DEFAULT_INDEX + ) + + +@pytest.mark.parametrize( + "version", + ["latest", ">=0.3.0", "0.3", "0.3.0;echo bad", "0.3.0rc1", "0.3.0.dev", "0.3.0+local"], +) +def test_rejects_non_exact_or_unsupported_override(tmp_path, monkeypatch, version): + monkeypatch.setenv(runtime.VERSION_ENV, version) + with pytest.raises(ValueError, match="Invalid MSSQL_PYTHON_RS_VERSION"): + runtime.resolve_runtime(_pin(tmp_path)) + + +@pytest.mark.parametrize("version", ["", "0.4.0.dev1", "0.4.0rc1", ">=0.3.0"]) +def test_committed_pin_must_be_a_release(tmp_path, version): + with pytest.raises(ValueError, match="exact released"): + runtime.resolve_runtime(_pin(tmp_path, version)) + + +@pytest.mark.parametrize("override", [None, "0.4.0.dev20261028187000"]) +def test_setup_metadata_uses_same_effective_runtime(monkeypatch, override): + if override: + monkeypatch.setenv(runtime.VERSION_ENV, override) + captured = {} + monkeypatch.chdir(ROOT) + monkeypatch.setattr("setuptools.setup", lambda **kwargs: captured.update(kwargs)) + runpy.run_path(str(ROOT / "setup.py")) + selected = runtime.resolve_runtime() + assert f"mssql-python-rs=={selected.version}" in captured["install_requires"] + + +@pytest.mark.parametrize("override", [None, "0.4.0.dev20261028187000"]) +def test_generated_package_metadata_matches_selection(tmp_path, monkeypatch, override): + if override: + monkeypatch.setenv(runtime.VERSION_ENV, override) + subprocess.run( + [sys.executable, "setup.py", "egg_info", "--egg-base", str(tmp_path)], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + metadata = tmp_path / "mssql_python.egg-info" + requirements = (metadata / "requires.txt").read_text(encoding="utf-8").splitlines() + assert f"mssql-python-rs=={runtime.resolve_runtime().version}" in requirements + sources = (metadata / "SOURCES.txt").read_text(encoding="utf-8") + assert "eng/scripts/mssql_python_rs.py" in sources + assert "eng/versions/mssql-python-rs.version" in sources + + +def test_verify_rejects_runtime_replaced_by_later_pip_install(monkeypatch): + monkeypatch.setattr(runtime.importlib.metadata, "version", lambda _name: "0.3.0") + with pytest.raises(ValueError, match="expected 0.4.0.dev1"): + runtime.verify_installed(runtime.Runtime("0.4.0.dev1", runtime.DEV_INDEX, True)) + + +def test_verify_imports_native_module(monkeypatch): + monkeypatch.setattr(runtime.importlib.metadata, "version", lambda _name: "0.3.0") + imports = [] + + def import_module(name): + imports.append(name) + return SimpleNamespace(__file__="test-native-module") + + monkeypatch.setattr(runtime.importlib, "import_module", import_module) + runtime.verify_installed(runtime.Runtime("0.3.0", runtime.DEFAULT_INDEX, False)) + assert imports == ["mssql_py_core"] + + +@pytest.mark.parametrize("cross_arch", [False, True]) +def test_installer_selects_index_and_preserves_cross_arch_behavior( + tmp_path, monkeypatch, cross_arch +): + monkeypatch.setenv(runtime.VERSION_ENV, "0.4.0.dev1") + monkeypatch.setenv( + "PIP_EXTRA_INDEX_URL", # DevSkim: ignore DS205001 - test removal from child environment + "https://wrong.example/simple/", + ) + monkeypatch.setenv("PIP_CERT", "corporate-ca.pem") + monkeypatch.setattr(installer.platform, "machine", lambda: "AMD64") + monkeypatch.setattr(installer.sys, "platform", "win32") + commands = [] + verified = [] + + def run(command, *, env, check): + assert check is True + assert "PIP_EXTRA_INDEX_URL" not in env # DevSkim: ignore DS205001 - assert removal + assert env["PIP_CERT"] == "corporate-ca.pem" + commands.append(command) + if "download" in command: + target = Path(command[command.index("--dest") + 1]) + name = "mssql_python_rs-0.4.0.dev1-cp310-abi3-win_arm64.whl" + with zipfile.ZipFile(target / name, "w") as wheel: + wheel.writestr( + "mssql_python_rs-0.4.0.dev1.dist-info/METADATA", + "Name: mssql-python-rs\nVersion: 0.4.0.dev1\n", + ) + + monkeypatch.setattr(installer.subprocess, "run", run) + monkeypatch.setattr(installer, "verify_installed", verified.append) + installer.install_runtime("arm64" if cross_arch else "", tmp_path / "downloads") + download = commands[0] + assert download[download.index("--index-url") + 1] == runtime.DEV_INDEX + assert download[-1] == "mssql-python-rs==0.4.0.dev1" + if cross_arch: + assert download[download.index("--platform") + 1] == "win_arm64" + assert len(commands) == 1 + assert verified == [] + else: + assert len(commands) == 2 + assert "--no-deps" in commands[1] + assert verified[0].version == "0.4.0.dev1" + assert not list((tmp_path / "downloads").iterdir()) + + +def test_installer_does_not_fall_back_after_download_failure(monkeypatch): + commands = [] + + def run(command, **kwargs): + commands.append(command) + raise subprocess.CalledProcessError(1, command) + + monkeypatch.setattr(installer.subprocess, "run", run) + with pytest.raises(subprocess.CalledProcessError): + installer.install_runtime() + assert len(commands) == 1 + + +def test_resolver_cli_writes_effective_artifact_pin(tmp_path, monkeypatch): + monkeypatch.setenv(runtime.VERSION_ENV, "0.4.0.dev1") + destination = tmp_path / "mssql-python-rs.version" + result = subprocess.run( + [ + sys.executable, + str(ROOT / "eng/scripts/mssql_python_rs.py"), + "--write-version-file", + str(destination), + ], + capture_output=True, + text=True, + check=True, + ) + assert destination.read_text(encoding="ascii") == "0.4.0.dev1\n" + assert json.loads(result.stdout)["version_override"] is True diff --git a/tests/test_041_rs_nightly_selection.py b/tests/test_041_rs_nightly_selection.py new file mode 100644 index 000000000..d834b8b74 --- /dev/null +++ b/tests/test_041_rs_nightly_selection.py @@ -0,0 +1,373 @@ +import io +import json +import subprocess +import sys +import urllib.error +from dataclasses import replace +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if not all( + (ROOT / "eng" / "scripts" / name).is_file() + for name in ( + "resolve_pr_rs_runtime.py", + "download_mssql_python_rs_wheels.py", + "mssql_python_rs.py", + "mssql_python_build_safety.py", + ) +): + pytest.skip("Nightly selection requires a complete source checkout", allow_module_level=True) + +from eng.scripts import resolve_pr_rs_runtime as resolver +from eng.scripts.mssql_python_rs import DEFAULT_INDEX, DEV_INDEX, VERSION_ENV + +VERSION = "0.4.0.dev20261026" +DIGEST = "abcdef01" * 8 +PLATFORMS = ( + "win_amd64", + "win_arm64", + "manylinux_2_28_x86_64", + "manylinux_2_28_aarch64", + "manylinux_2_34_x86_64", + "manylinux_2_34_aarch64", + "musllinux_1_2_x86_64", + "musllinux_1_2_aarch64", + "macosx_15_0_universal2", +) + + +@pytest.fixture(autouse=True) +def clear_override(monkeypatch): + monkeypatch.delenv(VERSION_ENV, raising=False) + + +def _anchor(version=VERSION, platform="win_amd64", *, hash_value=DIGEST, yanked=False): + name = f"mssql_python_rs-{version}-cp310-abi3-{platform}.whl" + attribute = ' data-yanked="broken"' if yanked else "" + fragment = f"#sha256={hash_value}" if hash_value is not None else "" + return f'{name}' + + +def _page(version=VERSION, platforms=PLATFORMS, **kwargs): + return ( + "" + + "".join(_anchor(version, p, **kwargs) for p in platforms) + + "" + ) + + +def _serve(monkeypatch, page): + requests = [] + + def urlopen(url, timeout): + requests.append((url, timeout)) + return io.BytesIO(page.encode("utf-8")) + + monkeypatch.setattr(resolver.urllib.request, "urlopen", urlopen) + return requests + + +def test_selects_all_nine_platforms_with_hash_and_url_evidence(monkeypatch): + requests = _serve(monkeypatch, _page()) + selection = resolver.resolve_pr_runtime() + assert selection.version == VERSION + assert selection.index_url == DEV_INDEX + assert selection.channel == "nightly" + assert selection.version_override is False + assert resolver.EXPECTED_PLATFORMS == set(PLATFORMS) + assert len(selection.wheel_sha256) == len(selection.wheel_urls) == 9 + for name, digest in selection.wheel_sha256.items(): + assert digest == DIGEST + assert selection.wheel_urls[name] == f"https://files.example.test/{name}" + assert requests == [(DEV_INDEX + "mssql-python-rs/", 120)] + + +@pytest.mark.parametrize( + ("versions", "expected"), + [ + (["99.99.99.dev20261025", VERSION], VERSION), + (["0.9.9.dev20261026", "0.10.0.dev20261026"], "0.10.0.dev20261026"), + (["9.0.0.dev20261026", "10.0.0.dev20261026"], "10.0.0.dev20261026"), + (["0.0.9.dev20261026", "0.0.10.dev20261026"], "0.0.10.dev20261026"), + ([VERSION, "99.0.0.dev20261026186010", "999.0.0"], VERSION), + (["0.1.0.dev20240229", "0.1.0.dev20240301"], "0.1.0.dev20240301"), + ], +) +def test_date_first_then_numeric_base_ordering(versions, expected): + assert ( + resolver.select_nightly("".join(_page(version) for version in versions)).version == expected + ) + + +@pytest.mark.parametrize("version", ["1.0.0.dev20260229", "1.0.0.dev20261301", "1.0.0.dev20261000"]) +def test_rejects_invalid_calendar_dates(version): + with pytest.raises(ValueError, match="Invalid nightly calendar date"): + resolver.select_nightly(_page(version)) + + +@pytest.mark.parametrize( + "page", + [ + "", + "Please log in", + _page("0.3.0"), + _page("0.4.0.dev20261026186010"), + _page("0.4.0.dev1"), + _page("0.4.0.dev2026102"), + ], +) +def test_no_nightly_fails_with_exact_override_remedy(page): + with pytest.raises(ValueError, match="No scheduled nightly.*MSSQL_PYTHON_RS_VERSION"): + resolver.select_nightly(page) + + +@pytest.mark.parametrize( + "page", + ['wheel", _anchor()[:-4], ""], +) +def test_rejects_malformed_html(page): + with pytest.raises(ValueError, match="Malformed wheel index"): + resolver.select_nightly(page) + + +def test_rejects_duplicate_href_attributes(): + page = _page().replace('href="', 'href="ignored" href="', 1) + with pytest.raises(ValueError, match="Malformed wheel index anchor"): + resolver.select_nightly(page) + + +def test_http_error_is_visible_and_not_retried(monkeypatch): + requests = [] + + def urlopen(url, timeout): + requests.append(url) + raise urllib.error.HTTPError(url, 503, "Unavailable", {}, None) + + monkeypatch.setattr(resolver.urllib.request, "urlopen", urlopen) + with pytest.raises(urllib.error.HTTPError, match="503"): + resolver.resolve_pr_runtime() + assert requests == [DEV_INDEX + "mssql-python-rs/"] + + +@pytest.mark.parametrize("yanked", [False, True]) +def test_newest_partial_or_yanked_never_falls_back_to_complete_older(yanked): + page = _page("99.0.0.dev20261025") + _page(platforms=PLATFORMS[:1], yanked=yanked) + with pytest.raises(ValueError, match="Newest nightly.*(incomplete|yanked)"): + resolver.select_nightly(page) + + +def test_fully_yanked_newest_never_falls_back(): + with pytest.raises(ValueError, match="Newest nightly.*yanked"): + resolver.select_nightly(_page("0.3.0.dev20261025") + _page(yanked=True)) + + +def test_partial_yank_is_not_treated_as_complete(): + page = _page(platforms=PLATFORMS[1:]) + _anchor(yanked=True) + with pytest.raises(ValueError, match="yanked"): + resolver.select_nightly(page) + + +@pytest.mark.parametrize( + "hash_value", [None, "", "bad", "g" * 64, DIGEST + "&sha256=" + DIGEST, DIGEST + "&sha256="] +) +def test_rejects_missing_invalid_or_duplicate_hashes(hash_value): + page = _page(platforms=PLATFORMS[1:]) + _anchor(hash_value=hash_value) + with pytest.raises(ValueError, match="Missing or invalid SHA-256"): + resolver.select_nightly(page) + + +def test_uppercase_hash_is_normalized(): + assert set(resolver.select_nightly(_page(hash_value=DIGEST.upper())).wheel_sha256.values()) == { + DIGEST + } + + +def test_duplicate_wheel_links_fail(): + with pytest.raises(ValueError, match="Duplicate wheel filename"): + resolver.select_nightly(_page() + _anchor()) + + +@pytest.mark.parametrize( + "replacement", + [ + "http://files.example.test/", + "//", + "https://:443/", + "https://user:password@files.example.test/", + ], +) +def test_rejects_invalid_wheel_urls(replacement): + page = _page().replace("https://files.example.test/", replacement) + with pytest.raises(ValueError, match="HTTPS"): + resolver.select_nightly(page) + + +@pytest.mark.parametrize("index", ["http://example.test/simple/", "https:///simple/"]) +def test_rejects_insecure_or_missing_index_host(index): + with pytest.raises(ValueError, match="HTTPS"): + resolver.select_nightly(_page(), index) + + +def test_relative_wheel_urls_resolve_against_package_index(): + selection = resolver.select_nightly(_page().replace("https://files.example.test/", "")) + assert all( + url.startswith(DEV_INDEX + "mssql-python-rs/") for url in selection.wheel_urls.values() + ) + + +@pytest.mark.parametrize( + "tags", ["cp311-abi3-win_amd64", "cp310-abi3-unexpected", "cp310-cp310-win_amd64"] +) +def test_rejects_unexpected_newest_wheel_tags(tags): + page = _page().replace("cp310-abi3-win_amd64", tags) + with pytest.raises(ValueError, match="Unexpected newest nightly wheel tags"): + resolver.select_nightly(page) + + +def test_malformed_newest_wheel_name_does_not_fall_back(): + page = _page("0.3.0.dev20261025") + _anchor().replace("-cp310-abi3-win_amd64", "") + with pytest.raises(ValueError, match="Unexpected newest nightly wheel tags"): + resolver.select_nightly(page) + + +def test_rejects_ambiguous_numeric_version_spellings(): + with pytest.raises(ValueError, match="Ambiguous nightly"): + resolver.select_nightly(_page() + _page("00.4.0.dev20261026")) + + +@pytest.mark.parametrize("source", ["environment", "argument"]) +@pytest.mark.parametrize("version", ["0.3.0", "0.4.0.dev20261026186010", " 0.4.0.dev1 "]) +def test_exact_overrides_do_not_access_network(monkeypatch, source, version): + def unexpected_network(*args, **kwargs): + pytest.fail("Exact override must not access the network") + + monkeypatch.setattr(resolver.urllib.request, "urlopen", unexpected_network) + if source == "environment": + monkeypatch.setenv(VERSION_ENV, version) + selection = resolver.resolve_pr_runtime( + **({"version": version} if source == "argument" else {}) + ) + assert selection.version == version.strip() + assert selection.index_url == (DEV_INDEX if ".dev" in version else DEFAULT_INDEX) + assert selection.channel == "override" + assert selection.version_override + assert selection.wheel_sha256 == selection.wheel_urls == {} + + +def test_explicit_version_takes_precedence_over_environment(monkeypatch): + monkeypatch.setenv(VERSION_ENV, "invalid") + assert resolver.resolve_pr_runtime(version="0.3.0").version == "0.3.0" + + +@pytest.mark.parametrize( + "version", ["latest", ">=0.3.0", "0.3.0\n##vso[bad]", "0.3.0;echo bad", "0.3.0rc1"] +) +def test_invalid_override_fails_before_network(monkeypatch, version): + def unexpected_network(*args, **kwargs): + pytest.fail("Invalid override must not access the network") + + monkeypatch.setattr(resolver.urllib.request, "urlopen", unexpected_network) + monkeypatch.setenv(VERSION_ENV, version) + with pytest.raises(ValueError, match="Invalid MSSQL_PYTHON_RS_VERSION"): + resolver.resolve_pr_runtime() + + +@pytest.mark.parametrize("value", ["", " "]) +def test_empty_environment_selects_feed_not_committed_release(monkeypatch, value): + monkeypatch.setenv(VERSION_ENV, value) + _serve(monkeypatch, _page()) + assert resolver.resolve_pr_runtime().version == VERSION + + +def test_explicit_empty_version_selects_nightly_despite_environment(monkeypatch): + monkeypatch.setenv(VERSION_ENV, "0.3.0") + _serve(monkeypatch, _page()) + assert resolver.resolve_pr_runtime(version="").version == VERSION + + +def test_writes_reproducible_selection_files(tmp_path): + selection = resolver.select_nightly(_page()) + output = resolver.write_selection(selection, tmp_path / "selection") + assert (output / "mssql-python-rs.version").read_bytes() == (VERSION + "\n").encode("ascii") + receipt = json.loads((output / "runtime-selection.json").read_text(encoding="utf-8")) + assert receipt == { + "version": VERSION, + "index_url": DEV_INDEX, + "channel": "nightly", + "version_override": False, + "wheel_sha256": selection.wheel_sha256, + "wheel_urls": selection.wheel_urls, + } + assert sorted(path.name for path in output.iterdir()) == [ + "mssql-python-rs.version", + "runtime-selection.json", + ] + + +def test_output_directory_uses_shared_safety_helper(monkeypatch): + monkeypatch.chdir(ROOT) + with pytest.raises(ValueError, match="current working directory"): + resolver.write_selection(resolver.select_nightly(_page()), ROOT) + + +def test_output_artifact_directory_is_rejected_without_partial_write(tmp_path): + output = tmp_path / "selection" + (output / "mssql-python-rs.version").mkdir(parents=True) + with pytest.raises(ValueError, match="Unsafe selection artifact"): + resolver.write_selection(resolver.select_nightly(_page()), output) + assert not (output / "runtime-selection.json").exists() + + +@pytest.mark.parametrize("version", ["bad", "0.3.0\n##vso[bad]", "0.3.0\n"]) +def test_artifact_writer_rejects_unvalidated_version(tmp_path, version): + selection = replace(resolver.select_nightly(_page()), version=version) + with pytest.raises(ValueError): + resolver.write_selection(selection, tmp_path / "selection") + assert not (tmp_path / "selection").exists() + + +def test_nightly_cli_logs_feed_validation_and_output_variable(monkeypatch, tmp_path, capsys): + _serve(monkeypatch, _page()) + monkeypatch.setattr( + sys, "argv", ["resolve_pr_rs_runtime.py", "--output-dir", str(tmp_path / "out")] + ) + resolver.main() + stdout = capsys.readouterr().out + assert "Selected feed-validated nightly:" in stdout + assert "successful build" not in stdout + assert f"##vso[task.setvariable variable=version;isOutput=true]{VERSION}\n" in stdout + assert (tmp_path / "out" / "mssql-python-rs.version").read_text() == VERSION + "\n" + + +def test_failed_selection_emits_no_output_variable_or_files(monkeypatch, tmp_path, capsys): + _serve(monkeypatch, _page("0.4.0.dev20261026186010")) + output = tmp_path / "out" + monkeypatch.setattr(sys, "argv", ["resolve_pr_rs_runtime.py", "--output-dir", str(output)]) + with pytest.raises(ValueError, match="No scheduled nightly"): + resolver.main() + assert "##vso[" not in capsys.readouterr().out + assert not output.exists() + + +def test_standalone_cli_supports_override_and_writes_files(tmp_path, monkeypatch): + monkeypatch.setenv(VERSION_ENV, "0.4.0.dev20261026186010") + output = tmp_path / "out" + result = subprocess.run( + [ + sys.executable, + str(ROOT / "eng" / "scripts" / "resolve_pr_rs_runtime.py"), + "--output-dir", + str(output), + ], + capture_output=True, + text=True, + check=True, + ) + assert "Selected exact override:" in result.stdout + assert ( + "##vso[task.setvariable variable=version;isOutput=true]0.4.0.dev20261026186010" + in result.stdout + ) + assert json.loads((output / "runtime-selection.json").read_text())["channel"] == "override"