FIX: Use pinned Rust releases with explicit dev-build overrides

CONTRIBUTING.md

@@ -15,6 +15,70 @@ or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any addi
 
 ## Before Contributing
 
+### Selecting the Rust Runtime
+
+Local builds and official/release pipelines install the exact released
+`mssql-python-rs` version in
+[`eng/versions/mssql-python-rs.version`](eng/versions/mssql-python-rs.version)
+from public PyPI. The runtime is a separate dependency, not embedded in the
+`mssql-python` wheel. No NuGet transport package is needed.
+
+To test a Python feature against merged but unreleased Rust changes, select the
+exact version from a successful `mssql-rs` dev or nightly publishing run:
+
+```powershell
+# Windows, with the development environment activated
+$env:MSSQL_PYTHON_RS_VERSION = "0.4.0.dev20261028187000"  # Replace with a published build
+.\eng\scripts\install-mssql-py-core.ps1
+```
+
+```bash
+# Linux or macOS, with the development environment activated
+export MSSQL_PYTHON_RS_VERSION=0.4.0.dev20261028187000  # Replace with a published build
+bash eng/scripts/install-mssql-py-core.sh
+```
+
+Keep the environment variable set while building/installing `mssql-python` and
+running tests. Its generated dependency metadata then uses the same selected
+version. Development versions come from the anonymous Azure Artifacts PyPI
+index; clean releases come from public PyPI. A missing requested wheel fails;
+the installer never falls back to another version. Cross-architecture Windows
+builds validate the target wheel without installing it into the host Python.
+
+The GitHub PR-validation checks run in Azure DevOps using
+`eng/pipelines/pr-validation-pipeline.yml`. This validation pipeline (including
+its main-branch CI runs) defaults to the latest published nightly from the Azure
+Python feed, not the stable PyPI pin. It resolves the version once before the
+matrix and passes that exact version to every build/test job. The
+`rs-runtime-selection` artifact records the version, source index and nightly
+wheel hashes.
+
+Nightlies use `X.Y.Z.devYYYYMMDD`; per-merge development builds append a build ID
+and are not selected automatically. Selection requires all nine platform wheels
+for the newest nightly, with valid SHA-256 links and no yanked files. Missing or
+incomplete publication fails validation rather than falling back to an older
+nightly or a stable release. Feed contents establish wheel publication, not
+the overall status or source commit of the producer's Azure build.
+
+Azure build/test pipelines expose the explicit selection as `rsVersionOverride`.
+For PR validation, leave it empty for the nightly, or provide an exact version
+to reproduce a run or test newer per-merge Rust changes. Official/release builds
+retain the committed stable PyPI pin and reject overrides. Other test pipelines
+retain their existing stable default unless explicitly overridden. Record the
+exact dev version in the feature's PR so another developer can reproduce the
+tests. Before running tests after other package installations, verify the runtime
+has not changed:
+
+```console
+python eng/scripts/mssql_python_rs.py --verify
+```
+
+Feature-test artifacts are not eligible for public release, even if an override
+selects a stable version. Once the required Rust runtime is released on PyPI,
+update the committed pin and validate with no override before releasing Python.
+Unset `MSSQL_PYTHON_RS_VERSION` to return to the committed release pin and rerun
+the installer.
+
 ### Install Local Formatting Hooks
 
 From the repository root, with your Python 3.10+ development virtual environment

MANIFEST.in

@@ -1 +1,2 @@
-include eng/versions/mssql-python-rs.version
+include eng/versions/mssql-python-rs.version
+include eng/scripts/mssql_python_rs.py

OneBranchPipelines/build-release-package-pipeline.yml

@@ -51,6 +51,11 @@ schedules:
 # PIPELINE PARAMETERS
 # =========================
 parameters:
+  - name: rsVersionOverride
+    displayName: 'Exact mssql-python-rs test version (NonOfficial only)'
+    type: string
+    default: ''
+
   # OneBranch build type determines compliance level
   # - Official: Production builds with full SDL compliance, all security scanning enabled
   # - NonOfficial: Development/test builds with reduced security scanning
@@ -182,6 +187,8 @@ parameters:
 # PIPELINE VARIABLES
 # =========================
 variables:
+  - name: MSSQL_PYTHON_RS_VERSION
+    value: ${{ parameters.rsVersionOverride }}
   # Determine effective build type: scheduled builds are Official, manual/PR builds use parameter
   # Build.Reason values: Schedule, Manual, IndividualCI, PullRequest, BatchedCI
   - name: effectiveOneBranchType

OneBranchPipelines/conda-release-pipeline.yml

@@ -165,7 +165,6 @@ extends:
                 parameters:
                   mssqlPythonVersion: '$(Provenance.mssqlPythonVersion)'
                   releaseVersions: '$(Provenance.releaseVersions)'
-                  rsTransportVersion: '$(Provenance.rsTransportVersion)'
 
               # Dry run (publishToConda=false): enumerate every package and print the exact
               # `anaconda upload` plan WITHOUT uploading, so a validate-only run shows precisely
@@ -250,7 +249,6 @@ extends:
                 ob_outputDirectory: '$(Build.ArtifactStagingDirectory)'
                 mssqlPythonVersion: $[ dependencies.ValidateConda.outputs['Provenance.mssqlPythonVersion'] ]
                 releaseVersions: $[ dependencies.ValidateConda.outputs['Provenance.releaseVersions'] ]
-                rsTransportVersion: $[ dependencies.ValidateConda.outputs['Provenance.rsTransportVersion'] ]
               steps:
                 - checkout: self
                   fetchDepth: 1
@@ -260,4 +258,3 @@ extends:
                     condaLabel: '${{ parameters.condaLabel }}'
                     mssqlPythonVersion: '$(mssqlPythonVersion)'
                     releaseVersions: '$(releaseVersions)'
-                    rsTransportVersion: '$(rsTransportVersion)'

OneBranchPipelines/dummy-release-pipeline.yml

@@ -202,14 +202,14 @@ extends:
                     inputs:
                       targetType: 'filePath'
                       filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1'
-                      arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version"'
+                      arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version" -ReceiptFile "$(Build.SourcesDirectory)\artifacts\dist\transport.json"'
 
                   - task: PowerShell@2
                     displayName: '[TEST] Validate mssql-python-rs wheel dependency'
                     inputs:
                       targetType: 'filePath'
                       filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-rs-wheel-dependency.ps1'
-                      arguments: '-WheelsDir "$(Build.SourcesDirectory)\artifacts\dist" -VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version"'
+                      arguments: '-WheelsDir "$(Build.SourcesDirectory)\artifacts\dist" -VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version" -ReceiptFile "$(Build.SourcesDirectory)\artifacts\dist\transport.json"'
 
                   # Guard: if the released mssql-python wheel declares an
                   # mssql-python-odbc dependency, it MUST be the exact pinned version.

OneBranchPipelines/jobs/consolidate-artifacts-job.yml

@@ -125,16 +125,22 @@ jobs:
         displayName: 'Consolidate Windows symbols (optional)'
         continueOnError: true
 
-      # Stage the actual RS payload using THIS producer source's immutable pins.
-      # Downstream Conda jobs consume this artifact, never their checkout's NuGet pin.
+      # Carry the effective runtime selection with the artifact, including test overrides.
       - bash: |
           set -e
           python3 "$(Build.SourcesDirectory)/eng/scripts/download_mssql_python_rs_wheels.py" \
             --output-dir "$(ob_outputDirectory)/rs-wheels"
-          cp "$(Build.SourcesDirectory)/eng/versions/mssql-python-rs.version" "$(ob_outputDirectory)/dist/"
-          cp "$(Build.SourcesDirectory)/eng/versions/mssql-python-rs-nuget.version" "$(ob_outputDirectory)/dist/"
+          python3 -c "import json,pathlib; root=pathlib.Path('$(ob_outputDirectory)'); receipt=json.loads((root/'rs-wheels/transport.json').read_text()); (root/'dist/mssql-python-rs.version').write_text(receipt['distribution_version']+'\n'); (root/'dist/transport.json').write_bytes((root/'rs-wheels/transport.json').read_bytes())"
           echo "mssql-python-rs version: $(cat "$(ob_outputDirectory)/dist/mssql-python-rs.version")"
-        displayName: 'Stage source-pinned mssql-python-rs wheels and transport receipt'
+        displayName: 'Stage effective mssql-python-rs wheels, version and receipt'
+
+      - task: PowerShell@2
+        displayName: 'Validate every wheel uses the effective runtime version'
+        inputs:
+          pwsh: true
+          targetType: filePath
+          filePath: OneBranchPipelines/scripts/validate-rs-wheel-dependency.ps1
+          arguments: '-WheelsDir "$(ob_outputDirectory)/dist" -VersionFile "$(ob_outputDirectory)/dist/mssql-python-rs.version" -ReceiptFile "$(ob_outputDirectory)/dist/transport.json"'
 
       # Verify consolidation
       - bash: |

OneBranchPipelines/official-release-pipeline.yml

@@ -202,14 +202,14 @@ extends:
                     inputs:
                       targetType: 'filePath'
                       filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1'
-                      arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version"'
+                      arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version" -ReceiptFile "$(Build.SourcesDirectory)\artifacts\dist\transport.json"'
 
                   - task: PowerShell@2
                     displayName: 'Validate mssql-python-rs wheel dependency'
                     inputs:
                       targetType: 'filePath'
                       filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-rs-wheel-dependency.ps1'
-                      arguments: '-WheelsDir "$(Build.SourcesDirectory)\artifacts\dist" -VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version"'
+                      arguments: '-WheelsDir "$(Build.SourcesDirectory)\artifacts\dist" -VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-python-rs.version" -ReceiptFile "$(Build.SourcesDirectory)\artifacts\dist\transport.json"'
 
                   # Guard: if the released mssql-python wheel declares an
                   # mssql-python-odbc dependency, it MUST be the exact pinned version.

OneBranchPipelines/scripts/validate-release-versions.ps1

@@ -3,47 +3,62 @@
 
 <#
 .SYNOPSIS
-    Validates that a version file contains a stable (non-prerelease) version.
-
+    Gates Official builds and releases on an exact public PyPI runtime release.
 .DESCRIPTION
-    Reads a single .version file and rejects any version containing
-    dev, alpha, beta, or rc tags. Intended to gate official releases.
-
-.PARAMETER VersionFile
-    Path to a .version file. Defaults to eng/versions/mssql-python-rs.version
-    relative to the repository root.
-
-.EXAMPLE
-    # Run from repo root (uses default path):
-    .\eng\scripts\validate-release-versions.ps1
-
-    # Explicit path:
-    .\OneBranchPipelines\scripts\validate-release-versions.ps1 -VersionFile C:\work\mssql-python\eng\versions\mssql-python-rs.version
+    BuildOnly rejects overrides and non-release source pins without network access.
+    Release validation additionally requires the selected artifact's transport
+    receipt and verifies the exact version exists on public PyPI before ESRP.
 #>
-
 param(
-    [string]$VersionFile
+    [string]$VersionFile,
+    [string]$ReceiptFile,
+    [switch]$BuildOnly
 )
 
 $ErrorActionPreference = 'Stop'
 
 if (-not $VersionFile) {
-    $repoRoot = (Resolve-Path "$PSScriptRoot\..\..").Path
-    $VersionFile = Join-Path $repoRoot 'eng\versions\mssql-python-rs.version'
+    $VersionFile = Join-Path $PSScriptRoot '../../eng/versions/mssql-python-rs.version'
+}
+
+if (-not [string]::IsNullOrWhiteSpace($env:MSSQL_PYTHON_RS_VERSION)) {
+    throw 'Official builds and releases forbid MSSQL_PYTHON_RS_VERSION overrides, including stable versions.'
 }
 
-if (-not (Test-Path $VersionFile)) {
-    Write-Error "Version file not found: $VersionFile"
-    exit 1
+$version = (Get-Content -LiteralPath $VersionFile -Raw).Trim()
+if ($version -cnotmatch '^[0-9]+\.[0-9]+\.[0-9]+$') {
+    throw "Official builds and releases require an exact stable X.Y.Z runtime version, got '$version'."
 }
 
-$version = (Get-Content $VersionFile -Raw).Trim()
-$name = [System.IO.Path]::GetFileNameWithoutExtension($VersionFile)
+if (-not $BuildOnly) {
+    if (-not $ReceiptFile) {
+        $ReceiptFile = Join-Path (Split-Path -Parent $VersionFile) 'transport.json'
+    }
+    $receipt = Get-Content -LiteralPath $ReceiptFile -Raw | ConvertFrom-Json
+    if ($receipt.version_override -isnot [bool] -or $receipt.version_override) {
+        throw 'Official releases require a receipt with version_override=false.'
+    }
+    if ($receipt.distribution_version -cne $version) {
+        throw 'The staged runtime version and transport receipt disagree.'
+    }
+    if ($receipt.index_url -cnotmatch '^https://pypi\.org/simple/?$') {
+        throw 'Official releases require the public PyPI runtime source.'
+    }
+    if (-not $receipt.wheel_sha256 -or @($receipt.wheel_sha256.PSObject.Properties).Count -eq 0) {
+        throw 'The runtime receipt must record wheel SHA256 hashes.'
+    }
+    foreach ($hash in $receipt.wheel_sha256.PSObject.Properties) {
+        if ($hash.Name -notlike '*.whl' -or $hash.Value -cnotmatch '^[a-f0-9]{64}$') {
+            throw 'The runtime receipt contains an invalid wheel SHA256 entry.'
+        }
+    }
 
-if ($version -match '(dev|alpha|beta|rc)') {
-    Write-Host "FAIL: $name version '$version' is a pre-release ($($Matches[1]))" -ForegroundColor Red
-    Write-Error "$name version is pre-release. Official releases require stable versions."
-    exit 1
+    $release = Invoke-RestMethod -Uri "https://pypi.org/pypi/mssql-python-rs/$version/json" -Method Get
+    if ($release.info.version -cne $version -or
+        $release.info.name -notmatch '^mssql[-_]python[-_]rs$' -or
+        @($release.urls | Where-Object { $_.packagetype -eq 'bdist_wheel' -and -not $_.yanked }).Count -eq 0) {
+        throw "mssql-python-rs==$version has no matching public PyPI wheel release."
+    }
 }
 
-Write-Host "OK: $name version '$version'" -ForegroundColor Green
+Write-Host "OK: mssql-python-rs==$version is eligible for Official $(if ($BuildOnly) { 'build' } else { 'release' })."

OneBranchPipelines/scripts/validate-rs-wheel-dependency.ps1

@@ -3,7 +3,9 @@ param(
     [string]$WheelsDir,
 
     [Parameter(Mandatory = $true)]
-    [string]$VersionFile
+    [string]$VersionFile,
+
+    [string]$ReceiptFile
 )
 
 $ErrorActionPreference = 'Stop'
@@ -12,6 +14,12 @@ $expectedVersion = (Get-Content $VersionFile -Raw).Trim()
 if (-not $expectedVersion) {
     throw "Version file is empty: $VersionFile"
 }
+if ($ReceiptFile) {
+    $receipt = Get-Content -LiteralPath $ReceiptFile -Raw | ConvertFrom-Json
+    if ($receipt.distribution_version -cne $expectedVersion) {
+        throw 'The wheel dependency version and runtime receipt disagree.'
+    }
+}
 
 $wheels = @(Get-ChildItem $WheelsDir -Filter 'mssql_python-*.whl' -File)
 if ($wheels.Count -eq 0) {

OneBranchPipelines/stages/build-linux-single-stage.yml

@@ -74,6 +74,15 @@ stages:
         steps:
           - checkout: self
             fetchDepth: 0
+
+          - ${{ if eq(parameters.oneBranchType, 'Official') }}:
+              - task: PowerShell@2
+                displayName: 'Reject test overrides and prereleases before Official build'
+                inputs:
+                  pwsh: true
+                  targetType: filePath
+                  filePath: OneBranchPipelines/scripts/validate-release-versions.ps1
+                  arguments: '-BuildOnly'
 
           # Docker is pre-installed now
           # Verify it's working and start the daemon
@@ -127,7 +136,7 @@ stages:
                 exit 1
               fi
 
-              docker run -d --name build-$(LINUX_TAG)-$(ARCH) \
+              docker run -e MSSQL_PYTHON_RS_VERSION -d --name build-$(LINUX_TAG)-$(ARCH) \
                 --platform $(DOCKER_PLATFORM) \
                 -v $(Build.SourcesDirectory):/workspace \
                 -w /workspace \
@@ -314,6 +323,7 @@ stages:
                     $PY -m pip install -q --require-hashes -r /workspace/eng/requirements-test-linux.txt;
                     echo "Installing: $WHEEL";
                     $PY -m pip install -q "$WHEEL";
+                    $PY /workspace/eng/scripts/mssql_python_rs.py --verify;
 
                     # Step 6: Verify package imports correctly
                     echo "Verifying package installation...";
@@ -330,7 +340,9 @@ stages:
                     mkdir -p $TEST_DIR/OneBranchPipelines/stages;
                     cp /workspace/eng/scripts/run-mssql-odbc-tests.sh /workspace/eng/scripts/verify_mssql_odbc_provider.py $TEST_DIR/eng/scripts/;
                     cp /workspace/eng/pipelines/mssql-odbc-daily-validation-pipeline.yml $TEST_DIR/eng/pipelines/;
-                    cp /workspace/eng/versions/mssql-python-rs-nuget.version $TEST_DIR/eng/versions/;
+                    cp -r /workspace/eng/pipelines/. $TEST_DIR/eng/pipelines/;
+                    cp -r /workspace/OneBranchPipelines/. $TEST_DIR/OneBranchPipelines/;
+                    cp /workspace/eng/versions/mssql-python-rs.version $TEST_DIR/eng/versions/;
                     cp /workspace/OneBranchPipelines/stages/build-linux-single-stage.yml $TEST_DIR/OneBranchPipelines/stages/;
                     cp /workspace/pytest.ini $TEST_DIR/ || echo "WARNING: No pytest.ini";
 
@@ -395,6 +407,7 @@ stages:
                     $PY -m pip install -q --require-hashes -r /workspace/eng/requirements-test-linux.txt;
                     echo "Installing: $WHEEL";
                     $PY -m pip install -q "$WHEEL";
+                    $PY /workspace/eng/scripts/mssql_python_rs.py --verify;
 
                     # Step 6: Verify package imports correctly
                     echo "Verifying package installation...";
@@ -411,7 +424,9 @@ stages:
                     mkdir -p $TEST_DIR/OneBranchPipelines/stages;
                     cp /workspace/eng/scripts/run-mssql-odbc-tests.sh /workspace/eng/scripts/verify_mssql_odbc_provider.py $TEST_DIR/eng/scripts/;
                     cp /workspace/eng/pipelines/mssql-odbc-daily-validation-pipeline.yml $TEST_DIR/eng/pipelines/;
-                    cp /workspace/eng/versions/mssql-python-rs-nuget.version $TEST_DIR/eng/versions/;
+                    cp -r /workspace/eng/pipelines/. $TEST_DIR/eng/pipelines/;
+                    cp -r /workspace/OneBranchPipelines/. $TEST_DIR/OneBranchPipelines/;
+                    cp /workspace/eng/versions/mssql-python-rs.version $TEST_DIR/eng/versions/;
                     cp /workspace/OneBranchPipelines/stages/build-linux-single-stage.yml $TEST_DIR/OneBranchPipelines/stages/;
                     cp /workspace/pytest.ini $TEST_DIR/ || echo "WARNING: No pytest.ini";
 

OneBranchPipelines/stages/build-macos-single-stage.yml

@@ -77,6 +77,15 @@ stages:
           # fetchDepth: 0 = full git history (needed for version tagging)
           - checkout: self
             fetchDepth: 0
+
+          - ${{ if eq(parameters.oneBranchType, 'Official') }}:
+              - task: PowerShell@2
+                displayName: 'Reject test overrides and prereleases before Official build'
+                inputs:
+                  pwsh: true
+                  targetType: filePath
+                  filePath: OneBranchPipelines/scripts/validate-release-versions.ps1
+                  arguments: '-BuildOnly'
 
           # =========================
           # PYTHON INSTALLATION
@@ -217,7 +226,7 @@ stages:
           # Tests use localhost:1433 connection (SA user with password from variable)
           # -v = verbose output (show test names and results)
           - script: |
-              python -m pytest -v
+              python eng/scripts/mssql_python_rs.py --verify && python -m pytest -v
             displayName: 'Run pytests'
             env:
               # Connection string uses localhost (SQL Server container exposed on port 1433)