diff --git a/OneBranchPipelines/conda-release-pipeline.yml b/OneBranchPipelines/conda-release-pipeline.yml index a2611e809..1fab7af02 100644 --- a/OneBranchPipelines/conda-release-pipeline.yml +++ b/OneBranchPipelines/conda-release-pipeline.yml @@ -8,15 +8,11 @@ # step: required subdirs + full Python matrix + one version), and -- when publishToConda=true -- publishes the SELF-CONTAINED # mssql-python conda package to Anaconda.org via anaconda-client (conda-publish-step; ESRP # has no Conda ContentType). There is a single self-contained package (it vendors the ODBC -# Driver 18 payload); no companion. When publishToConda=false (the default), the same gate -# runs and a DRY-RUN step prints the exact `anaconda upload` plan WITHOUT uploading -- so a -# validate-only run of THIS pipeline replaces the old separate dummy pipeline. -# -# TEMPLATE by mode: publishToConda=false (dry run / dummy) extends the NonOfficial template -# (no Production Service-Tree classification required, so a validate-only run passes the -# inventory-compliance gate); publishToConda=true (real release) extends the Official template -# (validated production publish -- the definition must be classified Production to -# upload). Review the dry-run output, then explicitly select publishToConda=true. +# Driver 18 payload); no companion. When publishToConda=false (the default), the same +# official container runs publisher provisioning and audits without publishing secrets, +# then deliberately fails at a read-only Anaconda authentication probe with a random +# invalid token. Only a confirmed HTTP 401 is the expected rehearsal outcome. +# No upload, promotion or cleanup is attempted in rehearsal, even if authentication succeeds. # An administrator must enable the native Exclusive lock check on 'Anaconda Publishing'. # The production stage uses that shared resource throughout publication and recovery. # Keep its credentials, pipeline authorizations and other checks restricted. @@ -31,7 +27,7 @@ parameters: - name: publishToConda displayName: 'Publish Conda Packages to Anaconda.org (PRODUCTION)' type: boolean - default: false # Safety: default to a validate-only dry run. + default: false # Safety: default to a secretless, invalid-token publisher rehearsal. - name: condaChannel displayName: 'Anaconda.org channel/org to publish to' type: string @@ -72,13 +68,8 @@ resources: trigger: none extends: - # NonOfficial for a validate-only dry run (publishToConda=false): no Production Service-Tree - # classification is required, so the inventory-compliance gate passes. Official for a real - # publish (publishToConda=true): the governed production path. - ${{ if eq(parameters.publishToConda, true) }}: - template: 'v2/OneBranch.Official.CrossPlat.yml@templates' - ${{ else }}: - template: 'v2/OneBranch.NonOfficial.CrossPlat.yml@templates' + # Rehearse the actual managed container/network path, not a NonOfficial substitute. + template: 'v2/OneBranch.Official.CrossPlat.yml@templates' parameters: featureFlags: WindowsHostVersion: @@ -98,8 +89,7 @@ extends: exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' publishLogs: enabled: true - # Official-only heavy gates (binskim break + TSA). The NonOfficial dry run skips them, - # matching the old dummy pipeline's minimal SDL. + # Publication-only compliance outputs; both modes retain managed container isolation. ${{ if eq(parameters.publishToConda, true) }}: binskim: enabled: true @@ -167,97 +157,33 @@ extends: releaseVersions: '$(Provenance.releaseVersions)' rsTransportVersion: '$(Provenance.rsTransportVersion)' - # Dry run (publishToConda=false): enumerate every package and print the exact - # `anaconda upload` plan WITHOUT uploading, so a validate-only run shows precisely - # what a real publish WOULD do. Reuses the conda tree the gate already downloaded - # into $(Build.SourcesDirectory)/conda-artifacts. This replaces the old dummy pipeline. - - ${{ if eq(parameters.publishToConda, false) }}: - - task: PowerShell@2 - displayName: 'Dry-run publish plan (validate-only): enumerate packages + print the anaconda upload plan (NO upload)' - env: - CONDA_CHANNEL: '${{ parameters.condaChannel }}' - CONDA_LABEL: '${{ parameters.condaLabel }}' - inputs: - targetType: 'inline' - script: | - $ErrorActionPreference = 'Stop' - - $root = "$(Build.SourcesDirectory)/conda-artifacts/conda" - if (-not (Test-Path $root)) { - Write-Error "Consolidated conda tree not found at $root. Did the readiness gate download it?" - exit 1 - } - - $pkgs = @(Get-ChildItem -Path $root -Recurse -Include *.conda, *.tar.bz2) - if ($pkgs.Count -eq 0) { Write-Error "No conda packages found under $root."; exit 1 } - $identifierPattern = '^[A-Za-z0-9][A-Za-z0-9._-]*$' - if ($env:CONDA_CHANNEL -notmatch $identifierPattern) { - Write-Error "Invalid condaChannel '$env:CONDA_CHANNEL'. Use letters, digits, dot, underscore, or hyphen; the first character must be alphanumeric." - exit 1 - } - if ($env:CONDA_LABEL -notmatch $identifierPattern) { - Write-Error "Invalid condaLabel '$env:CONDA_LABEL'. Use letters, digits, dot, underscore, or hyphen; the first character must be alphanumeric." - exit 1 - } - - # mssql-python bindings only (the self-contained model ships no companion). - function Get-Binding($items) { @($items | Where-Object { $_.Name -like 'mssql-python-*' -and $_.Name -notlike 'mssql-python-odbc-*' }) } - $bySubdir = $pkgs | Group-Object { $_.Directory.Name } | Sort-Object Name - - Write-Host "=====================================" - Write-Host "[DRY-RUN CONDA RELEASE] discovered $($pkgs.Count) package(s) across $($bySubdir.Count) subdir(s):" - foreach ($g in $bySubdir) { - Write-Host (" {0,-14} {1} mssql-python package(s)" -f $g.Name, (Get-Binding $g.Group).Count) - } - - # A self-contained release ships ONLY mssql-python; flag (don't fail) a stray companion. - $stray = @($pkgs | Where-Object { $_.Name -like 'mssql-python-odbc-*' }) - if ($stray.Count -gt 0) { - Write-Host "##vso[task.logissue type=warning]$($stray.Count) stray mssql-python-odbc package(s) present; the self-contained model ships only mssql-python." - } - - Write-Host "`nanaconda upload PLAN (DRY-RUN - NOT executed):" - Write-Host " target channel/org : $env:CONDA_CHANNEL" - Write-Host " target label : $env:CONDA_LABEL" - foreach ($p in ($pkgs | Sort-Object FullName)) { - $subdir = Split-Path -Leaf (Split-Path -Parent $p.FullName) - $sha256 = (Get-FileHash -LiteralPath $p.FullName -Algorithm SHA256).Hash.ToLowerInvariant() - Write-Host " SHA256: $sha256 $subdir/$($p.Name)" - Write-Host " WOULD RUN: anaconda upload --user $env:CONDA_CHANNEL --label $env:CONDA_LABEL --skip-existing $subdir/$($p.Name)" - } - - Write-Host "`n=====================================" - Write-Host "DRY-RUN COMPLETE: no 'anaconda upload' was executed (publishToConda=false)." - Write-Host "Review this dry run and verify the shared publishing-resource checks before selecting publishToConda=true." - Write-Host "=====================================" - - # PRODUCTION publish (releaseJob) -- runs ONLY when publishToConda=true and - # ONLY after the ValidateConda gate succeeds, so an incomplete set is never - # uploaded. anaconda-client reads ANACONDA_API_TOKEN from the env (never the - # command line), so the token never appears in the logs. - - ${{ if eq(parameters.publishToConda, true) }}: - - job: PublishConda - displayName: 'Publish conda packages to Anaconda.org' - dependsOn: ValidateConda - timeoutInMinutes: 60 - cancelTimeoutInMinutes: 5 - templateContext: - type: releaseJob - isProduction: true - pool: - type: windows - variables: - ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' - mssqlPythonVersion: $[ dependencies.ValidateConda.outputs['Provenance.mssqlPythonVersion'] ] - releaseVersions: $[ dependencies.ValidateConda.outputs['Provenance.releaseVersions'] ] - rsTransportVersion: $[ dependencies.ValidateConda.outputs['Provenance.rsTransportVersion'] ] - steps: - - checkout: self - fetchDepth: 1 - - template: /OneBranchPipelines/steps/conda-publish-step.yml@self - parameters: - condaChannel: '${{ parameters.condaChannel }}' - condaLabel: '${{ parameters.condaLabel }}' - mssqlPythonVersion: '$(mssqlPythonVersion)' - releaseVersions: '$(releaseVersions)' - rsTransportVersion: '$(rsTransportVersion)' + # Both modes share provisioning and audits; only production receives the group. + - job: PublishConda + displayName: 'Publish or rehearse conda packages at Anaconda.org' + dependsOn: ValidateConda + timeoutInMinutes: 60 + cancelTimeoutInMinutes: 5 + templateContext: + type: releaseJob + isProduction: ${{ parameters.publishToConda }} + pool: + type: windows + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' + mssqlPythonVersion: $[ dependencies.ValidateConda.outputs['Provenance.mssqlPythonVersion'] ] + releaseVersions: $[ dependencies.ValidateConda.outputs['Provenance.releaseVersions'] ] + rsTransportVersion: $[ dependencies.ValidateConda.outputs['Provenance.rsTransportVersion'] ] + steps: + - checkout: self + # Match OneBranch's injected checkout; restricted commands cannot relocate it. + path: s + fetchDepth: 1 + - template: /OneBranchPipelines/steps/conda-publish-step.yml@self + parameters: + publishToConda: ${{ parameters.publishToConda }} + condaChannel: '${{ parameters.condaChannel }}' + condaLabel: '${{ parameters.condaLabel }}' + mssqlPythonVersion: '$(mssqlPythonVersion)' + releaseVersions: '$(releaseVersions)' + rsTransportVersion: '$(rsTransportVersion)' diff --git a/OneBranchPipelines/steps/conda-build-validate-step-posix.yml b/OneBranchPipelines/steps/conda-build-validate-step-posix.yml index 0ddfbb158..a440cae91 100644 --- a/OneBranchPipelines/steps/conda-build-validate-step-posix.yml +++ b/OneBranchPipelines/steps/conda-build-validate-step-posix.yml @@ -93,12 +93,13 @@ steps: --stage-dir "$(ob_outputDirectory)/conda" \ --conda-subdir "${{ parameters.condaSubdir }}" \ --conda-target-subdir "${CONDA_TARGET_SUBDIR:-}" \ - --python-versions "${PYTHON_VERSIONS:-}" + --python-versions "${MSSQL_CONDA_PYTHON_VERSIONS:-}" workingDirectory: $(Build.SourcesDirectory) env: # Surface the caller's value inputs as env vars (validated in-script above) rather than # interpolating them into the script text -- repo secret-handling convention. - PYTHON_VERSIONS: ${{ parameters.pythonVersions }} + # Avoid the shared PYTHON_VERSIONS pipeline variable overwriting this input. + MSSQL_CONDA_PYTHON_VERSIONS: ${{ parameters.pythonVersions }} CONDA_TARGET_SUBDIR: ${{ parameters.condaTargetSubdir }} displayName: 'Conda build + validate (${{ parameters.condaSubdir }})' continueOnError: ${{ parameters.continueOnError }} diff --git a/OneBranchPipelines/steps/conda-publish-step.yml b/OneBranchPipelines/steps/conda-publish-step.yml index 9c406726a..0dc352594 100644 --- a/OneBranchPipelines/steps/conda-publish-step.yml +++ b/OneBranchPipelines/steps/conda-publish-step.yml @@ -20,14 +20,19 @@ # no separate companion to order; we still refuse to publish an incomplete or # mislabeled set. # -# The caller MUST: +# For production publication, the caller MUST: # - run this ONLY after the conda-release-step.yml readiness gate has passed, and # - keep the entire step in a stage consuming the shared publishing variable group # with its administrator-configured native Exclusive lock check, and # - supply ANACONDA_API_TOKEN (variable group 'Anaconda Publishing') to the job. # This template never puts the token on the command line (anaconda-client reads it # from the env), so it never appears in the logs. +# Rehearsal shares provisioning/audits but uses only a random invalid token and a +# read-only authentication probe; the production credential group must be omitted. parameters: + - name: publishToConda + type: boolean + default: true # Consolidated conda artifact name (see consolidate-conda-artifacts-job.yml). - name: condaArtifactName type: string @@ -69,10 +74,6 @@ parameters: - name: pythonVersions type: string default: '3.10,3.11,3.12,3.13,3.14,3.15' - # The publisher hash lock targets CPython 3.12 / Windows x64; other runtimes reject. - - name: pythonVersion - type: string - default: '3.12' # Optional display-name prefix (e.g. '[TEST] ' for the dummy pipeline). - name: labelPrefix type: string @@ -90,14 +91,12 @@ steps: artifactName: '${{ parameters.condaArtifactName }}' targetPath: '$(Build.SourcesDirectory)/conda-artifacts' - - task: UsePythonVersion@0 - displayName: '${{ parameters.labelPrefix }}Use Python ${{ parameters.pythonVersion }}' - inputs: - versionSpec: '${{ parameters.pythonVersion }}' - addToPath: true - - task: PowerShell@2 - displayName: '${{ parameters.labelPrefix }}Install anaconda-client' + name: PublisherBootstrap + displayName: '${{ parameters.labelPrefix }}Verify image Python and install isolated anaconda-client' + target: + container: windows_build_container + commands: restricted env: ANACONDA_API_TOKEN: '' SYSTEM_ACCESSTOKEN: '' @@ -105,46 +104,76 @@ steps: targetType: 'inline' script: | $ErrorActionPreference = 'Stop' - python -c "import sys, sysconfig; assert sys.implementation.name == 'cpython' and sys.version_info[:2] == (3, 12) and sys.platform == 'win32' and sysconfig.get_platform() == 'win-amd64', 'Publisher lock requires CPython 3.12 Windows x64'" - if ($LASTEXITCODE -ne 0) { throw "Publisher runtime does not match the reviewed hash lock." } + $baseDirectory = Join-Path ([Environment]::GetFolderPath('LocalApplicationData')) 'Programs\Python\Python312' + $basePython = Join-Path $baseDirectory 'python.exe' + if (-not (Test-Path -LiteralPath $basePython -PathType Leaf)) { + throw "Required image Python is missing: $basePython. No installation or fallback is permitted." + } + @' + import json, pathlib, sys, sysconfig + root = pathlib.Path(sys.argv[1]).resolve() + print("PUBLISHER_IMAGE_PYTHON: " + json.dumps({ + "executable": sys.executable, "version": sys.version, + "prefix": sys.prefix, "base_prefix": sys.base_prefix, + "platform": sysconfig.get_platform(), + })) + assert sys.implementation.name == "cpython" and sys.version_info[:2] == (3, 12) and sys.platform == "win32" and sysconfig.get_platform() == "win-amd64", "Publisher lock requires CPython 3.12 Windows x64" + assert pathlib.Path(sys.executable).resolve() == root / "python.exe" and pathlib.Path(sys.prefix).resolve() == pathlib.Path(sys.base_prefix).resolve() == root, "Image Python executable/prefix mismatch" + import ssl, venv, encodings, _ssl, ensurepip, xml.etree.ElementTree as et + misplaced = {m.__name__: str(pathlib.Path(m.__file__).resolve()) for m in (ssl, venv, encodings, _ssl, ensurepip, et) if not pathlib.Path(m.__file__).resolve().is_relative_to(root)} + assert not misplaced, f"Image Python standard library outside {root}: {misplaced}" + ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + print("PUBLISHER_BASE_STDLIB_OK:", root) + '@ | & $basePython -I - "$baseDirectory" + if ($LASTEXITCODE -ne 0) { throw "Image Python failed runtime/stdlib checks: $basePython. No installation or fallback is permitted." } $lock = "$(Build.SourcesDirectory)/conda/requirements-publish.txt" if (-not (Test-Path -LiteralPath $lock -PathType Leaf)) { throw "Publisher dependency lock is missing." } # A fresh venv avoids trusting already-installed packages or their startup hooks. # Only the later publishing task maps credentials into its environment. - $publisherEnvironment = Join-Path "$(Agent.TempDirectory)" ("conda-publisher-" + [guid]::NewGuid().ToString('N')) + $publisherWorkspace = Join-Path "$(Agent.TempDirectory)" ("conda-publisher-" + [guid]::NewGuid().ToString('N')) + if (Test-Path -LiteralPath $publisherWorkspace) { throw "Publisher workspace already exists." } + New-Item -ItemType Directory -Path $publisherWorkspace | Out-Null + $publisherEnvironment = Join-Path $publisherWorkspace 'venv' if (Test-Path -LiteralPath $publisherEnvironment) { throw "Publisher environment already exists." } - python -m venv "$publisherEnvironment" + & $basePython -I -m venv "$publisherEnvironment" if ($LASTEXITCODE -ne 0) { throw "Could not create the isolated publisher environment." } $publisherPython = Join-Path $publisherEnvironment 'Scripts\python.exe' - & $publisherPython -m pip --isolated install --disable-pip-version-check --require-hashes --only-binary=:all: ` + & $publisherPython -I -c "import pathlib, sys; assert sys.prefix != sys.base_prefix and pathlib.Path(sys.base_prefix).resolve()==pathlib.Path(sys.argv[1]).parent.resolve() and pathlib.Path(sys.executable).resolve()==pathlib.Path(sys.prefix).resolve()/'Scripts'/'python.exe', 'Publisher venv does not belong to the verified image interpreter'; print('PUBLISHER_VENV_OK:', sys.executable)" "$basePython" + if ($LASTEXITCODE -ne 0) { throw "Publisher virtual environment has an unexpected interpreter." } + & $publisherPython -I -m pip --isolated install --disable-pip-version-check --require-hashes --only-binary=:all: ` --index-url "https://packagefeedproxy.microsoft.io/pypi/simple/" ` -r "$lock" if ($LASTEXITCODE -ne 0) { Write-Error "Failed to install the complete hash-locked publisher dependency closure." exit 1 } - & $publisherPython -m pip --isolated check + & $publisherPython -I -m pip --isolated check if ($LASTEXITCODE -ne 0) { throw "Publisher dependency closure is inconsistent." } - & $publisherPython -c "import importlib.metadata as m; assert m.version('anaconda-client') == '1.14.1'; assert m.version('zstandard') == '0.23.0'; from binstar_client.utils import get_server_api; import zstandard; print('pinned publisher APIs ready')" + & $publisherPython -I -c "import importlib.metadata as m; assert m.version('anaconda-client') == '1.14.1'; assert m.version('zstandard') == '0.23.0'; from binstar_client.utils import get_server_api; import zstandard; print('pinned publisher APIs ready')" if ($LASTEXITCODE -ne 0) { Write-Error "Pinned anaconda-client version/API verification failed." exit 1 } $env:ANACONDA_CLIENT_FORCE_STANDALONE = '1' - & $publisherPython -m binstar_client.scripts.cli upload --help | Out-Null + & $publisherPython -I -m binstar_client.scripts.cli upload --help | Out-Null if ($LASTEXITCODE -ne 0) { Write-Error "Pinned anaconda-client upload module is unavailable." exit 1 } - Write-Host "##vso[task.prependpath]$(Join-Path $publisherEnvironment 'Scripts')" + Write-Host "##vso[task.setvariable variable=pythonExecutable;isOutput=true;isReadOnly=true]$publisherPython" - task: PowerShell@2 - displayName: '${{ parameters.labelPrefix }}Publish conda packages to anaconda.org/${{ parameters.condaChannel }}' + displayName: '${{ parameters.labelPrefix }}Publish or rehearse conda packages at anaconda.org/${{ parameters.condaChannel }}' env: # The API receives this token explicitly; the CLI inherits it after the # competing BINSTAR_API_TOKEN is removed below. Keep it off command lines. # Supplied by the caller from the 'Anaconda Publishing' variable group. - ANACONDA_API_TOKEN: $(ANACONDA_API_TOKEN) + ${{ if eq(parameters.publishToConda, true) }}: + ANACONDA_API_TOKEN: $(ANACONDA_API_TOKEN) + ${{ else }}: + ANACONDA_API_TOKEN: '' + SYSTEM_ACCESSTOKEN: '' + MSSQL_CONDA_PUBLISHER_PYTHON: '$(PublisherBootstrap.pythonExecutable)' ANACONDA_CLIENT_FORCE_STANDALONE: '1' BINSTAR_CONFIG_DIR: '$(Agent.TempDirectory)/anaconda-client-$(Build.BuildId)' # Queue-time values are surfaced as env vars rather than interpolated into the script text. @@ -156,12 +185,23 @@ steps: RS_TRANSPORT_VERSION: '${{ parameters.rsTransportVersion }}' REQUIRED_SUBDIRS: '${{ parameters.requiredSubdirs }}' ALLOWED_SUBDIRS: '${{ parameters.allowedSubdirs }}' - PYTHON_VERSIONS: '${{ parameters.pythonVersions }}' + # Avoid the shared PYTHON_VERSIONS pipeline variable overwriting this input. + MSSQL_CONDA_PYTHON_VERSIONS: '${{ parameters.pythonVersions }}' inputs: targetType: 'inline' workingDirectory: '$(Build.SourcesDirectory)' script: | $ErrorActionPreference = 'Stop' + $pythonExecutable = $env:MSSQL_CONDA_PUBLISHER_PYTHON + $workspacePattern = '^' + [regex]::Escape("$(Agent.TempDirectory)".TrimEnd('\')) + '\\conda-publisher-[0-9a-f]{32}\\venv\\Scripts\\python\.exe$' + if ([string]::IsNullOrWhiteSpace($pythonExecutable) -or + [IO.Path]::GetFullPath($pythonExecutable) -notmatch $workspacePattern -or + -not (Test-Path -LiteralPath $pythonExecutable -PathType Leaf)) { + throw "Publisher interpreter output is missing or outside this job's isolated workspace." + } + foreach ($variable in @('PYTHONHOME', 'PYTHONPATH')) { + [Environment]::SetEnvironmentVariable($variable, $null, 'Process') + } # Refuse to publish without an explicit target channel (protects the dummy # pipeline, whose default channel is empty, from ever hitting production). @@ -188,7 +228,11 @@ steps: exit 1 } - if ([string]::IsNullOrWhiteSpace($env:ANACONDA_API_TOKEN)) { + if ('${{ parameters.publishToConda }}' -eq 'False') { + # Do not trust task-env precedence over public variables in rehearsal. + [Environment]::SetEnvironmentVariable('ANACONDA_API_TOKEN', $null, 'Process') + } + elseif ([string]::IsNullOrWhiteSpace($env:ANACONDA_API_TOKEN)) { Write-Error "ANACONDA_API_TOKEN is not set. Add the 'Anaconda Publishing' variable group to the job." exit 1 } @@ -203,7 +247,7 @@ steps: New-Item -ItemType Directory -Force -Path $clientConfigDir | Out-Null @('url: https://api.anaconda.org', 'ssl_verify: true') | Set-Content -Path (Join-Path $clientConfigDir 'config.yaml') -Encoding ASCII - python -c "from binstar_client.utils import get_config; c=get_config(); assert c['url']=='https://api.anaconda.org'; assert c['ssl_verify'] is True; print('Anaconda endpoint configuration verified')" + & $pythonExecutable -c "from binstar_client.utils import get_config; c=get_config(); assert c['url']=='https://api.anaconda.org'; assert c['ssl_verify'] is True; print('Anaconda endpoint configuration verified')" if ($LASTEXITCODE -ne 0) { Write-Error "Failed to verify isolated Anaconda endpoint configuration." exit 1 @@ -217,12 +261,12 @@ steps: Write-Error "Conda release metadata gate not found at $metadataGate." exit 1 } - python -m eng.conda_tools validate ` + & $pythonExecutable -m eng.conda_tools validate ` --release-versions ` --root "$root" ` --required-subdirs "$env:REQUIRED_SUBDIRS" ` --allowed-subdirs "$env:ALLOWED_SUBDIRS" ` - --pythons "$env:PYTHON_VERSIONS" ` + --pythons "$env:MSSQL_CONDA_PYTHON_VERSIONS" ` --mssql-python-version "$env:MSSQL_PYTHON_VERSION" if ($LASTEXITCODE -ne 0) { Write-Error "Credential-boundary Conda release metadata validation failed." @@ -238,7 +282,7 @@ steps: $pkgs = @(Get-ChildItem -Path $root -Recurse -File -Include *.conda, *.tar.bz2 | Sort-Object FullName) $packagePaths = @($pkgs | ForEach-Object { $_.FullName }) $stagingLabel = "$($env:CONDA_LABEL)_staging_$(Build.BuildId)" - python -m eng.conda_tools promote ` + & $pythonExecutable -m eng.conda_tools promote ` --owner "$env:CONDA_CHANNEL" ` --staging-label "$stagingLabel" ` --target-label "$env:CONDA_LABEL" ` @@ -250,6 +294,21 @@ steps: exit 1 } + # Audit this job's independent download, not only the earlier validation job's copy. + foreach ($audit in @('elf', 'pe', 'macho')) { + & $pythonExecutable -m eng.conda_tools $audit --root "$root" + if ($LASTEXITCODE -ne 0) { + Write-Error "$audit native artifact audit failed before upload." + exit 1 + } + } + + if ('${{ parameters.publishToConda }}' -eq 'False') { + & $pythonExecutable -c "from eng.conda_tools.publication import rehearse_auth; rehearse_auth()" + if ($LASTEXITCODE -eq 0) { throw "Rehearsal must reject the invalid token." } + exit $LASTEXITCODE + } + # Staged-then-promoted publish: upload the FULL set to a BUILD-UNIQUE staging # label first; only after every package lands AND its bytes are verified do we # begin adding the public label. Failed uploads/promotions clean this run's staging @@ -270,7 +329,6 @@ steps: AttemptedPaths = [System.Collections.Generic.List[string]]::new() TerminationUncertain = $false } - $pythonExecutable = (Get-Command python -CommandType Application -ErrorAction Stop | Select-Object -First 1).Source function Invoke-Anaconda($argsList) { for ($attempt = 1; $attempt -le 3; $attempt++) { $remainingMs = $uploadBudgetMs - $uploadState.Clock.ElapsedMilliseconds @@ -367,7 +425,7 @@ steps: Invoke-Anaconda @('upload', '--user', "$env:CONDA_CHANNEL", '--label', $stagingLabel, '--skip-existing', "$($p.FullName)") } - python -m eng.conda_tools promote ` + & $pythonExecutable -m eng.conda_tools promote ` --owner "$env:CONDA_CHANNEL" ` --staging-label "$stagingLabel" ` --target-label "$env:CONDA_LABEL" ` @@ -386,7 +444,7 @@ steps: $attemptedPaths = @($uploadState.AttemptedPaths.ToArray()) if ($attemptedPaths.Count -gt 0) { try { - python -m eng.conda_tools promote ` + & $pythonExecutable -m eng.conda_tools promote ` --owner "$env:CONDA_CHANNEL" ` --staging-label "$stagingLabel" ` --target-label "$env:CONDA_LABEL" ` diff --git a/OneBranchPipelines/steps/conda-release-step.yml b/OneBranchPipelines/steps/conda-release-step.yml index 3e3cce161..79a946aae 100644 --- a/OneBranchPipelines/steps/conda-release-step.yml +++ b/OneBranchPipelines/steps/conda-release-step.yml @@ -77,10 +77,6 @@ parameters: - name: rsTransportVersion type: string default: '' - # The audit dependency hash targets CPython 3.12 / Windows x64, like the publisher. - - name: pythonVersion - type: string - default: '3.12' # Optional display-name prefix (e.g. '[TEST] ' for the dummy pipeline). - name: labelPrefix type: string @@ -98,12 +94,7 @@ steps: artifactName: '${{ parameters.condaArtifactName }}' targetPath: '$(Build.SourcesDirectory)/conda-artifacts' - - task: UsePythonVersion@0 - displayName: '${{ parameters.labelPrefix }}Use Python ${{ parameters.pythonVersion }} for release validation' - inputs: - versionSpec: '${{ parameters.pythonVersion }}' - addToPath: true - + # The caller already selected Python 3.12 before its provenance gate. - task: PowerShell@2 displayName: '${{ parameters.labelPrefix }}Install hash-locked release-audit dependency' inputs: @@ -157,7 +148,8 @@ steps: env: REQUIRED_SUBDIRS: '${{ parameters.requiredSubdirs }}' ALLOWED_SUBDIRS: '${{ parameters.allowedSubdirs }}' - PYTHON_VERSIONS: '${{ parameters.pythonVersions }}' + # Avoid the shared PYTHON_VERSIONS pipeline variable overwriting this input. + MSSQL_CONDA_PYTHON_VERSIONS: '${{ parameters.pythonVersions }}' MSSQL_PYTHON_VERSION: '${{ parameters.mssqlPythonVersion }}' RELEASE_VERSIONS: '${{ parameters.releaseVersions }}' RS_TRANSPORT_VERSION: '${{ parameters.rsTransportVersion }}' @@ -178,7 +170,7 @@ steps: exit 1 } - foreach ($policy in @('REQUIRED_SUBDIRS', 'ALLOWED_SUBDIRS', 'PYTHON_VERSIONS')) { + foreach ($policy in @('REQUIRED_SUBDIRS', 'ALLOWED_SUBDIRS', 'MSSQL_CONDA_PYTHON_VERSIONS')) { if ([string]::IsNullOrWhiteSpace([Environment]::GetEnvironmentVariable($policy))) { Write-Error "$policy is empty. Refusing to weaken the release matrix." exit 1 @@ -189,7 +181,7 @@ steps: '--root', $root, '--required-subdirs', $env:REQUIRED_SUBDIRS, '--allowed-subdirs', $env:ALLOWED_SUBDIRS, - '--pythons', $env:PYTHON_VERSIONS + '--pythons', $env:MSSQL_CONDA_PYTHON_VERSIONS ) if (-not [string]::IsNullOrWhiteSpace($env:MSSQL_PYTHON_VERSION)) { $gateArgs += @('--mssql-python-version', $env:MSSQL_PYTHON_VERSION.Trim()) diff --git a/conda/README.md b/conda/README.md index efbd5fb92..795216a57 100644 --- a/conda/README.md +++ b/conda/README.md @@ -147,6 +147,16 @@ Windows ARM64 3.15 uses `microsoft`, then `conda-forge`, for both build and veri existing channel policies and the ARM64 3.12 manual ABI pin. The blocking full dependency solve remains; every 3.15 package must pass the `python_abi 3.15.* *_cp315` native audit. +Windows builds pass the package target through conda-build's `target_platform` +variant, leaving its build process native (win-64 on the x64 agent). Only that +command's environment removes `CONDA_SUBDIR`, avoiding the ARM64 `/machine` +launcher on Windows Server 2022. Verification keeps `CONDA_SUBDIR=win-arm64` +for both target environment creation and the full dependency dry-run. + +POSIX steps pass the requested Python matrix through `MSSQL_CONDA_PYTHON_VERSIONS` +so the shared `PYTHON_VERSIONS` pipeline variable cannot replace it. Custom subsets +and empty-input wheel auto-detection remain supported. + ### Python 3.15 packaging Conda-forge's Windows Python 3.15 declares `lib/python/site-packages` instead of @@ -215,13 +225,56 @@ access. `promote --check-local-only` needs neither a publishing token nor an Anaconda client; actual publication and recovery require restricted credentials and the shared protected release stage described below. Module placement is not a new authorization boundary. The PowerShell upload process deadlines, attempted-file -tracking, process-tree termination, and dry-run plan remain in their pipeline tasks. +tracking and process-tree termination remain in their pipeline tasks. Network publication and recovery require an explicit nonblank `ANACONDA_API_TOKEN`, not an ambient or cached login. The publishing task removes `BINSTAR_API_TOKEN` only from its process and children so the CLI uploader uses the same reviewed credential. User/machine environment settings and saved credentials are not modified. -Only trusted, reviewed release revisions may be queued, including validate-only feature +The official Windows publisher invokes the image's existing interpreter by its +full path: `%LOCALAPPDATA%\Programs\Python\Python312\python.exe` (the account's +LocalApplicationData special folder, not a PATH search). It requires CPython 3.12 +Windows x64, without unnecessarily pinning the patch version. Before creating any +environment, it checks executable/prefix/base-prefix coherence and required +SSL/native, encoding, venv and XML module ownership. A missing or unusable image +interpreter fails explicitly; there is no automatic installation, uninstall, +registry repair, runtime download or fallback to another interpreter. + +The bootstrap creates a unique job-owned venv under `Agent.TempDirectory` and +verifies that it belongs to the checked image interpreter. Only that venv's absolute +path is passed to the publishing task through a read-only task output. Dependencies +still use the unchanged hash lock and explicit package-feed-proxy `--index-url` +with pip's `--isolated` mode. Both production and rehearsal use this same code. + +With `publishToConda=false` (the default), the release pipeline now rehearses the +publisher instead of only printing an upload plan. Both modes use the Official +template, Windows 2022 managed container, network isolation, independent producer +download, Python 3.12 bootstrap, hash-locked dependencies and metadata/native audits. +The publisher checkout explicitly uses `path: s`, matching OneBranch's injected +checkout so the lock, Python modules and artifacts share `Build.SourcesDirectory` +on the host and in the container; restricted task commands remain enabled. + +Rehearsal omits `Anaconda Publishing` and its production lock, marks the job +nonproduction and uses a newly generated invalid token, never a supplied or cached +credential. After all shared gates pass, the pinned client makes only an authenticated +`GET https://api.anaconda.org/user`. The expected result is a **failed task** with +`EXPECTED_REHEARSAL_AUTH_REJECTION` and HTTP 401. Network/TLS/setup errors, other HTTP +statuses and unexpectedly accepted credentials are failures, not successful rehearsals. +No rehearsal path can reach upload, promotion or cleanup. This tests the authentication +boundary, not valid-token authorization or successful publication/recovery. + +To rehearse a reviewed PR branch, queue the existing release definition with that +exact branch/commit, `publishToConda=false` and the intended recorded Conda producer +run. Inspect actual hosted logs through the expected rejection; a compiled preview +or a local interpreter check does not establish hosted image/venv/dependency coverage. +Resource or policy denials must be resolved by the owner, never by weakening the +production checks. `publishToConda=true` retains the main-branch provenance guard, +publishing credential group and shared exclusive lock. +The old upload-plan-only task and duplicate interpreter selection in the validation +job are removed. Superseded MSI cleanup and temporary installer diagnostics are removed; +the short base-library check remains. Independent publisher artifact rechecks are retained. + +Only trusted, reviewed release revisions may be queued, including rehearsal feature branches: their YAML and Python run with `System.AccessToken`. YAML provenance checks cannot sandbox an untrusted queued revision or protect that token from it. Administrators must restrict pipeline editing/queueing and source access, minimize the job identity's @@ -238,8 +291,8 @@ must fail before the stage receives the publishing credential. Preserve existing permissions and approvals, and restrict permission to administer or bypass the check. Do not grant broader pipeline access as part of configuring it. -The existing `publishToConda` condition omits this group for validate-only runs, so -trusted feature-branch validation remains separate from credentialed publication. +The existing `publishToConda` condition omits this group for rehearsal runs, so +trusted feature-branch rehearsal remains separate from credentialed publication. Neither a checkout-local branch check nor a secondary pinned checkout can replace the resource-side control: editable YAML could bypass either. These external checks are not installed by this repository; production must remain unapproved until the @@ -255,7 +308,7 @@ create the external check. The lock spans validation, staged uploads, the initia label snapshot, promotion, rollback and staging cleanup until the stage ends. All publishers and recovery paths for `microsoft/mssql-python` must consume that same resource, across versions and target labels; separate groups would not serialize -them. Keep the group and lock behavior absent from validate-only runs. +them. Keep the group and lock behavior absent from rehearsal runs. Recover by rerunning the original protected release stage with its original artifact inputs, preserving the build-specific staging label and reacquiring the shared lock. diff --git a/eng/conda_tools/build.py b/eng/conda_tools/build.py index f68392cb0..36d80afc6 100644 --- a/eng/conda_tools/build.py +++ b/eng/conda_tools/build.py @@ -302,6 +302,11 @@ def build_packages( croot = os.path.join(os.path.dirname(os.path.abspath(bld)), "croot") if os.path.isdir(croot): shutil.rmtree(croot) + build_env = env.copy() + if target_subdir.startswith("win-"): + # CONDA_SUBDIR also changes conda-build's build machine, selecting a /machine + # launcher on x64 -> ARM64. Keep the native process; target only the host variant. + build_env.pop("CONDA_SUBDIR", None) for py in pyvers: channels = ["microsoft", "conda-forge"] if target_subdir == "win-arm64" and py != "3.15": @@ -325,10 +330,12 @@ def build_packages( "--croot", croot, ] + if target_subdir.startswith("win-"): + cmd += ["--variants", json.dumps({"target_platform": target_subdir})] for channel in channels: cmd += ["-c", channel] cmd.append("--override-channels") - environment.run(cmd, env=env, what=f"conda-build mssql-python (py {py})") + environment.run(cmd, env=build_env, what=f"conda-build mssql-python (py {py})") # A local channel is only valid if it ALSO carries noarch/repodata.json (even empty) -- # conda-build wrote it only for the built subdir. Create it directly (miniforge has no diff --git a/eng/conda_tools/environment.py b/eng/conda_tools/environment.py index 1268ddd0f..aa43dbb19 100644 --- a/eng/conda_tools/environment.py +++ b/eng/conda_tools/environment.py @@ -186,9 +186,9 @@ def build_env( env["MSSQL_ODBC_VERSION"] = odbc_ver env["MSSQL_RS_VERSION"] = rs_ver or "" if cross_target_subdir: - # conda-build AND the verify `conda create` honor CONDA_SUBDIR -> the packages are - # stamped for the target subdir and the import check runs the target Python where the - # host can execute it (natively / Rosetta 2 / QEMU binfmt). + # Verification must solve for the target, not the native build machine. + # Windows conda-build uses a separate env copy and target_platform variant; + # POSIX builds retain CONDA_SUBDIR for their emulated target execution. env["CONDA_SUBDIR"] = cross_target_subdir _log(f"Cross-targeting conda subdir: CONDA_SUBDIR={cross_target_subdir}") if cross_target_subdir.endswith("aarch64") and os.path.isdir("/usr/aarch64-linux-gnu"): diff --git a/eng/conda_tools/provenance.py b/eng/conda_tools/provenance.py index 126b500e9..072c00d6c 100644 --- a/eng/conda_tools/provenance.py +++ b/eng/conda_tools/provenance.py @@ -205,7 +205,7 @@ def execute() -> None: transport = result["rsTransportVersion"] print(f"##vso[task.setvariable variable=rsTransportVersion;isOutput=true]{transport}") if publish == "false": - print("Validate-only: provenance verified; package readiness is gated separately.") + print("Non-publishing run: provenance verified; publisher rehearsal is gated separately.") def cli() -> int: diff --git a/eng/conda_tools/publication.py b/eng/conda_tools/publication.py index bba59b9dc..668481f6b 100644 --- a/eng/conda_tools/publication.py +++ b/eng/conda_tools/publication.py @@ -467,6 +467,39 @@ def run_verification() -> set[str]: ) +def rehearse_auth() -> None: + """Deliberately fail at a read-only authentication boundary; never publish.""" + from uuid import uuid4 + + token = f"mssql-python-invalid-rehearsal-{uuid4().hex}" + os.environ["ANACONDA_API_TOKEN"] = token + os.environ.pop("BINSTAR_API_TOKEN", None) + + from binstar_client.errors import Unauthorized # type: ignore[import-not-found] + from binstar_client.utils import get_server_api # type: ignore[import-not-found] + + api = get_server_api(token=token, config={"url": _ANACONDA_API_URL, "ssl_verify": True}) + request = api.session.request + + def rehearsal_request(*args: Any, **kwargs: Any) -> Any: + kwargs.update(timeout=(15, 60), allow_redirects=False) + return request(*args, **kwargs) + + api.session.request = rehearsal_request + try: + api.user() + except Unauthorized as error: + if len(error.args) != 2 or error.args[1] != 401: + raise + print( + "EXPECTED_REHEARSAL_AUTH_REJECTION: GET https://api.anaconda.org/user HTTP 401; " + "no upload, promotion or cleanup attempted.", + flush=True, + ) + raise SystemExit(1) from None + raise RuntimeError("Rehearsal token unexpectedly accepted; no publication attempted.") + + def add_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument("--owner", required=True) parser.add_argument("--staging-label", required=True) diff --git a/tests/test_027_conda_release_metadata.py b/tests/test_027_conda_release_metadata.py index 7d599683f..1e08a7472 100644 --- a/tests/test_027_conda_release_metadata.py +++ b/tests/test_027_conda_release_metadata.py @@ -13,6 +13,7 @@ import importlib.util import io import json +import re import sys import tarfile import types @@ -531,6 +532,112 @@ def test_conda_python_defaults_are_synchronized(): ) in build_source +@pytest.mark.parametrize("selected", ["3.10,3.11,3.12,3.13,3.14,3.15", "3.12,3.15", ""]) +def test_posix_python_selection_survives_public_pipeline_variables(selected): + source = (_ROOT / "OneBranchPipelines/steps/conda-build-validate-step-posix.yml").read_text( + encoding="utf-8" + ) + parameters = {"pythonVersions": selected, "condaTargetSubdir": ""} + step_env = { + name: parameters[parameter] + for name, parameter in re.findall( + r"^ (\w+): \$\{\{ parameters\.(\w+) \}\}$", source, re.MULTILINE + ) + } + # The agent overlays public pipeline variables after the explicit step environment. + step_env.update(PYTHON_VERSIONS="3.10,3.11,3.12,3.13,3.14") + argument = re.search(r'--python-versions "\$\{(\w+):-\}"', source) + assert argument is not None + assert step_env.get(argument[1], "") == selected + + +def _run_conda_powershell(script, env=None): + import shutil + import subprocess + + shell = shutil.which("powershell") or shutil.which("pwsh") + assert shell is not None, "The Windows Conda tasks require PowerShell" + return subprocess.run( + [ + shell, + "-NoProfile", + "-NonInteractive", + "-Command", + "$ErrorActionPreference = 'Stop'\n" + script, + ], + env=env, + capture_output=True, + text=True, + timeout=30, + ) + + +@pytest.fixture(params=["release", "publish"]) +def conda_matrix_step(request): + source = (_ROOT / f"OneBranchPipelines/steps/conda-{request.param}-step.yml").read_text( + encoding="utf-8" + ) + parameters = dict( + re.findall(r"- name: (\w+)\n type: string\n default: '([^']*)'", source) + ) + step = source.rsplit(" - task: PowerShell@2\n", 1)[1] + environment = dict( + re.findall(r"^ (\w+): '\$\{\{ parameters\.(\w+) \}\}'$", step, re.MULTILINE) + ) + return request.param, step, parameters, environment + + +@pytest.mark.skipif(sys.platform != "win32", reason="Release tasks run on Windows") +@pytest.mark.parametrize("selected", [None, "3.12,3.15", "", " \t "]) +def test_release_matrix_powershell_arguments(conda_matrix_step, selected): + import os + import textwrap + + kind, step, parameters, environment = conda_matrix_step + if selected is not None: + parameters["pythonVersions"] = selected + env = dict(os.environ) + env.update({name: parameters[parameter] for name, parameter in environment.items()}) + env["PYTHON_VERSIONS"] = "3.10,3.11,3.12,3.13,3.14" + if kind == "release": + start = step.index(" foreach ($policy in @(") + script = step[start:] + else: + start = step.index(" & $pythonExecutable -m eng.conda_tools validate `") + script = step[start : step.index(" $promoter =")] + # Execute the production guard/argument construction, but never publish or contact a service. + result = _run_conda_powershell( + """ +$root = 'unused' +$pythonExecutable = 'python' +function python { + Write-Output ('GATE_ARGS:' + (ConvertTo-Json -InputObject @($args) -Compress)) + $global:LASTEXITCODE = 0 +} +""" + textwrap.dedent(script), + env=env, + ) + expected = parameters["pythonVersions"] + if kind == "release" and not expected.strip(): + assert result.returncode != 0 + assert "is empty. Refusing to weaken the release matrix." in result.stderr + assert "GATE_ARGS:" not in result.stdout + else: + assert result.returncode == 0, result.stdout + result.stderr + calls = [ + json.loads(line.removeprefix("GATE_ARGS:")) + for line in result.stdout.splitlines() + if line.startswith("GATE_ARGS:") + ] + assert len(calls) == 1 + assert calls[0][:3] == ["-m", "eng.conda_tools", "validate"] + arguments = cli.parser().parse_args(calls[0][2:]) + assert arguments.pythons == expected + assert arguments.required_subdirs == parameters["requiredSubdirs"] + assert arguments.allowed_subdirs == parameters["allowedSubdirs"] + assert "--release-versions" in calls[0] + + def test_parse_subdir_pythons(): assert vcr._parse_subdir_pythons("") == {} assert vcr._parse_subdir_pythons("a=3.10;b=3.11,3.12") == { @@ -1733,46 +1840,276 @@ def test_publication_lock_covers_the_complete_release_stage(): assert "- job: PublishConda" in jobs -def test_dry_release_excludes_the_complete_publication_job(): +def test_dry_release_shares_the_official_publisher_without_credentials(): pipeline = (_ROOT / "OneBranchPipelines" / "conda-release-pipeline.yml").read_text() - assert " default: false # Safety: default to a validate-only dry run." in pipeline - guarded_job = pipeline.split( - " - ${{ if eq(parameters.publishToConda, true) }}:\n" - " - job: PublishConda\n", - 1, - )[1] - assert " dependsOn: ValidateConda\n" in guarded_job - assert " type: releaseJob\n isProduction: true\n" in guarded_job + assert "default: false # Safety: default to a secretless" in pipeline + assert "OneBranch.NonOfficial" not in pipeline + guarded_job = pipeline.split(" - job: PublishConda\n", 1)[1] + assert " dependsOn: ValidateConda\n" in guarded_job + assert ( + " type: releaseJob\n" + " isProduction: ${{ parameters.publishToConda }}\n" + ) in guarded_job template = "/OneBranchPipelines/steps/conda-publish-step.yml@self" assert pipeline.count(template) == 1 assert template in guarded_job + assert " publishToConda: ${{ parameters.publishToConda }}" in guarded_job + assert "path: s" in guarded_job + assert ( + "WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest'" + in guarded_job.split(" steps:\n", 1)[0] + ) + assert pipeline.count("task: UsePythonVersion@0") == 1 publisher = (_ROOT / "OneBranchPipelines" / "steps" / "conda-publish-step.yml").read_text() + assert ( + " ${{ if eq(parameters.publishToConda, true) }}:\n" + " ANACONDA_API_TOKEN: $(ANACONDA_API_TOKEN)\n" + " ${{ else }}:\n" + " ANACONDA_API_TOKEN: ''\n" + ) in publisher + assert publisher.index("foreach ($audit") < publisher.index("rehearse_auth()") + assert publisher.index("rehearse_auth()") < publisher.index("Invoke-Anaconda @('upload'") assert "Invoke-Anaconda @('upload'" in publisher - assert "python -m eng.conda_tools promote `" in publisher + assert "& $pythonExecutable -m eng.conda_tools promote `" in publisher assert "--cleanup-staging `" in publisher assert "\nstages:" not in publisher and "\njobs:" not in publisher +def test_publisher_reuses_verified_image_python_without_fallback(): + source = (_ROOT / "OneBranchPipelines/steps/conda-publish-step.yml").read_text(encoding="utf-8") + bootstrap, publisher = source.split(" - task: PowerShell@2\n")[1:] + assert "container: windows_build_container\n commands: restricted" in bootstrap + assert '\'@ | & $basePython -I - "$baseDirectory"' in bootstrap + assert "sys.version_info[:2] == (3, 12)" in bootstrap + assert 'sysconfig.get_platform() == "win-amd64"' in bootstrap + assert "for m in (ssl, venv, encodings, _ssl, ensurepip, et)" in bootstrap + assert bootstrap.index("PUBLISHER_BASE_STDLIB_OK") < bootstrap.index("-I -m venv") + assert bootstrap.index("PUBLISHER_VENV_OK") < bootstrap.index("-m pip --isolated install") + assert "variable=pythonExecutable;isOutput=true;isReadOnly=true" in bootstrap + assert "MSSQL_CONDA_PUBLISHER_PYTHON: '$(PublisherBootstrap.pythonExecutable)'" in publisher + assert not re.search(r"^\s+python ", publisher, re.MULTILINE) + assert "UsePythonVersion" not in source + + +@pytest.mark.skipif(sys.platform != "win32", reason="Publisher preparation runs on Windows") +@pytest.mark.parametrize("outcome", ["success", "missing", "invalid-runtime"]) +def test_publisher_checks_image_before_creating_environment(outcome): + import os + import textwrap + + source = (_ROOT / "OneBranchPipelines/steps/conda-publish-step.yml").read_text() + bootstrap = source.split(" - task: PowerShell@2\n")[1] + script = textwrap.dedent(bootstrap.split(" script: |\n", 1)[1]).split("$lock =", 1)[0] + health = script.split("@'\n", 1)[1].split("\n'@ |", 1)[0] + compile(health, "", "exec") + result = _run_conda_powershell( + """ +Import-Module Microsoft.PowerShell.Management +function Join-Path { + param($Path, $ChildPath) + if ($ChildPath -eq 'python.exe') { return 'mock-python' } + Microsoft.PowerShell.Management\\Join-Path $Path $ChildPath +} +function Test-Path { + param($LiteralPath, $PathType) + if ($PathType -ne 'Leaf') { throw 'Wrong image executable check' } + return $env:TEST_OUTCOME -ne 'missing' +} +function mock-python { + $input | Out-Null + if ($args[0] -ne '-I' -or $args[1] -ne '-' -or + -not $args[2].EndsWith('\\Programs\\Python\\Python312')) { throw 'Wrong image interpreter arguments' } + Write-Host 'IMAGE_HEALTH_CHECK' + $global:LASTEXITCODE = [int]($env:TEST_OUTCOME -eq 'invalid-runtime') +} +""" + script + "\nWrite-Host 'RUNTIME_BOUNDARY'", + env=dict(os.environ, TEST_OUTCOME=outcome), + ) + assert ("IMAGE_HEALTH_CHECK" in result.stdout) == (outcome != "missing"), result.stderr + assert ("RUNTIME_BOUNDARY" in result.stdout) == (outcome == "success") + assert (result.returncode == 0) == (outcome == "success"), result.stderr + if outcome != "success": + assert "No installation or fallback is permitted" in result.stderr + + +@pytest.mark.skipif(sys.platform != "win32", reason="Publisher interpreter handoff runs on Windows") +@pytest.mark.parametrize("outcome", ["valid", "blank", "outside", "directory"]) +def test_publisher_interpreter_output_is_job_owned(tmp_path, outcome): + import os + import textwrap + + source = (_ROOT / "OneBranchPipelines/steps/conda-publish-step.yml").read_text() + start = source.index(" $pythonExecutable = $env:MSSQL_CONDA_PUBLISHER_PYTHON") + end = source.index(" foreach ($variable in @('PYTHONHOME', 'PYTHONPATH'))") + script = textwrap.dedent(source[start:end]).replace("$(Agent.TempDirectory)", str(tmp_path)) + path = tmp_path / ("conda-publisher-" + "a" * 32) / "venv" / "Scripts" / "python.exe" + if outcome == "outside": + path = tmp_path / "unexpected.exe" + path.parent.mkdir(parents=True, exist_ok=True) + if outcome == "directory": + path.mkdir() + else: + path.write_bytes(b"") + result = _run_conda_powershell( + script + "\nWrite-Host 'INTERPRETER_ACCEPTED'", + env=dict(os.environ, MSSQL_CONDA_PUBLISHER_PYTHON="" if outcome == "blank" else str(path)), + ) + assert (result.returncode == 0) == (outcome == "valid"), result.stdout + result.stderr + assert ("INTERPRETER_ACCEPTED" in result.stdout) == (outcome == "valid") + + +@pytest.mark.parametrize("outcome", [401, 403, 500, "network", "accepted"]) +def test_rehearsal_auth_rejects_only_401_without_writes(monkeypatch, capsys, outcome): + import os + + class Unauthorized(Exception): + pass + + requests = [] + clients = [] + + def request(*args, **kwargs): + requests.append((args, kwargs)) + if outcome == "network": + raise OSError("network unavailable") + if outcome != "accepted": + raise Unauthorized("test response", outcome) + return {"login": "unexpected"} + + api = types.SimpleNamespace(session=types.SimpleNamespace(request=request)) + # Requests Session.get supplies this default at call time. + api.user = lambda: api.session.request( + "GET", "https://api.anaconda.org/user", allow_redirects=True + ) + + def get_server_api(**kwargs): + clients.append(kwargs) + return api + + monkeypatch.setitem(sys.modules, "binstar_client", types.ModuleType("binstar_client")) + monkeypatch.setitem( + sys.modules, "binstar_client.errors", types.SimpleNamespace(Unauthorized=Unauthorized) + ) + monkeypatch.setitem( + sys.modules, "binstar_client.utils", types.SimpleNamespace(get_server_api=get_server_api) + ) + monkeypatch.setenv("ANACONDA_API_TOKEN", "ambient-must-not-be-used") + monkeypatch.setenv("BINSTAR_API_TOKEN", "competing-must-not-be-used") + expected = { + 401: SystemExit, + 403: Unauthorized, + 500: Unauthorized, + "network": OSError, + "accepted": RuntimeError, + }[outcome] + with pytest.raises(expected) as error: + promoter.rehearse_auth() + if outcome == 401: + assert error.value.code == 1 + assert ("EXPECTED_REHEARSAL_AUTH_REJECTION:" in capsys.readouterr().out) == (outcome == 401) + assert len(clients) == 1 + token = clients[0]["token"] + assert re.fullmatch(r"mssql-python-invalid-rehearsal-[0-9a-f]{32}", token) + assert os.environ["ANACONDA_API_TOKEN"] == token + assert "BINSTAR_API_TOKEN" not in os.environ + assert clients[0]["config"] == {"url": "https://api.anaconda.org", "ssl_verify": True} + assert requests == [ + (("GET", "https://api.anaconda.org/user"), {"timeout": (15, 60), "allow_redirects": False}) + ] + + @pytest.mark.skipif(sys.platform != "win32", reason="Publishing task runs on Windows") -@pytest.mark.parametrize("token", [None, "", " \t ", "synthetic-reviewed-token"]) -@pytest.mark.parametrize("other_token", [None, "", "synthetic-other-token"]) -def test_upload_task_uses_only_reviewed_token(tmp_path, token, other_token): +@pytest.mark.parametrize( + "publish,auth_exit", [("True", 0), ("False", 0), ("False", 1), ("False", 7)] +) +def test_rehearsal_never_falls_through_to_publication(publish, auth_exit): + import textwrap + + source = (_ROOT / "OneBranchPipelines/steps/conda-publish-step.yml").read_text() + script = source[ + source.index( + " if ('${{ parameters.publishToConda }}' -eq 'False')", + source.index("foreach ($audit"), + ) : source.index(" # Staged-then-promoted publish:") + ] + script = textwrap.dedent(script).replace("${{ parameters.publishToConda }}", publish) + result = _run_conda_powershell( + "$pythonExecutable = 'python'\n" + f"function python {{ Write-Output 'AUTH_PROBE'; $global:LASTEXITCODE = {auth_exit} }}\n" + + script + + "\nWrite-Output 'PUBLICATION_BOUNDARY'", + ) + assert ("AUTH_PROBE" in result.stdout) == (publish == "False") + assert ("PUBLICATION_BOUNDARY" in result.stdout) == (publish == "True") + assert (result.returncode == 0) == (publish == "True") + + +@pytest.mark.skipif(sys.platform != "win32", reason="Publishing task runs on Windows") +@pytest.mark.parametrize("failed_audit", ["", "elf", "pe", "macho"]) +def test_publisher_native_audits_fail_closed_before_upload(failed_audit): + import os + import textwrap + + source = (_ROOT / "OneBranchPipelines/steps/conda-publish-step.yml").read_text() + start = source.index(" foreach ($audit in @('elf', 'pe', 'macho'))") + end = source.index(" # Staged-then-promoted publish:") + assert source.index("--check-local-only") < start < end < source.index("Invoke-Anaconda @(") + result = _run_conda_powershell( + """ +$root = 'artifact path with spaces' +$pythonExecutable = 'python' +function python { + Write-Output ('AUDIT_ARGS:' + (ConvertTo-Json -InputObject @($args) -Compress)) + $global:LASTEXITCODE = [int]($args[2] -eq $env:TEST_FAILED_AUDIT) +} +""" + textwrap.dedent(source[start:end]), + env=dict(os.environ, TEST_FAILED_AUDIT=failed_audit), + ) + calls = [ + json.loads(line.removeprefix("AUDIT_ARGS:")) + for line in result.stdout.splitlines() + if line.startswith("AUDIT_ARGS:") + ] + audits = ["elf", "pe", "macho"] + expected = audits[: audits.index(failed_audit) + 1] if failed_audit else audits + assert calls == [ + ["-m", "eng.conda_tools", audit, "--root", "artifact path with spaces"] + for audit in expected + ] + assert (result.returncode != 0) == bool(failed_audit), result.stdout + result.stderr + if failed_audit: + assert "native artifact audit failed before upload" in result.stderr + + +@pytest.mark.skipif(sys.platform != "win32", reason="Publishing task runs on Windows") +@pytest.mark.parametrize( + "publish,token,other_token", + [ + ("True", None, "synthetic-other-token"), + ("True", "", None), + ("True", " \t ", ""), + ("True", "synthetic-reviewed-token", None), + ("True", "synthetic-reviewed-token", "synthetic-other-token"), + ("False", None, None), + ("False", "synthetic-reviewed-token", "synthetic-other-token"), + ], +) +def test_upload_task_uses_only_reviewed_token(tmp_path, token, other_token, publish): import os - import shutil - import subprocess import textwrap - shell = shutil.which("powershell") or shutil.which("pwsh") - assert shell is not None, "The Windows publishing task requires PowerShell" template = (_ROOT / "OneBranchPipelines" / "steps" / "conda-publish-step.yml").read_text() - start = template.index(" if ([string]::IsNullOrWhiteSpace($env:ANACONDA_API_TOKEN))") + start = template.index(" if ('${{ parameters.publishToConda }}' -eq 'False')") end = template.index(" if ([string]::IsNullOrWhiteSpace($env:BINSTAR_CONFIG_DIR))") - guard = textwrap.dedent(template[start:end]) + guard = textwrap.dedent(template[start:end]).replace( + "${{ parameters.publishToConda }}", publish + ) probe = tmp_path / "inspect_environment.py" + expected_token = "synthetic-reviewed-token" if publish == "True" else None probe.write_text( "import os\n" 'assert "BINSTAR_API_TOKEN" not in os.environ\n' - 'assert os.environ["ANACONDA_API_TOKEN"] == "synthetic-reviewed-token"\n' + f'assert os.environ.get("ANACONDA_API_TOKEN") == {expected_token!r}\n' 'print("PUBLISHER_TOKEN_OK")\n', encoding="utf-8", ) @@ -1782,27 +2119,19 @@ def test_upload_task_uses_only_reviewed_token(tmp_path, token, other_token): env.pop(name, None) else: env[name] = value - result = subprocess.run( - [ - shell, - "-NoProfile", - "-NonInteractive", - "-Command", - "$ErrorActionPreference = 'Stop'\n" + guard + """ + result = _run_conda_powershell( + guard + """ $process = Start-Process -FilePath $env:TEST_PUBLISHER_PYTHON ` -ArgumentList ('"' + $env:TEST_PUBLISHER_PROBE + '"') -NoNewWindow -PassThru $null = $process.Handle $process.WaitForExit() exit $process.ExitCode """, - ], env=env, - capture_output=True, - text=True, - timeout=30, ) - assert result.returncode == (0 if token and token.strip() else 1), result.stderr - if token and token.strip(): + accepted = publish == "False" or bool(token and token.strip()) + assert result.returncode == (0 if accepted else 1), result.stderr + if accepted: assert result.stdout.strip() == "PUBLISHER_TOKEN_OK" else: assert "ANACONDA_API_TOKEN is not set" in result.stderr diff --git a/tests/test_034_conda_verify_cwd.py b/tests/test_034_conda_verify_cwd.py index befe385c4..9f018bc49 100644 --- a/tests/test_034_conda_verify_cwd.py +++ b/tests/test_034_conda_verify_cwd.py @@ -15,6 +15,7 @@ """ import importlib.machinery +import json import os import subprocess import sys @@ -1217,6 +1218,59 @@ def _capture_run(command, **_kwargs): ] +@pytest.mark.parametrize("target_subdir", ["win-64", "win-arm64", "osx-arm64", "linux-aarch64"]) +@pytest.mark.parametrize("python_version", ["3.12", "3.15"]) +def test_conda_build_target_does_not_change_verification_environment( + target_subdir, python_version, tmp_path, monkeypatch +): + env = environment.build_env("1.16.0", "18.6.2.2", "wheels", target_subdir, "0.3.0") + original = env.copy() + calls = [] + + def capture(command, **kwargs): + calls.append((command, kwargs["env"].copy())) + + monkeypatch.setattr(environment, "run", capture) + build.build_packages( + "conda", + "builder", + str(tmp_path / "recipe"), + [python_version], + str(tmp_path / "bld"), + target_subdir, + env, + ) + command, build_env = calls.pop() + if target_subdir.startswith("win-"): + assert "CONDA_SUBDIR" not in build_env + assert json.loads(command[command.index("--variants") + 1]) == { + "target_platform": target_subdir + } + assert build_env == {key: value for key, value in original.items() if key != "CONDA_SUBDIR"} + else: + assert "--variants" not in command + assert build_env == original + assert env == original + + monkeypatch.setattr(environment, "run_ok", lambda *_args, **_kwargs: 0) + monkeypatch.setattr(environment, "run_capture", lambda *_args, **_kwargs: (0, "")) + verify._verify_impl( + "conda", + "local-channel", + str(tmp_path), + [python_version], + "1.16.0", + target_subdir, + True, + env, + ) + creates = [(cmd, call_env) for cmd, call_env in calls if cmd[1:2] == ["create"]] + assert len(creates) == (2 if target_subdir == "win-arm64" else 1) + assert all(call_env == original for _, call_env in creates) + if target_subdir == "win-arm64": + assert any("--dry-run" in cmd for cmd, _ in creates) + + @pytest.mark.parametrize( "target,pyvers,spec", [ diff --git a/tests/test_036_conda_provenance.py b/tests/test_036_conda_provenance.py index 445b13cbc..31e4ff9e1 100644 --- a/tests/test_036_conda_provenance.py +++ b/tests/test_036_conda_provenance.py @@ -990,7 +990,7 @@ def test_provenance_cli_executes_same_chain_in_both_modes(monkeypatch, chain, ca output = capsys.readouterr().out result = json.loads(output.splitlines()[0].split(": ", 1)[1]) assert result["wheel"]["run"] == 173176 - assert ("Validate-only:" in output) == (publish == "false") + assert ("Non-publishing run:" in output) == (publish == "false") directive = "##vso[task.setvariable variable=releaseVersions;isOutput=true]" assert ( json.loads(