Enable publish of Windows releases through Azure Pipelines (GH-14720)

.azure-pipelines/windows-release.yml

 name: Release_$(Build.SourceBranchName)_$(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr)
 
+variables:
+    __RealSigningCertificate: 'Python Software Foundation'
 # QUEUE TIME VARIABLES
-# variables:
 #   GitRemote: python
 #   SourceTag:
 #   DoPGO: true
@@ -13,6 +14,9 @@ name: Release_$(Build.SourceBranchName)_$(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr)
 #   DoEmbed: true
 #   DoMSI: true
 #   DoPublish: false
+#   PyDotOrgUsername: ''
+#   PyDotOrgServer: ''
+#   BuildToPublish: ''
 
 trigger: none
 pr: none
@@ -20,18 +24,21 @@ pr: none
 stages:
 - stage: Build
   displayName: Build binaries
+  condition: and(succeeded(), not(variables['BuildToPublish']))
   jobs:
   - template: windows-release/stage-build.yml
 
 - stage: Sign
   displayName: Sign binaries
   dependsOn: Build
+  condition: and(succeeded(), not(variables['BuildToPublish']))
   jobs:
   - template: windows-release/stage-sign.yml
 
 - stage: Layout
   displayName: Generate layouts
   dependsOn: Sign
+  condition: and(succeeded(), not(variables['BuildToPublish']))
   jobs:
   - template: windows-release/stage-layout-full.yml
   - template: windows-release/stage-layout-embed.yml
@@ -39,11 +46,13 @@ stages:
 
 - stage: Pack
   dependsOn: Layout
+  condition: and(succeeded(), not(variables['BuildToPublish']))
   jobs:
   - template: windows-release/stage-pack-nuget.yml
 
 - stage: Test
   dependsOn: Pack
+  condition: and(succeeded(), not(variables['BuildToPublish']))
   jobs:
   - template: windows-release/stage-test-embed.yml
   - template: windows-release/stage-test-nuget.yml
@@ -51,46 +60,70 @@ stages:
 - stage: Layout_MSIX
   displayName: Generate MSIX layouts
   dependsOn: Sign
-  condition: and(succeeded(), eq(variables['DoMSIX'], 'true'))
+  condition: and(succeeded(), and(eq(variables['DoMSIX'], 'true'), not(variables['BuildToPublish'])))
   jobs:
   - template: windows-release/stage-layout-msix.yml
 
 - stage: Pack_MSIX
   displayName: Package MSIX
   dependsOn: Layout_MSIX
+  condition: and(succeeded(), not(variables['BuildToPublish']))
   jobs:
   - template: windows-release/stage-pack-msix.yml
 
 - stage: Build_MSI
   displayName: Build MSI installer
   dependsOn: Sign
-  condition: and(succeeded(), eq(variables['DoMSI'], 'true'))
+  condition: and(succeeded(), and(eq(variables['DoMSI'], 'true'), not(variables['BuildToPublish'])))
   jobs:
   - template: windows-release/stage-msi.yml
 
 - stage: Test_MSI
   displayName: Test MSI installer
   dependsOn: Build_MSI
+  condition: and(succeeded(), not(variables['BuildToPublish']))
   jobs:
   - template: windows-release/stage-test-msi.yml
 
 - stage: PublishPyDotOrg
   displayName: Publish to python.org
   dependsOn: ['Test_MSI', 'Test']
-  condition: and(succeeded(), eq(variables['DoPublish'], 'true'))
+  condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish'])))
   jobs:
   - template: windows-release/stage-publish-pythonorg.yml
 
 - stage: PublishNuget
   displayName: Publish to nuget.org
   dependsOn: Test
-  condition: and(succeeded(), eq(variables['DoPublish'], 'true'))
+  condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish'])))
   jobs:
   - template: windows-release/stage-publish-nugetorg.yml
 
 - stage: PublishStore
   displayName: Publish to Store
   dependsOn: Pack_MSIX
-  condition: and(succeeded(), eq(variables['DoPublish'], 'true'))
+  condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish'])))
+  jobs:
+  - template: windows-release/stage-publish-store.yml
+
+
+- stage: PublishExistingPyDotOrg
+  displayName: Publish existing build to python.org
+  dependsOn: []
+  condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish']))
+  jobs:
+  - template: windows-release/stage-publish-pythonorg.yml
+
+- stage: PublishExistingNuget
+  displayName: Publish existing build to nuget.org
+  dependsOn: []
+  condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish']))
+  jobs:
+  - template: windows-release/stage-publish-nugetorg.yml
+
+- stage: PublishExistingStore
+  displayName: Publish existing build to Store
+  dependsOn: []
+  condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish']))
   jobs:
   - template: windows-release/stage-publish-store.yml

.azure-pipelines/windows-release/gpg-sign.yml

+parameters:
+  GPGKeyFile: $(GPGKey)
+  GPGPassphrase: $(GPGPassphrase)
+  Files: '*'
+  WorkingDirectory: $(Build.BinariesDirectory)
+
+steps:
+- task: DownloadSecureFile@1
+  name: gpgkey
+  inputs:
+    secureFile: ${{ parameters.GPGKeyFile }}
+  displayName: 'Download GPG key'
+
+- powershell: |
+    git clone https://github.com/python/cpython-bin-deps --branch gpg --single-branch --depth 1 --progress -v "gpg"
+    gpg/gpg2.exe --import "$(gpgkey.secureFilePath)"
+    (gci -File ${{ parameters.Files }}).FullName | %{
+        gpg/gpg2.exe -ba --batch --passphrase ${{ parameters.GPGPassphrase }} $_
+        "Made signature for $_"
+    }
+  displayName: 'Generate GPG signatures'
+  workingDirectory: ${{ parameters.WorkingDirectory }}
+
+- powershell: |
+    $p = gps "gpg-agent" -EA 0
+    if ($p) { $p.Kill() }
+  displayName: 'Kill GPG agent'
+  condition: true

.azure-pipelines/windows-release/stage-publish-nugetorg.yml

@@ -14,13 +14,26 @@ jobs:
 
   - task: DownloadBuildArtifacts@0
     displayName: 'Download artifact: nuget'
+    condition: and(succeeded(), not(variables['BuildToPublish']))
     inputs:
       artifactName: nuget
       downloadPath: $(Build.BinariesDirectory)
 
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: nuget'
+    condition: and(succeeded(), variables['BuildToPublish'])
+    inputs:
+      artifactName: nuget
+      downloadPath: $(Build.BinariesDirectory)
+      buildType: specific
+      project: cpython
+      pipeline: Windows-Release
+      buildVersionToDownload: specific
+      buildId: $(BuildToPublish)
+
   - task: NuGetCommand@2
     displayName: Push packages
-    condition: and(succeeded(), eq(variables['SigningCertificate'], 'Python Software Foundation'))
+    condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate']))
     inputs:
       command: push
       packagesToPush: $(Build.BinariesDirectory)\nuget\*.nupkg'

.azure-pipelines/windows-release/stage-publish-pythonorg.yml

@@ -4,31 +4,151 @@ jobs:
   condition: and(succeeded(), and(eq(variables['DoMSI'], 'true'), eq(variables['DoEmbed'], 'true')))
 
   pool:
-    vmName: win2016-vs2017
+    #vmName: win2016-vs2017
+    name: 'Windows Release'
 
   workspace:
     clean: all
 
   steps:
-  - checkout: none
+  - template: ./checkout.yml
 
-  - task: DownloadBuildArtifacts@0
+  - task: UsePythonVersion@0
+    displayName: 'Use Python 3.6 or later'
+    inputs:
+      versionSpec: '>=3.6'
+
+  - task: DownloadPipelineArtifact@1
     displayName: 'Download artifact: Doc'
+    condition: and(succeeded(), not(variables['BuildToPublish']))
     inputs:
       artifactName: Doc
-      downloadPath: $(Build.BinariesDirectory)
+      targetPath: $(Build.BinariesDirectory)\Doc
 
-  - task: DownloadBuildArtifacts@0
+  - task: DownloadPipelineArtifact@1
     displayName: 'Download artifact: msi'
+    condition: and(succeeded(), not(variables['BuildToPublish']))
     inputs:
       artifactName: msi
-      downloadPath: $(Build.BinariesDirectory)
+      targetPath: $(Build.BinariesDirectory)\msi
 
   - task: DownloadBuildArtifacts@0
     displayName: 'Download artifact: embed'
+    condition: and(succeeded(), not(variables['BuildToPublish']))
+    inputs:
+      artifactName: embed
+      downloadPath: $(Build.BinariesDirectory)
+
+
+  - task: DownloadPipelineArtifact@1
+    displayName: 'Download artifact from $(BuildToPublish): Doc'
+    condition: and(succeeded(), variables['BuildToPublish'])
+    inputs:
+      artifactName: Doc
+      targetPath: $(Build.BinariesDirectory)\Doc
+      buildType: specific
+      project: cpython
+      pipeline: 21
+      buildVersionToDownload: specific
+      buildId: $(BuildToPublish)
+
+  - task: DownloadPipelineArtifact@1
+    displayName: 'Download artifact from $(BuildToPublish): msi'
+    condition: and(succeeded(), variables['BuildToPublish'])
+    inputs:
+      artifactName: msi
+      targetPath: $(Build.BinariesDirectory)\msi
+      buildType: specific
+      project: cpython
+      pipeline: 21
+      buildVersionToDownload: specific
+      buildId: $(BuildToPublish)
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact from $(BuildToPublish): embed'
+    condition: and(succeeded(), variables['BuildToPublish'])
     inputs:
       artifactName: embed
       downloadPath: $(Build.BinariesDirectory)
+      buildType: specific
+      project: cpython
+      pipeline: Windows-Release
+      buildVersionToDownload: specific
+      buildId: $(BuildToPublish)
+
+
+  - template: ./gpg-sign.yml
+    parameters:
+      GPGKeyFile: 'python-signing.key'
+      Files: 'doc\htmlhelp\*.chm, msi\*\*, embed\*.zip'
 
-  # TODO: eq(variables['SigningCertificate'], 'Python Software Foundation')
-  # If we are not real-signed, DO NOT PUBLISH
+  - powershell: >
+      $(Build.SourcesDirectory)\Tools\msi\uploadrelease.ps1
+      -build msi
+      -user $(PyDotOrgUsername)
+      -server $(PyDotOrgServer)
+      -doc_htmlhelp doc\htmlhelp
+      -embed embed
+      -skippurge
+      -skiptest
+      -skiphash
+    condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate']))
+    workingDirectory: $(Build.BinariesDirectory)
+    displayName: 'Upload files to python.org'
+
+  - powershell: >
+      python
+      "$(Build.SourcesDirectory)\Tools\msi\purge.py"
+      (gci msi\*\python-*.exe | %{ $_.Name -replace 'python-(.+?)(-|\.exe).+', '$1' } | select -First 1)
+    workingDirectory: $(Build.BinariesDirectory)
+    displayName: 'Purge CDN'
+
+  - powershell: |
+      $failures = 0
+      gci "msi\*\*-webinstall.exe" -File | %{
+          $d = mkdir "tests\$($_.BaseName)" -Force
+          gci $d -r -File | del
+          $ic = copy $_ $d -PassThru
+          "Checking layout for $($ic.Name)"
+          Start-Process -wait $ic "/passive", "/layout", "$d\layout", "/log", "$d\log\install.log"
+          if (-not $?) {
+              Write-Error "Failed to validate layout of $($inst.Name)"
+              $failures += 1
+          }
+      }
+      if ($failures) {
+        Write-Error "Failed to validate $failures installers"
+        exit 1
+      }
+    #condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate']))
+    workingDirectory: $(Build.BinariesDirectory)
+    displayName: 'Test layouts'
+
+  - powershell: |
+      $hashes = gci doc\htmlhelp\python*.chm, msi\*\*.exe, embed\*.zip | `
+          Sort-Object Name | `
+          Format-Table Name, @{
+            Label="MD5";
+            Expression={(Get-FileHash $_ -Algorithm MD5).Hash}
+          }, Length -AutoSize | `
+          Out-String -Width 4096
+      $d = mkdir "$(Build.ArtifactStagingDirectory)\hashes" -Force
+      $hashes | Out-File "$d\hashes.txt" -Encoding ascii
+      $hashes
+    workingDirectory: $(Build.BinariesDirectory)
+    displayName: 'Generate hashes'
+
+  - powershell: |
+      "Copying:"
+      (gci msi\*\python*.asc, doc\htmlhelp\*.asc, embed\*.asc).FullName
+      $d = mkdir "$(Build.ArtifactStagingDirectory)\hashes" -Force
+      move msi\*\python*.asc, doc\htmlhelp\*.asc, embed\*.asc $d -Force
+      gci msi -Directory | %{ move "msi\$_\*.asc" (mkdir "$d\$_" -Force) }
+    workingDirectory: $(Build.BinariesDirectory)
+    displayName: 'Copy GPG signatures for build'
+
+  - task: PublishPipelineArtifact@0
+    displayName: 'Publish Artifact: hashes'
+    inputs:
+      targetPath: '$(Build.ArtifactStagingDirectory)\hashes'
+      artifactName: hashes

.azure-pipelines/windows-release/stage-publish-store.yml

@@ -14,9 +14,22 @@ jobs:
 
   - task: DownloadBuildArtifacts@0
     displayName: 'Download artifact: msixupload'
+    condition: and(succeeded(), not(variables['BuildToPublish']))
     inputs:
       artifactName: msixupload
       downloadPath: $(Build.BinariesDirectory)
 
-  # TODO: eq(variables['SigningCertificate'], 'Python Software Foundation')
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: msixupload'
+    condition: and(succeeded(), variables['BuildToPublish'])
+    inputs:
+      artifactName: msixupload
+      downloadPath: $(Build.BinariesDirectory)
+      buildType: specific
+      project: cpython
+      pipeline: Windows-Release
+      buildVersionToDownload: specific
+      buildId: $(BuildToPublish)
+
+  # TODO: eq(variables['SigningCertificate'], variables['__RealSigningCertificate'])
   # If we are not real-signed, DO NOT PUBLISH

Tools/msi/uploadrelease.ps1

@@ -164,5 +164,4 @@ if (-not $skiphash) {
         Out-String -Width 4096
     $hashes | clip
     $hashes
-    popd
 }