From 0cdf9d7260611155d8f8bf90e7403c1c7bbc611c Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 1 Apr 2021 23:29:19 +0100 Subject: [PATCH 1/5] Drop all existing CI --- .azure-pipelines/jobs/package.yml | 36 ------ .azure-pipelines/jobs/test-windows.yml | 53 -------- .azure-pipelines/jobs/test.yml | 38 ------ .azure-pipelines/linux.yml | 11 -- .azure-pipelines/scripts/New-RAMDisk.ps1 | 74 ----------- .azure-pipelines/steps/run-tests-windows.yml | 54 -------- .azure-pipelines/steps/run-tests.yml | 25 ---- .azure-pipelines/windows.yml | 11 -- .github/workflows/linting.yml | 53 -------- .github/workflows/macos.yml | 127 ------------------- .travis.yml | 32 ----- tools/travis/install.sh | 7 - tools/travis/run.sh | 65 ---------- tools/travis/setup.sh | 6 - 14 files changed, 592 deletions(-) delete mode 100644 .azure-pipelines/jobs/package.yml delete mode 100644 .azure-pipelines/jobs/test-windows.yml delete mode 100644 .azure-pipelines/jobs/test.yml delete mode 100644 .azure-pipelines/linux.yml delete mode 100644 .azure-pipelines/scripts/New-RAMDisk.ps1 delete mode 100644 .azure-pipelines/steps/run-tests-windows.yml delete mode 100644 .azure-pipelines/steps/run-tests.yml delete mode 100644 .azure-pipelines/windows.yml delete mode 100644 .github/workflows/linting.yml delete mode 100644 .github/workflows/macos.yml delete mode 100644 .travis.yml delete mode 100755 tools/travis/install.sh delete mode 100755 tools/travis/run.sh delete mode 100755 tools/travis/setup.sh diff --git a/.azure-pipelines/jobs/package.yml b/.azure-pipelines/jobs/package.yml deleted file mode 100644 index bdb0254a1ba..00000000000 --- a/.azure-pipelines/jobs/package.yml +++ /dev/null @@ -1,36 +0,0 @@ -parameters: - vmImage: - -jobs: -- job: Package - dependsOn: - - Test_Primary - - Test_Secondary - pool: - vmImage: ${{ parameters.vmImage }} - - steps: - - task: UsePythonVersion@0 - displayName: Use Python 3 latest - inputs: - versionSpec: '3' - - - bash: | - git config --global user.email "distutils-sig@python.org" - git config --global user.name "pip" - displayName: Setup Git credentials - - - bash: pip install nox - displayName: Install dependencies - - - bash: nox -s prepare-release -- 99.9 - displayName: Prepare dummy release - - - bash: nox -s build-release -- 99.9 - displayName: Generate distributions for the dummy release - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: dist' - inputs: - pathtoPublish: dist - artifactName: dist diff --git a/.azure-pipelines/jobs/test-windows.yml b/.azure-pipelines/jobs/test-windows.yml deleted file mode 100644 index 99cd8a836bd..00000000000 --- a/.azure-pipelines/jobs/test-windows.yml +++ /dev/null @@ -1,53 +0,0 @@ -parameters: - vmImage: - -jobs: -- job: Test_Primary - displayName: Tests / - - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - matrix: - "3.6": # lowest Python version - python.version: '3.6' - python.architecture: x64 - "3.8": # current - python.version: '3.8' - python.architecture: x64 - maxParallel: 6 - - steps: - - template: ../steps/run-tests-windows.yml - parameters: - runIntegrationTests: true - -- job: Test_Secondary - displayName: Tests / - # Don't run integration tests for these runs - # Run after Test_Primary so we don't devour time and jobs if tests are going to fail - dependsOn: Test_Primary - - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - matrix: - "3.7": - python.version: '3.7' - python.architecture: x64 - # This is for Windows, so test x86 builds - "3.6-x86": - python.version: '3.6' - python.architecture: x86 - "3.7-x86": - python.version: '3.7' - python.architecture: x86 - "3.8-x86": - python.version: '3.8' - python.architecture: x86 - maxParallel: 6 - - steps: - - template: ../steps/run-tests-windows.yml - parameters: - runIntegrationTests: false diff --git a/.azure-pipelines/jobs/test.yml b/.azure-pipelines/jobs/test.yml deleted file mode 100644 index a3a0ef80b6d..00000000000 --- a/.azure-pipelines/jobs/test.yml +++ /dev/null @@ -1,38 +0,0 @@ -parameters: - vmImage: - -jobs: -- job: Test_Primary - displayName: Tests / - - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - matrix: - "3.6": # lowest Python version - python.version: '3.6' - python.architecture: x64 - "3.8": - python.version: '3.8' - python.architecture: x64 - maxParallel: 2 - - steps: - - template: ../steps/run-tests.yml - -- job: Test_Secondary - displayName: Tests / - # Run after Test_Primary so we don't devour time and jobs if tests are going to fail - dependsOn: Test_Primary - - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - matrix: - "3.7": - python.version: '3.7' - python.architecture: x64 - maxParallel: 4 - - steps: - - template: ../steps/run-tests.yml diff --git a/.azure-pipelines/linux.yml b/.azure-pipelines/linux.yml deleted file mode 100644 index e5598074344..00000000000 --- a/.azure-pipelines/linux.yml +++ /dev/null @@ -1,11 +0,0 @@ -variables: - CI: true - -jobs: -- template: jobs/test.yml - parameters: - vmImage: ubuntu-16.04 - -- template: jobs/package.yml - parameters: - vmImage: ubuntu-16.04 diff --git a/.azure-pipelines/scripts/New-RAMDisk.ps1 b/.azure-pipelines/scripts/New-RAMDisk.ps1 deleted file mode 100644 index 21b1a573a49..00000000000 --- a/.azure-pipelines/scripts/New-RAMDisk.ps1 +++ /dev/null @@ -1,74 +0,0 @@ -[CmdletBinding()] -param( - [Parameter(Mandatory=$true, - HelpMessage="Drive letter to use for the RAMDisk")] - [String]$drive, - [Parameter(HelpMessage="Size to allocate to the RAMDisk")] - [UInt64]$size=1GB -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version Latest - -Write-Output "Installing FS-iSCSITarget-Server" -Install-WindowsFeature -Name FS-iSCSITarget-Server - -Write-Output "Starting MSiSCSI" -Start-Service MSiSCSI -$retry = 10 -do { - $service = Get-Service MSiSCSI - if ($service.Status -eq "Running") { - break; - } - $retry-- - Start-Sleep -Milliseconds 500 -} until ($retry -eq 0) - -$service = Get-Service MSiSCSI -if ($service.Status -ne "Running") { - throw "MSiSCSI is not running" -} - -Write-Output "Configuring Firewall" -Get-NetFirewallServiceFilter -Service MSiSCSI | Enable-NetFirewallRule - -Write-Output "Configuring RAMDisk" -# Must use external-facing IP address, otherwise New-IscsiTargetPortal is -# unable to connect. -$ip = ( - Get-NetIPAddress -AddressFamily IPv4 | - Where-Object {$_.IPAddress -ne "127.0.0.1"} -)[0].IPAddress -if ( - -not (Get-IscsiServerTarget -ComputerName localhost | Where-Object {$_.TargetName -eq "ramdisks"}) -) { - New-IscsiServerTarget ` - -ComputerName localhost ` - -TargetName ramdisks ` - -InitiatorId IPAddress:$ip -} - -$newVirtualDisk = New-IscsiVirtualDisk ` - -ComputerName localhost ` - -Path ramdisk:local$drive.vhdx ` - -Size $size -Add-IscsiVirtualDiskTargetMapping ` - -ComputerName localhost ` - -TargetName ramdisks ` - -Path ramdisk:local$drive.vhdx - -Write-Output "Connecting to iSCSI" -New-IscsiTargetPortal -TargetPortalAddress $ip -Get-IscsiTarget | Where-Object {!$_.IsConnected} | Connect-IscsiTarget - -Write-Output "Configuring disk" -$newDisk = Get-IscsiConnection | - Get-Disk | - Where-Object {$_.SerialNumber -eq $newVirtualDisk.SerialNumber} - -Set-Disk -InputObject $newDisk -IsOffline $false -Initialize-Disk -InputObject $newDisk -PartitionStyle MBR -New-Partition -InputObject $newDisk -UseMaximumSize -DriveLetter $drive - -Format-Volume -DriveLetter $drive -NewFileSystemLabel Temp -FileSystem NTFS diff --git a/.azure-pipelines/steps/run-tests-windows.yml b/.azure-pipelines/steps/run-tests-windows.yml deleted file mode 100644 index 39282a3cc80..00000000000 --- a/.azure-pipelines/steps/run-tests-windows.yml +++ /dev/null @@ -1,54 +0,0 @@ -parameters: - runIntegrationTests: - -steps: -- task: UsePythonVersion@0 - displayName: Use Python $(python.version) - inputs: - versionSpec: '$(python.version)' - architecture: '$(python.architecture)' - -- task: PowerShell@2 - inputs: - filePath: .azure-pipelines/scripts/New-RAMDisk.ps1 - arguments: "-Drive R -Size 1GB" - displayName: Setup RAMDisk - -- powershell: | - mkdir R:\Temp - $acl = Get-Acl "R:\Temp" - $rule = New-Object System.Security.AccessControl.FileSystemAccessRule( - "Everyone", "FullControl", "ContainerInherit,ObjectInherit", "None", "Allow" - ) - $acl.AddAccessRule($rule) - Set-Acl "R:\Temp" $acl - displayName: Set RAMDisk Permissions - -- bash: pip install --upgrade 'virtualenv<20' setuptools tox - displayName: Install Tox - -- script: tox -e py -- -m unit -n auto --junit-xml=junit/unit-test.xml - env: - TEMP: "R:\\Temp" - displayName: Tox run unit tests - -- ${{ if eq(parameters.runIntegrationTests, 'true') }}: - - powershell: | - # Fix Git SSL errors - pip install certifi tox - python -m certifi > cacert.txt - $env:GIT_SSL_CAINFO = $(Get-Content cacert.txt) - - # Shorten paths to get under MAX_PATH or else integration tests will fail - # https://bugs.python.org/issue18199 - $env:TEMP = "R:\Temp" - - tox -e py -- -m integration -n auto --durations=5 --junit-xml=junit/integration-test.xml - displayName: Tox run integration tests - -- task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFiles: junit/*.xml - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml deleted file mode 100644 index 5b9a9c50c89..00000000000 --- a/.azure-pipelines/steps/run-tests.yml +++ /dev/null @@ -1,25 +0,0 @@ -steps: -- task: UsePythonVersion@0 - displayName: Use Python $(python.version) - inputs: - versionSpec: '$(python.version)' - -- bash: pip install --upgrade 'virtualenv<20' setuptools tox - displayName: Install Tox - -- script: tox -e py -- -m unit -n auto --junit-xml=junit/unit-test.xml - displayName: Tox run unit tests - -# Run integration tests in two groups so we will fail faster if there is a failure in the first group -- script: tox -e py -- -m integration -n auto --durations=5 -k "not test_install" --junit-xml=junit/integration-test-group0.xml - displayName: Tox run Group 0 integration tests - -- script: tox -e py -- -m integration -n auto --durations=5 -k "test_install" --junit-xml=junit/integration-test-group1.xml - displayName: Tox run Group 1 integration tests - -- task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFiles: junit/*.xml - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() diff --git a/.azure-pipelines/windows.yml b/.azure-pipelines/windows.yml deleted file mode 100644 index f56b8f50486..00000000000 --- a/.azure-pipelines/windows.yml +++ /dev/null @@ -1,11 +0,0 @@ -variables: - CI: true - -jobs: -- template: jobs/test-windows.yml - parameters: - vmImage: vs2017-win2016 - -- template: jobs/package.yml - parameters: - vmImage: vs2017-win2016 diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml deleted file mode 100644 index 71459d660e8..00000000000 --- a/.github/workflows/linting.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Linting - -on: - push: - pull_request: - schedule: - # Run every Friday at 18:02 UTC - - cron: 2 18 * * 5 - -jobs: - lint: - name: ${{ matrix.os }} - runs-on: ${{ matrix.os }}-latest - env: - TOXENV: lint,docs,vendoring - - strategy: - matrix: - os: - - Ubuntu - - Windows - - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.9 - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - # Setup Caching - - name: pip cache - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('tools/requirements/tests.txt') }}-${{ hashFiles('tools/requirements/docs.txt') }}-${{ hashFiles('tox.ini') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- - - - name: Set PY (for pre-commit cache) - run: echo "PY=$(python -c 'import hashlib, sys;print(hashlib.sha256(sys.version.encode()+sys.executable.encode()).hexdigest())')" >> $GITHUB_ENV - - name: pre-commit cache - uses: actions/cache@v1 - with: - path: ~/.cache/pre-commit - key: pre-commit|2020-02-14|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} - - # Get the latest tox - - name: Install tox - run: python -m pip install tox - - # Main check - - run: python -m tox diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml deleted file mode 100644 index de226389d26..00000000000 --- a/.github/workflows/macos.yml +++ /dev/null @@ -1,127 +0,0 @@ -name: MacOS - -on: - push: - pull_request: - schedule: - # Run every Friday at 18:02 UTC - - cron: 2 18 * * 5 - -jobs: - dev-tools: - name: Quality Check - runs-on: macos-latest - - steps: - # Caches - - name: pip cache - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('tools/requirements/tests.txt') }}-${{ hashFiles('tools/requirements/docs.txt') }}-${{ hashFiles('tox.ini') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- - - name: Set PY (for pre-commit cache) - run: echo "PY=$(python -c 'import hashlib, sys;print(hashlib.sha256(sys.version.encode()+sys.executable.encode()).hexdigest())')" >> $GITHUB_ENV - - name: pre-commit cache - uses: actions/cache@v1 - with: - path: ~/.cache/pre-commit - key: pre-commit|2020-02-14|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} - - # Setup - - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Install tox - run: python -m pip install tox - - # Main check - - run: python -m tox -e "lint,docs" - - packaging: - name: Packaging - runs-on: macos-latest - - steps: - # Caches - - name: pip cache - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('tools/requirements/tests.txt') }}-${{ hashFiles('tools/requirements/docs.txt') }}-${{ hashFiles('tox.ini') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- - # Setup - - name: Set up git credentials - run: | - git config --global user.email "pypa-dev@googlegroups.com" - git config --global user.name "pip" - - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Install tox and nox - run: python -m pip install tox nox - - # Main check - - name: Check vendored packages - run: python -m tox -e "vendoring" - - - name: Prepare dummy release - run: nox -s prepare-release -- 99.9 - - - name: Generate distributions for the dummy release - run: nox -s build-release -- 99.9 - - tests: - name: Tests / ${{ matrix.python }} - runs-on: macos-latest - - needs: dev-tools - - strategy: - fail-fast: false - matrix: - python: [3.6, 3.7, 3.8, 3.9] - - steps: - # Caches - - name: pip cache - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('tools/requirements/tests.txt') }}-${{ hashFiles('tools/requirements/docs.txt') }}-${{ hashFiles('tox.ini') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- - # Setup - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python }} - - - name: Install tox - run: python -m pip install tox 'virtualenv<20' - - # Main check - - name: Run unit tests - run: >- - python -m tox -e py -- - -m unit - --verbose - --numprocesses auto - - - name: Run integration tests - run: >- - python -m tox -e py -- - -m integration - --verbose - --numprocesses auto - --durations=5 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 6610b6eb019..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,32 +0,0 @@ -language: python -cache: pip -dist: xenial -python: 3.9 -addons: - apt: - packages: - - bzr - -stages: -- primary -- secondary - -jobs: - include: - # Basic Checks - - stage: primary - env: TOXENV=docs - - env: TOXENV=lint - - env: TOXENV=vendoring - - # Complete checking for ensuring compatibility - # PyPy - - stage: secondary - env: GROUP=1 - python: pypy3.6-7.3.1 - - env: GROUP=2 - python: pypy3.6-7.3.1 - -before_install: tools/travis/setup.sh -install: travis_retry tools/travis/install.sh -script: tools/travis/run.sh diff --git a/tools/travis/install.sh b/tools/travis/install.sh deleted file mode 100755 index 3b12d69a26b..00000000000 --- a/tools/travis/install.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e -set -x - -pip install --upgrade setuptools -pip install --upgrade tox tox-venv -pip freeze --all diff --git a/tools/travis/run.sh b/tools/travis/run.sh deleted file mode 100755 index df8f03e7a57..00000000000 --- a/tools/travis/run.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash -set -e - -# Short circuit test runs if there are no code changes involved. -if [[ $TOXENV != docs ]] || [[ $TOXENV != lint ]]; then - if [[ "$TRAVIS_PULL_REQUEST" == "false" ]] - then - echo "This is not a PR -- will do a complete build." - else - # Pull requests are slightly complicated because $TRAVIS_COMMIT_RANGE - # may include more changes than desired if the history is convoluted. - # Instead, explicitly fetch the base branch and compare against the - # merge-base commit. - git fetch -q origin +refs/heads/$TRAVIS_BRANCH - changes=$(git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD)) - echo "Files changed:" - echo "$changes" - if ! echo "$changes" | grep -qvE '(\.rst$)|(^docs)|(^news)|(^\.github)' - then - echo "Code was not changed -- skipping build." - exit - fi - fi -fi - -# Export the correct TOXENV when not provided. -echo "Determining correct TOXENV..." -if [[ -z "$TOXENV" ]]; then - if [[ ${TRAVIS_PYTHON_VERSION} == pypy* ]]; then - export TOXENV=pypy - else - # We use the syntax ${string:index:length} to make 2.7 -> py27 - _major=${TRAVIS_PYTHON_VERSION:0:1} - _minor=${TRAVIS_PYTHON_VERSION:2:1} - export TOXENV="py${_major}${_minor}" - fi -fi -echo "TOXENV=${TOXENV}" - -if [[ -z "$NEW_RESOLVER" ]]; then - RESOLVER_SWITCH='' -else - RESOLVER_SWITCH='--new-resolver' -fi - -# Print the commands run for this test. -set -x -if [[ "$GROUP" == "1" ]]; then - # Unit tests - tox -- --use-venv -m unit -n auto - # Integration tests (not the ones for 'pip install') - tox -- -m integration -n auto --durations=5 -k "not test_install" \ - --use-venv $RESOLVER_SWITCH -elif [[ "$GROUP" == "2" ]]; then - # Separate Job for running integration tests for 'pip install' - tox -- -m integration -n auto --durations=5 -k "test_install" \ - --use-venv $RESOLVER_SWITCH -elif [[ "$GROUP" == "3" ]]; then - # Separate Job for tests that fail with the new resolver - tox -- -m fails_on_new_resolver -n auto --durations=5 \ - --use-venv $RESOLVER_SWITCH --new-resolver-runtests -else - # Non-Testing Jobs should run once - tox -fi diff --git a/tools/travis/setup.sh b/tools/travis/setup.sh deleted file mode 100755 index c52ce5f167e..00000000000 --- a/tools/travis/setup.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -e - -echo "Setting Git Credentials..." -git config --global user.email "distutils-sig@python.org" -git config --global user.name "pip" From c022615961b3b6fa5b1a5aa6b6f620c8eee7f7da Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 1 Apr 2021 23:48:39 +0100 Subject: [PATCH 2/5] Add a single GitHub Actions workflow for CI --- .github/workflows/ci.yml | 189 +++++++++++++++++++++++++++++++++++ tools/ci/New-RAMDisk.ps1 | 74 ++++++++++++++ tools/requirements/tests.txt | 9 +- 3 files changed, 267 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 tools/ci/New-RAMDisk.ps1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000..950239b6aab --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,189 @@ +name: CI + +on: + push: + branches: [master] + tags: + # Tags for all potential release numbers till 2030. + - "2[0-9].[0-3]" # 20.0 -> 29.3 + - "2[0-9].[0-3].[0-9]+" # 20.0.0 -> 29.3.[0-9]+ + pull_request: + schedule: + - cron: 0 0 * * MON # Run every Monday at 00:00 UTC + +jobs: + determine-changes: + runs-on: ubuntu-latest + outputs: + tests: ${{ steps.filter.outputs.tests }} + vendoring: ${{ steps.filter.outputs.vendoring }} + steps: + # For pull requests it's not necessary to checkout the code + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + vendoring: + # Anything that's touching "vendored code" + - "src/pip/_vendor/**" + - "pyproject.toml" + tests: + # Anything that's touching testable stuff + - ".github/workflows/ci.yml" + - "tools/requirements/tests.txt" + - "src/**" + - "tests/**" + + pre-commit: + name: pre-commit + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - uses: pre-commit/action@v2.0.0 + with: + extra_args: --hook-stage=manual + + packaging: + name: packaging + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - name: Set up git credentials + run: | + git config --global user.email "pypa-dev@googlegroups.com" + git config --global user.name "pip" + + - run: pip install nox + - run: nox -s prepare-release -- 99.9 + - run: nox -s build-release -- 99.9 + + vendoring: + name: vendoring + runs-on: ubuntu-latest + + needs: [determine-changes] + if: ${{ needs.determine-changes.outputs.vendoring == 'true' }} + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + + - run: pip install vendoring + - run: vendoring sync . --verbose + - run: git diff --exit-code + + tests-unix: + name: tests / ${{ matrix.python }} / ${{ matrix.os }} + runs-on: ${{ matrix.os }}-latest + + needs: [pre-commit, packaging, determine-changes] + if: ${{ needs.determine-changes.outputs.tests == 'true' }} + + strategy: + fail-fast: true + matrix: + os: [Ubuntu, MacOS] + python: + - 3.6 + - 3.7 + - 3.8 + - 3.9 + include: + - os: Ubuntu + python: pypy3 + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + + - run: pip install tox 'virtualenv<20' + + # Main check + - name: Run unit tests + run: >- + tox -e py -- + -m unit + --verbose --numprocesses auto --showlocals + - name: Run integration tests + run: >- + tox -e py -- + -m integration + --verbose --numprocesses auto --showlocals + --durations=5 + + tests-windows: + name: tests / ${{ matrix.python }} / ${{ matrix.os }} / ${{ matrix.group }} + runs-on: ${{ matrix.os }}-latest + + needs: [pre-commit, packaging, determine-changes] + if: ${{ needs.determine-changes.outputs.tests == 'true' }} + + strategy: + fail-fast: true + matrix: + os: [Windows] + python: + - 3.6 + - 3.7 + - 3.8 + - 3.9 + group: [1, 2] + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + + # We use a RAMDisk on Windows, since filesystem IO is a big slowdown + # for our tests. + - name: Create a RAMDisk + run: ./tools/ci/New-RAMDisk.ps1 -Drive R -Size 1GB + + - name: Setup RAMDisk permissions + run: | + mkdir R:\Temp + $acl = Get-Acl "R:\Temp" + $rule = New-Object System.Security.AccessControl.FileSystemAccessRule( + "Everyone", "FullControl", "ContainerInherit,ObjectInherit", "None", "Allow" + ) + $acl.AddAccessRule($rule) + Set-Acl "R:\Temp" $acl + + - run: pip install tox 'virtualenv<20' + env: + TEMP: "R:\\Temp" + + # Main check + - name: Run unit tests + if: matrix.group == 1 + run: >- + tox -e py -- + -m unit + --verbose --numprocesses auto --showlocals + env: + TEMP: "R:\\Temp" + + - name: Run integration tests (group 1) + if: matrix.group == 1 + run: >- + tox -e py -- + -m integration -k "not test_install" + --verbose --numprocesses auto --showlocals + env: + TEMP: "R:\\Temp" + + - name: Run integration tests (group 2) + if: matrix.group == 2 + run: >- + tox -e py -- + -m integration -k "test_install" + --verbose --numprocesses auto --showlocals + env: + TEMP: "R:\\Temp" diff --git a/tools/ci/New-RAMDisk.ps1 b/tools/ci/New-RAMDisk.ps1 new file mode 100644 index 00000000000..21b1a573a49 --- /dev/null +++ b/tools/ci/New-RAMDisk.ps1 @@ -0,0 +1,74 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory=$true, + HelpMessage="Drive letter to use for the RAMDisk")] + [String]$drive, + [Parameter(HelpMessage="Size to allocate to the RAMDisk")] + [UInt64]$size=1GB +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version Latest + +Write-Output "Installing FS-iSCSITarget-Server" +Install-WindowsFeature -Name FS-iSCSITarget-Server + +Write-Output "Starting MSiSCSI" +Start-Service MSiSCSI +$retry = 10 +do { + $service = Get-Service MSiSCSI + if ($service.Status -eq "Running") { + break; + } + $retry-- + Start-Sleep -Milliseconds 500 +} until ($retry -eq 0) + +$service = Get-Service MSiSCSI +if ($service.Status -ne "Running") { + throw "MSiSCSI is not running" +} + +Write-Output "Configuring Firewall" +Get-NetFirewallServiceFilter -Service MSiSCSI | Enable-NetFirewallRule + +Write-Output "Configuring RAMDisk" +# Must use external-facing IP address, otherwise New-IscsiTargetPortal is +# unable to connect. +$ip = ( + Get-NetIPAddress -AddressFamily IPv4 | + Where-Object {$_.IPAddress -ne "127.0.0.1"} +)[0].IPAddress +if ( + -not (Get-IscsiServerTarget -ComputerName localhost | Where-Object {$_.TargetName -eq "ramdisks"}) +) { + New-IscsiServerTarget ` + -ComputerName localhost ` + -TargetName ramdisks ` + -InitiatorId IPAddress:$ip +} + +$newVirtualDisk = New-IscsiVirtualDisk ` + -ComputerName localhost ` + -Path ramdisk:local$drive.vhdx ` + -Size $size +Add-IscsiVirtualDiskTargetMapping ` + -ComputerName localhost ` + -TargetName ramdisks ` + -Path ramdisk:local$drive.vhdx + +Write-Output "Connecting to iSCSI" +New-IscsiTargetPortal -TargetPortalAddress $ip +Get-IscsiTarget | Where-Object {!$_.IsConnected} | Connect-IscsiTarget + +Write-Output "Configuring disk" +$newDisk = Get-IscsiConnection | + Get-Disk | + Where-Object {$_.SerialNumber -eq $newVirtualDisk.SerialNumber} + +Set-Disk -InputObject $newDisk -IsOffline $false +Initialize-Disk -InputObject $newDisk -PartitionStyle MBR +New-Partition -InputObject $newDisk -UseMaximumSize -DriveLetter $drive + +Format-Volume -DriveLetter $drive -NewFileSystemLabel Temp -FileSystem NTFS diff --git a/tools/requirements/tests.txt b/tools/requirements/tests.txt index f760f004ad6..7badf2a27ff 100644 --- a/tools/requirements/tests.txt +++ b/tools/requirements/tests.txt @@ -1,5 +1,4 @@ ---use-feature=2020-resolver -cryptography==2.8 +cryptography freezegun pretend pytest @@ -7,7 +6,7 @@ pytest-cov pytest-rerunfailures pytest-xdist scripttest -setuptools>=39.2.0 # Needed for `setuptools.wheel.Wheel` support. -https://github.com/pypa/virtualenv/archive/legacy.zip#egg=virtualenv -werkzeug==0.16.0 +setuptools +virtualenv < 20.0 +werkzeug wheel From 876df5a57fe46a2831995164836bf2410a37307b Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 2 Apr 2021 01:16:35 +0100 Subject: [PATCH 3/5] Remove PyPy3 for now --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 950239b6aab..c9e3a16c4a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,9 +92,6 @@ jobs: - 3.7 - 3.8 - 3.9 - include: - - os: Ubuntu - python: pypy3 steps: - uses: actions/checkout@v2 From 1ab662f3302a5d8e3592113d1e1a006df628c80d Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 2 Apr 2021 01:25:40 +0100 Subject: [PATCH 4/5] Trim the number of Windows jobs we run --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9e3a16c4a3..420bbfab227 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,8 +127,9 @@ jobs: os: [Windows] python: - 3.6 - - 3.7 - - 3.8 + # Commented out, since Windows tests are expensively slow. + # - 3.7 + # - 3.8 - 3.9 group: [1, 2] From 879cee8a20d439d1f4e15b121d64cfaf1c3168cf Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Fri, 2 Apr 2021 01:32:05 +0100 Subject: [PATCH 5/5] Update note that CI docs are out of date --- docs/html/development/ci.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/html/development/ci.rst b/docs/html/development/ci.rst index 991c66c736a..ac51e9ffa05 100644 --- a/docs/html/development/ci.rst +++ b/docs/html/development/ci.rst @@ -1,7 +1,8 @@ .. note:: - This section of the documentation is currently being written. pip - developers welcome your help to complete this documentation. If + This section of the documentation is currently out of date. + + pip developers welcome your help to update this documentation. If you're interested in helping out, please let us know in the `tracking issue`_, or just submit a pull request and mention it in that tracking issue.