diff --git a/.azure-pipelines/jobs/dev-tools.yml b/.azure-pipelines/jobs/dev-tools.yml new file mode 100644 index 00000000000..549d2ed46c5 --- /dev/null +++ b/.azure-pipelines/jobs/dev-tools.yml @@ -0,0 +1,16 @@ +parameters: + vmImage: + +jobs: +- job: Developer_Tooling + steps: + - task: UsePythonVersion@0 + displayName: Use Python 3 latest + inputs: + versionSpec: '3' + + - bash: pip install tox + displayName: Install dependencies + + - bash: tox -e lint,docs,vendoring + displayName: Run the development tools diff --git a/.azure-pipelines/jobs/test-windows.yml b/.azure-pipelines/jobs/test-windows.yml index 1a933a6934b..a007a969eb8 100644 --- a/.azure-pipelines/jobs/test-windows.yml +++ b/.azure-pipelines/jobs/test-windows.yml @@ -3,15 +3,13 @@ parameters: jobs: - job: Test_Primary + dependsOn: Developer_Tooling displayName: Test Primary pool: vmImage: ${{ parameters.vmImage }} strategy: matrix: - Python27-x86: - python.version: '2.7' - python.architecture: x86 Python27-x64: python.version: '2.7' python.architecture: x64 @@ -29,6 +27,14 @@ jobs: Python38-x64: python.version: '3.8' python.architecture: x64 + PyPy2: + python.version: '2.7' + python.architecture: x64 + python.flavor: pypy + PyPy3: + python.version: '3.5' + python.architecture: x64 + python.flavor: pypy maxParallel: 6 steps: @@ -48,6 +54,9 @@ jobs: strategy: matrix: # This is for Windows, so test x86 builds + Python27-x86: + python.version: '2.7' + python.architecture: x86 Python35-x86: python.version: '3.5' python.architecture: x86 diff --git a/.azure-pipelines/jobs/test.yml b/.azure-pipelines/jobs/test.yml deleted file mode 100644 index cadf99a51f7..00000000000 --- a/.azure-pipelines/jobs/test.yml +++ /dev/null @@ -1,41 +0,0 @@ -parameters: - vmImage: - -jobs: -- job: Test_Primary - displayName: Test Primary - - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - matrix: - Python27: - python.version: '2.7' - python.architecture: x64 - Python36: - python.version: '3.6' - python.architecture: x64 - maxParallel: 2 - - steps: - - template: ../steps/run-tests.yml - -- job: Test_Secondary - displayName: Test Secondary - # Run after Test_Primary so we don't devour time and jobs if tests are going to fail - dependsOn: Test_Primary - - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - matrix: - Python35: - python.version: '3.5' - python.architecture: x64 - Python37: - python.version: '3.7' - python.architecture: x64 - maxParallel: 3 - - steps: - - template: ../steps/run-tests.yml diff --git a/.azure-pipelines/linux.yml b/.azure-pipelines/linux.yml deleted file mode 100644 index 6965a15fc6d..00000000000 --- a/.azure-pipelines/linux.yml +++ /dev/null @@ -1,8 +0,0 @@ -jobs: -- template: jobs/test.yml - parameters: - vmImage: ubuntu-16.04 - -- template: jobs/package.yml - parameters: - vmImage: ubuntu-16.04 diff --git a/.azure-pipelines/macos.yml b/.azure-pipelines/macos.yml deleted file mode 100644 index 85c2a0246af..00000000000 --- a/.azure-pipelines/macos.yml +++ /dev/null @@ -1,8 +0,0 @@ -jobs: -- template: jobs/test.yml - parameters: - vmImage: macos-10.14 - -- template: jobs/package.yml - parameters: - vmImage: macos-10.14 diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml deleted file mode 100644 index 64163a5be8b..00000000000 --- a/.azure-pipelines/steps/run-tests.yml +++ /dev/null @@ -1,25 +0,0 @@ -steps: -- task: UsePythonVersion@0 - displayName: Use Python $(python.version) - inputs: - versionSpec: '$(python.version)' - -- bash: pip install --upgrade setuptools tox - displayName: Install Tox - -- script: tox -e py -- -m unit -n auto --junit-xml=junit/unit-test.xml - displayName: Tox run unit tests - -# Run integration tests in two groups so we will fail faster if there is a failure in the first group -- script: tox -e py -- -m integration -n auto --duration=5 -k "not test_install" --junit-xml=junit/integration-test-group0.xml - displayName: Tox run Group 0 integration tests - -- script: tox -e py -- -m integration -n auto --duration=5 -k "test_install" --junit-xml=junit/integration-test-group1.xml - displayName: Tox run Group 1 integration tests - -- task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFiles: junit/*.xml - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() diff --git a/.azure-pipelines/windows.yml b/.azure-pipelines/windows.yml index 9d1bf5385d0..ced172e5583 100644 --- a/.azure-pipelines/windows.yml +++ b/.azure-pipelines/windows.yml @@ -1,4 +1,9 @@ jobs: + +- template: jobs/dev-tools.yml + parameters: + vmImage: vs2017-win2016 + - template: jobs/test-windows.yml parameters: vmImage: vs2017-win2016 diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml new file mode 100644 index 00000000000..4fbe34fba9e --- /dev/null +++ b/.github/workflows/macos.yml @@ -0,0 +1,62 @@ +name: MacOS + +on: + push: + pull_request: + schedule: + # Run every Friday at 18:02 UTC + # https://crontab.guru/#2_18_*_*_5 + - cron: 2 18 * * 5 + +jobs: + dev-tools: + name: Developer Tooling + runs-on: macos-latest + + env: + TOXENV: lint,docs,vendoring + + steps: + + # Setup Python and git. + - uses: actions/checkout@master + + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Pre-configure global Git settings + run: | + git config --global user.email "pypa-dev@googlegroups.com" + git config --global user.name "pip" + + # Set up caching + - name: set PY + run: echo "::set-env name=PY::$(python -VV | sha256sum | cut -d' ' -f1)" + + - name: Storing pip's cache + uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('tools/requirements/tests.txt') }}-${{ hashFiles('tools/requirements/docs.txt') }}-${{ hashFiles('tox.ini') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + + - name: Storing pre-commit's cache + uses: actions/cache@v1 + with: + path: ~/.cache/pre-commit + key: pre-commit|2020-02-14|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} + + # Update tooling + - name: Update setuptools + run: python -m pip install --upgrade setuptools + - name: Install tox + run: python -m pip install --upgrade tox + - name: Log the list of packages + run: python -m pip freeze --all + + # Run checks with tox + - name: Run tox + run: python -m tox diff --git a/.github/workflows/python-linters.yml b/.github/workflows/python-linters.yml deleted file mode 100644 index 0b9bb0a2e44..00000000000 --- a/.github/workflows/python-linters.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: >- - 🤖 - Code quality - -on: - push: - pull_request: - schedule: - # Run every Friday at 18:02 UTC - # https://crontab.guru/#2_18_*_*_5 - - cron: 2 18 * * 5 - -jobs: - linters: - name: >- - ${{ matrix.env.TOXENV }}/${{ matrix.python-version }}@${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - # max-parallel: 5 - matrix: - python-version: - - 3.8 - os: - - ubuntu-latest - - windows-latest - - macos-latest - env: - - TOXENV: docs - - TOXENV: lint - - TOXENV: vendoring - - env: - TOX_PARALLEL_NO_SPINNER: 1 - - steps: - - uses: actions/checkout@master - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Log Python version - run: >- - python --version - - name: Log Python location - run: >- - which python - - name: Log Python env - run: >- - python -m sysconfig - - name: Pip cache - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('tools/requirements/tests.txt') }}-${{ hashFiles('tools/requirements/docs.txt') }}-${{ hashFiles('tox.ini') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- - - name: set PY - run: echo "::set-env name=PY::$(python -VV | sha256sum | cut -d' ' -f1)" - - uses: actions/cache@v1 - with: - path: ~/.cache/pre-commit - key: pre-commit|2020-02-14|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} - - name: Pre-configure global Git settings - run: | - git config --global user.email "pypa-dev@googlegroups.com" - git config --global user.name "pip" - - name: Update setuptools - run: >- - python -m pip install --upgrade setuptools - - name: Install tox - run: >- - python -m pip install --upgrade tox tox-venv - - name: Log the list of packages - run: >- - python -m pip freeze --all - - name: 'Initialize tox envs: ${{ matrix.env.TOXENV }}' - run: >- - python -m - tox - --parallel auto - --notest - --skip-missing-interpreters false - env: ${{ matrix.env }} - - name: Pre-fetch pre-commit hooks - # This is to separate test step from deps install - if: matrix.env.TOXENV == 'lint' - run: >- - .tox/lint/${{ runner.os == 'Windows' && 'Scripts' || 'bin' }}/python -m - pre_commit - install-hooks - - name: Test with tox - run: >- - python -m - tox - --parallel auto - env: ${{ matrix.env }} diff --git a/.travis.yml b/.travis.yml index 7259b473a45..e81bb482979 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,11 +13,9 @@ stages: jobs: include: - # Basic Checks + # Developer Tooling - stage: primary - env: TOXENV=docs - - env: TOXENV=lint - - env: TOXENV=vendoring + env: TOXENV=lint,docs,vendoring # Latest CPython - env: GROUP=1 python: 2.7 diff --git a/docs/html/development/ci.rst b/docs/html/development/ci.rst index dbd80eb6cef..d0ea74a8ae9 100644 --- a/docs/html/development/ci.rst +++ b/docs/html/development/ci.rst @@ -34,28 +34,30 @@ and on different architectures: - x64 - x86 -so 42 hypothetical interpreters. +so 42 hypothetical combinations. Checks ====== -``pip`` CI runs different kind of tests: +``pip`` CI runs various "checks": -- lint (defined in ``.pre-commit-config.yaml``) -- docs -- vendoring (is the ``src/_internal/_vendor`` directory cleanly vendored) -- unit tests (present in ``tests/unit``) -- "integration" tests (mostly present in ``tests/functional``) -- package (test the packaging steps) +- Development tooling + - docs (does the documentation build correctly?) + - lint (automated code quality checks, run with ``pre-commit``) + - vendoring (is ``src/pip/_vendor`` correctly constructed?) +- Tests + - unit tests (present in ``tests/unit``) + - "integration" tests (mostly present in ``tests/functional``) +- Packaging (test the packaging steps) -Since lint, docs, vendoring and package tests only need to run on a pip -developer/contributor machine, they only need to be tested on the x64 variant -of the 3 different operating systems, and when an interpreter needs to be -specified it's ok to require the latest CPython interpreter. +We run development tooling checks on the latest CPython (x64), on every +OS. This helps ensure they'll also run on a pip developer/contributor's +machine regardless of the OS they use. -So only unit tests and integration tests would need to be run with the different -interpreters. +We try to run the tests on as many interpreter-OS-architecture +combinations as we can, without having a significant slowdown on our +productivity due to CI wait times. Services @@ -65,13 +67,20 @@ pip test suite and checks are distributed on three different platforms that provides free executors for open source packages: - `Travis CI`_ (Used for Linux) - - `Azure DevOps CI`_ (Linux, MacOS & Windows tests) - - `GitHub Actions`_ (Linux, MacOS & Windows tests) + - `Azure DevOps CI`_ (Used for Windows) + - `GitHub Actions`_ (Used for MacOS) .. _`Travis CI`: https://travis-ci.org/ .. _`Azure DevOps CI`: https://azure.microsoft.com/en-us/services/devops/ .. _`GitHub Actions`: https://github.com/features/actions +TODO + +- how many workers we get per-CI-service. +- add links to "most relevant resources" for each CI-service. +- describe how our runs are set up for short-circuiting on failures in + linting / documentation / vendoring / unit tests / integration tests. + Current run tests ================= @@ -82,18 +91,18 @@ Developer tasks ======== =============== ================ ================== ============ OS docs lint vendoring packages ======== =============== ================ ================== ============ -Linux Travis, Github Travis, Github Travis, Github Azure -Windows Azure -MacOS Azure +Linux Travis Travis Travis +Windows Azure Azure Azure Azure +MacOS GitHub GitHub GitHub ======== =============== ================ ================== ============ Actual testing -------------- +------------------------------+---------------+-----------------+ -| **interpreter** | **unit** | **integration** | +| **combination** | **unit** | **integration** | +-----------+----------+-------+---------------+-----------------+ -| | | CP2.7 | Azure | Azure | +| | | CP2.7 | Azure | | | | +-------+---------------+-----------------+ | | | CP3.5 | Azure | | | | +-------+---------------+-----------------+ @@ -117,9 +126,9 @@ Actual testing | | +-------+---------------+-----------------+ | | | CP3.8 | Azure | Azure | | | +-------+---------------+-----------------+ -| | | PyPy | | | +| | | PyPy | Azure | Azure | | | +-------+---------------+-----------------+ -| | | PyPy3 | | | +| | | PyPy3 | Azure | Azure | +-----------+----------+-------+---------------+-----------------+ | | | CP2.7 | | | | | +-------+---------------+-----------------+ @@ -135,13 +144,13 @@ Actual testing | | +-------+---------------+-----------------+ | | | PyPy3 | | | | Linux +----------+-------+---------------+-----------------+ -| | | CP2.7 | Travis,Azure | Travis,Azure | +| | | CP2.7 | Travis | Travis | | | +-------+---------------+-----------------+ -| | | CP3.5 | Travis,Azure | Travis,Azure | +| | | CP3.5 | Travis | Travis | | | +-------+---------------+-----------------+ -| | | CP3.6 | Travis,Azure | Travis,Azure | +| | | CP3.6 | Travis | Travis | | | +-------+---------------+-----------------+ -| | x64 | CP3.7 | Travis,Azure | Travis,Azure | +| | x64 | CP3.7 | Travis | Travis | | | +-------+---------------+-----------------+ | | | CP3.8 | Travis | Travis | | | +-------+---------------+-----------------+ @@ -163,13 +172,13 @@ Actual testing | | +-------+---------------+-----------------+ | | | PyPy3 | | | | MacOS +----------+-------+---------------+-----------------+ -| | | CP2.7 | Azure | Azure | +| | | CP2.7 | | | | | +-------+---------------+-----------------+ -| | | CP3.5 | Azure | Azure | +| | | CP3.5 | | | | | +-------+---------------+-----------------+ -| | | CP3.6 | Azure | Azure | +| | | CP3.6 | | | | | +-------+---------------+-----------------+ -| | x64 | CP3.7 | Azure | Azure | +| | x64 | CP3.7 | | | | | +-------+---------------+-----------------+ | | | CP3.8 | | | | | +-------+---------------+-----------------+