Speedup testsuite - venv creation

.travis.yml

@@ -30,7 +30,6 @@ matrix:
       python: nightly
   allow_failures:
     - python: nightly
-    - python: pypy3
 
 install: travis_retry .travis/install.sh
 script: .travis/run.sh

tests/conftest.py

@@ -8,7 +8,6 @@
 import six
 
 import pip._internal
-from pip._internal.utils import appdirs
 from tests.lib import SRC_DIR, TestData
 from tests.lib.path import Path
 from tests.lib.scripttest import PipTestEnvironment
@@ -124,18 +123,9 @@ def isolate(tmpdir):
         )
 
 
-@pytest.fixture
-def virtualenv(tmpdir, monkeypatch, isolate):
-    """
-    Return a virtual environment which is unique to each test function
-    invocation created inside of a sub directory of the test function's
-    temporary directory. The returned object is a
-    ``tests.lib.venv.VirtualEnvironment`` object.
-    """
-    # Force shutil to use the older method of rmtree that didn't use the fd
-    # functions. These seem to fail on Travis (and only on Travis).
-    monkeypatch.setattr(shutil, "_use_fd_functions", False, raising=False)
-
+@pytest.yield_fixture(scope='session')
+def virtualenv_template(tmpdir_factory):
+    tmpdir = Path(str(tmpdir_factory.mktemp('virtualenv')))
     # Copy over our source tree so that each virtual environment is self
     # contained
     pip_src = tmpdir.join("pip_src").abspath
@@ -147,21 +137,33 @@ def virtualenv(tmpdir, monkeypatch, isolate):
             "tests", "pip.egg-info", "build", "dist", ".tox", ".git",
         ),
     )
-
     # Create the virtual environment
     venv = VirtualEnvironment.create(
-        tmpdir.join("workspace", "venv"),
+        tmpdir.join("venv_orig"),
         pip_source_dir=pip_src,
+        relocatable=True,
     )
+    # Rename original virtualenv directory to make sure
+    # it's not reused by mistake from one of the copies.
+    venv_template = tmpdir / "venv_template"
+    os.rename(venv.location, venv_template)
+    yield venv_template
+    tmpdir.rmtree(noerrors=True)
 
-    # Clean out our cache: creating the venv injects wheels into it.
-    if os.path.exists(appdirs.user_cache_dir("pip")):
-        shutil.rmtree(appdirs.user_cache_dir("pip"))
 
-    # Undo our monkeypatching of shutil
-    monkeypatch.undo()
-
-    return venv
+@pytest.yield_fixture
+def virtualenv(virtualenv_template, tmpdir, isolate):
+    """
+    Return a virtual environment which is unique to each test function
+    invocation created inside of a sub directory of the test function's
+    temporary directory. The returned object is a
+    ``tests.lib.venv.VirtualEnvironment`` object.
+    """
+    venv_location = tmpdir.join("workspace", "venv")
+    shutil.copytree(virtualenv_template, venv_location, symlinks=True)
+    venv = VirtualEnvironment(venv_location)
+    yield venv
+    venv_location.rmtree(noerrors=True)
 
 
 @pytest.fixture

tests/lib/__init__.py

@@ -9,6 +9,7 @@
 import shutil
 
 import scripttest
+import six
 import virtualenv
 
 from tests.lib.path import Path, curdir
@@ -35,6 +36,15 @@ def path_to_url(path):
     return 'file://' + url
 
 
+# workaround for https://github.com/pypa/virtualenv/issues/306
+def virtualenv_lib_path(venv_home, venv_lib):
+    if not hasattr(sys, "pypy_version_info"):
+        return venv_lib
+    version_fmt = '{0}' if six.PY3 else '{0}.{1}'
+    version_dir = version_fmt.format(*sys.version_info)
+    return os.path.join(venv_home, 'lib-python', version_dir)
+
+
 def create_file(path, contents=None):
     """Create a file on the path, with the given contents
     """
@@ -262,11 +272,8 @@ def __init__(self, base_path, *args, **kwargs):
         path_locations = virtualenv.path_locations(_virtualenv)
         # Make sure we have test.lib.path.Path objects
         venv, lib, include, bin = map(Path, path_locations)
-        # workaround for https://github.com/pypa/virtualenv/issues/306
-        if hasattr(sys, "pypy_version_info"):
-            lib = os.path.join(venv, 'lib-python', pyversion)
         self.venv_path = venv
-        self.lib_path = lib
+        self.lib_path = virtualenv_lib_path(venv, lib)
         self.include_path = include
         self.bin_path = bin
 

tests/lib/venv.py

@@ -1,50 +1,39 @@
 from __future__ import absolute_import
 
-import os
-import subprocess
-import sys
+import distutils
 
 import virtualenv as _virtualenv
 
+from . import virtualenv_lib_path
 from .path import Path
 
-# On Python < 3.3 we don't have subprocess.DEVNULL
-try:
-    DEVNULL = subprocess.DEVNULL
-except AttributeError:
-    DEVNULL = open(os.devnull, "wb")
-
 
 class VirtualEnvironment(object):
     """
     An abstraction around virtual environments, currently it only uses
     virtualenv but in the future it could use pyvenv.
     """
 
-    def __init__(self, location, *args, **kwargs):
+    def __init__(self, location, system_site_packages=False):
         self.location = Path(location)
-        self.pip_source_dir = kwargs.pop("pip_source_dir")
-        self._system_site_packages = kwargs.pop("system_site_packages", False)
-
+        self._system_site_packages = system_site_packages
         home, lib, inc, bin = _virtualenv.path_locations(self.location)
-        # workaround for https://github.com/pypa/virtualenv/issues/306
-        if hasattr(sys, "pypy_version_info"):
-            lib = os.path.join(home, 'lib-python', sys.version[:3])
-        self.lib = Path(lib)
+        self.lib = Path(virtualenv_lib_path(home, lib))
         self.bin = Path(bin)
 
-        super(VirtualEnvironment, self).__init__(*args, **kwargs)
-
     def __repr__(self):
         return "<VirtualEnvironment {0}>".format(self.location)
 
     @classmethod
-    def create(cls, location, clear=False, pip_source_dir=None):
-        obj = cls(location, pip_source_dir=pip_source_dir)
-        obj._create(clear=clear)
+    def create(cls, location, clear=False,
+               pip_source_dir=None, relocatable=False):
+        obj = cls(location)
+        obj._create(clear=clear,
+                    pip_source_dir=pip_source_dir,
+                    relocatable=relocatable)
         return obj
 
-    def _create(self, clear=False):
+    def _create(self, clear=False, pip_source_dir=None, relocatable=False):
         # Create the actual virtual environment
         _virtualenv.create_environment(
             self.location,
@@ -53,23 +42,13 @@ def _create(self, clear=False):
             no_pip=True,
             no_wheel=True,
         )
-
-        # Install our development version of pip install the virtual
-        # environment
-        cmd = [self.bin.join("python"), "setup.py", "install", "--no-compile"]
-        p = subprocess.Popen(
-            cmd,
-            cwd=self.pip_source_dir,
-            stderr=subprocess.STDOUT,
-            stdout=DEVNULL,
-        )
-        p.communicate()
-        if p.returncode != 0:
-            raise subprocess.CalledProcessError(
-                p.returncode,
-                cmd,
-                output=p.stdout,
-            )
+        _virtualenv.install_wheel([pip_source_dir or '.'],
+                                  self.bin.join("python"))
+        if relocatable:
+            _virtualenv.make_environment_relocatable(self.location)
+        # FIXME: some tests rely on 'easy-install.pth' being already present.
+        site_package = distutils.sysconfig.get_python_lib(prefix=self.location)
+        Path(site_package).join('easy-install.pth').touch()
 
     def clear(self):
         self._create(clear=True)