Use --no-deps when installing compiled requirements files (#2752)

Author: relud

.circleci/config.yml

@@ -34,7 +34,7 @@ jobs:
           command: |
             python3.8 -m venv venv/
             venv/bin/pip install pip-tools --constraint requirements.in
-            venv/bin/pip-sync
+            venv/bin/pip-sync --pip-args=--no-deps
       - run:
           name: Yamllint Test
           command: PATH="venv/bin:$PATH" yamllint -c .yamllint.yaml .
@@ -176,7 +176,8 @@ jobs:
             apt update
             apt install -y maven default-jdk-headless
             mvn dependency:copy-dependencies
-            venv/bin/pip-sync requirements.txt java-requirements.txt
+            venv/bin/pip-sync --pip-args=--no-deps requirements.txt \
+              java-requirements.txt
       - run:
           name: PyTest Integration Test
           # Google's client libraries will check for
@@ -497,8 +498,8 @@ jobs:
               bigquery-etl-main
 
             cd bigquery-etl-main
-            pip install -r requirements.txt
-            pip install -r java-requirements.txt
+            pip install --no-deps -r requirements.txt
+            pip install --no-deps -r java-requirements.txt
             mvn dependency:copy-dependencies
             ./script/bqetl generate all \
               --target-project moz-fx-data-shared-prod

Dockerfile

@@ -13,9 +13,10 @@ WORKDIR /app
 FROM base AS python-deps
 RUN apt-get update -qqy && apt-get install -qqy gcc libc-dev
 COPY java-requirements.txt ./
-RUN pip install -r java-requirements.txt
+RUN pip install --no-deps -r java-requirements.txt
 COPY requirements.txt ./
-RUN pip install -r requirements.txt
+# use --no-deps to work around https://github.com/pypa/pip/issues/9644
+RUN pip install --no-deps -r requirements.txt
 
 # download java dependencies in separate stage because it requires maven
 FROM base AS java-deps

README.md

@@ -49,7 +49,8 @@ venv/bin/pre-commit install
 4. Download java dependencies
 ```bash
 mvn dependency:copy-dependencies
-venv/bin/pip-sync requirements.txt java-requirements.txt
+# specify `<(echo mozilla-bigquery-etl)` to retain bqetl from `./bqetl bootstrap`
+venv/bin/pip-sync --pip-args=--no-deps requirements.txt java-requirements.txt <(echo mozilla-bigquery-etl)
 ```
 
 Finally, if you are using Visual Studio Code, you may also wish to use our recommended defaults:

bqetl

@@ -29,7 +29,7 @@ if [ "$CMD" == "bootstrap" ]; then
     # see e.g. https://github.com/scipy/scipy/issues/13102#issuecomment-788160041
     export SYSTEM_VERSION_COMPAT=1
 
-    venv/bin/pip install -r requirements.txt
+    venv/bin/pip install --no-deps -r requirements.txt
     venv/bin/pip install -e .
     echo "bqetl configured! It should now be ready for use."
     exit 0

docs/cookbooks/common_workflows.md

@@ -227,7 +227,7 @@ python3 -m venv venv/
 source venv/bin/activate
 
 # If not installed:
-pip install pip-tools
+pip install pip-tools --constraint requirements.in
 
 # Add the dependency to requirements.in e.g. Jinja2.
 echo Jinja2==2.11.1 >> requirements.in

docs/cookbooks/testing.md

@@ -10,7 +10,7 @@ python3.8 -m venv venv/
 ./venv/bin/pip install pip-tools -c requirements.in
 
 # install python dependencies with pip-sync (provided by pip-tools)
-./venv/bin/pip-sync
+./venv/bin/pip-sync --pip-args=--no-deps requirements.txt java-requirements.txt
 
 # install java dependencies with maven
 mvn dependency:copy-dependencies

requirements.in

@@ -5,10 +5,8 @@ click==8.0.4
 gcsfs==2022.2.0
 gcloud==0.18.3
 gitpython==3.1.27
-google-api-core==1.31.5  # transitive dep that needs dependabot updates
 google-cloud-bigquery==2.34.0
 google-cloud-storage==2.1.0
-googleapis-common-protos==1.54.0  # transitive dep that needs dependabot updates
 importlib-resources>=1.4.0  # jsonschema dep for python<3.9
 Jinja2==3.0.3
 jsonschema==4.4.0

script/glam/README.md

@@ -17,7 +17,7 @@ python3 -m venv venv
 source venv/bin/activate
 
 # install dependencies
-pip install -r requirements.txt
+pip install --no-deps -r requirements.txt
 # install bqetl command-line tool
 pip install -e .
 ```