diff --git a/.circleci/config.yml b/.circleci/config.yml index 5786b6ea3c7..5c260cf3439 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,7 +34,7 @@ jobs: command: | python3.8 -m venv venv/ venv/bin/pip install pip-tools --constraint requirements.in - venv/bin/pip-sync + venv/bin/pip-sync --pip-args=--no-deps - run: name: Yamllint Test command: PATH="venv/bin:$PATH" yamllint -c .yamllint.yaml . @@ -176,7 +176,8 @@ jobs: apt update apt install -y maven default-jdk-headless mvn dependency:copy-dependencies - venv/bin/pip-sync requirements.txt java-requirements.txt + venv/bin/pip-sync --pip-args=--no-deps requirements.txt \ + java-requirements.txt - run: name: PyTest Integration Test # Google's client libraries will check for @@ -497,8 +498,8 @@ jobs: bigquery-etl-main cd bigquery-etl-main - pip install -r requirements.txt - pip install -r java-requirements.txt + pip install --no-deps -r requirements.txt + pip install --no-deps -r java-requirements.txt mvn dependency:copy-dependencies ./script/bqetl generate all \ --target-project moz-fx-data-shared-prod diff --git a/Dockerfile b/Dockerfile index 745d14c0d9d..c7838dadd10 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,9 +13,10 @@ WORKDIR /app FROM base AS python-deps RUN apt-get update -qqy && apt-get install -qqy gcc libc-dev COPY java-requirements.txt ./ -RUN pip install -r java-requirements.txt +RUN pip install --no-deps -r java-requirements.txt COPY requirements.txt ./ -RUN pip install -r requirements.txt +# use --no-deps to work around https://github.com/pypa/pip/issues/9644 +RUN pip install --no-deps -r requirements.txt # download java dependencies in separate stage because it requires maven FROM base AS java-deps diff --git a/README.md b/README.md index 4d4ef8aef4a..f6a4a048257 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,8 @@ venv/bin/pre-commit install 4. Download java dependencies ```bash mvn dependency:copy-dependencies -venv/bin/pip-sync requirements.txt java-requirements.txt +# specify `<(echo mozilla-bigquery-etl)` to retain bqetl from `./bqetl bootstrap` +venv/bin/pip-sync --pip-args=--no-deps requirements.txt java-requirements.txt <(echo mozilla-bigquery-etl) ``` Finally, if you are using Visual Studio Code, you may also wish to use our recommended defaults: diff --git a/bqetl b/bqetl index 037d5d289b0..6a6f1b0a460 100755 --- a/bqetl +++ b/bqetl @@ -29,7 +29,7 @@ if [ "$CMD" == "bootstrap" ]; then # see e.g. https://github.com/scipy/scipy/issues/13102#issuecomment-788160041 export SYSTEM_VERSION_COMPAT=1 - venv/bin/pip install -r requirements.txt + venv/bin/pip install --no-deps -r requirements.txt venv/bin/pip install -e . echo "bqetl configured! It should now be ready for use." exit 0 diff --git a/docs/cookbooks/common_workflows.md b/docs/cookbooks/common_workflows.md index 8fefd8a50f3..e8545d197a6 100644 --- a/docs/cookbooks/common_workflows.md +++ b/docs/cookbooks/common_workflows.md @@ -227,7 +227,7 @@ python3 -m venv venv/ source venv/bin/activate # If not installed: -pip install pip-tools +pip install pip-tools --constraint requirements.in # Add the dependency to requirements.in e.g. Jinja2. echo Jinja2==2.11.1 >> requirements.in diff --git a/docs/cookbooks/testing.md b/docs/cookbooks/testing.md index 8c7b22bcbcc..4ec3c6319dd 100644 --- a/docs/cookbooks/testing.md +++ b/docs/cookbooks/testing.md @@ -10,7 +10,7 @@ python3.8 -m venv venv/ ./venv/bin/pip install pip-tools -c requirements.in # install python dependencies with pip-sync (provided by pip-tools) -./venv/bin/pip-sync +./venv/bin/pip-sync --pip-args=--no-deps requirements.txt java-requirements.txt # install java dependencies with maven mvn dependency:copy-dependencies diff --git a/requirements.in b/requirements.in index 928fd64390e..86276f7c91e 100644 --- a/requirements.in +++ b/requirements.in @@ -5,10 +5,8 @@ click==8.0.4 gcsfs==2022.2.0 gcloud==0.18.3 gitpython==3.1.27 -google-api-core==1.31.5 # transitive dep that needs dependabot updates google-cloud-bigquery==2.34.0 google-cloud-storage==2.1.0 -googleapis-common-protos==1.54.0 # transitive dep that needs dependabot updates importlib-resources>=1.4.0 # jsonschema dep for python<3.9 Jinja2==3.0.3 jsonschema==4.4.0 diff --git a/script/glam/README.md b/script/glam/README.md index 5d7bc905dd0..942bb197ef8 100644 --- a/script/glam/README.md +++ b/script/glam/README.md @@ -17,7 +17,7 @@ python3 -m venv venv source venv/bin/activate # install dependencies -pip install -r requirements.txt +pip install --no-deps -r requirements.txt # install bqetl command-line tool pip install -e . ```