Rewrite repo-build and package-build in Python (toltec-dev#218)

(See toltec-dev#211.)

The new build scripts are backward-compatible with existing recipes. Apart from some minor improvements on generated maintainer scripts, there are no changes in the final .ipk files.

* Rewrite to Python 3.8
    - Create `toltec` Python support library (in `scripts/toltec`)
    - Remove superseded `opkg` Python library (in `scripts/opkg`)
    - Add Python linting and formatting checks
    - Replace `scripts/repo-build` (Bash script) with `scripts/repo_build.py`
    - Replace `scripts/package-build` (Bash script) with `scripts/package_build.py`
    - Update Makefile and workflows to work with the new scripts
* Package changes
    - Update koreader to not use the `$recipedir` variable (not documented, not supported by the new scripts, and should not be used anymore)
    - Update zoneinfo-utils to fix use of an undefined variable
* Documentation changes
    - Add info on how required Python version and modules
    - Document custom fields and functions support
* New features and improvements
    - Improve support for custom fields and functions in recipes
    - Remove unnecessary generated install scripts, reduce size of generated scripts
    - Parameterize the <https://toltec-dev.org> value in workflows, using a `REMOTE_HTTP` secret.
    - Rename the `REMOTE` secret to `REMOTE_SSH` for clarity
* Additional checks
    - Check that all custom declarations start with `_` to guard against typos on optional standard declarations
    - Check that timestamp fields contain a valid ISO-8601 value
    - Check that pkgver fields contain a valid version number
    - Run all Bash scripts with `set -euo pipefail` flags enabled

Author: matteodelabre

.github/actions/setup/action.yml

@@ -6,7 +6,7 @@ runs:
           shell: bash
           run: |
             sudo apt-get update -yq
-            sudo apt-get install -yq bsdtar tree python3-docutils
+            sudo apt-get install -yq bsdtar python3-docutils
         - name: Install shfmt and Shellcheck
           shell: bash
           run: |
@@ -35,3 +35,8 @@ runs:
             chmod a+x shellcheck
             sudo chown root:root shellcheck
             sudo mv shellcheck "$install_dir"
+        - name: Install Python dependencies
+          shell: bash
+          run: |
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt

.github/workflows/pr.yml

@@ -8,7 +8,11 @@ jobs:
         steps:
             - name: Checkout the Git repository
               uses: actions/checkout@v2
-            - name: Setup Toltec environment
+            - name: Setup Python
+              uses: actions/setup-python@v2
+              with:
+                python-version: '3.8'
+            - name: Setup Toltec dependencies
               uses: ./.github/actions/setup
             - name: Check formatting
               run: make format
@@ -21,10 +25,14 @@ jobs:
         steps:
             - name: Checkout the Git repository
               uses: actions/checkout@v2
-            - name: Setup Toltec environment
+            - name: Setup Python
+              uses: actions/setup-python@v2
+              with:
+                python-version: '3.8'
+            - name: Setup Toltec dependencies
               uses: ./.github/actions/setup
             - name: Build packages
-              run: remote_repo='https://toltec-dev.org/${{ github.base_ref }}' make repo-new
+              run: make repo-new FLAGS='--remote-repo ${{ secrets.REMOTE_HTTP }}/${{ github.base_ref }}'
             - name: Save the build output
               uses: actions/upload-artifact@v2
               with:

.github/workflows/stable.yml

@@ -10,17 +10,23 @@ jobs:
         steps:
             - name: Checkout the Git repository
               uses: actions/checkout@v2
+            - name: Setup Python
+              uses: actions/setup-python@v2
+              with:
+                python-version: '3.8'
             - name: Setup Toltec dependencies
               uses: ./.github/actions/setup
             - name: Build packages
-              run: remote_repo="https://toltec-dev.org/stable" make repo
+              run: |
+                make repo FLAGS='--remote-repo ${{ secrets.REMOTE_HTTP }}/stable'
+                ./scripts/repo-build-web package build/repo
             - name: Sync packages with the remote repository
               uses: ./.github/actions/sync-repository
               with:
                 local-path: build/repo/
                 ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
                 ssh-known-hosts: ${{ secrets.SSH_KNOWN_HOSTS }}
-                remote-path: ${{ secrets.REMOTE }}:/srv/toltec/stable
+                remote-path: ${{ secrets.REMOTE_SSH }}:/srv/toltec/stable
             - name: Build website
               run: make web
             - name: Sync website with the remote repository
@@ -29,7 +35,7 @@ jobs:
                 local-path: build/web/
                 ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
                 ssh-known-hosts: ${{ secrets.SSH_KNOWN_HOSTS }}
-                remote-path: ${{ secrets.REMOTE }}:/srv/toltec/web
+                remote-path: ${{ secrets.REMOTE_SSH }}:/srv/toltec/web
             - name: Send notification to Discord
               continue-on-error: true
               uses: ./.github/actions/discord-send

.github/workflows/testing.yml

@@ -10,18 +10,23 @@ jobs:
         steps:
             - name: Checkout the Git repository
               uses: actions/checkout@v2
+            - name: Setup Python
+              uses: actions/setup-python@v2
+              with:
+                python-version: '3.8'
             - name: Setup Toltec dependencies
               uses: ./.github/actions/setup
             - name: Build packages
               run: |
-                remote_repo="https://toltec-dev.org/testing" make repo
+                make repo FLAGS='--remote-repo ${{ secrets.REMOTE_HTTP }}/testing'
+                ./scripts/repo-build-web package build/repo
             - name: Sync packages with the remote repository
               uses: ./.github/actions/sync-repository
               with:
                 local-path: build/repo/
                 ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
                 ssh-known-hosts: ${{ secrets.SSH_KNOWN_HOSTS }}
-                remote-path: ${{ secrets.REMOTE }}:/srv/toltec/testing
+                remote-path: ${{ secrets.REMOTE_SSH }}:/srv/toltec/testing
             - name: Send notification to Discord
               continue-on-error: true
               uses: ./.github/actions/discord-send

.mypy.ini

@@ -0,0 +1,3 @@
+[mypy]
+[mypy-docker.*]
+ignore_missing_imports = True

.pylintrc

@@ -0,0 +1,3 @@
+[SIMILARITIES]
+# Do not consider duplicate imports in several files as duplicated code
+ignore-imports=yes

Makefile

@@ -47,35 +47,45 @@ web:
 	./scripts/web-build web build/web
 
 repo:
-	./scripts/repo-build package build/package build/repo "$$remote_repo"
+	./scripts/repo_build.py $(FLAGS)
 
 repo-local:
-	./scripts/repo-build -l package build/package build/repo "$$remote_repo"
+	./scripts/repo_build.py --local $(FLAGS)
 
 repo-new:
-	./scripts/repo-build -n package build/package build/repo "$$remote_repo"
+	./scripts/repo_build.py --no-fetch $(FLAGS)
 
 repo-check:
 	./scripts/repo-check build/repo
 
 $(RECIPES): %:
-	./scripts/package-build package/"${@}" build/package/"${@}"
+	./scripts/package_build.py $(FLAGS) "$(@)"
 
-$(RECIPES_PUSH): %:
-	ssh root@"${HOST}" mkdir -p .cache/opkg
-	scp build/package/"$(@:%-push=%)"/*/*.ipk root@"${HOST}":.cache/opkg
+push: %:
+	rsync --rsync-path /opt/bin/rsync \
+	      --archive --verbose --compress --delete \
+	      build/repo/ \
+	      root@"$(HOST)":~/.cache/toltec/
 
 format:
-	@echo "==> Checking the formatting of shell scripts"
+	@echo "==> Checking Bash formatting"
 	shfmt -d .
+	@echo "==> Checking Python formatting"
+	black --line-length 80 --check --diff scripts
 
 format-fix:
-	@echo "==> Fixing the formatting of shell scripts"
+	@echo "==> Fixing Bash formatting"
 	shfmt -l -w .
+	@echo "==> Fixing Python formatting"
+	black --line-length 80 scripts
 
 lint:
-	@echo "==> Linting shell scripts"
+	@echo "==> Linting Bash scripts"
 	shellcheck $$(shfmt -f .)
+	@echo "==> Typechecking Python files"
+	MYPYPATH=scripts mypy --disallow-untyped-defs scripts
+	@echo "==> Linting Python files"
+	PYTHONPATH=: pylint scripts
 	@echo "==> Verifying that the bootstrap checksum is correct"
 	./scripts/bootstrap/checksum-check
 

docs/building.md

@@ -18,7 +18,9 @@ Before running the build, make sure you have all the required dependencies:
 
 * Docker
 * bsdtar
-* tree
+* Python ⩾ 3.8
+
+You’ll also need all the Python modules listed in [requirements.txt](../requirements.txt) (install them by running `pip install --user -r requirements.txt` or using a [virtual environment](https://docs.python.org/3/tutorial/venv.html)).
 
 To start the build, run `make repo-local`.
 This will be a long process, so you may want to grab a cup of coffee.

docs/contributing.md

@@ -52,15 +52,15 @@ Before proposing your package, make sure that it conforms to the rules on how to
 
 ### Style Guide
 
-All contributions must follow the project’s [style guide](../.editorconfig).
-Shell scripts must also comply with [Shellcheck](https://github.com/koalaman/shellcheck).
 Sticking to a common set of conventions makes it easier for everyone to read the source code and reduces the time spent reviewing little formatting details.
 
-The code style for shell scripts will automatically be checked when you submit your pull request.
-You may also check it manually by running `make format` (or `make format-fix` to automatically fix any issues) at the root of the repository (you need to have [shfmt](https://github.com/mvdan/sh) installed on your computer for this to work).
+All contributions must follow the project’s [style guide](../.editorconfig).
+The code style for shell scripts and Python code will automatically be checked for pull requests and can be checked locally using `make format`.
+Use `make format-fix` to automatically reformat your code to fit the style guide.
 
-Compliance of shell scripts with Shellcheck will also automatically be checked.
-To check it manually, run `make lint` at the root of the repository (you need to have Shellcheck installed on your computer for this to work).
+Shell scripts must comply with [Shellcheck](https://github.com/koalaman/shellcheck).
+Python code must have valid type annotations (typechecked using [mypy](http://mypy-lang.org/)) and be free of [pylint](https://www.pylint.org/) errors.
+Compliance with those tools will automatically be checked for pull requests and can be checked locally using `make lint`.
 
 ### Code of Conduct
 

docs/package.md

@@ -20,6 +20,7 @@ Sourcing a recipe must have no side effects: the metadata section can only execu
 
 At the top of the file is a block of fields that define metadata about the package.
 For consistency, declare those fields in the same order they are described below.
+You can also declare custom variables to reduce repetition, but make sure to prefix their name with `_`.
 
 > **Note:** The field names and semantics are inspired both by the [Debian control file format](https://www.debian.org/doc/debian-policy/ch-controlfields.html) and the [Arch Linux PKGBUILD format](https://wiki.archlinux.org/index.php/PKGBUILD).
 
@@ -246,6 +247,22 @@ Each entry can either be a local path relative to the recipe file or a full URL
 Archive files whose names end in `.zip`, `.tar.gz`, `.tar.xz`, or `.tar.bz` will be automatically extracted in place, with all container directories stripped.
 You can disable this behavior by adding the archive name to the `noextract` array.
 
+#### `flags`
+
+<table>
+    <tr>
+        <th>Required?</th>
+        <td>No, defaults to <code>()</code></th>
+    </tr>
+    <tr>
+        <th>Type</th>
+        <td>Array of strings</td>
+    </tr>
+</table>
+
+Set of flags that affect the build process.
+Currently, the only available flag is `nostrip`, which disables the automatic removal of unneeded symbols from binaries.
+
 #### `noextract`
 
 <table>
@@ -284,6 +301,7 @@ This array must have exactly as many elements as the `source` array.
 ### Prepare Section
 
 The prepare section contains the `prepare()` function in which the source files may be prepared for the building step that follows.
+This function has access to all the metadata fields declared above.
 Common tasks include patching sources, extracting archives, and moving downloaded sources to the right location.
 
 ### Build Section
@@ -307,7 +325,7 @@ The `package()` function populates the `$pkgdir` directory with the files and di
 The install section can contain additional functions to customize the behavior of the package when it is installed, removed, or upgraded on the device.
 Those functions are `preinstall()`, `configure()`, `preremove()`, `postremove()`, `preupgrade()` and `postupgrade()`.
 Unlike the previous functions, all the install functions **run in the context of the target device.**
-They have access to all the metadata fields, but not to other functions.
+They have access to all the metadata fields and to custom functions whose name starts with `_`.
 They can also use functions from the [install library](../scripts/install-lib).
 
 When installing a new package, the following happens:

package/koreader/package

@@ -18,12 +18,14 @@ source=(
     koreader.draft
     KOReader.oxide
     rm2-support.patch
+    koreader
 )
 sha256sums=(
     38db8f3895472828d7c1d63fc54db426135a8dbf8425633e299a1a9abfdb69b2
     SKIP
     SKIP
     SKIP
+    SKIP
 )
 
 prepare() {
@@ -42,5 +44,5 @@ package() {
     install -D -m 644 -t "$pkgdir"/opt/etc/draft/ "$srcdir"/koreader.draft
     install -D -m 644 -t "$pkgdir"/opt/usr/share/applications/ "$srcdir"/KOReader.oxide
     install -D -m 644 -t "$pkgdir"/opt/etc/draft/icons/ "$srcdir"/resources/koreader.png
-    install -D -m 755 -t "$pkgdir"/opt/bin/ "$recipedir"/koreader
+    install -D -m 755 -t "$pkgdir"/opt/bin/ "$srcdir"/koreader
 }

package/zoneinfo-utils/package

@@ -40,11 +40,11 @@ build() {
 
     shopt -s extglob
     cp -r ../* .
-    make LFLAGS="${LDFLAGS}"
+    make VERSION="$_tzver"
 
     popd > /dev/null
 
-    make LFLAGS="${LDFLAGS}" CC=arm-linux-gnueabihf-cc
+    make VERSION="$_tzver" CC=arm-linux-gnueabihf-cc
 }
 
 package() {

requirements.txt

@@ -0,0 +1,25 @@
+appdirs==1.4.4
+astroid==2.4.2
+black==20.8b1
+certifi==2020.12.5
+chardet==4.0.0
+click==7.1.2
+docker==4.4.1
+idna==2.10
+isort==5.7.0
+lazy-object-proxy==1.4.3
+mccabe==0.6.1
+mypy==0.790
+mypy-extensions==0.4.3
+pathspec==0.8.1
+pylint==2.6.0
+python-dateutil==2.8.1
+regex==2020.11.13
+requests==2.25.1
+six==1.15.0
+toml==0.10.2
+typed-ast==1.4.2
+typing-extensions==3.7.4.3
+urllib3==1.26.2
+websocket-client==0.57.0
+wrapt==1.12.1