tooling: Update build scripts for new build process

Author: matteodelabre

.github/actions/publish-packages/Dockerfile

@@ -5,43 +5,25 @@ on:
             - testing
 jobs:
     images:
-        name: Publish images
+        name: Build and publish images
         runs-on: ubuntu-latest
-        strategy:
-            matrix:
-                image: [
-                    base,
-                    qt,
-                    python,
-                    rust
-                ]
-            # Build images one at a time, since we can’t specify fine-grained
-            # dependencies and all images depend on the 'base' one
-            max-parallel: 1
         steps:
             - name: Checkout the Git repository
               uses: actions/checkout@v2
-            - name: Publish images
-              uses: docker/build-push-action@v1
-              with:
-                  tags: testing
-                  build_args: BASE=docker.pkg.github.com/${{ github.repository }}/base:testing
-                  username: ${{ github.actor }}
-                  password: ${{ secrets.GITHUB_TOKEN }}
-                  registry: docker.pkg.github.com
-                  repository: ${{ github.repository }}/${{ matrix.image }}
-                  path: image/${{ matrix.image }}
+            - name: Build and publish images
+              run: |
+                  echo ${{ secrets.CR_PAT }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
+                  ./scripts/build-images -p image
     packages:
         needs: images
-        name: Publish packages
+        name: Build and publish packages
         runs-on: ubuntu-latest
         steps:
             - name: Checkout the Git repository
               uses: actions/checkout@v2
-            - name: Publish packages
-              uses: ./.github/actions/publish-packages
-              env:
-                  REMOTE_PATH: /srv/toltec/testing
-                  REMOTE: ${{ secrets.REMOTE }}
-                  SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
-                  SSH_KNOWN_HOSTS: ${{ secrets.SSH_KNOWN_HOSTS }}
+            - name: Build packages
+              run: ./scripts/build-repo package build
+            - uses: actions/upload-artifact@v2
+              with:
+                name: repo
+                path: build/repo

.github/actions/publish-packages/README.md

@@ -1,2 +1,3 @@
 private
 build
+__pycache__

.github/workflows/publish-testing.yml

@@ -1,33 +1,23 @@
-PACKAGE?=draft
 HOST?=10.11.99.1
-
-default: repo
-repo: docker_images
-	bash scripts/build-repo-in-docker
-
-# Use `make <app>` to build any package individually
-# Use `make push_<app>` to push any package to .cache/opkg/ on the rM
 PACKAGES=$(shell ls package/)
-PUSH_PACKAGES=$(foreach app, $(PACKAGES), push_$(app))
+PUSH_PACKAGES=$(foreach app, $(PACKAGES), push-$(app))
 
-$(PACKAGES): %:
-	PACKAGE=$(@) make package
+help:
+	@echo "Available recipes: ${PACKAGES}"
+	@echo
+	@echo "Use 'make repo' to build all the packages and the index file"
+	@echo "Use 'make <recipe>' to build any package individually"
+	@echo "Use 'make push-<recipe>' to push any built package to .cache/opkg on the reMarkable"
 
-$(PUSH_PACKAGES): %:
-	PACKAGE=$(@:push_%=%) make push_package
+repo:
+	./scripts/build-repo package build
 
-package: docker_images
-	echo "BUILDING ${PACKAGE}"
-	PACKAGE=${PACKAGE} bash scripts/build-package-in-docker
-
-push_package:
-	ssh root@${HOST} mkdir -p .cache/opkg/
-	PACKAGE=${PACKAGE} scp artifacts/package/${PACKAGE}/*.ipk root@${HOST}:.cache/opkg/
+$(PACKAGES): %:
+	./scripts/build-package package/"$(@)" build/packages/"$(@)"
 
-docker_images:
-	docker build --tag toltec:base . -f image/base/Dockerfile
-	docker build --tag toltec:rm . -f image/rm_toolchain/Dockerfile --build-arg "BASE=toltec:base"
-	docker build --tag toltec:build . -f image/build/Dockerfile
 
-.PHONY: docker package
+$(PUSH_PACKAGES): %:
+	ssh root@"${HOST}" mkdir -p .cache/opkg/
+	scp build/packages/"$(@:push-%=%)"/*.ipk root@"${HOST}":.cache/opkg
 
+.PHONY: help repo $(PACKAGES) $(PUSH_PACKAGES)

.gitignore

@@ -10,20 +10,19 @@ Unchanged images will not be rebuilt.
 
 Options:
 
+    -h              Show this help message.
     -p              Publish each built image to the GitHub Container Registry
                     (you need to be logged in to ghcr.io with the appropriate
-                    permissions for this to work).
+                    permissions for this to work)."
 
-    -h              Show this help message."
 
-publishflag=
 helpflag=
-imagesdir=
+publishflag=
 
-while getopts ph name; do
+while getopts hp name; do
     case $name in
-        p) publishflag=1 ;;
         h) helpflag=1 ;;
+        p) publishflag=1 ;;
         *) error "Invalid option. Use the -h flag for more information." ;;
     esac
 done
@@ -67,12 +66,14 @@ docker-build base
 popd
 
 for imagepath in "$imagesdir"/*; do
-    imagename="$(basename "$imagepath")"
-
-    if [[ $imagename != base ]]; then
-        pushd "$imagepath"
-        docker-build "$imagename"
-        popd
+    if [[ -d $imagepath ]]; then
+        imagename="$(basename "$imagepath")"
+
+        if [[ $imagename != base ]]; then
+            pushd "$imagepath"
+            docker-build "$imagename"
+            popd
+        fi
     fi
 done
 

Makefile

@@ -29,11 +29,11 @@ if [[ -n $helpflag ]]; then
 fi
 
 if [[ $# -eq 0 ]]; then
-    error "Missing RECIPEDIR and WORKDIR arguments. Use the --help flag for more information."
+    error "Missing RECIPEDIR and WORKDIR arguments. Use the -h flag for more information."
 fi
 
 if [[ $# -eq 1 ]]; then
-    error "Missing WORKDIR argument. Use the --help flag for more information."
+    error "Missing WORKDIR argument. Use the -h flag for more information."
 fi
 
 if [[ $# -gt 2 ]]; then
@@ -45,7 +45,7 @@ workdir="$2"
 
 # Create working directories
 [[ -d $workdir ]] && files="$(ls -qA -- "$workdir")" && [[ -n $files ]] \
-    && error "Working directory exists and is not empty"
+    && error "Working directory '$workdir' exists and is not empty"
 mkdir -p "$workdir"
 
 srcdir="$workdir"/src

scripts/build-images

@@ -1,55 +1,87 @@
 #!/usr/bin/env bash
 
+set -e
 source "${BASH_SOURCE%/*}"/lib
-usage="$0 PACKAGESDIR WORKDIR
 
-Build all packages described in PACKAGESDIR.
-The WORKDIR must be a non-existent directory in which the build is performed.
+usage="$0 [OPTION]... RECIPESDIR WORKDIR
 
-Arguments:
+Build new or updated packages defined in RECIPESDIR and create a package index.
+The WORKDIR must be a non-existent directory in which the builds are performed.
 
-    -h, --help      Show this help message."
+Options:
 
-if [[ $1 == "-h" ]] || [[ $1 == "--help" ]]; then
+    -h              Show this help message.
+    -f              Force building all the packages, even those for which
+                    an already built version exists on the remote."
+
+helpflag=
+forceflag=
+
+while getopts hf name; do
+    case $name in
+        h) helpflag=1 ;;
+        f) forceflag=1 ;;
+        *) error "Invalid option. Use the -h flag for more information." ;;
+    esac
+done
+
+shift $((OPTIND - 1))
+
+if [[ -n $helpflag ]]; then
     echo "$usage"
     exit
 fi
 
-if [[ $# -lt 2 ]]; then
-    error "Missing arguments. Use the --help flag for more information."
+if [[ $# -eq 0 ]]; then
+    error "Missing RECIPESDIR and WORKDIR arguments. Use the -h flag for more information."
 fi
 
-pkgsdir="$1"
-workdir="$2"
+if [[ $# -eq 1 ]]; then
+    error "Missing WORKDIR argument. Use the -h flag for more information."
+fi
+
+if [[ $# -gt 2 ]]; then
+    error "Extraneous arguments. Use the -h flag for more information."
+fi
 
-[[ -d $workdir ]] && files="$(ls -qA -- "$workdir")" && [[ -n $files ]] \
-    && error "Working directory exists and is not empty"
-mkdir -p "$workdir"
+recipesdir="$1"
+workdir="$2"
 
-pkgsworkdir="$workdir"/package
-mkdir "$pkgsworkdir"
+pkgsworkdir="$workdir"/packages
+mkdir -p "$pkgsworkdir"
 
 repodir="$workdir"/repo
-mkdir "$repodir"
+
+[[ -d $repodir ]] && files="$(ls -qA -- "$repodir")" && [[ -n $files ]] \
+    && error "Repository directory '$repodir' exists and is not empty"
+mkdir -p "$repodir"
+
+# URL to the remote repository
+branch="$(git branch --show-current)"
+remote="https://toltec.delab.re/$branch"
 
 # Define the SOURCE_DATE_EPOCH to the date of latest commit
 # See <https://reproducible-builds.org/docs/source-date-epoch/>
 SOURCE_DATE_EPOCH="$(git log -1 --pretty=%ct)"
 export SOURCE_DATE_EPOCH
 
-# Build each individual package
-for pkgpath in "$pkgsdir"/*
-do
-    pkgname="$(basename "$pkgpath")"
-    pkgworkdir="$pkgsworkdir"/"$pkgname"
-    status "Building $pkgname in $pkgworkdir"
+# Build each package or get it from the remote server
+for recipedir in "$recipesdir"/*; do
+    load-recipe "$recipedir"
+    arname="$(archive-name "$recipedir")"
 
-    "${BASH_SOURCE%/*}"/build-package "$pkgpath" "$pkgworkdir"
-    mv "$pkgworkdir"/*.ipk "$repodir"
+    if [[ -n $forceflag ]] || ! rcurl "$remote"/"$arname" -o "$repodir"/"$arname"; then
+        pkgworkdir="$pkgsworkdir"/"$pkgname"
+        status "Building $pkgname in $pkgworkdir"
+        "${BASH_SOURCE%/*}"/build-package "$recipedir" "$pkgworkdir"
+        mv "$pkgworkdir"/*.ipk "$repodir"
+    else
+        status "Reusing already built $pkgname from remote"
+    fi
 done
 
 # Build packages index
-opkg-make-index --checksum sha256 -p "$repodir"/Packages "$repodir"
+"${BASH_SOURCE%/*}"/opkg/opkg-make-index --checksum sha256 -p "$repodir"/Packages "$repodir"
 
 # Set atime and mtime to fixed timestamp for all files
 find "$repodir" -exec touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" {} +

scripts/build-package

@@ -29,6 +29,11 @@ rtar() {
         "$@"
 }
 
+# Curl command with flags suitable for scripting
+rcurl() {
+    curl --tlsv1.2 --proto '=https' "$@" --fail --silent
+}
+
 # Load information from a package recipe
 #
 # Arguments: