Skip to content

Commit 3946cbd

Browse files
koalamankoalaman
committed
Upgrade docker build images
1 parent c4b7b79 commit 3946cbd

File tree

12 files changed

+195
-78
lines changed

12 files changed

+195
-78
lines changed

‎build/README.md

+4
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,7 @@ This makes it simple to build any release without exotic hardware or software.
1111

1212
An image can be built and tagged using `build_builder`,
1313
and run on a source tarball using `run_builder`.
14+
15+
Tip: Are you developing an image that relies on QEmu usermode emulation?
16+
It's easy to accidentally depend on binfmt\_misc on the host OS.
17+
Do a `echo 0 | sudo tee /proc/sys/fs/binfmt_misc/status` before testing.

‎build/darwin.aarch64/build

-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ set -xe
44
tar xzv --strip-components=1
55
chmod +x striptests && ./striptests
66
mkdir "$TARGETNAME"
7-
cabal update
87
( IFS=';'; cabal build $CABALOPTS )
98
find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \;
109
ls -l "$TARGETNAME"

‎build/darwin.x86_64/build

-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ set -xe
44
tar xzv --strip-components=1
55
chmod +x striptests && ./striptests
66
mkdir "$TARGETNAME"
7-
cabal update
87
( IFS=';'; cabal build $CABALOPTS )
98
find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \;
109
ls -l "$TARGETNAME"

‎build/linux.aarch64/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.9.0.0/cabal-in
2828

2929
# Due to an apparent cabal bug, we specify our options directly to cabal
3030
# It won't reuse caches if ghc-options are specified in ~/.cabal/config
31-
ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections -optc-fPIC;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg"
31+
ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections -optc-fPIC;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg;-c;hashable -arch-native"
3232

3333
# Prebuild the dependencies
3434
RUN cabal update && IFS=';' && cabal install --dependencies-only $CABALOPTS ShellCheck

‎build/linux.aarch64/build

-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ set -xe
44
tar xzv --strip-components=1
55
chmod +x striptests && ./striptests
66
mkdir "$TARGETNAME"
7-
cabal update
87
( IFS=';'; cabal build $CABALOPTS --enable-executable-static )
98
find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \;
109
ls -l "$TARGETNAME"

‎build/linux.armv6hf/Dockerfile

+24-42
Original file line numberDiff line numberDiff line change
@@ -1,60 +1,42 @@
1-
# I've again spent days trying to get a working armv6hf compiler going.
2-
# God only knows how many recompilations of GCC, GHC, libraries, and
3-
# ShellCheck itself, has gone into it.
4-
#
5-
# I tried Debian's toolchain. I tried my custom one built according to
6-
# RPi `gcc -v`. I tried GHC9, glibc, musl, registerised vs not, but
7-
# nothing has yielded an armv6hf binary that does not immediately
8-
# segfault on qemu-arm-static or the RPi itself.
9-
#
10-
# I then tried the same but with armv7hf. Same story.
11-
#
12-
# Emulating the entire userspace with balenalib again? Very strange build
13-
# failures where programs would fail to execute with > ~100 arguments.
14-
#
15-
# Finally, creating our own appears to work when using a custom QEmu
16-
# patched to follow execve calls.
17-
#
18-
# PS: $100 bounty for getting a RPi1 compatible static build going
19-
# with cross-compilation, similar to what the aarch64 build does.
20-
#
21-
22-
FROM ubuntu:20.04
1+
# This Docker file uses a custom QEmu fork with patches to follow execve
2+
# to build all of ShellCheck emulated.
3+
4+
FROM ubuntu:24.04
235

246
ENV TARGETNAME linux.armv6hf
257

268
# Build QEmu with execve follow support
279
USER root
2810
ENV DEBIAN_FRONTEND noninteractive
2911
RUN apt-get update
30-
RUN apt-get install -y build-essential git ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev
31-
WORKDIR /build
32-
RUN git clone --depth 1 https://github.com/koalaman/qemu
33-
RUN cd qemu && ./configure --static && cd build && ninja qemu-arm
34-
RUN cp qemu/build/qemu-arm /build/qemu-arm-static
12+
RUN apt-get install -y --no-install-recommends build-essential git ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev python3-setuptools ca-certificates debootstrap
13+
WORKDIR /qemu
14+
RUN git clone --depth 1 https://github.com/koalaman/qemu .
15+
RUN ./configure --static --disable-werror && cd build && ninja qemu-arm
3516
ENV QEMU_EXECVE 1
3617

18+
# Convenience utility
19+
COPY scutil /bin/scutil
20+
COPY scutil /chroot/bin/scutil
21+
RUN chmod +x /bin/scutil /chroot/bin/scutil
22+
3723
# Set up an armv6 userspace
3824
WORKDIR /
39-
RUN apt-get install -y debootstrap qemu-user-static
40-
# We expect this to fail if the host doesn't have binfmt qemu support
41-
RUN qemu-debootstrap --arch armhf bullseye pi http://mirrordirector.raspbian.org/raspbian || [ -e /pi/etc/issue ]
42-
RUN cp /build/qemu-arm-static /pi/usr/bin/qemu-arm-static
43-
RUN printf > /bin/pirun '%s\n' '#!/bin/sh' 'chroot /pi /usr/bin/qemu-arm-static /usr/bin/env "$@"' && chmod +x /bin/pirun
44-
# If the debootstrap process didn't finish, continue it
45-
RUN [ ! -e /pi/debootstrap ] || pirun '/debootstrap/debootstrap' --second-stage
25+
RUN debootstrap --arch armhf --variant=minbase --foreign bookworm /chroot http://mirrordirector.raspbian.org/raspbian
26+
RUN cp /qemu/build/qemu-arm /chroot/bin/qemu
27+
RUN scutil emu /debootstrap/debootstrap --second-stage
4628

4729
# Install deps in the chroot
48-
RUN pirun apt-get update
49-
RUN pirun apt-get install -y ghc cabal-install
30+
RUN scutil emu apt-get update
31+
RUN scutil emu apt-get install -y --no-install-recommends ghc cabal-install
32+
RUN scutil emu cabal update
5033

5134
# Finally we can build the current dependencies. This takes hours.
5235
ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--gcc-options;-Os -Wl,--gc-sections -ffunction-sections -fdata-sections"
53-
RUN pirun cabal update
54-
RUN IFS=";" && pirun cabal install --dependencies-only $CABALOPTS ShellCheck
55-
RUN IFS=';' && pirun cabal install $CABALOPTS --lib fgl
36+
# Generated with `cabal freeze --constraint 'hashable -arch-native'`
37+
COPY cabal.project.freeze /chroot/etc
38+
RUN IFS=";" && scutil install_from_freeze /chroot/etc/cabal.project.freeze emu cabal install $CABALOPTS
5639

5740
# Copy the build script
58-
WORKDIR /pi/scratch
59-
COPY build /pi/usr/bin
60-
ENTRYPOINT ["/bin/pirun", "/usr/bin/build"]
41+
COPY build /chroot/bin
42+
ENTRYPOINT ["/bin/scutil", "emu", "/bin/build"]

‎build/linux.armv6hf/build

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
#!/bin/sh
22
set -xe
3-
cd /scratch
3+
mkdir /scratch && cd /scratch
44
{
55
tar xzv --strip-components=1
6+
cp /etc/cabal.project.freeze .
67
chmod +x striptests && ./striptests
78
mkdir "$TARGETNAME"
89
# This script does not cabal update because compiling anything new is slow

‎build/linux.armv6hf/cabal.project.freeze

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
active-repositories: hackage.haskell.org:merge
2+
constraints: any.Diff ==0.5,
3+
any.OneTuple ==0.4.2,
4+
any.QuickCheck ==2.14.3,
5+
QuickCheck -old-random +templatehaskell,
6+
any.StateVar ==1.2.2,
7+
any.aeson ==2.2.3.0,
8+
aeson +ordered-keymap,
9+
any.array ==0.5.4.0,
10+
any.assoc ==1.1.1,
11+
assoc -tagged,
12+
any.base ==4.15.1.0,
13+
any.base-orphans ==0.9.2,
14+
any.bifunctors ==5.6.2,
15+
bifunctors +tagged,
16+
any.binary ==0.8.8.0,
17+
any.bytestring ==0.10.12.1,
18+
any.character-ps ==0.1,
19+
any.comonad ==5.0.8,
20+
comonad +containers +distributive +indexed-traversable,
21+
any.containers ==0.6.4.1,
22+
any.contravariant ==1.5.5,
23+
contravariant +semigroups +statevar +tagged,
24+
any.data-array-byte ==0.1.0.1,
25+
any.data-fix ==0.3.3,
26+
any.deepseq ==1.4.5.0,
27+
any.directory ==1.3.6.2,
28+
any.distributive ==0.6.2.1,
29+
distributive +semigroups +tagged,
30+
any.dlist ==1.0,
31+
dlist -werror,
32+
any.exceptions ==0.10.4,
33+
any.fgl ==5.8.2.0,
34+
fgl +containers042,
35+
any.filepath ==1.4.2.1,
36+
any.foldable1-classes-compat ==0.1,
37+
foldable1-classes-compat +tagged,
38+
any.generically ==0.1.1,
39+
any.ghc-bignum ==1.1,
40+
any.ghc-boot-th ==9.0.2,
41+
any.ghc-prim ==0.7.0,
42+
any.hashable ==1.4.6.0,
43+
hashable -arch-native +integer-gmp -random-initial-seed,
44+
any.indexed-traversable ==0.1.4,
45+
any.indexed-traversable-instances ==0.1.2,
46+
any.integer-conversion ==0.1.1,
47+
any.integer-logarithms ==1.0.3.1,
48+
integer-logarithms -check-bounds +integer-gmp,
49+
any.mtl ==2.2.2,
50+
any.network-uri ==2.6.4.2,
51+
any.parsec ==3.1.14.0,
52+
any.pretty ==1.1.3.6,
53+
any.primitive ==0.9.0.0,
54+
any.process ==1.6.13.2,
55+
any.random ==1.2.1.2,
56+
any.regex-base ==0.94.0.2,
57+
any.regex-tdfa ==1.3.2.2,
58+
regex-tdfa +doctest -force-o2,
59+
any.rts ==1.0.2,
60+
any.scientific ==0.3.8.0,
61+
scientific -integer-simple,
62+
any.semialign ==1.3.1,
63+
semialign +semigroupoids,
64+
any.semigroupoids ==6.0.1,
65+
semigroupoids +comonad +containers +contravariant +distributive +tagged +unordered-containers,
66+
any.splitmix ==0.1.0.5,
67+
splitmix -optimised-mixer,
68+
any.stm ==2.5.0.0,
69+
any.strict ==0.5,
70+
any.tagged ==0.8.8,
71+
tagged +deepseq +transformers,
72+
any.template-haskell ==2.17.0.0,
73+
any.text ==1.2.5.0,
74+
any.text-iso8601 ==0.1.1,
75+
any.text-short ==0.1.6,
76+
text-short -asserts,
77+
any.th-abstraction ==0.7.0.0,
78+
any.th-compat ==0.1.5,
79+
any.these ==1.2.1,
80+
any.time ==1.9.3,
81+
any.time-compat ==1.9.7,
82+
any.transformers ==0.5.6.2,
83+
any.transformers-compat ==0.7.2,
84+
transformers-compat -five +five-three -four +generic-deriving +mtl -three -two,
85+
any.unix ==2.7.2.2,
86+
any.unordered-containers ==0.2.20,
87+
unordered-containers -debug,
88+
any.uuid-types ==1.0.6,
89+
any.vector ==0.13.1.0,
90+
vector +boundschecks -internalchecks -unsafechecks -wall,
91+
any.vector-stream ==0.1.0.1,
92+
any.witherable ==0.5
93+
index-state: hackage.haskell.org 2024-06-18T02:21:19Z

‎build/linux.armv6hf/scutil

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#!/bin/dash
2+
# Various ShellCheck build utility functions
3+
4+
# Generally set a ulimit to avoid QEmu using too much memory
5+
ulimit -v "$((10*1024*1024))"
6+
# If we happen to invoke or run under QEmu, make sure to follow execve.
7+
# This requires a patched QEmu.
8+
export QEMU_EXECVE=1
9+
10+
# Retry a command until it succeeds
11+
# Usage: scutil retry 3 mycmd
12+
retry() {
13+
n="$1"
14+
ret=1
15+
shift
16+
while [ "$n" -gt 0 ]
17+
do
18+
"$@"
19+
ret=$?
20+
[ "$ret" = 0 ] && break
21+
n=$((n-1))
22+
done
23+
return "$ret"
24+
}
25+
26+
# Install all dependencies from a freeze file
27+
# Usage: scutil install_from_freeze /path/cabal.project.freeze cabal install
28+
install_from_freeze() {
29+
linefeed=$(printf '\nx')
30+
linefeed=${linefeed%x}
31+
flags=$(
32+
sed 's/constraints:/&\n /' "$1" |
33+
grep -vw -e rts -e base |
34+
sed -n -e 's/^ *\([^,]*\).*/\1/p' |
35+
sed -e 's/any\.\([^ ]*\) ==\(.*\)/\1-\2/; te; s/.*/--constraint\n&/; :e')
36+
shift
37+
# shellcheck disable=SC2086
38+
( IFS=$linefeed; set -x; "$@" $flags )
39+
}
40+
41+
# Run a command under emulation.
42+
# This assumes the correct emulator is named 'qemu' and the chroot is /chroot
43+
# Usage: scutil emu echo "Hello World"
44+
emu() {
45+
chroot /chroot /bin/qemu /usr/bin/env "$@"
46+
}
47+
48+
"$@"

‎build/linux.riscv64/Dockerfile

+20-29
Original file line numberDiff line numberDiff line change
@@ -10,46 +10,37 @@ ENV DEBIAN_FRONTEND noninteractive
1010
RUN apt-get update -y
1111

1212
# Install qemu
13-
RUN apt-get install -y --no-install-recommends build-essential ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev curl ca-certificates python3-virtualenv git python3-setuptools
13+
RUN apt-get install -y --no-install-recommends build-essential ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev curl ca-certificates python3-virtualenv git python3-setuptools debootstrap
1414
WORKDIR /qemu
1515
RUN git clone --depth 1 https://github.com/koalaman/qemu .
16-
#RUN git clone https://github.com/balena-io/qemu .
17-
# Release 7.0.0
18-
#RUN git checkout 639d1d8903f65d74eb04c49e0df7a4b2f014cd86
1916
RUN ./configure --target-list=riscv64-linux-user --static --disable-system --disable-pie --disable-werror
2017
RUN cd build && ninja qemu-riscv64
2118
ENV QEMU_EXECVE 1
2219

20+
# Convenience utility
21+
COPY scutil /bin/scutil
22+
# We have to copy to /usr/bin because debootstrap will try to symlink /bin and fail if it exists
23+
COPY scutil /chroot/usr/bin/scutil
24+
RUN chmod +x /bin/scutil /chroot/usr/bin/scutil
2325

2426
# Set up a riscv64 userspace
25-
RUN apt-get install -y --no-install-recommends debootstrap
26-
RUN debootstrap --arch=riscv64 --foreign noble /rvfs http://ports.ubuntu.com/ubuntu-ports
27-
RUN cp /qemu/build/qemu-riscv64 /rvfs/usr/bin/qemu-riscv64-static
28-
29-
# Command to run riscv binaries in the chroot. The Haskell runtime allocates 1TB
30-
# vspace up front and QEmu has a RAM cost per vspace, so use ulimit to allocate
31-
# less and reduce RAM usage.
32-
RUN printf > /bin/rv '%s\n' '#!/bin/sh' 'ulimit -v $((10*1024*1024)); chroot /rvfs /usr/bin/qemu-riscv64-static /usr/bin/env "$@"'
33-
RUN chmod +x /bin/rv
34-
RUN [ ! -e /rvfs/debootstrap ] || rv '/debootstrap/debootstrap' --second-stage
27+
WORKDIR /
28+
RUN debootstrap --arch=riscv64 --variant=minbase --components=main,universe --foreign noble /chroot http://ports.ubuntu.com/ubuntu-ports
29+
RUN cp /qemu/build/qemu-riscv64 /chroot/bin/qemu
30+
RUN scutil emu /debootstrap/debootstrap --second-stage
3531

3632
# Install deps in the chroot
37-
RUN printf > /rvfs/etc/apt/sources.list '%s\n' 'deb http://ports.ubuntu.com/ubuntu-ports noble main universe'
38-
RUN rv apt-get update -y
39-
RUN rv apt-get install -y --no-install-recommends ghc cabal-install
33+
RUN scutil emu apt-get update
34+
RUN scutil emu apt-get install -y --no-install-recommends ghc cabal-install
35+
RUN scutil emu cabal update
4036

41-
RUN rv cabal update
4237
# Generated with: cabal freeze -c 'hashable -arch-native'. We put it in /etc so cabal won't find it.
43-
COPY cabal.project.freeze /rvfs/etc
44-
# Awful hack to install everything from the freeze file
45-
# This basically turns 'any.tagged ==0.8.8' into tagged-0.8.8 to install by version,
46-
# and adds a -c before 'hashable -arch-native +integer-gmp' to make it a flag constraint.
47-
RUN < /rvfs/etc/cabal.project.freeze sed 's/constraints:/&\n /' | grep -vw rts | sed -n -e 's/^ *\([^,]*\).*/\1/p' | sed -e 's/any\.\([^ ]*\) ==\(.*\)/\1-\2/; te; s/.*/-c\n&/; :e' > /tmp/preinstall-flags
48-
# Finally we can build the current dependencies. This takes hours.
49-
# There's apparently a random segfault during assembly, so retry a few times.
50-
RUN set -x; IFS=${IFS# }; f() { rv cabal install --keep-going $(cat /tmp/preinstall-flags); }; for i in $(seq 5); do f; ret=$?; [ $ret = 0 ] && break; done; exit $ret
38+
COPY cabal.project.freeze /chroot/etc
39+
40+
# Build all dependencies from the freeze file. The emulator segfaults at random,
41+
# so retry a few times.
42+
RUN scutil install_from_freeze /chroot/etc/cabal.project.freeze retry 5 emu cabal install --keep-going
5143

5244
# Copy the build script
53-
WORKDIR /rvfs/scratch
54-
COPY build /rvfs/usr/bin/build
55-
ENTRYPOINT ["/bin/rv", "/usr/bin/build"]
45+
COPY build /chroot/bin/build
46+
ENTRYPOINT ["/bin/scutil", "emu", "/bin/build"]

‎build/linux.riscv64/build

+3-1
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,16 @@
22
set -xe
33
IFS=';'
44
{
5+
mkdir -p /tmp/scratch
6+
cd /tmp/scratch
57
tar xzv --strip-components=1
68
chmod +x striptests && ./striptests
79
# Use a freeze file to ensure we use the same dependencies we cached during
810
# the docker image build. We don't want to spend time compiling anything new.
911
cp /etc/cabal.project.freeze .
1012
mkdir "$TARGETNAME"
1113
# Retry in case of random segfault
12-
cabal build --enable-executable-static || cabal build --enable-executable-static
14+
scutil retry 3 cabal build --enable-executable-static
1315
find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \;
1416
ls -l "$TARGETNAME"
1517
"$TARGET-strip" -s "$TARGETNAME/shellcheck"

‎build/windows.x86_64/build

-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ set -xe
88
tar xzv --strip-components=1
99
chmod +x striptests && ./striptests
1010
mkdir "$TARGETNAME"
11-
cabal update
1211
( IFS=';'; cabal build $CABALOPTS )
1312
find dist*/ -name shellcheck.exe -type f -ls -exec mv {} "$TARGETNAME/" \;
1413
ls -l "$TARGETNAME"

0 commit comments

Comments
 (0)