Skip to content

Commit 673d251

Browse files
luqmanaluqmana
authored
ci_download_softnpu_machinery: don't skip just because file exists locally (#3403)
Ran into this fun issue again where `ci_download_softnpu_machinery` would skip updating the softnpu binaries if it found them in `out/softnpu` already. That's all well and good until the pinned commit actually gets updated and you've run `install_runner_prerequisites.sh` thinking that's enough. To anyone who's softnpu deploy doesn't seem to be working and `/softnpu.log` in the `softnpu` zone is spamming: ``` Dec 28 10:34:48.723 WARN got sidecar egress port of 0 Dec 28 10:34:48.723 WARN this is probably a p4 program bug Dec 28 10:34:48.723 WARN dropping packet ``` You're running an old copy of `softnpu` is why. Fix that by using the published sha256 hashes to verify any local copy that exists.
1 parent 13fe7fc commit 673d251

File tree

2 files changed

+168
-26
lines changed

2 files changed

+168
-26
lines changed

‎tools/ci_download_softnpu_machinery

Lines changed: 38 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -7,34 +7,46 @@
77
# - the sidecar-lite precompiled P4 program
88
#
99

10+
set -euo pipefail
1011

11-
# This is the softnpu ASIC emulator
12-
if [[ ! -f out/softnpu/softnpu ]]; then
13-
echo "fetching softnpu"
14-
# This comes from a separate repo from the below artifacts,
15-
SOFTNPU_COMMIT="88f5f1334364e5580fe778c44ac0746a35927351"
16-
COMMIT_URL="https://buildomat.eng.oxide.computer/public/file/oxidecomputer/softnpu/image/$SOFTNPU_COMMIT"
17-
curl -OL "$COMMIT_URL/softnpu"
18-
chmod +x softnpu
19-
mkdir -p out/softnpu
20-
mv softnpu out/softnpu/
21-
fi
22-
23-
# Commit and base URL that's pinned for softnpu tools
24-
SIDECAR_LITE_COMMIT="3fff53ae549ab1348b680845693e66b224bb5d2f"
25-
COMMIT_URL="https://buildomat.eng.oxide.computer/public/file/oxidecomputer/sidecar-lite/release/$SIDECAR_LITE_COMMIT"
12+
TOOLS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
13+
14+
OUT_DIR="out/softnpu"
15+
16+
# Pinned commit for softnpu ASIC simulator
17+
SOFTNPU_REPO="softnpu"
18+
SOFTNPU_COMMIT="88f5f1334364e5580fe778c44ac0746a35927351"
19+
20+
# Pinned commit for softnpu tools
21+
SIDECAR_LITE_REPO="sidecar-lite"
22+
SIDECAR_LITE_SERIES="release"
23+
SIDECAR_LITE_COMMIT="56abef043a9e2ba0363fea82528d7a9e86d3c0b0"
24+
25+
# This is the softnpu ASIC simulator
26+
echo "fetching softnpu"
27+
mkdir -p out/softnpu
28+
$TOOLS_DIR/ensure_buildomat_artifact.sh \
29+
-O $OUT_DIR \
30+
"softnpu" \
31+
"$SOFTNPU_REPO" \
32+
"$SOFTNPU_COMMIT"
33+
chmod +x $OUT_DIR/softnpu
2634

2735
# This is an ASIC administration program.
28-
if [[ ! -f out/softnpu/scadm ]]; then
29-
echo "fetching scadm"
30-
curl -OL "$COMMIT_URL/scadm"
31-
chmod +x scadm
32-
mv scadm out/softnpu/
33-
fi
36+
echo "fetching scadm"
37+
$TOOLS_DIR/ensure_buildomat_artifact.sh \
38+
-O $OUT_DIR \
39+
-s "$SIDECAR_LITE_SERIES" \
40+
"scadm" \
41+
"$SIDECAR_LITE_REPO" \
42+
"$SIDECAR_LITE_COMMIT"
43+
chmod +x $OUT_DIR/scadm
3444

3545
# Fetch the pre-compiled sidecar_lite p4 program
36-
if [[ ! -f out/softnpu/libsidecar_lite.so ]]; then
37-
echo "fetching libsidecar_lite.so"
38-
curl -OL "$COMMIT_URL/libsidecar_lite.so"
39-
mv libsidecar_lite.so out/softnpu/
40-
fi
46+
echo "fetching libsidecar_lite.so"
47+
$TOOLS_DIR/ensure_buildomat_artifact.sh \
48+
-O $OUT_DIR \
49+
-s "$SIDECAR_LITE_SERIES" \
50+
"libsidecar_lite.so" \
51+
"$SIDECAR_LITE_REPO" \
52+
"$SIDECAR_LITE_COMMIT"

‎tools/ensure_buildomat_artifact.sh

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
#!/bin/bash
2+
#
3+
# Ensure a buildomat artifact is downloaded and available locally.
4+
5+
6+
set -o errexit
7+
set -o nounset
8+
set -o pipefail
9+
10+
# Published buildomat artifacts are available at a predictable URL:
11+
# https://buildomat.eng.oxide.computer/public/file/ORG/REPO/SERIES/HASH/ARTIFACT
12+
BUILDOMAT_BASE_URL="https://buildomat.eng.oxide.computer/public/file"
13+
14+
# Default value for optional flags
15+
ORG="oxidecomputer"
16+
SERIES="image"
17+
CHECK_HASH=true
18+
OUTDIR="$PWD"
19+
20+
function usage() {
21+
cat <<EOF
22+
Usage: $0 [OPTIONS] <ARTIFACT> <REPO> <COMMIT>
23+
24+
REPO: The repository that published the artifact.
25+
ARTIFACT: The name of the artifact.
26+
COMMIT: The commit the artifact was published from.
27+
28+
Options:
29+
-o <ORG> Org containing the repository. [default: $ORG]
30+
-s <SERIES> The series artifact was published to. [default: $SERIES]
31+
-f Disable artifact validation.
32+
-O <OUTDIR> Directory to output artifact to. [default: $OUTDIR]
33+
-h Print help and exit
34+
35+
By default, this script expects a SHA256 hash to be published alongside
36+
the artifact with the same name but with a '.sha256.txt' suffix. This hash
37+
is used to validate the downloaded artifact as well as to check if we can
38+
skip downloading the artifact if it already exists locally. To disable this
39+
behavior, pass the -f flag.
40+
41+
Note that if hash validation is disabled, we'll always download the artifact
42+
even if it already exists locally.
43+
EOF
44+
}
45+
46+
function validate_file_hash() {
47+
local file="$1"
48+
local hash="$2"
49+
echo "$hash $file" | shasum -a 256 --check --status
50+
}
51+
52+
function main() {
53+
# Parse flags
54+
local opt
55+
while getopts "o:s:fO:h" opt; do
56+
case $opt in
57+
o) ORG="$OPTARG" ;;
58+
s) SERIES="$OPTARG" ;;
59+
f) CHECK_HASH=false ;;
60+
O) OUTDIR="$OPTARG" ;;
61+
62+
h)
63+
usage
64+
exit 0
65+
;;
66+
*)
67+
usage
68+
exit 1
69+
;;
70+
esac
71+
done
72+
shift $((OPTIND-1))
73+
74+
# Grab required arguments
75+
if [[ $# -ne 3 ]]; then
76+
usage
77+
exit 1
78+
fi
79+
80+
ARTIFACT="$1"
81+
REPO="$2"
82+
COMMIT="$3"
83+
84+
ARTIFACT_URL="$BUILDOMAT_BASE_URL/$ORG/$REPO/$SERIES/$COMMIT/$ARTIFACT"
85+
ARTIFACT_OUT="$OUTDIR/$ARTIFACT"
86+
87+
echo "Ensuring $ORG/$REPO/$SERIES/$ARTIFACT in $OUTDIR"
88+
echo " (commit: $COMMIT)"
89+
90+
local hash=""
91+
# If hash checking is enabled, grab the expected hash
92+
if $CHECK_HASH; then
93+
local hash_url="$ARTIFACT_URL.sha256.txt"
94+
echo "Getting hash for $ARTIFACT"
95+
hash="$(curl --silent --show-error --fail --location "$hash_url")"
96+
echo " (hash: $hash)"
97+
fi
98+
99+
# Check if the artifact already exists
100+
if [[ -f "$ARTIFACT_OUT" ]]; then
101+
if $CHECK_HASH; then
102+
# If the artifact exists and has the correct hash, we're done.
103+
if validate_file_hash "$ARTIFACT_OUT" "$hash"; then
104+
echo "$ARTIFACT already exists with correct hash"
105+
exit 0
106+
else
107+
echo "$ARTIFACT already exists but has incorrect hash, re-downloading"
108+
fi
109+
else
110+
echo "$ARTIFACT already exists but hash validation disabled, re-downloading"
111+
fi
112+
fi
113+
114+
# Either the artifact doesn't exist or it has the wrong hash. Download it.
115+
mkdir -p "$OUTDIR"
116+
curl --silent --show-error --fail --location --output "$ARTIFACT_OUT" "$ARTIFACT_URL"
117+
118+
# If hash checking is enabled, validate the downloaded artifact.
119+
if $CHECK_HASH; then
120+
if ! validate_file_hash "$ARTIFACT_OUT" "$hash"; then
121+
echo "Downloaded artifact failed verification"
122+
exit 1
123+
fi
124+
fi
125+
126+
echo "$ARTIFACT downloaded successfully"
127+
128+
}
129+
130+
main "$@"

0 commit comments

Comments
 (0)