Skip to content

Commit 6a98753

Browse files
lschuermannlschuermann
committed
lib/fetch-{newlib,libc++}: implement locking to avoid parallel unzips racing each other
1 parent 227dcbc commit 6a98753

File tree

2 files changed

+96
-46
lines changed

2 files changed

+96
-46
lines changed

‎lib/fetch-libc++.sh

Lines changed: 48 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -32,27 +32,52 @@ fi
3232

3333
FOUND=0
3434

35-
# Try from each mirror until we successfully download a .zip file.
36-
for MIRROR in ${MIRRORS[@]}; do
37-
URL=$MIRROR/$ZIP_FILE
38-
echo "Fetching libc++ from ${MIRROR}..."
39-
echo " Fetching ${URL}..."
40-
# Note: There must be two space characters for `shasum` (sha256sum doesn't care)
41-
wget -O $ZIP_FILE "$URL" && (echo "$GCC_SHA $ZIP_FILE" | $CHECK_SHA_CMD)
42-
if [ $? -ne 0 ]; then
43-
echo " WARNING: Fetching libc++ from mirror $MIRROR failed!" >&2
44-
else
45-
FOUND=1
46-
break
47-
fi
48-
done
35+
# We must ensure that multiple parallel fetch and unzip operations
36+
# don't trample over each other, which we do by obtaining a write-lock
37+
# on the ZIP file that's being downloaded / extracted.
38+
#
39+
# This will also truncate any already downloaded ZIP file, which is
40+
# fine because we'll overwrite it anyways.
41+
NONBLOCK_LOCK_ACQ_FAIL=0
42+
while true; do
43+
: >> ${ZIP_FILE}
44+
{
45+
if [ $NONBLOCK_LOCK_ACQ_FAIL -eq 0 ]; then
46+
flock -n $fd || NONBLOCK_LOCK_ACQ_FAIL=1
47+
if [ $NONBLOCK_LOCK_ACQ_FAIL -ne 0 ]; then
48+
# Try again, blocking this time:
49+
echo "Could not acquire non-blocking lock on ${ZIP_FILE}, waiting for lock to be released..." >&2
50+
continue
51+
fi
52+
else
53+
flock $fd
54+
fi
55+
echo "Acquired lock on file ${ZIP_FILE}" >&2
4956

50-
if [[ $FOUND -ne 0 ]]; then
51-
echo "Unpacking $ZIP_FILE..."
52-
# -n: never overwrite existing files, -q: quiet mode
53-
unzip -n -q $ZIP_FILE
54-
echo "Done upacking $ZIP_FILE..."
55-
else
56-
echo "ERROR: Unable to find tock-libc++"
57-
exit -1
58-
fi
57+
# Try from each mirror until we successfully download a .zip file.
58+
for MIRROR in ${MIRRORS[@]}; do
59+
URL=$MIRROR/$ZIP_FILE
60+
echo "Fetching libc++ from ${MIRROR}..."
61+
echo " Fetching ${URL}..."
62+
# Note: There must be two space characters for `shasum` (sha256sum doesn't care)
63+
wget -O $ZIP_FILE "$URL" && (echo "$GCC_SHA $ZIP_FILE" | $CHECK_SHA_CMD)
64+
if [ $? -ne 0 ]; then
65+
echo " WARNING: Fetching libc++ from mirror $MIRROR failed!" >&2
66+
else
67+
FOUND=1
68+
break
69+
fi
70+
done
71+
72+
if [[ $FOUND -ne 0 ]]; then
73+
echo "Unpacking $ZIP_FILE..."
74+
# -n: don't overwrite existing files, -q: quiet mode
75+
unzip -n -q $ZIP_FILE
76+
echo "Done upacking $ZIP_FILE..."
77+
exit 0
78+
else
79+
echo "ERROR: Unable to find tock-libc++"
80+
exit -1
81+
fi
82+
} {fd}<${ZIP_FILE}
83+
done

‎lib/fetch-newlib.sh

Lines changed: 48 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -30,27 +30,52 @@ fi
3030

3131
FOUND=0
3232

33-
# Try from each mirror until we successfully download a .zip file.
34-
for MIRROR in ${MIRRORS[@]}; do
35-
URL=$MIRROR/$ZIP_FILE
36-
echo "Fetching newlib from ${MIRROR}..."
37-
echo " Fetching ${URL}..."
38-
# Note: There must be two space characters for `shasum` (sha256sum doesn't care)
39-
wget -O $ZIP_FILE "$URL" && (echo "$NEWLIB_SHA $ZIP_FILE" | $CHECK_SHA_CMD)
40-
if [ $? -ne 0 ]; then
41-
echo " WARNING: Fetching newlib from mirror $MIRROR failed!" >&2
42-
else
43-
FOUND=1
44-
break
45-
fi
46-
done
33+
# We must ensure that multiple parallel fetch and unzip operations
34+
# don't trample over each other, which we do by obtaining a write-lock
35+
# on the ZIP file that's being downloaded / extracted.
36+
#
37+
# This will also truncate any already downloaded ZIP file, which is
38+
# fine because we'll overwrite it anyways.
39+
NONBLOCK_LOCK_ACQ_FAIL=0
40+
while true; do
41+
: >> ${ZIP_FILE}
42+
{
43+
if [ $NONBLOCK_LOCK_ACQ_FAIL -eq 0 ]; then
44+
flock -n $fd || NONBLOCK_LOCK_ACQ_FAIL=1
45+
if [ $NONBLOCK_LOCK_ACQ_FAIL -ne 0 ]; then
46+
# Try again, blocking this time:
47+
echo "Could not acquire non-blocking lock on ${ZIP_FILE}, waiting for lock to be released..." >&2
48+
continue
49+
fi
50+
else
51+
flock $fd
52+
fi
53+
echo "Acquired lock on file ${ZIP_FILE}" >&2
4754

48-
if [[ $FOUND -ne 0 ]]; then
49-
echo "Unpacking $ZIP_FILE..."
50-
# -n: never overwrite existing files, -q: quiet mode
51-
unzip -n -q $ZIP_FILE
52-
echo "Done upacking $ZIP_FILE..."
53-
else
54-
echo "ERROR: Unable to find tock-newlib"
55-
exit -1
56-
fi
55+
# Try from each mirror until we successfully download a .zip file.
56+
for MIRROR in ${MIRRORS[@]}; do
57+
URL=$MIRROR/$ZIP_FILE
58+
echo "Fetching newlib from ${MIRROR}..."
59+
echo " Fetching ${URL}..."
60+
# Note: There must be two space characters for `shasum` (sha256sum doesn't care)
61+
wget -O $ZIP_FILE "$URL" && (echo "$NEWLIB_SHA $ZIP_FILE" | $CHECK_SHA_CMD)
62+
if [ $? -ne 0 ]; then
63+
echo " WARNING: Fetching newlib from mirror $MIRROR failed!" >&2
64+
else
65+
FOUND=1
66+
break
67+
fi
68+
done
69+
70+
if [[ $FOUND -ne 0 ]]; then
71+
echo "Unpacking $ZIP_FILE..."
72+
# -n: don't overwrite existing files, -q: quiet mode
73+
unzip -n -q $ZIP_FILE
74+
echo "Done upacking $ZIP_FILE..."
75+
exit 0
76+
else
77+
echo "ERROR: Unable to find tock-newlib"
78+
exit -1
79+
fi
80+
} {fd}<${ZIP_FILE}
81+
done

0 commit comments

Comments
 (0)