Skip to content

Commit 3d006d6

Browse files
krnowakkrnowak
committed
build_toolchains: Break dep loop and handle more dependencies
Switching to a selinux profile caused more USE flags to be enabled (selinux, audit, caps), thus more dependencies to be pulled. More dependencies caused two things: - cyclic dependencies appeared - sys-apps/baselayout is being pulled in Cyclic dependencies need to be handled in a similar way it was done in build_packages, thus factor out the code doing it into a separate and reusable part. The dependency on baselayout needs to be handled by installing the package as a first thing in $ROOT, followed by a more careful way of copying things from $SYSROOT to $ROOT (due to split-usr differences), followed by installing the rest of the packages.
1 parent 99d44ca commit 3d006d6

File tree

4 files changed

+222
-90
lines changed

4 files changed

+222
-90
lines changed

‎build_library/break_dep_loop.sh

+137
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
# Goo to attempt to resolve dependency loops on individual packages.
2+
# If this becomes insufficient we will need to move to a full multi-stage
3+
# bootstrap process like we do with the SDK via catalyst.
4+
#
5+
# Called like:
6+
#
7+
# break_dep_loop [-v] [PKG_USE_PAIR]…
8+
#
9+
# Pass -v for verbose output.
10+
#
11+
# PKG_USE_PAIR consists of two arguments: a package name (for example:
12+
# sys-fs/lvm2), and a comma-separated list of USE flags to clear (for
13+
# example: udev,systemd).
14+
#
15+
# Env vars:
16+
#
17+
# BDL_ROOT, BDL_PORTAGEQ, BDL_EQUERY, BDL_EMERGE, BDL_INFO
18+
break_dep_loop() {
19+
local bdl_root=${BDL_ROOT:-/}
20+
local bdl_portageq=${BDL_PORTAGEQ:-portageq}
21+
local bdl_equery=${BDL_EQUERY:-equery}
22+
local bdl_emerge=${BDL_EMERGE:-emerge}
23+
local bdl_info=${BDL_INFO:-echo}
24+
local conf_dir="${bdl_root%/}/etc/portage"
25+
local flag_file="${conf_dir}/package.use/break_dep_loop"
26+
local force_flag_file="${conf_dir}/profile/package.use.force/break_dep_loop"
27+
28+
local verbose=
29+
if [[ ${1:-} = '-v' ]]; then
30+
verbose=x
31+
shift
32+
fi
33+
34+
# Be sure to clean up use flag hackery from previous failed runs
35+
sudo rm -f "${flag_file}" "${force_flag_file}"
36+
37+
if [[ ${#} -eq 0 ]]; then
38+
return 0
39+
fi
40+
41+
function bdl_call() {
42+
local output_var_name=${1}; shift
43+
if [[ ${output_var_name} = '-' ]]; then
44+
local throw_away
45+
output_var_name=throw_away
46+
fi
47+
local -n output_ref=${output_var_name}
48+
if [[ -n ${verbose} ]]; then
49+
"${bdl_info}" "${*@Q}"
50+
fi
51+
local -i rv=0
52+
output_ref=$("${@}") || rv=${?}
53+
if [[ -n ${verbose} ]]; then
54+
"${bdl_info}" "output: ${output_ref}"
55+
"${bdl_info}" "exit status: ${rv}"
56+
fi
57+
return ${rv}
58+
}
59+
60+
# Temporarily compile/install packages with flags disabled. If a binary
61+
# package is available use it regardless of its version or use flags.
62+
local pkg use_flags disabled_flags
63+
local -a flags
64+
local -a pkgs args flag_file_entries pkg_summaries
65+
local -A per_pkg_flags=()
66+
while [[ $# -gt 1 ]]; do
67+
pkg=${1}
68+
use_flags=${2}
69+
shift 2
70+
71+
mapfile -t flags <<<"${use_flags//,/$'\n'}"
72+
disabled_flags="${flags[*]/#/-}"
73+
74+
pkgs+=( "${pkg}" )
75+
per_pkg_flags["${pkg}"]=${use_flags}
76+
flag_file_entries+=( "${pkg} ${disabled_flags}" )
77+
args+=( "--buildpkg-exclude=${pkg}" )
78+
pkg_summaries+=( "${pkg}[${disabled_flags}]" )
79+
done
80+
unset pkg use_flags disabled_flags flags
81+
82+
# If packages are already installed we have nothing to do
83+
local pkg any_package_uninstalled=
84+
for pkg in "${pkgs[@]}"; do
85+
if ! bdl_call - "${bdl_portageq}" has_version "${bdl_root}" "${pkg}"; then
86+
any_package_uninstalled=x
87+
break
88+
fi
89+
done
90+
if [[ -z ${any_package_uninstalled} ]]; then
91+
if [[ -n ${verbose} ]]; then
92+
"${bdl_info}" "all packages (${pkgs[*]}) are installed already, skipping"
93+
fi
94+
return 0
95+
fi
96+
unset pkg any_package_uninstalled
97+
98+
# Likewise, nothing to do if the flags aren't actually enabled.
99+
local pkg any_flag_enabled= equery_output flag flags_str
100+
local -a flags grep_args
101+
for pkg in "${pkgs[@]}"; do
102+
bdl_call equery_output "${bdl_equery}" -q uses "${pkg}"
103+
flags_str=${per_pkg_flags["${pkg}"]}
104+
mapfile -t flags <<<"${flags_str//,/$'\n'}"
105+
for flag in "${flags[@]}"; do
106+
grep_args+=( -e "${flag/#/+}" )
107+
done
108+
if bdl_call - grep --quiet --line-regexp --fixed-strings "${grep_args[@]}" <<<"${equery_output}"; then
109+
any_flag_enabled=x
110+
break
111+
fi
112+
done
113+
if [[ -z ${any_flag_enabled} ]]; then
114+
if [[ -n ${verbose} ]]; then
115+
"${bdl_info}" "all packages (${pkgs[*]}) has all the desired USE flags already disabled, skipping"
116+
fi
117+
return 0
118+
fi
119+
unset pkg any_flag_enabled equery_output flag flags_str flags grep_args
120+
121+
"${bdl_info}" "Merging ${pkg_summaries[*]}"
122+
sudo mkdir -p "${flag_file%/*}" "${force_flag_file%/*}"
123+
printf '%s\n' "${flag_file_entries[@]}" | sudo tee "${flag_file}" >/dev/null
124+
cp -a "${flag_file}" "${force_flag_file}"
125+
if [[ -n ${verbose} ]]; then
126+
"${bdl_info}" "contents of ${flag_file@Q}:"
127+
"${bdl_info}" "$(<"${flag_file}")"
128+
"${bdl_info}" "${bdl_emerge}" --rebuild-if-unbuilt=n "${args[@]}" "${pkgs[@]}"
129+
fi
130+
# rebuild-if-unbuilt is disabled to prevent portage from needlessly
131+
# rebuilding zlib for some unknown reason, in turn triggering more rebuilds.
132+
"${bdl_emerge}" \
133+
--rebuild-if-unbuilt=n \
134+
"${args[@]}" "${pkgs[@]}"
135+
sudo rm -f "${flag_file}" "${force_flag_file}"
136+
unset bdl_call
137+
}

‎build_library/catalyst_toolchains.sh

+71-6
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
set -e
44
source /tmp/chroot-functions.sh
55
source /tmp/toolchain_util.sh
6+
source /tmp/break_dep_loop.sh
67

78
# A note on packages:
89
# The default PKGDIR is /usr/portage/packages
@@ -28,13 +29,77 @@ build_target_toolchain() {
2829
local ROOT="/build/${board}"
2930
local SYSROOT="/usr/$(get_board_chost "${board}")"
3031

31-
mkdir -p "${ROOT}/usr"
32-
cp -at "${ROOT}" "${SYSROOT}"/lib*
33-
cp -at "${ROOT}"/usr "${SYSROOT}"/usr/include "${SYSROOT}"/usr/lib*
32+
function btt_emerge() {
33+
# --root is required because run_merge overrides ROOT=
34+
PORTAGE_CONFIGROOT="$ROOT" run_merge --root="$ROOT" --sysroot="$ROOT" "${@}"
35+
}
3436

35-
# --root is required because run_merge overrides ROOT=
36-
PORTAGE_CONFIGROOT="$ROOT" \
37-
run_merge -u --root="$ROOT" --sysroot="$ROOT" "${TOOLCHAIN_PKGS[@]}"
37+
# install baselayout first - with the selinux profile, this is
38+
# pulled into the dependency chain
39+
btt_emerge --oneshot --nodeps sys-apps/baselayout
40+
41+
# copy libraries from sysroot to root - sysroot seems to be
42+
# split-usr, whereas root does not, so take this into account
43+
(
44+
shopt -s nullglob
45+
local d f
46+
local -a files
47+
for d in "${SYSROOT}"/lib* "${SYSROOT}"/usr/lib*; do
48+
if [[ ! -d ${d} ]]; then
49+
continue
50+
fi
51+
files=( "${d}"/* )
52+
if [[ ${#files[@]} -gt 0 ]]; then
53+
f=${d##*/}
54+
cp -at "${ROOT}/usr/${f}" "${files[@]}"
55+
fi
56+
done
57+
)
58+
cp -at "${ROOT}"/usr "${SYSROOT}"/usr/include
59+
60+
local -a args_for_bdl=()
61+
if [[ -n ${clst_VERBOSE} ]]; then
62+
args_for_bdl+=(-v)
63+
fi
64+
function btt_bdl_portageq() {
65+
ROOT=${ROOT} SYSROOT=${ROOT} PORTAGE_CONFIGROOT=${ROOT} portageq "${@}"
66+
}
67+
function btt_bdl_equery() {
68+
ROOT=${ROOT} SYSROOT=${ROOT} PORTAGE_CONFIGROOT=${ROOT} equery "${@}"
69+
}
70+
# Breaking the following loops here:
71+
#
72+
# glibc[nscd] -> libcap[pam] -> sys-libs/pam -> libcrypt -> libxcrypt[system] -> glibc
73+
# glibc[nscd] -> audit[python] -> python -> libcrypt -> libxcrypt[system] -> glibc
74+
# glibc[selinux] -> libselinux[python] -> python -> libcrypt -> libxcrypt[system] -> glibc
75+
# systemd[cryptsetup] -> cryptsetup[udev] -> libudev[systemd] -> systemd
76+
# systemd[cryptsetup] -> cryptsetup -> lvm2[udev] -> libudev[systemd] -> systemd
77+
# systemd[cryptsetup] -> cryptsetup -> lvm2[lvm,systemd] -> systemd
78+
# systemd[cryptsetup] -> cryptsetup -> tmpfiles[systemd] -> systemd
79+
# systemd[curl] -> curl -> nghttp2[systemd] -> systemd
80+
# importd requires curl, so needs to be disabled too
81+
# systemd[tpm] -> tpm2-tss -> tmpfiles[systemd] -> systemd
82+
# util-linux[audit] -> audit[python] -> python -> util-linux
83+
# util-linux[cryptsetup] -> cryptsetup -> util-linux
84+
# util-linux[pam] -> sys-libs/pam[audit] -> sys-process/audit[python] -> python -> util-linux
85+
# su requires pam, so needs to be disabled too
86+
# util-linux[selinux] -> libselinux[python] -> python -> util-linux
87+
# util-linux[systemd] -> systemd -> util-linux
88+
# util-linux[udev] -> libudev[systemd] -> systemd -> util-linux
89+
args_for_bdl+=(
90+
sys-apps/systemd cryptsetup,curl,importd,tpm
91+
sys-apps/util-linux audit,cryptsetup,pam,selinux,su,systemd,udev
92+
sys-libs/glibc nscd,selinux
93+
)
94+
BDL_ROOT=${ROOT} \
95+
BDL_PORTAGEQ=btt_bdl_portageq \
96+
BDL_EQUERY=btt_bdl_equery \
97+
BDL_EMERGE=btt_emerge \
98+
break_dep_loop "${args_for_bdl[@]}"
99+
unset btt_bdl_portageq btt_bdl_equery
100+
101+
btt_emerge --changed_use --update --deep "${TOOLCHAIN_PKGS[@]}"
102+
unset btt_emerge
38103
}
39104

40105
configure_crossdev_overlay / /usr/local/portage/crossdev

‎build_packages

+11-82
Original file line numberDiff line numberDiff line change
@@ -173,89 +173,12 @@ if [[ ${#WORKON_PKGS[@]} -gt 0 ]]; then
173173
)
174174
fi
175175

176-
# Goo to attempt to resolve dependency loops on individual packages.
177-
# If this becomes insufficient we will need to move to a full multi-stage
178-
# bootstrap process like we do with the SDK via catalyst.
179-
#
180-
# Called like:
181-
#
182-
# break_dep_loop [PKG_USE_PAIR]…
183-
#
184-
# PKG_USE_PAIR consists of two arguments: a package name (for example:
185-
# sys-fs/lvm2), and a comma-separated list of USE flags to clear (for
186-
# example: udev,systemd).
187-
break_dep_loop() {
188-
local -a pkgs
189-
local -a all_flags
190-
local -a args
191-
local flag_file="${BOARD_ROOT}/etc/portage/package.use/break_dep_loop"
192-
local -a flag_file_entries
193-
local -a pkg_summaries
194-
195-
# Be sure to clean up use flag hackery from previous failed runs
196-
sudo rm -f "${flag_file}"
197-
198-
if [[ $# -eq 0 ]]; then
199-
return 0
200-
fi
201-
202-
# Temporarily compile/install packages with flags disabled. If a binary
203-
# package is available use it regardless of its version or use flags.
204-
local pkg
205-
local -a flags
206-
local disabled_flags
207-
while [[ $# -gt 0 ]]; do
208-
pkg="${1}"
209-
pkgs+=("${pkg}")
210-
flags=( ${2//,/ } )
211-
all_flags+=("${flags[@]}")
212-
disabled_flags="${flags[@]/#/-}"
213-
flag_file_entries+=("${pkg} ${disabled_flags}")
214-
args+=("--buildpkg-exclude=${pkg}")
215-
pkg_summaries+=("${pkg}[${disabled_flags}]")
216-
shift 2
217-
done
218-
219-
# If packages are already installed we have nothing to do
220-
local any_package_uninstalled=0
221-
for pkg in "${pkgs[@]}"; do
222-
if ! portageq-"${BOARD}" has_version "${BOARD_ROOT}" "${pkgs[@]}"; then
223-
any_package_uninstalled=1
224-
break
225-
fi
226-
done
227-
if [[ ${any_package_uninstalled} -eq 0 ]]; then
228-
return 0
229-
fi
230-
231-
# Likewise, nothing to do if the flags aren't actually enabled.
232-
local any_flag_enabled=0
233-
for pkg in "${pkgs[@]}"; do
234-
if pkg_use_enabled "${pkg}" "${all_flags[@]}"; then
235-
any_flag_enabled=1
236-
break
237-
fi
238-
done
239-
if [[ ${any_flag_enabled} -eq 0 ]]; then
240-
return 0
241-
fi
242-
243-
info "Merging ${pkg_summaries[@]}"
244-
sudo mkdir -p "${flag_file%/*}"
245-
sudo_clobber "${flag_file}" <<<"${flag_file_entries[0]}"
246-
local entry
247-
for entry in "${flag_file_entries[@]:1}"; do
248-
sudo_append "${flag_file}" <<<"${entry}"
249-
done
250-
# rebuild-if-unbuilt is disabled to prevent portage from needlessly
251-
# rebuilding zlib for some unknown reason, in turn triggering more rebuilds.
252-
sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" \
253-
--rebuild-if-unbuilt=n \
254-
"${args[@]}" "${pkgs[@]}"
255-
sudo rm -f "${flag_file}"
256-
}
257-
258176
if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_FALSE}" ]]; then
177+
. "${BUILD_LIBRARY_DIR}/break_dep_loop.sh" || exit 1
178+
179+
function bp_bdl_emerge() {
180+
sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "${@}"
181+
}
259182
# Breaking the following loops here:
260183
#
261184
# util-linux[udev] -> virtual/udev -> systemd -> util-linux
@@ -266,12 +189,18 @@ if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_FALSE}" ]]; then
266189
# lvm2[systemd] -> systemd[cryptsetup] -> cryptsetup -> lvm2
267190
# systemd[cryptsetup] -> cryptsetup[udev] -> virtual/udev -> systemd
268191
# curl[http2] -> nghttp2[systemd] -> systemd[curl] -> curl
192+
BDL_ROOT=${BOARD_ROOT} \
193+
BDL_PORTAGEQ=portageq-"${BOARD}" \
194+
BDL_EQUERY=equery-"${BOARD}" \
195+
BDL_EMERGE=bp_bdl_emerge \
196+
BDL_INFO=info \
269197
break_dep_loop sys-apps/util-linux udev,systemd,cryptsetup \
270198
sys-fs/cryptsetup udev \
271199
sys-fs/lvm2 udev,systemd \
272200
sys-apps/systemd cryptsetup,tpm \
273201
net-misc/curl http2 \
274202
net-libs/nghttp2 systemd
203+
unset bp_bdl_emerge
275204
fi
276205

277206
if [[ "${FLAGS_only_resolve_circular_deps}" -eq "${FLAGS_TRUE}" ]]; then

‎build_toolchains

+3-2
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,10 @@ catalyst_init "$@"
4949

5050
ROOT_OVERLAY="${TEMPDIR}/stage4-${ARCH}-$FLAGS_version-overlay"
5151

52-
# toolchain_util.sh is required by catalyst_toolchains.sh
52+
# toolchain_util.sh and break_dep_loop.sh are required by
53+
# catalyst_toolchains.sh
5354
mkdir -p "${ROOT_OVERLAY}/tmp"
54-
cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
55+
cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${BUILD_LIBRARY_DIR}/break_dep_loop.sh" "${ROOT_OVERLAY}/tmp"
5556
create_provenance_overlay "${ROOT_OVERLAY}"
5657

5758
catalyst_build

0 commit comments

Comments
 (0)