Auto merge of rust-lang#117771 - tmandry:fuchsia-gha, r=<try>

Build Fuchsia in CI

This is very much in a draft state but I wanted to put it up now to get early feedback. It would also be nice if we could test it on actual CI builders.

Fittingly, it is failing right now due to discovering an ICE in clippy (looks like fixed in rust-lang/rust-clippy#11760), probably more fallout from recent type system changes. Other recent regressions this would have caught include

- rust-lang#117455 and rust-lang#117493
- rust-lang#117602

This ends up not sharing very much at all with cargotest. Fuchsia has its own tool to manage checkouts and its own build system. What it requires is a fully "install"ed toolchain with a host and fuchsia target. We share logic from the dist-various-2 builder to build the fuchsia target.

Right now this runs clippy and skips linking a bunch of targets, since most issues we catch are in the frontend. In theory we could probably get the build CPU time down quite a bit with this approach, but right now some linked targets are creeping into the dependencies anyway and we don't have a good way of preventing that yet.

The approach is basically to get a checkout at a pinned commit and then run a [script](https://fuchsia-review.git.corp.google.com/c/fuchsia/+/943833/6/scripts/rust/build_fuchsia_from_rust_ci.sh) at a predetermined location. I would like to update that pin every few weeks. For now we cherry-pick some build changes but that's temporary. Partial checkouts are used to minimize clone time, but we don't filter out prebuilt packages.

r? `@Mark-Simulacrum`

Based on discussion in [this Zulip thread](https://rust-lang.zulipchat.com/#narrow/stream/242791-t-infra/topic/Putting.20Fuchsia.20in.20crater).

Author: bors

.github/workflows/ci.yml

@@ -288,6 +288,9 @@ jobs:
           - name: x86_64-gnu-aux
             os: ubuntu-20.04-4core-16gb
             env: {}
+          - name: x86_64-gnu-integration
+            os: ubuntu-20.04-16core-64gb
+            env: {}
           - name: x86_64-gnu-debug
             os: ubuntu-20.04-8core-32gb
             env: {}
@@ -583,6 +586,12 @@ jobs:
             env:
               CODEGEN_BACKENDS: "llvm,cranelift"
             os: ubuntu-20.04-16core-64gb
+          - name: x86_64-gnu-integration
+            os: ubuntu-20.04-16core-64gb
+            env: {}
+          - name: dist-various-2
+            os: ubuntu-20.04-8core-32gb
+            env: {}
     timeout-minutes: 600
     runs-on: "${{ matrix.os }}"
     steps:

src/bootstrap/src/core/build_steps/compile.rs

@@ -1722,7 +1722,7 @@ impl Step for Assemble {
             let dst_exe = exe("rust-lld", target_compiler.host);
             builder.copy(&lld_install.join("bin").join(&src_exe), &libdir_bin.join(&dst_exe));
             let self_contained_lld_dir = libdir_bin.join("gcc-ld");
-            t!(fs::create_dir(&self_contained_lld_dir));
+            t!(fs::create_dir_all(&self_contained_lld_dir));
             let lld_wrapper_exe = builder.ensure(crate::core::build_steps::tool::LldWrapper {
                 compiler: build_compiler,
                 target: target_compiler.host,

src/ci/docker/host-x86_64/dist-various-2/Dockerfile

@@ -76,8 +76,8 @@ RUN env \
     rm -rf /build/*
 
 WORKDIR /tmp
-COPY host-x86_64/dist-various-2/shared.sh /tmp/
-COPY host-x86_64/dist-various-2/build-fuchsia-toolchain.sh /tmp/
+COPY scripts/shared.sh /tmp/
+COPY scripts/build-fuchsia-toolchain.sh /tmp/
 RUN /tmp/build-fuchsia-toolchain.sh
 COPY host-x86_64/dist-various-2/build-solaris-toolchain.sh /tmp/
 RUN /tmp/build-solaris-toolchain.sh x86_64  amd64   solaris-i386  pc

src/ci/docker/host-x86_64/x86_64-gnu-integration/Dockerfile

@@ -0,0 +1,68 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  build-essential \
+  g++ \
+  make \
+  ninja-build \
+  file \
+  curl \
+  ca-certificates \
+  python3 \
+  git \
+  cmake \
+  libssl-dev \
+  sudo \
+  xz-utils \
+  pkg-config \
+  unzip \
+  && rm -rf /var/lib/apt/lists/*
+
+# Duplicated in dist-various-2 Dockerfile.
+# FIXME: Move to canonical triple
+ENV \
+    AR_x86_64_fuchsia=x86_64-unknown-fuchsia-ar \
+    CC_x86_64_fuchsia=x86_64-unknown-fuchsia-clang \
+    CFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \
+    CXX_x86_64_fuchsia=x86_64-unknown-fuchsia-clang++ \
+    CXXFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \
+    LDFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -L/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib"
+
+WORKDIR /tmp
+COPY scripts/shared.sh /tmp/
+COPY scripts/build-fuchsia-toolchain.sh /tmp/
+RUN /tmp/build-fuchsia-toolchain.sh
+
+ENV CARGO_TARGET_X86_64_FUCHSIA_AR /usr/local/bin/llvm-ar
+ENV CARGO_TARGET_X86_64_FUCHSIA_RUSTFLAGS \
+  -C panic=abort \
+  -C force-unwind-tables=yes \
+  -C link-arg=--sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot \
+  -Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot/lib \
+  -Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib
+
+ENV TARGETS=x86_64-fuchsia
+ENV TARGETS=$TARGETS,x86_64-unknown-linux-gnu
+
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
+
+ENV RUST_INSTALL_DIR /checkout/obj/install
+RUN mkdir -p $RUST_INSTALL_DIR/etc
+
+ENV RUST_CONFIGURE_ARGS \
+  --prefix=$RUST_INSTALL_DIR \
+  --sysconfdir=etc \
+  --enable-lld \
+  --llvm-libunwind=in-tree \
+  --enable-extended \
+  --disable-docs \
+  --set target.x86_64-fuchsia.cc=/usr/local/bin/clang \
+  --set target.x86_64-fuchsia.cxx=/usr/local/bin/clang++ \
+  --set target.x86_64-fuchsia.ar=/usr/local/bin/llvm-ar \
+  --set target.x86_64-fuchsia.ranlib=/usr/local/bin/llvm-ranlib \
+  --set target.x86_64-fuchsia.linker=/usr/local/bin/ld.lld
+ENV SCRIPT \
+    python3 ../x.py install --target $TARGETS compiler/rustc library/std clippy && \
+    bash ../src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh

src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+# Downloads and builds the Fuchsia operating system using a toolchain installed
+# in $RUST_INSTALL_DIR.
+
+set -euf -o pipefail
+
+INTEGRATION_SHA=06ae16d18bd8e4db9a3fc062f678a170025d9f1a
+PICK_REFS=(
+    refs/changes/14/948614/1
+    refs/changes/33/943833/21
+)
+
+checkout=fuchsia
+jiri=.jiri_root/bin/jiri
+
+set -x
+
+# This script will:
+# - create a directory named "fuchsia" if it does not exist
+# - download "jiri" to "fuchsia/.jiri_root/bin"
+curl -s "https://fuchsia.googlesource.com/jiri/+/HEAD/scripts/bootstrap_jiri?format=TEXT" \
+    | base64 --decode \
+    | bash -s $checkout
+
+cd $checkout
+
+$jiri init \
+    -partial=true \
+    -analytics-opt=false \
+    .
+
+$jiri import \
+    -name=integration \
+    -revision=$INTEGRATION_SHA \
+    -overwrite=true \
+    flower \
+    "https://fuchsia.googlesource.com/integration"
+
+if [ -d ".git" ]; then
+    # Wipe out any local changes if we're reusing a checkout.
+    git checkout --force JIRI_HEAD
+fi
+
+$jiri update -autoupdate=false
+
+echo integration commit = $(git -C integration rev-parse HEAD)
+
+for git_ref in "${PICK_REFS[@]}"; do
+    git fetch https://fuchsia.googlesource.com/fuchsia $git_ref
+    git cherry-pick --no-commit FETCH_HEAD
+done
+
+bash scripts/rust/build_fuchsia_from_rust_ci.sh

src/ci/docker/host-x86_64/dist-various-2/build-fuchsia-toolchain.sh renamed to src/ci/docker/scripts/build-fuchsia-toolchain.sh

@@ -4,13 +4,13 @@ set -ex
 source shared.sh
 
 FUCHSIA_SDK_URL=https://chrome-infra-packages.appspot.com/dl/fuchsia/sdk/core/linux-amd64
-FUCHSIA_SDK_ID=4xjxrGUrDbQ6_zJwj6cDN1IbWsWV5aCQXC_zO_Hu0XkC
-FUCHSIA_SDK_SHA256=e318f1ac652b0db43aff32708fa70337521b5ac595e5a0905c2ff33bf1eed179
+FUCHSIA_SDK_ID=MrhQwtmP8CpZre-i_PNOREcThbUcrX3bA-45d6WQr-cC
+FUCHSIA_SDK_SHA256=32b850c2d98ff02a59adefa2fcf34e44471385b51cad7ddb03ee3977a590afe7
 FUCHSIA_SDK_USR_DIR=/usr/local/core-linux-amd64-fuchsia-sdk
 CLANG_DOWNLOAD_URL=\
 https://chrome-infra-packages.appspot.com/dl/fuchsia/third_party/clang/linux-amd64
-CLANG_DOWNLOAD_ID=vU0vNjSihOV4Q6taQYCpy03JXGiCyVwxen3rFMNMIgsC
-CLANG_DOWNLOAD_SHA256=bd4d2f3634a284e57843ab5a4180a9cb4dc95c6882c95c317a7deb14c34c220b
+CLANG_DOWNLOAD_ID=Tpc85d1ZwSlZ6UKl2d96GRUBGNA5JKholOKe24sRDr0C
+CLANG_DOWNLOAD_SHA256=4e973ce5dd59c12959e942a5d9df7a19150118d03924a86894e29edb8b110ebd
 
 install_clang() {
   mkdir -p clang_download

src/ci/docker/host-x86_64/dist-various-2/shared.sh renamed to src/ci/docker/scripts/shared.sh

@@ -1,5 +1,11 @@
-#!/usr/bin/env bash
-hide_output() {
+#!/bin/false
+# shellcheck shell=bash
+
+# This file is intended to be sourced with `. shared.sh` or
+# `source shared.sh`, hence the invalid shebang and not being
+# marked as an executable file in git.
+
+function hide_output {
   { set +x; } 2>/dev/null
   on_err="
 echo ERROR: An error was encountered with the build.
@@ -15,7 +21,8 @@ exit 1
   set -x
 }
 
-# Copied from ../../shared.sh
+# See https://unix.stackexchange.com/questions/82598
+# Duplicated in src/ci/shared.sh
 function retry {
   echo "Attempting with retry:" "$@"
   local n=1

src/ci/github-actions/ci.yml

@@ -470,6 +470,9 @@ jobs:
           - name: x86_64-gnu-aux
             <<: *job-linux-4c
 
+          - name: x86_64-gnu-integration
+            <<: *job-linux-16c
+
           - name: x86_64-gnu-debug
             <<: *job-linux-8c
 
@@ -734,6 +737,11 @@ jobs:
             env:
               CODEGEN_BACKENDS: llvm,cranelift
             <<: *job-linux-16c
+          - name: x86_64-gnu-integration
+            <<: *job-linux-16c
+          - name: dist-various-2
+            <<: *job-linux-8c
+
 
   master:
     name: master

src/ci/shared.sh

@@ -8,7 +8,7 @@
 export MIRRORS_BASE="https://ci-mirrors.rust-lang.org/rustc"
 
 # See https://unix.stackexchange.com/questions/82598
-# Duplicated in docker/dist-various-2/shared.sh
+# Duplicated in docker/scripts/shared.sh
 function retry {
   echo "Attempting with retry:" "$@"
   local n=1