Skip to content

Commit dbf83c4

Browse files
teh-cmcteh-cmc
authored
CI: Rerun CLI as a release asset (#3959)
Builds Rerun CLI for all supported platforms, and automatically adds them to release assets. ```sh $ cargo binstall --force rerun-cli INFO resolve: Resolving package: 'rerun-cli' INFO This will install the following binaries: INFO - rerun (bin-rerun-cli-x86_64-unknown-linux-gnu-GhCrateMeta -> /home/cmc/.cargo/bin/rerun) INFO Installing binaries... INFO Done in 2.849511699s $ which rerun /home/cmc/.cargo/bin/rerun $ file $(which rerun) /home/cmc/.cargo/bin/rerun: ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, BuildID[sha1]=77170f12c868bb5dd3858bbe0303f210c0edcc26, for GNU/Linux 3.2.0, with debug_info, not stripped $ rerun --version # the version will not match 0.9.1 because I manually uploaded a tip binary to the 0.9.1 release rerun-cli 0.10.0-alpha.7+dev [rustc 1.72.1 (d5c2e9c34 2023-09-13), LLVM 16.0.5] x86_64-unknown-linux-gnu 3959/merge 0c83403, built 2023-10-23T16:18:52Z ``` --- - Part of #3942 - Fixes #3901
1 parent e4c2608 commit dbf83c4

File tree

4 files changed

+295
-12
lines changed

4 files changed

+295
-12
lines changed

‎.github/workflows/on_pull_request.yml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,16 @@ jobs:
4444
UPLOAD_COMMIT_OVERRIDE: ${{ github.event.pull_request.head.sha }}
4545
secrets: inherit
4646

47+
build-rerun-cli-and-upload:
48+
name: "Build & Upload rerun-cli (Linux x64)"
49+
if: github.event.pull_request.head.repo.owner.login == 'rerun-io'
50+
uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
51+
with:
52+
CONCURRENCY: pr-${{ github.event.pull_request.number }}
53+
PLATFORM: linux
54+
UPLOAD_COMMIT_OVERRIDE: ${{ github.event.pull_request.head.sha }}
55+
secrets: inherit
56+
4757
bundle-and-upload-rerun_cpp:
4858
name: "Bundle and upload rerun_cpp"
4959
needs: [build-rerun_c-and-upload]

‎.github/workflows/on_push_main.yml

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,42 @@ jobs:
129129
PLATFORM: windows
130130
secrets: inherit
131131

132+
build-rerun-cli-and-upload-linux:
133+
needs: [checks]
134+
name: "Linux-x64: Build & Upload rerun-cli"
135+
uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
136+
with:
137+
CONCURRENCY: push-linux-${{ github.ref_name }}
138+
PLATFORM: linux
139+
secrets: inherit
140+
141+
build-rerun-cli-and-upload-macos-intel:
142+
needs: [checks]
143+
name: "Mac-Intel: Build & Upload rerun-cli"
144+
uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
145+
with:
146+
CONCURRENCY: push-macos-intel-${{ github.ref_name }}
147+
PLATFORM: macos-intel
148+
secrets: inherit
149+
150+
build-rerun-cli-and-upload-macos-arm:
151+
needs: [checks]
152+
name: "Mac-Arm: Build & Upload rerun-cli"
153+
uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
154+
with:
155+
CONCURRENCY: push-macos-arm-${{ github.ref_name }}
156+
PLATFORM: macos-arm
157+
secrets: inherit
158+
159+
build-rerun-cli-and-upload-windows:
160+
needs: [checks]
161+
name: "Windows-x64: Build & Upload rerun-cli"
162+
uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
163+
with:
164+
CONCURRENCY: push-windows-${{ github.ref_name }}
165+
PLATFORM: windows
166+
secrets: inherit
167+
132168
build-linux:
133169
needs: [checks]
134170
name: "Linux: Build/Test Wheels"

‎.github/workflows/reusable_build_and_upload_rerun_cli.yml

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
name: Reusable Rerun CLI build & upload
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
CONCURRENCY:
7+
required: true
8+
type: string
9+
PLATFORM:
10+
required: true
11+
type: string
12+
RELEASE_VERSION:
13+
required: false
14+
type: string
15+
default: "prerelease"
16+
UPLOAD_COMMIT_OVERRIDE:
17+
required: false
18+
type: string
19+
default: ""
20+
UPLOAD_COMMIT:
21+
required: false
22+
type: boolean
23+
default: true
24+
ADHOC_NAME:
25+
required: false
26+
type: string
27+
default: ""
28+
29+
workflow_dispatch:
30+
inputs:
31+
ADHOC_NAME:
32+
required: true
33+
type: string
34+
description: "Name of the adhoc build, used for upload directory"
35+
PLATFORM:
36+
type: choice
37+
options:
38+
- linux
39+
- windows
40+
- macos-arm
41+
- macos-intel
42+
description: "Platform to build for"
43+
required: true
44+
CONCURRENCY:
45+
required: false
46+
type: string
47+
default: "adhoc"
48+
RELEASE_VERSION:
49+
required: false
50+
type: string
51+
default: "prerelease"
52+
UPLOAD_COMMIT_OVERRIDE:
53+
required: false
54+
type: string
55+
default: ""
56+
UPLOAD_COMMIT:
57+
required: false
58+
type: boolean
59+
default: true
60+
61+
concurrency:
62+
group: ${{ inputs.CONCURRENCY }}-build-rerun-cli
63+
cancel-in-progress: true
64+
65+
env:
66+
# web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses
67+
# https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html
68+
# https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html
69+
RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings
70+
71+
# See https://github.com/ericseppanen/cargo-cranky/issues/8
72+
RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs
73+
74+
permissions:
75+
contents: "read"
76+
id-token: "write"
77+
78+
jobs:
79+
set-config:
80+
name: Set Config (${{ inputs.PLATFORM }})
81+
runs-on: ubuntu-latest-16-cores
82+
outputs:
83+
RUNNER: ${{ steps.set-config.outputs.runner }}
84+
TARGET: ${{ steps.set-config.outputs.target }}
85+
CONTAINER: ${{ steps.set-config.outputs.container }}
86+
BIN_NAME: ${{ steps.set-config.outputs.bin_name }}
87+
steps:
88+
- name: Set runner and target based on platform
89+
id: set-config
90+
run: |
91+
case "${{ inputs.PLATFORM }}" in
92+
linux)
93+
runner="ubuntu-latest-16-cores"
94+
target="x86_64-unknown-linux-gnu"
95+
container="{'image': 'rerunio/ci_docker:0.10.0'}"
96+
bin_name="rerun"
97+
;;
98+
windows)
99+
runner="windows-latest-8-cores"
100+
target="x86_64-pc-windows-msvc"
101+
container="null"
102+
bin_name="rerun.exe"
103+
;;
104+
macos-arm)
105+
runner="macos-latest"
106+
target="aarch64-apple-darwin"
107+
container="null"
108+
bin_name="rerun"
109+
;;
110+
macos-intel)
111+
runner="macos-latest"
112+
target="x86_64-apple-darwin"
113+
container="null"
114+
bin_name="rerun"
115+
;;
116+
*) echo "Invalid platform" && exit 1 ;;
117+
esac
118+
echo "runner=$runner" >> "$GITHUB_OUTPUT"
119+
echo "target=$target" >> "$GITHUB_OUTPUT"
120+
echo "container=$container" >> "$GITHUB_OUTPUT"
121+
echo "bin_name=$bin_name" >> "$GITHUB_OUTPUT"
122+
123+
build-rerun-cli:
124+
name: Build rerun-cli (${{ needs.set-config.outputs.RUNNER }})
125+
126+
needs: [set-config]
127+
128+
runs-on: ${{ needs.set-config.outputs.RUNNER }}
129+
container: ${{ fromJson(needs.set-config.outputs.CONTAINER) }}
130+
131+
steps:
132+
- name: Show context
133+
run: |
134+
echo "GITHUB_CONTEXT": $GITHUB_CONTEXT
135+
echo "JOB_CONTEXT": $JOB_CONTEXT
136+
echo "INPUTS_CONTEXT": $INPUTS_CONTEXT
137+
echo "ENV_CONTEXT": $ENV_CONTEXT
138+
env:
139+
ENV_CONTEXT: ${{ toJson(env) }}
140+
GITHUB_CONTEXT: ${{ toJson(github) }}
141+
JOB_CONTEXT: ${{ toJson(job) }}
142+
INPUTS_CONTEXT: ${{ toJson(inputs) }}
143+
144+
- uses: actions/checkout@v4
145+
with:
146+
ref: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.ref) || '' }}
147+
148+
- name: Set up Rust
149+
uses: ./.github/actions/setup-rust
150+
with:
151+
cache_key: "build-${{ inputs.PLATFORM }}"
152+
save_cache: false
153+
workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
154+
service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
155+
targets: ${{ needs.set-config.outputs.TARGET }}
156+
157+
- name: Build rerun-cli (release)
158+
uses: actions-rs/cargo@v1
159+
with:
160+
command: build
161+
args: --locked -p rerun-cli --no-default-features --features native_viewer,web_viewer --release --target ${{ needs.set-config.outputs.TARGET }}
162+
163+
- id: "auth"
164+
uses: google-github-actions/auth@v1
165+
with:
166+
workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
167+
service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
168+
169+
- name: Add SHORT_SHA env property with commit short sha
170+
shell: bash
171+
id: "short-sha"
172+
run: |
173+
if [ -z "${{ inputs.UPLOAD_COMMIT_OVERRIDE }}" ]; then
174+
USED_SHA=${{ github.sha }}
175+
else
176+
USED_SHA=${{ inputs.UPLOAD_COMMIT_OVERRIDE }}
177+
fi
178+
echo "SHORT_SHA=$(echo $USED_SHA | cut -c1-7)" >> $GITHUB_OUTPUT
179+
180+
- name: "Upload rerun-cli (commit)"
181+
if: ${{ inputs.UPLOAD_COMMIT }}
182+
uses: google-github-actions/upload-cloud-storage@v1
183+
with:
184+
path: "./target/${{ needs.set-config.outputs.TARGET }}/release/${{ needs.set-config.outputs.BIN_NAME }}"
185+
destination: "rerun-builds/commit/${{ steps.short-sha.outputs.SHORT_SHA }}/rerun-cli/${{ inputs.PLATFORM }}"
186+
parent: false
187+
188+
- name: "Upload rerun-cli (adhoc)"
189+
if: ${{ inputs.ADHOC_NAME != '' }}
190+
uses: google-github-actions/upload-cloud-storage@v1
191+
with:
192+
path: "./target/${{ needs.set-config.outputs.TARGET }}/release/${{ needs.set-config.outputs.BIN_NAME }}"
193+
destination: "rerun-builds/adhoc/${{inputs.ADHOC_NAME}}/rerun-cli/${{ inputs.PLATFORM }}"
194+
parent: false

‎scripts/ci/sync_release_assets.py

Lines changed: 55 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,13 @@
2626

2727

2828
def fetch_binary_assets(
29-
commit: str, *, do_wheels: bool = True, do_rerun_c: bool = True, do_rerun_cpp_sdk: bool = True
29+
tag: str,
30+
commit: str,
31+
*,
32+
do_wheels: bool = True,
33+
do_rerun_c: bool = True,
34+
do_rerun_cpp_sdk: bool = True,
35+
do_rerun_cli: bool = True,
3036
) -> Assets:
3137
"""Given a release ID, fetches all associated binary assets from our cloud storage (build.rerun.io)."""
3238
assets = dict()
@@ -39,33 +45,48 @@ def fetch_binary_assets(
3945
print(f" - wheels: {do_wheels}")
4046
print(f" - C libs: {do_rerun_c}")
4147
print(f" - C++ uber SDK: {do_rerun_cpp_sdk}")
48+
print(f" - CLI (Viewer): {do_rerun_cli}")
4249

4350
# Python wheels
4451
if do_wheels:
4552
wheel_blobs = list(bucket.list_blobs(prefix=f"commit/{commit_short}/wheels"))
4653
for blob in [bucket.get_blob(blob.name) for blob in wheel_blobs if blob.name.endswith(".whl")]:
4754
if blob is not None and blob.name is not None:
4855
name = blob.name.split("/")[-1]
56+
57+
if "macosx" in name:
58+
if "x86_64" in name:
59+
name = f"rerun_sdk-{tag}-aarch64-apple-darwin.whl"
60+
if "arm64" in name:
61+
name = f"rerun_sdk-{tag}-x86_64-apple-darwin.whl"
62+
63+
if "manylinux_2_31_x86_64" in name:
64+
if "x86_64" in name:
65+
name = f"rerun_sdk-{tag}-x86_64-unknown-linux-gnu.whl"
66+
67+
if "win_amd64" in name:
68+
name = f"rerun_sdk-{tag}-x86_64-pc-windows-msvc.whl"
69+
4970
print(f" Found Python wheel: {name} ")
5071
assets[name] = blob
5172

5273
# rerun_c
5374
if do_rerun_c:
5475
rerun_c_blobs = [
5576
(
56-
"librerun_c.x86_64-pc-windows-msvc.lib",
77+
f"rerun_c-{tag}-x86_64-pc-windows-msvc.lib",
5778
bucket.get_blob(f"commit/{commit_short}/rerun_c/windows/rerun_c.lib"),
5879
),
5980
(
60-
"librerun_c.x86_64-unknown-linux-gnu.a",
81+
f"librerun_c-{tag}-x86_64-unknown-linux-gnu.a",
6182
bucket.get_blob(f"commit/{commit_short}/rerun_c/linux/librerun_c.a"),
6283
),
6384
(
64-
"librerun_c.aarch64-apple-darwin.a",
85+
f"librerun_c-{tag}-aarch64-apple-darwin.a",
6586
bucket.get_blob(f"commit/{commit_short}/rerun_c/macos-arm/librerun_c.a"),
6687
),
6788
(
68-
"librerun_c.x86_64-apple-darwin.a",
89+
f"librerun_c-{tag}-x86_64-apple-darwin.a",
6990
bucket.get_blob(f"commit/{commit_short}/rerun_c/macos-intel/librerun_c.a"),
7091
),
7192
]
@@ -82,14 +103,33 @@ def fetch_binary_assets(
82103
name = blob.name.split("/")[-1]
83104
print(f" Found Rerun cross-platform bundle: {name}")
84105
assets[name] = blob
106+
# NOTE: Want a versioned one too.
107+
assets[f"rerun_cpp_sdk-{tag}-multiplatform.zip"] = blob
85108

86-
# rerun_cpp_sdk
87-
rerun_cpp_sdk_blob = bucket.get_blob(f"commit/{commit_short}/rerun_cpp_sdk.zip")
88-
for blob in [rerun_cpp_sdk_blob]:
89-
if blob is not None and blob.name is not None:
90-
name = blob.name.split("/")[-1]
91-
print(f" Found Rerun cross-platform bundle: {name} ({blob.size} bytes)")
92-
assets[name] = blob
109+
# rerun-cli
110+
if do_rerun_cli:
111+
rerun_cli_blobs = [
112+
(
113+
f"rerun-cli-{tag}-x86_64-pc-windows-msvc.exe",
114+
bucket.get_blob(f"commit/{commit_short}/rerun-cli/windows/rerun.exe"),
115+
),
116+
(
117+
f"rerun-cli-{tag}-x86_64-unknown-linux-gnu",
118+
bucket.get_blob(f"commit/{commit_short}/rerun-cli/linux/rerun"),
119+
),
120+
(
121+
f"rerun-cli-{tag}-aarch64-apple-darwin",
122+
bucket.get_blob(f"commit/{commit_short}/rerun-cli/macos-arm/rerun"),
123+
),
124+
(
125+
f"rerun-cli-{tag}-x86_64-apple-darwin",
126+
bucket.get_blob(f"commit/{commit_short}/rerun-cli/macos-intel/rerun"),
127+
),
128+
]
129+
for name, blob in rerun_cli_blobs:
130+
if blob is not None:
131+
print(f" Found Rerun CLI binary: {name}")
132+
assets[name] = blob
93133

94134
return assets
95135

@@ -134,6 +174,7 @@ def main() -> None:
134174
parser.add_argument("--no-wheels", action="store_true", help="Don't upload Python wheels")
135175
parser.add_argument("--no-rerun-c", action="store_true", help="Don't upload C libraries")
136176
parser.add_argument("--no-rerun-cpp-sdk", action="store_true", help="Don't upload C++ uber SDK")
177+
parser.add_argument("--no-rerun-cli", action="store_true", help="Don't upload CLI")
137178
args = parser.parse_args()
138179

139180
gh = Github(args.github_token, timeout=args.github_timeout)
@@ -146,10 +187,12 @@ def main() -> None:
146187
)
147188

148189
assets = fetch_binary_assets(
190+
release.tag_name,
149191
commit.sha,
150192
do_wheels=not args.no_wheels,
151193
do_rerun_c=not args.no_rerun_c,
152194
do_rerun_cpp_sdk=not args.no_rerun_cpp_sdk,
195+
do_rerun_cli=not args.no_rerun_cli,
153196
)
154197

155198
if args.remove:

0 commit comments

Comments
 (0)