Skip to content

Commit dfcc296

Browse files
SangJunBakSangJunBak
committed
Add deployment scripts and utilities for the Materialize debug tool
- I refactored mz_lsp_server's deploy_utils into a tarball_uploader. I plan to move the other binaries to use it too. - Tested macos and linux separately - Unclear if deploy_utils is necessary or if I should rename it. Not confident on where each variable lives as well. - Created a binary to run mz-debug
1 parent 6bed857 commit dfcc296

File tree

11 files changed

+411
-0
lines changed

11 files changed

+411
-0
lines changed

‎bin/mz-debug

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright Materialize, Inc. and contributors. All rights reserved.
4+
#
5+
# Use of this software is governed by the Business Source License
6+
# included in the LICENSE file at the root of this repository.
7+
#
8+
# As of the Change Date specified in that file, in accordance with
9+
# the Business Source License, use of this software will be governed
10+
# by the Apache License, Version 2.0.
11+
#
12+
# mz-debug -- build and run the Materialize debug tool.
13+
14+
set -euo pipefail
15+
16+
cd "$(dirname "$0")/.."
17+
18+
exec cargo run --bin mz-debug -- "$@"

‎ci/deploy_mz-debug/README.md

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Deploy the Materialize debug tool.
2+
3+
The CI process will build and deploy the LSP server to the materialize-binaries S3 bucket.
4+
You can try the process by running the following commands:
5+
6+
```bash
7+
# Set a tag version.
8+
export BUILDKITE_TAG=mz-debug-vx.y.z
9+
10+
# macOS
11+
bin/pyactivate -m ci.deploy_mz-debug.macos
12+
13+
# Linux
14+
bin/pyactivate -m ci.deploy_mz-debug.linux
15+
```
16+
17+
**Important Notes:**
18+
19+
- Update the version for `mz-debug`'s `Cargo.toml` to match the `BUILDKITE_TAG` version before deploying
20+
- When running on macOS, modify `linux.py` to use `target` instead of `target-xcompile`

‎ci/deploy_mz-debug/__init__.py

Whitespace-only changes.

‎ci/deploy_mz-debug/deploy_util.py

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Copyright Materialize, Inc. and contributors. All rights reserved.
2+
#
3+
# Use of this software is governed by the Business Source License
4+
# included in the LICENSE file at the root of this repository.
5+
#
6+
# As of the Change Date specified in that file, in accordance with
7+
# the Business Source License, use of this software will be governed
8+
# by the Apache License, Version 2.0.
9+
10+
import os
11+
12+
from materialize.mz_version import MzDebugVersion
13+
14+
TAG = os.environ["BUILDKITE_TAG"]
15+
MZ_DEBUG_VERSION = MzDebugVersion.parse(TAG)
16+
MZ_DEBUG_VERSION_STR = f"v{MZ_DEBUG_VERSION.str_without_prefix()}"

‎ci/deploy_mz-debug/docker.py

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# Copyright Materialize, Inc. and contributors. All rights reserved.
2+
#
3+
# Use of this software is governed by the Business Source License
4+
# included in the LICENSE file at the root of this repository.
5+
#
6+
# As of the Change Date specified in that file, in accordance with
7+
# the Business Source License, use of this software will be governed
8+
# by the Apache License, Version 2.0.
9+
10+
import os
11+
from pathlib import Path
12+
13+
from ci import tarball_uploader
14+
from materialize import mzbuild, ui
15+
from materialize.rustc_flags import Sanitizer
16+
from materialize.xcompile import Arch
17+
18+
from .deploy_util import MZ_DEBUG_VERSION
19+
20+
21+
def main() -> None:
22+
bazel = ui.env_is_truthy("CI_BAZEL_BUILD")
23+
bazel_remote_cache = os.getenv("CI_BAZEL_REMOTE_CACHE")
24+
25+
repos = [
26+
mzbuild.Repository(
27+
Path("."),
28+
Arch.X86_64,
29+
coverage=False,
30+
sanitizer=Sanitizer.none,
31+
bazel=bazel,
32+
bazel_remote_cache=bazel_remote_cache,
33+
),
34+
mzbuild.Repository(
35+
Path("."),
36+
Arch.AARCH64,
37+
coverage=False,
38+
sanitizer=Sanitizer.none,
39+
bazel=bazel,
40+
bazel_remote_cache=bazel_remote_cache,
41+
),
42+
]
43+
44+
print("--- Tagging Docker images")
45+
deps = [[repo.resolve_dependencies([repo.images["mz"]])["mz"]] for repo in repos]
46+
47+
mzbuild.publish_multiarch_images(f"v{MZ_DEBUG_VERSION.str_without_prefix()}", deps)
48+
if tarball_uploader.is_latest_version(MZ_DEBUG_VERSION):
49+
mzbuild.publish_multiarch_images("latest", deps)
50+
51+
52+
if __name__ == "__main__":
53+
main()

‎ci/deploy_mz-debug/linux.py

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# Copyright Materialize, Inc. and contributors. All rights reserved.
2+
#
3+
# Use of this software is governed by the Business Source License
4+
# included in the LICENSE file at the root of this repository.
5+
#
6+
# As of the Change Date specified in that file, in accordance with
7+
# the Business Source License, use of this software will be governed
8+
# by the Apache License, Version 2.0.
9+
10+
import os
11+
from pathlib import Path
12+
13+
from ci import tarball_uploader
14+
from ci.deploy.deploy_util import rust_version
15+
from materialize import mzbuild, spawn, ui
16+
from materialize.mz_version import MzDebugVersion
17+
from materialize.rustc_flags import Sanitizer
18+
19+
from . import deploy_util
20+
from .deploy_util import MZ_DEBUG_VERSION
21+
22+
23+
def main() -> None:
24+
bazel = ui.env_is_truthy("CI_BAZEL_BUILD")
25+
bazel_remote_cache = os.getenv("CI_BAZEL_REMOTE_CACHE")
26+
27+
repo = mzbuild.Repository(
28+
Path("."),
29+
coverage=False,
30+
sanitizer=Sanitizer.none,
31+
bazel=bazel,
32+
bazel_remote_cache=bazel_remote_cache,
33+
)
34+
target = f"{repo.rd.arch}-unknown-linux-gnu"
35+
36+
print("--- Checking version")
37+
assert (
38+
MzDebugVersion.parse_without_prefix(
39+
repo.rd.cargo_workspace.crates["mz-debug"].version_string
40+
)
41+
== MZ_DEBUG_VERSION
42+
)
43+
44+
print("--- Building mz-debug")
45+
# The bin/ci-builder uses target-xcompile as the volume and
46+
# is where the binary release will be available.
47+
path = Path("target-xcompile") / "release" / "mz-debug"
48+
spawn.runv(
49+
["cargo", "build", "--bin", "mz-debug", "--release"],
50+
env=dict(os.environ, RUSTUP_TOOLCHAIN=rust_version()),
51+
)
52+
mzbuild.chmod_x(path)
53+
54+
print(f"--- Uploading {target} binary tarball")
55+
uploader = tarball_uploader.TarballUploader(
56+
package_name="mz-debug",
57+
version=deploy_util.MZ_DEBUG_VERSION,
58+
)
59+
uploader.deploy_tarball(target, path)
60+
61+
62+
if __name__ == "__main__":
63+
main()

‎ci/deploy_mz-debug/macos.py

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Copyright Materialize, Inc. and contributors. All rights reserved.
2+
#
3+
# Use of this software is governed by the Business Source License
4+
# included in the LICENSE file at the root of this repository.
5+
#
6+
# As of the Change Date specified in that file, in accordance with
7+
# the Business Source License, use of this software will be governed
8+
# by the Apache License, Version 2.0.
9+
10+
import os
11+
from pathlib import Path
12+
13+
from ci.tarball_uploader import TarballUploader
14+
from materialize import spawn
15+
from materialize.mz_version import MzDebugVersion
16+
from materialize.xcompile import Arch
17+
18+
from ..deploy.deploy_util import rust_version
19+
from . import deploy_util
20+
21+
22+
def main() -> None:
23+
target = f"{Arch.host()}-apple-darwin"
24+
25+
print("--- Building mz-debug")
26+
spawn.runv(
27+
["cargo", "build", "--bin", "mz-debug", "--release"],
28+
env=dict(os.environ, RUSTUP_TOOLCHAIN=rust_version()),
29+
)
30+
31+
uploader = TarballUploader(
32+
package_name="mz-debug",
33+
version=deploy_util.MZ_DEBUG_VERSION,
34+
)
35+
36+
uploader.deploy_tarball(target, Path("target") / "release" / "mz-debug")
37+
38+
39+
if __name__ == "__main__":
40+
main()

‎ci/deploy_mz-debug/pipeline.template.yml

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# Copyright Materialize, Inc. and contributors. All rights reserved.
2+
#
3+
# Use of this software is governed by the Business Source License
4+
# included in the LICENSE file at the root of this repository.
5+
#
6+
# As of the Change Date specified in that file, in accordance with
7+
# the Business Source License, use of this software will be governed
8+
# by the Apache License, Version 2.0.
9+
10+
# Deploys are fast, do them quickly
11+
priority: 30
12+
13+
steps:
14+
- command: bin/ci-builder run stable bin/pyactivate -m ci.deploy_mz-debug.version
15+
timeout_in_minutes: 30
16+
concurrency: 1
17+
concurrency_group: deploy-mz-debug/version
18+
retry:
19+
manual:
20+
permit_on_passed: true
21+
22+
- id: linux-x86_64
23+
command: bin/ci-builder run stable bin/pyactivate -m ci.deploy_mz-debug.linux
24+
timeout_in_minutes: 30
25+
agents:
26+
queue: linux-x86_64-small
27+
concurrency: 1
28+
concurrency_group: deploy-mz-debug/linux/x86_64
29+
retry:
30+
manual:
31+
permit_on_passed: true
32+
33+
- id: linux-aarch64
34+
command: bin/ci-builder run stable bin/pyactivate -m ci.deploy_mz-debug.linux
35+
timeout_in_minutes: 30
36+
agents:
37+
queue: linux-aarch64-small
38+
concurrency: 1
39+
concurrency_group: deploy-mz-debug/linux/aarch64
40+
retry:
41+
manual:
42+
permit_on_passed: true
43+
44+
- command: bin/pyactivate -m ci.deploy_mz-debug.macos
45+
agents:
46+
queue: mac-x86_64
47+
timeout_in_minutes: 30
48+
concurrency: 1
49+
concurrency_group: deploy-mz-debug/macos/x86_64
50+
retry:
51+
manual:
52+
permit_on_passed: true
53+
54+
- command: bin/pyactivate -m ci.deploy_mz-debug.macos
55+
agents:
56+
queue: mac-aarch64
57+
timeout_in_minutes: 30
58+
concurrency: 1
59+
concurrency_group: deploy-mz-debug/macos/aarch64
60+
retry:
61+
manual:
62+
permit_on_passed: true

‎ci/deploy_mz-debug/version.py

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Copyright Materialize, Inc. and contributors. All rights reserved.
2+
#
3+
# Use of this software is governed by the Business Source License
4+
# included in the LICENSE file at the root of this repository.
5+
#
6+
# As of the Change Date specified in that file, in accordance with
7+
# the Business Source License, use of this software will be governed
8+
# by the Apache License, Version 2.0.
9+
10+
import boto3
11+
12+
from ci.tarball_uploader import BINARIES_BUCKET
13+
14+
from .deploy_util import MZ_DEBUG_VERSION
15+
16+
17+
def main() -> None:
18+
print("--- Uploading version file")
19+
boto3.client("s3").put_object(
20+
Body=f"{MZ_DEBUG_VERSION.str_without_prefix()}",
21+
Bucket=BINARIES_BUCKET,
22+
Key="mz-debug-latest.version",
23+
ContentType="text/plain",
24+
)
25+
26+
27+
if __name__ == "__main__":
28+
main()

0 commit comments

Comments
 (0)