Set up deployment pipeline for mz-debug

bin/mz-debug

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+#
+# mz-debug -- build and run the Materialize debug tool.
+
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+
+exec cargo run --bin mz-debug -- "$@"

ci/deploy_mz-debug/README.md

@@ -0,0 +1,20 @@
+# Deploy the Materialize debug tool.
+
+The CI process will build and deploy the Materialize debug tool to the materialize-binaries S3 bucket.
+You can try the process by running the following commands:
+
+```bash
+# Set a tag version.
+export BUILDKITE_TAG=mz-debug-vx.y.z
+
+# macOS
+bin/pyactivate -m ci.deploy_mz-debug.macos
+
+# Linux
+bin/pyactivate -m ci.deploy_mz-debug.linux
+```
+
+**Important Notes:**
+
+- Update the version for `mz-debug`'s `Cargo.toml` to match the `BUILDKITE_TAG` version before deploying
+- When running on macOS, modify `linux.py` to use `target` instead of `target-xcompile`

ci/deploy_mz-debug/deploy_util.py

@@ -0,0 +1,16 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+import os
+
+from materialize.mz_version import MzDebugVersion
+
+TAG = os.environ["BUILDKITE_TAG"]
+MZ_DEBUG_VERSION = MzDebugVersion.parse(TAG)
+MZ_DEBUG_VERSION_STR = f"v{MZ_DEBUG_VERSION.str_without_prefix()}"

ci/deploy_mz-debug/docker.py

@@ -0,0 +1,53 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+import os
+from pathlib import Path
+
+from ci import tarball_uploader
+from materialize import mzbuild, ui
+from materialize.rustc_flags import Sanitizer
+from materialize.xcompile import Arch
+
+from .deploy_util import MZ_DEBUG_VERSION
+
+
+def main() -> None:
+    bazel = ui.env_is_truthy("CI_BAZEL_BUILD")
+    bazel_remote_cache = os.getenv("CI_BAZEL_REMOTE_CACHE")
+
+    repos = [
+        mzbuild.Repository(
+            Path("."),
+            Arch.X86_64,
+            coverage=False,
+            sanitizer=Sanitizer.none,
+            bazel=bazel,
+            bazel_remote_cache=bazel_remote_cache,
+        ),
+        mzbuild.Repository(
+            Path("."),
+            Arch.AARCH64,
+            coverage=False,
+            sanitizer=Sanitizer.none,
+            bazel=bazel,
+            bazel_remote_cache=bazel_remote_cache,
+        ),
+    ]
+
+    print("--- Tagging Docker images")
+    deps = [[repo.resolve_dependencies([repo.images["mz"]])["mz"]] for repo in repos]
+
+    mzbuild.publish_multiarch_images(f"v{MZ_DEBUG_VERSION.str_without_prefix()}", deps)
+    if tarball_uploader.is_latest_version(MZ_DEBUG_VERSION):
+        mzbuild.publish_multiarch_images("latest", deps)
+
+
+if __name__ == "__main__":
+    main()

ci/deploy_mz-debug/linux.py

@@ -0,0 +1,63 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+import os
+from pathlib import Path
+
+from ci import tarball_uploader
+from ci.deploy.deploy_util import rust_version
+from materialize import mzbuild, spawn, ui
+from materialize.mz_version import MzDebugVersion
+from materialize.rustc_flags import Sanitizer
+
+from . import deploy_util
+from .deploy_util import MZ_DEBUG_VERSION
+
+
+def main() -> None:
+    bazel = ui.env_is_truthy("CI_BAZEL_BUILD")
+    bazel_remote_cache = os.getenv("CI_BAZEL_REMOTE_CACHE")
+
+    repo = mzbuild.Repository(
+        Path("."),
+        coverage=False,
+        sanitizer=Sanitizer.none,
+        bazel=bazel,
+        bazel_remote_cache=bazel_remote_cache,
+    )
+    target = f"{repo.rd.arch}-unknown-linux-gnu"
+
+    print("--- Checking version")
+    assert (
+        MzDebugVersion.parse_without_prefix(
+            repo.rd.cargo_workspace.crates["mz-debug"].version_string
+        )
+        == MZ_DEBUG_VERSION
+    )
+
+    print("--- Building mz-debug")
+    # The bin/ci-builder uses target-xcompile as the volume and
+    # is where the binary release will be available.
+    path = Path("target-xcompile") / "release" / "mz-debug"
+    spawn.runv(
+        ["cargo", "build", "--bin", "mz-debug", "--release"],
+        env=dict(os.environ, RUSTUP_TOOLCHAIN=rust_version()),
+    )
+    mzbuild.chmod_x(path)
+
+    print(f"--- Uploading {target} binary tarball")
+    uploader = tarball_uploader.TarballUploader(
+        package_name="mz-debug",
+        version=deploy_util.MZ_DEBUG_VERSION,
+    )
+    uploader.deploy_tarball(target, path)
+
+
+if __name__ == "__main__":
+    main()

ci/deploy_mz-debug/macos.py

@@ -0,0 +1,39 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+import os
+from pathlib import Path
+
+from ci.tarball_uploader import TarballUploader
+from materialize import spawn
+from materialize.xcompile import Arch
+
+from ..deploy.deploy_util import rust_version
+from . import deploy_util
+
+
+def main() -> None:
+    target = f"{Arch.host()}-apple-darwin"
+
+    print("--- Building mz-debug")
+    spawn.runv(
+        ["cargo", "build", "--bin", "mz-debug", "--release"],
+        env=dict(os.environ, RUSTUP_TOOLCHAIN=rust_version()),
+    )
+
+    uploader = TarballUploader(
+        package_name="mz-debug",
+        version=deploy_util.MZ_DEBUG_VERSION,
+    )
+
+    uploader.deploy_tarball(target, Path("target") / "release" / "mz-debug")
+
+
+if __name__ == "__main__":
+    main()

ci/deploy_mz-debug/pipeline.template.yml

@@ -0,0 +1,62 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+# Deploys are fast, do them quickly
+priority: 30
+
+steps:
+  - command: bin/ci-builder run stable bin/pyactivate -m ci.deploy_mz-debug.version
+    timeout_in_minutes: 30
+    concurrency: 1
+    concurrency_group: deploy-mz-debug/version
+    retry:
+      manual:
+        permit_on_passed: true
+
+  - id: linux-x86_64
+    command: bin/ci-builder run stable bin/pyactivate -m ci.deploy_mz-debug.linux
+    timeout_in_minutes: 30
+    agents:
+      queue: linux-x86_64-small
+    concurrency: 1
+    concurrency_group: deploy-mz-debug/linux/x86_64
+    retry:
+      manual:
+        permit_on_passed: true
+
+  - id: linux-aarch64
+    command: bin/ci-builder run stable bin/pyactivate -m ci.deploy_mz-debug.linux
+    timeout_in_minutes: 30
+    agents:
+      queue: linux-aarch64-small
+    concurrency: 1
+    concurrency_group: deploy-mz-debug/linux/aarch64
+    retry:
+      manual:
+        permit_on_passed: true
+
+  - command: bin/pyactivate -m ci.deploy_mz-debug.macos
+    agents:
+      queue: mac-x86_64
+    timeout_in_minutes: 30
+    concurrency: 1
+    concurrency_group: deploy-mz-debug/macos/x86_64
+    retry:
+      manual:
+        permit_on_passed: true
+
+  - command: bin/pyactivate -m ci.deploy_mz-debug.macos
+    agents:
+      queue: mac-aarch64
+    timeout_in_minutes: 30
+    concurrency: 1
+    concurrency_group: deploy-mz-debug/macos/aarch64
+    retry:
+      manual:
+        permit_on_passed: true

ci/deploy_mz-debug/version.py

@@ -0,0 +1,28 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+import boto3
+
+from ci.tarball_uploader import BINARIES_BUCKET
+
+from .deploy_util import MZ_DEBUG_VERSION
+
+
+def main() -> None:
+    print("--- Uploading version file")
+    boto3.client("s3").put_object(
+        Body=f"{MZ_DEBUG_VERSION.str_without_prefix()}",
+        Bucket=BINARIES_BUCKET,
+        Key="mz-debug-latest.version",
+        ContentType="text/plain",
+    )
+
+
+if __name__ == "__main__":
+    main()