From bbdc91eb452608a2881a81e4421cd540308d5099 Mon Sep 17 00:00:00 2001 From: Sang Jun Bak Date: Tue, 8 Apr 2025 13:23:29 -0400 Subject: [PATCH 1/5] Add deployment scripts and utilities for the Materialize debug tool - I refactored mz_lsp_server's deploy_utils into a tarball_uploader. I plan to move the other binaries to use it too. - Tested macos and linux separately - Unclear if deploy_utils is necessary or if I should rename it. Not confident on where each variable lives as well. - Created a binary to run mz-debug --- bin/mz-debug | 18 ++++ ci/deploy_mz-debug/README.md | 20 +++++ ci/deploy_mz-debug/__init__.py | 0 ci/deploy_mz-debug/deploy_util.py | 16 ++++ ci/deploy_mz-debug/docker.py | 53 ++++++++++++ ci/deploy_mz-debug/linux.py | 63 ++++++++++++++ ci/deploy_mz-debug/macos.py | 40 +++++++++ ci/deploy_mz-debug/pipeline.template.yml | 62 ++++++++++++++ ci/deploy_mz-debug/version.py | 28 ++++++ ci/tarball_uploader.py | 103 +++++++++++++++++++++++ misc/python/materialize/mz_version.py | 8 ++ 11 files changed, 411 insertions(+) create mode 100755 bin/mz-debug create mode 100644 ci/deploy_mz-debug/README.md create mode 100644 ci/deploy_mz-debug/__init__.py create mode 100644 ci/deploy_mz-debug/deploy_util.py create mode 100644 ci/deploy_mz-debug/docker.py create mode 100644 ci/deploy_mz-debug/linux.py create mode 100644 ci/deploy_mz-debug/macos.py create mode 100644 ci/deploy_mz-debug/pipeline.template.yml create mode 100644 ci/deploy_mz-debug/version.py create mode 100644 ci/tarball_uploader.py diff --git a/bin/mz-debug b/bin/mz-debug new file mode 100755 index 0000000000000..1b2757cc1f18d --- /dev/null +++ b/bin/mz-debug @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. +# +# mz-debug -- build and run the Materialize debug tool. + +set -euo pipefail + +cd "$(dirname "$0")/.." + +exec cargo run --bin mz-debug -- "$@" diff --git a/ci/deploy_mz-debug/README.md b/ci/deploy_mz-debug/README.md new file mode 100644 index 0000000000000..ac0ed07340081 --- /dev/null +++ b/ci/deploy_mz-debug/README.md @@ -0,0 +1,20 @@ +# Deploy the Materialize debug tool. + +The CI process will build and deploy the LSP server to the materialize-binaries S3 bucket. +You can try the process by running the following commands: + +```bash +# Set a tag version. +export BUILDKITE_TAG=mz-debug-vx.y.z + +# macOS +bin/pyactivate -m ci.deploy_mz-debug.macos + +# Linux +bin/pyactivate -m ci.deploy_mz-debug.linux +``` + +**Important Notes:** + +- Update the version for `mz-debug`'s `Cargo.toml` to match the `BUILDKITE_TAG` version before deploying +- When running on macOS, modify `linux.py` to use `target` instead of `target-xcompile` \ No newline at end of file diff --git a/ci/deploy_mz-debug/__init__.py b/ci/deploy_mz-debug/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/ci/deploy_mz-debug/deploy_util.py b/ci/deploy_mz-debug/deploy_util.py new file mode 100644 index 0000000000000..caa7eba8c8581 --- /dev/null +++ b/ci/deploy_mz-debug/deploy_util.py @@ -0,0 +1,16 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +import os + +from materialize.mz_version import MzDebugVersion + +TAG = os.environ["BUILDKITE_TAG"] +MZ_DEBUG_VERSION = MzDebugVersion.parse(TAG) +MZ_DEBUG_VERSION_STR = f"v{MZ_DEBUG_VERSION.str_without_prefix()}" \ No newline at end of file diff --git a/ci/deploy_mz-debug/docker.py b/ci/deploy_mz-debug/docker.py new file mode 100644 index 0000000000000..3e513fa0d97c7 --- /dev/null +++ b/ci/deploy_mz-debug/docker.py @@ -0,0 +1,53 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +import os +from pathlib import Path + +from ci import tarball_uploader +from materialize import mzbuild, ui +from materialize.rustc_flags import Sanitizer +from materialize.xcompile import Arch + +from .deploy_util import MZ_DEBUG_VERSION + + +def main() -> None: + bazel = ui.env_is_truthy("CI_BAZEL_BUILD") + bazel_remote_cache = os.getenv("CI_BAZEL_REMOTE_CACHE") + + repos = [ + mzbuild.Repository( + Path("."), + Arch.X86_64, + coverage=False, + sanitizer=Sanitizer.none, + bazel=bazel, + bazel_remote_cache=bazel_remote_cache, + ), + mzbuild.Repository( + Path("."), + Arch.AARCH64, + coverage=False, + sanitizer=Sanitizer.none, + bazel=bazel, + bazel_remote_cache=bazel_remote_cache, + ), + ] + + print("--- Tagging Docker images") + deps = [[repo.resolve_dependencies([repo.images["mz"]])["mz"]] for repo in repos] + + mzbuild.publish_multiarch_images(f"v{MZ_DEBUG_VERSION.str_without_prefix()}", deps) + if tarball_uploader.is_latest_version(MZ_DEBUG_VERSION): + mzbuild.publish_multiarch_images("latest", deps) + + +if __name__ == "__main__": + main() diff --git a/ci/deploy_mz-debug/linux.py b/ci/deploy_mz-debug/linux.py new file mode 100644 index 0000000000000..8615ce6cc0740 --- /dev/null +++ b/ci/deploy_mz-debug/linux.py @@ -0,0 +1,63 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +import os +from pathlib import Path + +from ci import tarball_uploader +from ci.deploy.deploy_util import rust_version +from materialize import mzbuild, spawn, ui +from materialize.mz_version import MzDebugVersion +from materialize.rustc_flags import Sanitizer + +from . import deploy_util +from .deploy_util import MZ_DEBUG_VERSION + + +def main() -> None: + bazel = ui.env_is_truthy("CI_BAZEL_BUILD") + bazel_remote_cache = os.getenv("CI_BAZEL_REMOTE_CACHE") + + repo = mzbuild.Repository( + Path("."), + coverage=False, + sanitizer=Sanitizer.none, + bazel=bazel, + bazel_remote_cache=bazel_remote_cache, + ) + target = f"{repo.rd.arch}-unknown-linux-gnu" + + print("--- Checking version") + assert ( + MzDebugVersion.parse_without_prefix( + repo.rd.cargo_workspace.crates["mz-debug"].version_string + ) + == MZ_DEBUG_VERSION + ) + + print("--- Building mz-debug") + # The bin/ci-builder uses target-xcompile as the volume and + # is where the binary release will be available. + path = Path("target-xcompile") / "release" / "mz-debug" + spawn.runv( + ["cargo", "build", "--bin", "mz-debug", "--release"], + env=dict(os.environ, RUSTUP_TOOLCHAIN=rust_version()), + ) + mzbuild.chmod_x(path) + + print(f"--- Uploading {target} binary tarball") + uploader = tarball_uploader.TarballUploader( + package_name="mz-debug", + version=deploy_util.MZ_DEBUG_VERSION, + ) + uploader.deploy_tarball(target, path) + + +if __name__ == "__main__": + main() diff --git a/ci/deploy_mz-debug/macos.py b/ci/deploy_mz-debug/macos.py new file mode 100644 index 0000000000000..04ad9d6fb53c4 --- /dev/null +++ b/ci/deploy_mz-debug/macos.py @@ -0,0 +1,40 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +import os +from pathlib import Path + +from ci.tarball_uploader import TarballUploader +from materialize import spawn +from materialize.mz_version import MzDebugVersion +from materialize.xcompile import Arch + +from ..deploy.deploy_util import rust_version +from . import deploy_util + + +def main() -> None: + target = f"{Arch.host()}-apple-darwin" + + print("--- Building mz-debug") + spawn.runv( + ["cargo", "build", "--bin", "mz-debug", "--release"], + env=dict(os.environ, RUSTUP_TOOLCHAIN=rust_version()), + ) + + uploader = TarballUploader( + package_name="mz-debug", + version=deploy_util.MZ_DEBUG_VERSION, + ) + + uploader.deploy_tarball(target, Path("target") / "release" / "mz-debug") + + +if __name__ == "__main__": + main() diff --git a/ci/deploy_mz-debug/pipeline.template.yml b/ci/deploy_mz-debug/pipeline.template.yml new file mode 100644 index 0000000000000..f3c85c80f8ea6 --- /dev/null +++ b/ci/deploy_mz-debug/pipeline.template.yml @@ -0,0 +1,62 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +# Deploys are fast, do them quickly +priority: 30 + +steps: + - command: bin/ci-builder run stable bin/pyactivate -m ci.deploy_mz-debug.version + timeout_in_minutes: 30 + concurrency: 1 + concurrency_group: deploy-mz-debug/version + retry: + manual: + permit_on_passed: true + + - id: linux-x86_64 + command: bin/ci-builder run stable bin/pyactivate -m ci.deploy_mz-debug.linux + timeout_in_minutes: 30 + agents: + queue: linux-x86_64-small + concurrency: 1 + concurrency_group: deploy-mz-debug/linux/x86_64 + retry: + manual: + permit_on_passed: true + + - id: linux-aarch64 + command: bin/ci-builder run stable bin/pyactivate -m ci.deploy_mz-debug.linux + timeout_in_minutes: 30 + agents: + queue: linux-aarch64-small + concurrency: 1 + concurrency_group: deploy-mz-debug/linux/aarch64 + retry: + manual: + permit_on_passed: true + + - command: bin/pyactivate -m ci.deploy_mz-debug.macos + agents: + queue: mac-x86_64 + timeout_in_minutes: 30 + concurrency: 1 + concurrency_group: deploy-mz-debug/macos/x86_64 + retry: + manual: + permit_on_passed: true + + - command: bin/pyactivate -m ci.deploy_mz-debug.macos + agents: + queue: mac-aarch64 + timeout_in_minutes: 30 + concurrency: 1 + concurrency_group: deploy-mz-debug/macos/aarch64 + retry: + manual: + permit_on_passed: true diff --git a/ci/deploy_mz-debug/version.py b/ci/deploy_mz-debug/version.py new file mode 100644 index 0000000000000..433eae56e9a5d --- /dev/null +++ b/ci/deploy_mz-debug/version.py @@ -0,0 +1,28 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +import boto3 + +from ci.tarball_uploader import BINARIES_BUCKET + +from .deploy_util import MZ_DEBUG_VERSION + + +def main() -> None: + print("--- Uploading version file") + boto3.client("s3").put_object( + Body=f"{MZ_DEBUG_VERSION.str_without_prefix()}", + Bucket=BINARIES_BUCKET, + Key="mz-debug-latest.version", + ContentType="text/plain", + ) + + +if __name__ == "__main__": + main() diff --git a/ci/tarball_uploader.py b/ci/tarball_uploader.py new file mode 100644 index 0000000000000..fedebaeae77d4 --- /dev/null +++ b/ci/tarball_uploader.py @@ -0,0 +1,103 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +# This file contains a class that can be used to upload a tarball to the materialize-binaries S3 bucket. + +import os +import tarfile +import tempfile +import time +from pathlib import Path + +import boto3 +import humanize + +from materialize import git +from materialize.mz_version import TypedVersionBase + +BINARIES_BUCKET = "materialize-binaries" + +class TarballUploader: + def __init__(self, package_name: str, version: TypedVersionBase): + self.package_name = package_name + self.version = version + self.version_str = f"v{version.str_without_prefix()}" + + def _create_tardir(self, name: str) -> tarfile.TarInfo: + """Takes a dir path and creates a TarInfo. Sets the + modification time, mode, and type of the dir.""" + d = tarfile.TarInfo(name) + d.mtime = int(time.time()) + d.mode = 0o40755 + d.type = tarfile.DIRTYPE + return d + + def _sanitize_tarinfo(self, tarinfo: tarfile.TarInfo) -> tarfile.TarInfo: + tarinfo.uid = tarinfo.gid = 0 + tarinfo.uname = tarinfo.gname = "root" + return tarinfo + + def _upload_redirect(self, key: str, to: str) -> None: + with tempfile.NamedTemporaryFile() as empty: + boto3.client("s3").upload_fileobj( + Fileobj=empty, + Bucket=BINARIES_BUCKET, + Key=key, + ExtraArgs={"WebsiteRedirectLocation": to}, + ) + + def _upload_tarball(self, tarball: Path, platform: str) -> None: + """Uploads the tarball (.tar) to the S3 bucket.""" + s3_object = f"{self.package_name}-{self.version_str}-{platform}.tar.gz" + boto3.client("s3").upload_file( + Filename=str(tarball), + Bucket=BINARIES_BUCKET, + Key=s3_object, + ) + if "aarch64" in platform: + self._upload_redirect( + f"{self.package_name}-{self.version_str}-{platform.replace('aarch64', 'arm64')}.tar.gz", + f"/{s3_object}", + ) + + def _upload_latest_redirect(self, platform: str) -> None: + """Uploads an S3 object for the latest version redirect.""" + self._upload_redirect( + f"{self.package_name}-latest-{platform}.tar.gz", + f"/{self.package_name}-{self.version_str}-{platform}.tar.gz", + ) + if "aarch64" in platform: + self._upload_latest_redirect(platform.replace("aarch64", "arm64")) + + def deploy_tarball(self, platform: str, binary_path: Path) -> None: + """Creates and uploads a tarball containing the binary.""" + tar_path = Path(f"{self.package_name}-{platform}.tar.gz") + with tarfile.open(str(tar_path), "x:gz") as f: + f.addfile(self._create_tardir("mz/bin")) + f.add( + str(binary_path), + arcname=f"mz/bin/{binary_path.name}", + filter=self._sanitize_tarinfo, + ) + + size = humanize.naturalsize(os.lstat(tar_path).st_size) + print(f"Tarball size: {size}") + + self._upload_tarball(tar_path, platform) + if is_latest_version(self.version): + self._upload_latest_redirect(platform) + + +def is_latest_version(version: TypedVersionBase) -> bool: + latest_version = max( + t + for t in git.get_version_tags(version_type=type(version)) + if t.prerelease is None + ) + return version == latest_version diff --git a/misc/python/materialize/mz_version.py b/misc/python/materialize/mz_version.py index 86aff21e9d39e..7ebc7462b65d5 100644 --- a/misc/python/materialize/mz_version.py +++ b/misc/python/materialize/mz_version.py @@ -131,3 +131,11 @@ class MzLspServerVersion(TypedVersionBase): @classmethod def get_prefix(cls) -> str: return "mz-lsp-server-v" + + +class MzDebugVersion(TypedVersionBase): + """Version of Materialize Debug Tool""" + + @classmethod + def get_prefix(cls) -> str: + return "mz-debug-v" From 6351f59108d2d37206092115786718932353b4eb Mon Sep 17 00:00:00 2001 From: Sang Jun Bak Date: Tue, 8 Apr 2025 13:37:31 -0400 Subject: [PATCH 2/5] Refactor deployment utilities into a centralized tarball uploader Fix lint errors Fix lint errors --- ci/deploy_mz-debug/README.md | 2 +- ci/deploy_mz-debug/deploy_util.py | 2 +- ci/deploy_mz-debug/macos.py | 1 - ci/deploy_mz/deploy_util.py | 84 ----------------------- ci/deploy_mz/docker.py | 4 +- ci/deploy_mz/linux.py | 7 +- ci/deploy_mz/macos.py | 7 +- ci/deploy_mz/version.py | 4 +- ci/deploy_mz_lsp_server/deploy_util.py | 95 -------------------------- ci/deploy_mz_lsp_server/linux.py | 7 +- ci/deploy_mz_lsp_server/macos.py | 7 +- ci/deploy_mz_lsp_server/version.py | 4 +- ci/tarball_uploader.py | 1 + 13 files changed, 35 insertions(+), 190 deletions(-) diff --git a/ci/deploy_mz-debug/README.md b/ci/deploy_mz-debug/README.md index ac0ed07340081..b214625e67fd8 100644 --- a/ci/deploy_mz-debug/README.md +++ b/ci/deploy_mz-debug/README.md @@ -17,4 +17,4 @@ bin/pyactivate -m ci.deploy_mz-debug.linux **Important Notes:** - Update the version for `mz-debug`'s `Cargo.toml` to match the `BUILDKITE_TAG` version before deploying -- When running on macOS, modify `linux.py` to use `target` instead of `target-xcompile` \ No newline at end of file +- When running on macOS, modify `linux.py` to use `target` instead of `target-xcompile` diff --git a/ci/deploy_mz-debug/deploy_util.py b/ci/deploy_mz-debug/deploy_util.py index caa7eba8c8581..5fd6dcc2bbda2 100644 --- a/ci/deploy_mz-debug/deploy_util.py +++ b/ci/deploy_mz-debug/deploy_util.py @@ -13,4 +13,4 @@ TAG = os.environ["BUILDKITE_TAG"] MZ_DEBUG_VERSION = MzDebugVersion.parse(TAG) -MZ_DEBUG_VERSION_STR = f"v{MZ_DEBUG_VERSION.str_without_prefix()}" \ No newline at end of file +MZ_DEBUG_VERSION_STR = f"v{MZ_DEBUG_VERSION.str_without_prefix()}" diff --git a/ci/deploy_mz-debug/macos.py b/ci/deploy_mz-debug/macos.py index 04ad9d6fb53c4..b3def2cc1b9ae 100644 --- a/ci/deploy_mz-debug/macos.py +++ b/ci/deploy_mz-debug/macos.py @@ -12,7 +12,6 @@ from ci.tarball_uploader import TarballUploader from materialize import spawn -from materialize.mz_version import MzDebugVersion from materialize.xcompile import Arch from ..deploy.deploy_util import rust_version diff --git a/ci/deploy_mz/deploy_util.py b/ci/deploy_mz/deploy_util.py index a820b5d511e73..11cc08525f9f1 100644 --- a/ci/deploy_mz/deploy_util.py +++ b/ci/deploy_mz/deploy_util.py @@ -8,93 +8,9 @@ # by the Apache License, Version 2.0. import os -import tarfile -import tempfile -import time -from pathlib import Path -import boto3 -import humanize - -from materialize import git from materialize.mz_version import MzCliVersion APT_BUCKET = "materialize-apt" -BINARIES_BUCKET = "materialize-binaries" TAG = os.environ["BUILDKITE_TAG"] MZ_CLI_VERSION = MzCliVersion.parse(TAG) - - -def _tardir(name: str) -> tarfile.TarInfo: - d = tarfile.TarInfo(name) - d.mtime = int(time.time()) - d.mode = 0o40755 - d.type = tarfile.DIRTYPE - return d - - -def _sanitize_tarinfo(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo: - tarinfo.uid = tarinfo.gid = 0 - tarinfo.uname = tarinfo.gname = "root" - return tarinfo - - -def upload_tarball(tarball: Path, platform: str, version: str) -> None: - s3_object = f"mz-{version}-{platform}.tar.gz" - boto3.client("s3").upload_file( - Filename=str(tarball), - Bucket=BINARIES_BUCKET, - Key=s3_object, - ) - if "aarch64" in platform: - upload_redirect( - f"mz-{version}-{platform.replace('aarch64', 'arm64')}.tar.gz", - f"/{s3_object}", - ) - - -def upload_redirect(key: str, to: str) -> None: - with tempfile.NamedTemporaryFile() as empty: - boto3.client("s3").upload_fileobj( - Fileobj=empty, - Bucket=BINARIES_BUCKET, - Key=key, - ExtraArgs={"WebsiteRedirectLocation": to}, - ) - - -def upload_latest_redirect(platform: str, version: str) -> None: - upload_redirect( - f"mz-latest-{platform}.tar.gz", - f"/mz-{version}-{platform}.tar.gz", - ) - if "aarch64" in platform: - upload_latest_redirect(platform.replace("aarch64", "arm64"), version) - - -def deploy_tarball(platform: str, mz: Path) -> None: - tar_path = Path(f"mz-{platform}.tar.gz") - with tarfile.open(str(tar_path), "x:gz") as f: - f.addfile(_tardir("mz")) - f.addfile(_tardir("mz/bin")) - f.add( - str(mz), - arcname="mz/bin/mz", - filter=_sanitize_tarinfo, - ) - - size = humanize.naturalsize(os.lstat(tar_path).st_size) - print(f"Tarball size: {size}") - - upload_tarball(tar_path, platform, f"v{MZ_CLI_VERSION.str_without_prefix()}") - if is_latest_version(): - upload_latest_redirect(platform, f"v{MZ_CLI_VERSION.str_without_prefix()}") - - -def is_latest_version() -> bool: - latest_version = max( - t - for t in git.get_version_tags(version_type=MzCliVersion) - if t.prerelease is None - ) - return MZ_CLI_VERSION == latest_version diff --git a/ci/deploy_mz/docker.py b/ci/deploy_mz/docker.py index 4bc6caaaad81b..a3a6e7dc34837 100644 --- a/ci/deploy_mz/docker.py +++ b/ci/deploy_mz/docker.py @@ -10,11 +10,11 @@ import os from pathlib import Path +from ci import tarball_uploader from materialize import mzbuild, ui from materialize.rustc_flags import Sanitizer from materialize.xcompile import Arch -from . import deploy_util from .deploy_util import MZ_CLI_VERSION @@ -45,7 +45,7 @@ def main() -> None: deps = [[repo.resolve_dependencies([repo.images["mz"]])["mz"]] for repo in repos] mzbuild.publish_multiarch_images(f"v{MZ_CLI_VERSION.str_without_prefix()}", deps) - if deploy_util.is_latest_version(): + if tarball_uploader.is_latest_version(MZ_CLI_VERSION): mzbuild.publish_multiarch_images("latest", deps) diff --git a/ci/deploy_mz/linux.py b/ci/deploy_mz/linux.py index 797d6bc5de045..cc51da760a1ea 100644 --- a/ci/deploy_mz/linux.py +++ b/ci/deploy_mz/linux.py @@ -10,6 +10,7 @@ import os from pathlib import Path +from ci import tarball_uploader from materialize import mzbuild, spawn, ui from materialize.mz_version import MzCliVersion from materialize.rustc_flags import Sanitizer @@ -61,7 +62,11 @@ def main() -> None: mzbuild.chmod_x(mz) print(f"--- Uploading {target} binary tarball") - deploy_util.deploy_tarball(target, mz) + uploader = tarball_uploader.TarballUploader( + package_name="mz", + version=deploy_util.MZ_CLI_VERSION, + ) + uploader.deploy_tarball(target, mz) print("--- Publishing Debian package") filename = f"mz_{MZ_CLI_VERSION.str_without_prefix()}_{repo.rd.arch.go_str()}.deb" diff --git a/ci/deploy_mz/macos.py b/ci/deploy_mz/macos.py index 56f9b3302f8bf..96ab53f6ec4bd 100644 --- a/ci/deploy_mz/macos.py +++ b/ci/deploy_mz/macos.py @@ -10,6 +10,7 @@ import os from pathlib import Path +from ci import tarball_uploader from materialize import spawn from materialize.xcompile import Arch @@ -27,7 +28,11 @@ def main() -> None: ) print(f"--- Uploading {target} binary tarball") - deploy_util.deploy_tarball(target, Path("target") / "release" / "mz") + uploader = tarball_uploader.TarballUploader( + package_name="mz", + version=deploy_util.MZ_CLI_VERSION, + ) + uploader.deploy_tarball(target, Path("target") / "release" / "mz") if __name__ == "__main__": diff --git a/ci/deploy_mz/version.py b/ci/deploy_mz/version.py index 59c68da7ef059..3175bbeba9918 100644 --- a/ci/deploy_mz/version.py +++ b/ci/deploy_mz/version.py @@ -9,7 +9,9 @@ import boto3 -from .deploy_util import BINARIES_BUCKET, MZ_CLI_VERSION +from ci.tarball_uploader import BINARIES_BUCKET + +from .deploy_util import MZ_CLI_VERSION def main() -> None: diff --git a/ci/deploy_mz_lsp_server/deploy_util.py b/ci/deploy_mz_lsp_server/deploy_util.py index 454e6fe55d753..2a591075d277a 100644 --- a/ci/deploy_mz_lsp_server/deploy_util.py +++ b/ci/deploy_mz_lsp_server/deploy_util.py @@ -8,103 +8,8 @@ # by the Apache License, Version 2.0. import os -import tarfile -import tempfile -import time -from pathlib import Path -import boto3 -import humanize - -from materialize import git from materialize.mz_version import MzLspServerVersion -BINARIES_BUCKET = "materialize-binaries" TAG = os.environ["BUILDKITE_TAG"] MZ_LSP_SERVER_VERSION = MzLspServerVersion.parse(TAG) - - -def _tardir(name: str) -> tarfile.TarInfo: - """Takes a dir path and creates a TarInfo. It sets the - modification time, mode, and type of the dir.""" - - d = tarfile.TarInfo(name) - d.mtime = int(time.time()) - d.mode = 0o40755 - d.type = tarfile.DIRTYPE - return d - - -def _sanitize_tarinfo(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo: - tarinfo.uid = tarinfo.gid = 0 - tarinfo.uname = tarinfo.gname = "root" - return tarinfo - - -def upload_tarball(tarball: Path, platform: str, version: str) -> None: - """Uploads the tarball (.tar) to the S3 binaries bucket.""" - s3_object = f"mz-lsp-server-{version}-{platform}.tar.gz" - boto3.client("s3").upload_file( - Filename=str(tarball), - Bucket=BINARIES_BUCKET, - Key=s3_object, - ) - if "aarch64" in platform: - upload_redirect( - f"mz-lsp-server-{version}-{platform.replace('aarch64', 'arm64')}.tar.gz", - f"/{s3_object}", - ) - - -def upload_redirect(key: str, to: str) -> None: - with tempfile.NamedTemporaryFile() as empty: - boto3.client("s3").upload_fileobj( - Fileobj=empty, - Bucket=BINARIES_BUCKET, - Key=key, - ExtraArgs={"WebsiteRedirectLocation": to}, - ) - - -def upload_latest_redirect(platform: str, version: str) -> None: - """Uploads an S3 object containing the latest version number of the package, - not the package itself. This is useful for determining the - latest available version number of the package. - As an example, mz (CLI) uses this information to check if there is a new update available. - """ - upload_redirect( - f"mz-lsp-server-latest-{platform}.tar.gz", - f"/mz-lsp-server-{version}-{platform}.tar.gz", - ) - if "aarch64" in platform: - upload_latest_redirect(platform.replace("aarch64", "arm64"), version) - - -def deploy_tarball(platform: str, lsp: Path) -> None: - tar_path = Path(f"mz-lsp-server-{platform}.tar.gz") - with tarfile.open(str(tar_path), "x:gz") as f: - f.addfile(_tardir("mz-lsp-server")) - f.addfile(_tardir("mz/bin")) - f.add( - str(lsp), - arcname="mz/bin/mz-lsp-server", - filter=_sanitize_tarinfo, - ) - - size = humanize.naturalsize(os.lstat(tar_path).st_size) - print(f"Tarball size: {size}") - - upload_tarball(tar_path, platform, f"v{MZ_LSP_SERVER_VERSION.str_without_prefix()}") - if is_latest_version(): - upload_latest_redirect( - platform, f"v{MZ_LSP_SERVER_VERSION.str_without_prefix()}" - ) - - -def is_latest_version() -> bool: - latest_version = max( - t - for t in git.get_version_tags(version_type=MzLspServerVersion) - if t.prerelease is None - ) - return MZ_LSP_SERVER_VERSION == latest_version diff --git a/ci/deploy_mz_lsp_server/linux.py b/ci/deploy_mz_lsp_server/linux.py index 08f3e43af7eb6..26d70fad33753 100644 --- a/ci/deploy_mz_lsp_server/linux.py +++ b/ci/deploy_mz_lsp_server/linux.py @@ -10,6 +10,7 @@ import os from pathlib import Path +from ci import tarball_uploader from ci.deploy.deploy_util import rust_version from materialize import mzbuild, spawn, ui from materialize.mz_version import MzLspServerVersion @@ -51,7 +52,11 @@ def main() -> None: mzbuild.chmod_x(path) print(f"--- Uploading {target} binary tarball") - deploy_util.deploy_tarball(target, path) + uploader = tarball_uploader.TarballUploader( + package_name="mz-lsp-server", + version=deploy_util.MZ_LSP_SERVER_VERSION, + ) + uploader.deploy_tarball(target, path) if __name__ == "__main__": diff --git a/ci/deploy_mz_lsp_server/macos.py b/ci/deploy_mz_lsp_server/macos.py index 860ad660299af..80fef7ac0eff3 100644 --- a/ci/deploy_mz_lsp_server/macos.py +++ b/ci/deploy_mz_lsp_server/macos.py @@ -10,6 +10,7 @@ import os from pathlib import Path +from ci import tarball_uploader from materialize import spawn from materialize.xcompile import Arch @@ -27,7 +28,11 @@ def main() -> None: ) print(f"--- Uploading {target} binary tarball") - deploy_util.deploy_tarball(target, Path("target") / "release" / "mz-lsp-server") + uploader = tarball_uploader.TarballUploader( + package_name="mz-lsp-server", + version=deploy_util.MZ_LSP_SERVER_VERSION, + ) + uploader.deploy_tarball(target, Path("target") / "release" / "mz-lsp-server") if __name__ == "__main__": diff --git a/ci/deploy_mz_lsp_server/version.py b/ci/deploy_mz_lsp_server/version.py index d0e894cbdb539..e046be7b357fd 100644 --- a/ci/deploy_mz_lsp_server/version.py +++ b/ci/deploy_mz_lsp_server/version.py @@ -9,7 +9,9 @@ import boto3 -from .deploy_util import BINARIES_BUCKET, MZ_LSP_SERVER_VERSION +from ci.tarball_uploader import BINARIES_BUCKET + +from .deploy_util import MZ_LSP_SERVER_VERSION def main() -> None: diff --git a/ci/tarball_uploader.py b/ci/tarball_uploader.py index fedebaeae77d4..745e8f93d378f 100644 --- a/ci/tarball_uploader.py +++ b/ci/tarball_uploader.py @@ -23,6 +23,7 @@ BINARIES_BUCKET = "materialize-binaries" + class TarballUploader: def __init__(self, package_name: str, version: TypedVersionBase): self.package_name = package_name From be228f3e7010dac8d4a6ed56c75e4017361ad81d Mon Sep 17 00:00:00 2001 From: Sang Jun Bak Date: Tue, 8 Apr 2025 22:37:43 -0400 Subject: [PATCH 3/5] Implement tests for debug tool - Implements cloud test and mzcompose test that checks if the tool generates a zip, which is a sign of success. --- ci/deploy_mz-debug/README.md | 2 +- ci/plugins/cloudtest/hooks/pre-exit | 5 ++ ci/plugins/mzcompose/hooks/pre-exit | 5 ++ src/mz-debug/src/kubectl_port_forwarder.rs | 4 +- test/cloudtest/test_mz_debug_tool.py | 64 +++++++++++++++++++ test/mz-debug/mzcompose | 14 ++++ test/mz-debug/mzcompose.py | 74 ++++++++++++++++++++++ 7 files changed, 166 insertions(+), 2 deletions(-) create mode 100644 test/cloudtest/test_mz_debug_tool.py create mode 100755 test/mz-debug/mzcompose create mode 100644 test/mz-debug/mzcompose.py diff --git a/ci/deploy_mz-debug/README.md b/ci/deploy_mz-debug/README.md index b214625e67fd8..5174aa365096c 100644 --- a/ci/deploy_mz-debug/README.md +++ b/ci/deploy_mz-debug/README.md @@ -1,6 +1,6 @@ # Deploy the Materialize debug tool. -The CI process will build and deploy the LSP server to the materialize-binaries S3 bucket. +The CI process will build and deploy the Materialize debug tool to the materialize-binaries S3 bucket. You can try the process by running the following commands: ```bash diff --git a/ci/plugins/cloudtest/hooks/pre-exit b/ci/plugins/cloudtest/hooks/pre-exit index 6eeeb26e67664..ab4b27fa0ef85 100755 --- a/ci/plugins/cloudtest/hooks/pre-exit +++ b/ci/plugins/cloudtest/hooks/pre-exit @@ -63,6 +63,11 @@ if [ -n "${CI_COVERAGE_ENABLED:-}" ]; then fi fi +ci_unimportant_heading "cloudtest: Cleaning up mz_debug files from test/cloudtest/test_mz_debug_tool.py" +if find mz_debug*; then + rm -rf mz_debug* +fi + ci_unimportant_heading "kail: Stopping instance..." docker logs kail > kail-output.log 2>&1 docker stop kail diff --git a/ci/plugins/mzcompose/hooks/pre-exit b/ci/plugins/mzcompose/hooks/pre-exit index 508ad26c9ae74..185d594a76d02 100755 --- a/ci/plugins/mzcompose/hooks/pre-exit +++ b/ci/plugins/mzcompose/hooks/pre-exit @@ -69,6 +69,11 @@ done # can be huge, clean up rm -rf cores +echo "Removing mz-debug files" +if find mz_debug*; then + rm -rf mz_debug* +fi + echo "Compressing parallel-workload-queries.log" bin/ci-builder run stable zstd --rm parallel-workload-queries.log || true diff --git a/src/mz-debug/src/kubectl_port_forwarder.rs b/src/mz-debug/src/kubectl_port_forwarder.rs index 5379fe6d972aa..4abd1a394efb1 100644 --- a/src/mz-debug/src/kubectl_port_forwarder.rs +++ b/src/mz-debug/src/kubectl_port_forwarder.rs @@ -167,7 +167,9 @@ pub async fn create_kubectl_port_forwarder( } } - Err(anyhow::anyhow!("No SQL port forwarding info found")) + Err(anyhow::anyhow!( + "No SQL port forwarding info found. Set --auto-port-forward to false and point --mz-connection-url to a Materialize instance." + )) } pub fn create_mz_connection_url( diff --git a/test/cloudtest/test_mz_debug_tool.py b/test/cloudtest/test_mz_debug_tool.py new file mode 100644 index 0000000000000..6f5dc151c23f9 --- /dev/null +++ b/test/cloudtest/test_mz_debug_tool.py @@ -0,0 +1,64 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +import glob +import subprocess + +from materialize import MZ_ROOT, spawn +from materialize.cloudtest import DEFAULT_K8S_CONTEXT_NAME, DEFAULT_K8S_NAMESPACE +from materialize.cloudtest.app.materialize_application import MaterializeApplication +from materialize.cloudtest.util.wait import wait + + +def test_successful_zip_creation(mz: MaterializeApplication) -> None: + # Wait until the Materialize instance is ready + wait( + condition="condition=Ready", + resource="pod", + label="cluster.environmentd.materialize.cloud/cluster-id=u1", + ) + + print("-- Port forwarding the internal SQL port") + subprocess.Popen( + [ + "kubectl", + "--context", + DEFAULT_K8S_CONTEXT_NAME, + "port-forward", + "pods/environmentd-0", + "6877:6877", + ] + ) + + print("-- Running mz-debug") + spawn.runv( + [ + "cargo", + "run", + "--bin", + "mz-debug", + "--", + "self-managed", + "--k8s-context", + DEFAULT_K8S_CONTEXT_NAME, + "--k8s-namespace", + DEFAULT_K8S_NAMESPACE, + # We need to disable auto-port-forwarding because auto-portforward expects a balancerd service we're not running the + # tool against our helm chart installation + "--auto-port-forward", + "false", + "--mz-connection-url", + "postgresql://mz_system@localhost:6877/materialize", + ], + cwd=MZ_ROOT, + ) + + print("-- Looking for mz-debug zip files") + zip_files = glob.glob(str(MZ_ROOT / "mz_debug*.zip")) + assert len(zip_files) > 0, "No mz-debug zip file was created" diff --git a/test/mz-debug/mzcompose b/test/mz-debug/mzcompose new file mode 100755 index 0000000000000..1f866645dabc8 --- /dev/null +++ b/test/mz-debug/mzcompose @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. +# +# mzcompose — runs Docker Compose with Materialize customizations. + +exec "$(dirname "$0")"/../../bin/pyactivate -m materialize.cli.mzcompose "$@" diff --git a/test/mz-debug/mzcompose.py b/test/mz-debug/mzcompose.py new file mode 100644 index 0000000000000..3c185983be573 --- /dev/null +++ b/test/mz-debug/mzcompose.py @@ -0,0 +1,74 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +""" +Basic test for mz-debug +""" + +from dataclasses import dataclass + +from materialize import spawn +from materialize.mzcompose.composition import Composition, WorkflowArgumentParser +from materialize.mzcompose.services.materialized import Materialized + +SERVICES = [ + Materialized(), +] + + +@dataclass +class TestCase: + name: str + dbt_env: dict[str, str] + materialized_options: list[str] + materialized_image: str | None = None + + +test_cases = [ + TestCase( + name="no-tls-cloud", + materialized_options=[], + dbt_env={}, + ), +] + + +def basic_test(c: Composition) -> None: + materialized = Materialized( + ports=[ + "6875:6875", + "6877:6877", + ] + ) + + with c.override(materialized): + c.down() + c.up("materialized") + + container_id = c.container_id("materialized") + if container_id is None: + raise ValueError("Failed to get materialized container ID") + + spawn.runv( + [ + "cargo", + "run", + "--bin", + "mz-debug", + "emulator", + "--docker-container-id", + container_id, + "--mz-connection-url", + "postgres://mz_system@localhost:6877/materialize", + ] + ) + + +def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: + basic_test(c) From bfd5dcd118bc8047e6d0b10d8afb43d1647a98ce Mon Sep 17 00:00:00 2001 From: Sang Jun Bak Date: Wed, 9 Apr 2025 13:25:07 -0400 Subject: [PATCH 4/5] Add pipeline to buildkite --- .../buildkite_insights/buildkite_api/buildkite_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/python/materialize/buildkite_insights/buildkite_api/buildkite_config.py b/misc/python/materialize/buildkite_insights/buildkite_api/buildkite_config.py index 0ff7de682021d..18e6200dabaac 100755 --- a/misc/python/materialize/buildkite_insights/buildkite_api/buildkite_config.py +++ b/misc/python/materialize/buildkite_insights/buildkite_api/buildkite_config.py @@ -14,6 +14,7 @@ "deploy", "deploy-mz-lsp-server", "deploy-mz", + "deploy-mz-debug", "deploy-website", "license", "nightly", From 829447a861bb7dd04ec27a0f3b2b1cffd2a9b7ff Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Thu, 10 Apr 2025 00:15:58 +0000 Subject: [PATCH 5/5] ci: Run mz-debug test --- ci/test/pipeline.template.yml | 10 ++++++++++ test/mz-debug/mzcompose.py | 6 +----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ci/test/pipeline.template.yml b/ci/test/pipeline.template.yml index bdc3821e65715..222469ad984fc 100644 --- a/ci/test/pipeline.template.yml +++ b/ci/test/pipeline.template.yml @@ -759,6 +759,16 @@ steps: queue: hetzner-aarch64-4cpu-8gb skip: "Version upgrade skips are allowed for LTS releases now" + - id: mz-debug + label: "mz-debug tool" + depends_on: build-aarch64 + timeout_in_minutes: 30 + plugins: + - ./ci/plugins/mzcompose: + composition: mz-debug + agents: + queue: hetzner-aarch64-4cpu-8gb + - id: deploy-website label: Deploy website depends_on: lint-docs diff --git a/test/mz-debug/mzcompose.py b/test/mz-debug/mzcompose.py index 3c185983be573..9680e861e9905 100644 --- a/test/mz-debug/mzcompose.py +++ b/test/mz-debug/mzcompose.py @@ -39,7 +39,7 @@ class TestCase: ] -def basic_test(c: Composition) -> None: +def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: materialized = Materialized( ports=[ "6875:6875", @@ -68,7 +68,3 @@ def basic_test(c: Composition) -> None: "postgres://mz_system@localhost:6877/materialize", ] ) - - -def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: - basic_test(c)