WIP

Author: SangJunBak

Diff for: Cargo.lock

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+#
+# mz-debug -- build and run the Materialize debug tool.
+
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+
+exec cargo run --bin mz-debug -- "$@"

Diff for: bin/mz-debug

@@ -0,0 +1,15 @@
+# Deploy Materialize Language Server Protocol (LSP) Server.
+
+The CI process will build and deploy the LSP server to the binaries S3 bucket.
+You can try the process by running the following commands:
+
+```bash
+# Set a tag version.
+export BUILDKITE_TAG=mz-debug-vx.y.z
+
+# macOS
+bin/pyactivate -m ci.deploy_mz_debug.macos
+
+# Linux
+bin/pyactivate -m ci.deploy_mz_debug.linux
+```

Diff for: ci/deploy_mz_debug/README.md

@@ -0,0 +1,109 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+import os
+import tarfile
+import tempfile
+import time
+from pathlib import Path
+
+import boto3
+import humanize
+
+from materialize import git
+from materialize.mz_version import MzDebugVersion
+
+BINARIES_BUCKET = "materialize-binaries"
+TAG = os.environ["BUILDKITE_TAG"]
+MZ_DEBUG_VERSION = MzDebugVersion.parse(TAG)
+
+
+def _tardir(name: str) -> tarfile.TarInfo:
+    """Takes a dir path and creates a TarInfo. It sets the
+    modification time, mode, and type of the dir."""
+
+    d = tarfile.TarInfo(name)
+    d.mtime = int(time.time())
+    d.mode = 0o40755
+    d.type = tarfile.DIRTYPE
+    return d
+
+
+def _sanitize_tarinfo(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo:
+    tarinfo.uid = tarinfo.gid = 0
+    tarinfo.uname = tarinfo.gname = "root"
+    return tarinfo
+
+
+def upload_tarball(tarball: Path, platform: str, version: str) -> None:
+    """Uploads the tarball (.tar) to the S3 binaries bucket."""
+    s3_object = f"mz-debug-{version}-{platform}.tar.gz"
+    boto3.client("s3").upload_file(
+        Filename=str(tarball),
+        Bucket=BINARIES_BUCKET,
+        Key=s3_object,
+    )
+    if "aarch64" in platform:
+        upload_redirect(
+            f"mz-debug-{version}-{platform.replace('aarch64', 'arm64')}.tar.gz",
+            f"/{s3_object}",
+        )
+
+
+def upload_redirect(key: str, to: str) -> None:
+    with tempfile.NamedTemporaryFile() as empty:
+        boto3.client("s3").upload_fileobj(
+            Fileobj=empty,
+            Bucket=BINARIES_BUCKET,
+            Key=key,
+            ExtraArgs={"WebsiteRedirectLocation": to},
+        )
+
+
+def upload_latest_redirect(platform: str, version: str) -> None:
+    """Uploads an S3 object containing the latest version number of the package,
+    not the package itself. This is useful for determining the
+    latest available version number of the package.
+    As an example, mz (CLI) uses this information to check if there is a new update available.
+    """
+    upload_redirect(
+        f"mz-debug-latest-{platform}.tar.gz",
+        f"/mz-debug-{version}-{platform}.tar.gz",
+    )
+    if "aarch64" in platform:
+        upload_latest_redirect(platform.replace("aarch64", "arm64"), version)
+
+
+def deploy_tarball(platform: str, lsp: Path) -> None:
+    tar_path = Path(f"mz-debug-{platform}.tar.gz")
+    with tarfile.open(str(tar_path), "x:gz") as f:
+        f.addfile(_tardir("mz/bin"))
+        f.add(
+            str(lsp),
+            arcname="mz/bin/mz-debug",
+            filter=_sanitize_tarinfo,
+        )
+
+    size = humanize.naturalsize(os.lstat(tar_path).st_size)
+    print(f"Tarball size: {size}")
+
+    upload_tarball(tar_path, platform, f"v{MZ_DEBUG_VERSION.str_without_prefix()}")
+    if is_latest_version():
+        upload_latest_redirect(
+            platform, f"v{MZ_DEBUG_VERSION.str_without_prefix()}"
+        )
+
+
+def is_latest_version() -> bool:
+    latest_version = max(
+        t
+        for t in git.get_version_tags(version_type=MzDebugVersion)
+        if t.prerelease is None
+    )
+    return MZ_DEBUG_VERSION == latest_version

Diff for: ci/deploy_mz_debug/__init__.py

@@ -0,0 +1,58 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+import os
+from pathlib import Path
+
+from ci.deploy.deploy_util import rust_version
+from materialize import mzbuild, spawn, ui
+from materialize.mz_version import MzDebugVersion
+from materialize.rustc_flags import Sanitizer
+
+from . import deploy_util
+from .deploy_util import MZ_DEBUG_VERSION
+
+
+def main() -> None:
+    bazel = ui.env_is_truthy("CI_BAZEL_BUILD")
+    bazel_remote_cache = os.getenv("CI_BAZEL_REMOTE_CACHE")
+
+    repo = mzbuild.Repository(
+        Path("."),
+        coverage=False,
+        sanitizer=Sanitizer.none,
+        bazel=bazel,
+        bazel_remote_cache=bazel_remote_cache,
+    )
+    target = f"{repo.rd.arch}-unknown-linux-gnu"
+
+    print("--- Checking version")
+    assert (
+        MzDebugVersion.parse_without_prefix(
+            repo.rd.cargo_workspace.crates["mz-debug"].version_string
+        )
+        == MZ_DEBUG_VERSION
+    )
+
+    print("--- Building mz-debug")
+    # The bin/ci-builder uses target-xcompile as the volume and
+    # is where the binary release will be available.
+    path = Path("target-xcompile") / "release" / "mz-debug"
+    spawn.runv(
+        ["cargo", "build", "--bin", "mz-debug", "--release"],
+        env=dict(os.environ, RUSTUP_TOOLCHAIN=rust_version()),
+    )
+    mzbuild.chmod_x(path)
+
+    print(f"--- Uploading {target} binary tarball")
+    deploy_util.deploy_tarball(target, path)
+
+
+if __name__ == "__main__":
+    main()

Diff for: ci/deploy_mz_debug/deploy_util.py

@@ -0,0 +1,34 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+import os
+from pathlib import Path
+
+from materialize import spawn
+from materialize.xcompile import Arch
+
+from ..deploy.deploy_util import rust_version
+from . import deploy_util
+
+
+def main() -> None:
+    target = f"{Arch.host()}-apple-darwin"
+
+    print("--- Building mz-debug")
+    spawn.runv(
+        ["cargo", "build", "--bin", "mz-debug", "--release"],
+        env=dict(os.environ, RUSTUP_TOOLCHAIN=rust_version()),
+    )
+
+    print(f"--- Uploading {target} binary tarball")
+    deploy_util.deploy_tarball(target, Path("target") / "release" / "mz-debug")
+
+
+if __name__ == "__main__":
+    main()

Diff for: ci/deploy_mz_debug/linux.py

@@ -0,0 +1,62 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+# Deploys are fast, do them quickly
+priority: 30
+
+steps:
+  - command: bin/ci-builder run stable bin/pyactivate -m ci.deploy_mz-debug.version
+    timeout_in_minutes: 30
+    concurrency: 1
+    concurrency_group: deploy-mz-debug/version
+    retry:
+      manual:
+        permit_on_passed: true
+
+  - id: linux-x86_64
+    command: bin/ci-builder run stable bin/pyactivate -m ci.deploy_mz-debug.linux
+    timeout_in_minutes: 30
+    agents:
+      queue: linux-x86_64-small
+    concurrency: 1
+    concurrency_group: deploy-mz-debug/linux/x86_64
+    retry:
+      manual:
+        permit_on_passed: true
+
+  - id: linux-aarch64
+    command: bin/ci-builder run stable bin/pyactivate -m ci.deploy_mz-debug.linux
+    timeout_in_minutes: 30
+    agents:
+      queue: linux-aarch64-small
+    concurrency: 1
+    concurrency_group: deploy-mz-debug/linux/aarch64
+    retry:
+      manual:
+        permit_on_passed: true
+
+  - command: bin/pyactivate -m ci.deploy_mz-debug.macos
+    agents:
+      queue: mac-x86_64
+    timeout_in_minutes: 30
+    concurrency: 1
+    concurrency_group: deploy-mz-debug/macos/x86_64
+    retry:
+      manual:
+        permit_on_passed: true
+
+  - command: bin/pyactivate -m ci.deploy_mz-debug.macos
+    agents:
+      queue: mac-aarch64
+    timeout_in_minutes: 30
+    concurrency: 1
+    concurrency_group: deploy-mz-debug/macos/aarch64
+    retry:
+      manual:
+        permit_on_passed: true

Diff for: ci/deploy_mz_debug/macos.py

@@ -0,0 +1,26 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+import boto3
+
+from .deploy_util import BINARIES_BUCKET, MZ_DEBUG_VERSION
+
+
+def main() -> None:
+    print("--- Uploading version file")
+    boto3.client("s3").put_object(
+        Body=f"{MZ_DEBUG_VERSION.str_without_prefix()}",
+        Bucket=BINARIES_BUCKET,
+        Key="mz-debug-latest.version",
+        ContentType="text/plain",
+    )
+
+
+if __name__ == "__main__":
+    main()

Diff for: ci/deploy_mz_debug/pipeline.template.yml

@@ -131,3 +131,11 @@ class MzLspServerVersion(TypedVersionBase):
     @classmethod
     def get_prefix(cls) -> str:
         return "mz-lsp-server-v"
+
+
+class MzDebugVersion(TypedVersionBase):
+    """Version of Materialize Debug Tool"""
+
+    @classmethod
+    def get_prefix(cls) -> str:
+        return "mz-debug-v"