Skip to content

Commit 1e1b601

Browse files
SangJunBakSangJunBak
committed
Refactor deployment utilities into a centralized tarball uploader
Fix lint errors Fix lint errors
1 parent dfcc296 commit 1e1b601

File tree

13 files changed

+35
-190
lines changed

13 files changed

+35
-190
lines changed

Diff for: ‎ci/deploy_mz-debug/README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,4 @@ bin/pyactivate -m ci.deploy_mz-debug.linux
1717
**Important Notes:**
1818

1919
- Update the version for `mz-debug`'s `Cargo.toml` to match the `BUILDKITE_TAG` version before deploying
20-
- When running on macOS, modify `linux.py` to use `target` instead of `target-xcompile`
20+
- When running on macOS, modify `linux.py` to use `target` instead of `target-xcompile`

Diff for: ‎ci/deploy_mz-debug/deploy_util.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,4 @@
1313

1414
TAG = os.environ["BUILDKITE_TAG"]
1515
MZ_DEBUG_VERSION = MzDebugVersion.parse(TAG)
16-
MZ_DEBUG_VERSION_STR = f"v{MZ_DEBUG_VERSION.str_without_prefix()}"
16+
MZ_DEBUG_VERSION_STR = f"v{MZ_DEBUG_VERSION.str_without_prefix()}"

Diff for: ‎ci/deploy_mz-debug/macos.py

-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212

1313
from ci.tarball_uploader import TarballUploader
1414
from materialize import spawn
15-
from materialize.mz_version import MzDebugVersion
1615
from materialize.xcompile import Arch
1716

1817
from ..deploy.deploy_util import rust_version

Diff for: ‎ci/deploy_mz/deploy_util.py

-84
Original file line numberDiff line numberDiff line change
@@ -8,93 +8,9 @@
88
# by the Apache License, Version 2.0.
99

1010
import os
11-
import tarfile
12-
import tempfile
13-
import time
14-
from pathlib import Path
1511

16-
import boto3
17-
import humanize
18-
19-
from materialize import git
2012
from materialize.mz_version import MzCliVersion
2113

2214
APT_BUCKET = "materialize-apt"
23-
BINARIES_BUCKET = "materialize-binaries"
2415
TAG = os.environ["BUILDKITE_TAG"]
2516
MZ_CLI_VERSION = MzCliVersion.parse(TAG)
26-
27-
28-
def _tardir(name: str) -> tarfile.TarInfo:
29-
d = tarfile.TarInfo(name)
30-
d.mtime = int(time.time())
31-
d.mode = 0o40755
32-
d.type = tarfile.DIRTYPE
33-
return d
34-
35-
36-
def _sanitize_tarinfo(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo:
37-
tarinfo.uid = tarinfo.gid = 0
38-
tarinfo.uname = tarinfo.gname = "root"
39-
return tarinfo
40-
41-
42-
def upload_tarball(tarball: Path, platform: str, version: str) -> None:
43-
s3_object = f"mz-{version}-{platform}.tar.gz"
44-
boto3.client("s3").upload_file(
45-
Filename=str(tarball),
46-
Bucket=BINARIES_BUCKET,
47-
Key=s3_object,
48-
)
49-
if "aarch64" in platform:
50-
upload_redirect(
51-
f"mz-{version}-{platform.replace('aarch64', 'arm64')}.tar.gz",
52-
f"/{s3_object}",
53-
)
54-
55-
56-
def upload_redirect(key: str, to: str) -> None:
57-
with tempfile.NamedTemporaryFile() as empty:
58-
boto3.client("s3").upload_fileobj(
59-
Fileobj=empty,
60-
Bucket=BINARIES_BUCKET,
61-
Key=key,
62-
ExtraArgs={"WebsiteRedirectLocation": to},
63-
)
64-
65-
66-
def upload_latest_redirect(platform: str, version: str) -> None:
67-
upload_redirect(
68-
f"mz-latest-{platform}.tar.gz",
69-
f"/mz-{version}-{platform}.tar.gz",
70-
)
71-
if "aarch64" in platform:
72-
upload_latest_redirect(platform.replace("aarch64", "arm64"), version)
73-
74-
75-
def deploy_tarball(platform: str, mz: Path) -> None:
76-
tar_path = Path(f"mz-{platform}.tar.gz")
77-
with tarfile.open(str(tar_path), "x:gz") as f:
78-
f.addfile(_tardir("mz"))
79-
f.addfile(_tardir("mz/bin"))
80-
f.add(
81-
str(mz),
82-
arcname="mz/bin/mz",
83-
filter=_sanitize_tarinfo,
84-
)
85-
86-
size = humanize.naturalsize(os.lstat(tar_path).st_size)
87-
print(f"Tarball size: {size}")
88-
89-
upload_tarball(tar_path, platform, f"v{MZ_CLI_VERSION.str_without_prefix()}")
90-
if is_latest_version():
91-
upload_latest_redirect(platform, f"v{MZ_CLI_VERSION.str_without_prefix()}")
92-
93-
94-
def is_latest_version() -> bool:
95-
latest_version = max(
96-
t
97-
for t in git.get_version_tags(version_type=MzCliVersion)
98-
if t.prerelease is None
99-
)
100-
return MZ_CLI_VERSION == latest_version

Diff for: ‎ci/deploy_mz/docker.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,11 @@
1010
import os
1111
from pathlib import Path
1212

13+
from ci import tarball_uploader
1314
from materialize import mzbuild, ui
1415
from materialize.rustc_flags import Sanitizer
1516
from materialize.xcompile import Arch
1617

17-
from . import deploy_util
1818
from .deploy_util import MZ_CLI_VERSION
1919

2020

@@ -45,7 +45,7 @@ def main() -> None:
4545
deps = [[repo.resolve_dependencies([repo.images["mz"]])["mz"]] for repo in repos]
4646

4747
mzbuild.publish_multiarch_images(f"v{MZ_CLI_VERSION.str_without_prefix()}", deps)
48-
if deploy_util.is_latest_version():
48+
if tarball_uploader.is_latest_version(MZ_CLI_VERSION):
4949
mzbuild.publish_multiarch_images("latest", deps)
5050

5151

Diff for: ‎ci/deploy_mz/linux.py

+6-1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import os
1111
from pathlib import Path
1212

13+
from ci import tarball_uploader
1314
from materialize import mzbuild, spawn, ui
1415
from materialize.mz_version import MzCliVersion
1516
from materialize.rustc_flags import Sanitizer
@@ -61,7 +62,11 @@ def main() -> None:
6162
mzbuild.chmod_x(mz)
6263

6364
print(f"--- Uploading {target} binary tarball")
64-
deploy_util.deploy_tarball(target, mz)
65+
uploader = tarball_uploader.TarballUploader(
66+
package_name="mz",
67+
version=deploy_util.MZ_CLI_VERSION,
68+
)
69+
uploader.deploy_tarball(target, mz)
6570

6671
print("--- Publishing Debian package")
6772
filename = f"mz_{MZ_CLI_VERSION.str_without_prefix()}_{repo.rd.arch.go_str()}.deb"

Diff for: ‎ci/deploy_mz/macos.py

+6-1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import os
1111
from pathlib import Path
1212

13+
from ci import tarball_uploader
1314
from materialize import spawn
1415
from materialize.xcompile import Arch
1516

@@ -27,7 +28,11 @@ def main() -> None:
2728
)
2829

2930
print(f"--- Uploading {target} binary tarball")
30-
deploy_util.deploy_tarball(target, Path("target") / "release" / "mz")
31+
uploader = tarball_uploader.TarballUploader(
32+
package_name="mz",
33+
version=deploy_util.MZ_CLI_VERSION,
34+
)
35+
uploader.deploy_tarball(target, Path("target") / "release" / "mz")
3136

3237

3338
if __name__ == "__main__":

Diff for: ‎ci/deploy_mz/version.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@
99

1010
import boto3
1111

12-
from .deploy_util import BINARIES_BUCKET, MZ_CLI_VERSION
12+
from ci.tarball_uploader import BINARIES_BUCKET
13+
14+
from .deploy_util import MZ_CLI_VERSION
1315

1416

1517
def main() -> None:

Diff for: ‎ci/deploy_mz_lsp_server/deploy_util.py

-95
Original file line numberDiff line numberDiff line change
@@ -8,103 +8,8 @@
88
# by the Apache License, Version 2.0.
99

1010
import os
11-
import tarfile
12-
import tempfile
13-
import time
14-
from pathlib import Path
1511

16-
import boto3
17-
import humanize
18-
19-
from materialize import git
2012
from materialize.mz_version import MzLspServerVersion
2113

22-
BINARIES_BUCKET = "materialize-binaries"
2314
TAG = os.environ["BUILDKITE_TAG"]
2415
MZ_LSP_SERVER_VERSION = MzLspServerVersion.parse(TAG)
25-
26-
27-
def _tardir(name: str) -> tarfile.TarInfo:
28-
"""Takes a dir path and creates a TarInfo. It sets the
29-
modification time, mode, and type of the dir."""
30-
31-
d = tarfile.TarInfo(name)
32-
d.mtime = int(time.time())
33-
d.mode = 0o40755
34-
d.type = tarfile.DIRTYPE
35-
return d
36-
37-
38-
def _sanitize_tarinfo(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo:
39-
tarinfo.uid = tarinfo.gid = 0
40-
tarinfo.uname = tarinfo.gname = "root"
41-
return tarinfo
42-
43-
44-
def upload_tarball(tarball: Path, platform: str, version: str) -> None:
45-
"""Uploads the tarball (.tar) to the S3 binaries bucket."""
46-
s3_object = f"mz-lsp-server-{version}-{platform}.tar.gz"
47-
boto3.client("s3").upload_file(
48-
Filename=str(tarball),
49-
Bucket=BINARIES_BUCKET,
50-
Key=s3_object,
51-
)
52-
if "aarch64" in platform:
53-
upload_redirect(
54-
f"mz-lsp-server-{version}-{platform.replace('aarch64', 'arm64')}.tar.gz",
55-
f"/{s3_object}",
56-
)
57-
58-
59-
def upload_redirect(key: str, to: str) -> None:
60-
with tempfile.NamedTemporaryFile() as empty:
61-
boto3.client("s3").upload_fileobj(
62-
Fileobj=empty,
63-
Bucket=BINARIES_BUCKET,
64-
Key=key,
65-
ExtraArgs={"WebsiteRedirectLocation": to},
66-
)
67-
68-
69-
def upload_latest_redirect(platform: str, version: str) -> None:
70-
"""Uploads an S3 object containing the latest version number of the package,
71-
not the package itself. This is useful for determining the
72-
latest available version number of the package.
73-
As an example, mz (CLI) uses this information to check if there is a new update available.
74-
"""
75-
upload_redirect(
76-
f"mz-lsp-server-latest-{platform}.tar.gz",
77-
f"/mz-lsp-server-{version}-{platform}.tar.gz",
78-
)
79-
if "aarch64" in platform:
80-
upload_latest_redirect(platform.replace("aarch64", "arm64"), version)
81-
82-
83-
def deploy_tarball(platform: str, lsp: Path) -> None:
84-
tar_path = Path(f"mz-lsp-server-{platform}.tar.gz")
85-
with tarfile.open(str(tar_path), "x:gz") as f:
86-
f.addfile(_tardir("mz-lsp-server"))
87-
f.addfile(_tardir("mz/bin"))
88-
f.add(
89-
str(lsp),
90-
arcname="mz/bin/mz-lsp-server",
91-
filter=_sanitize_tarinfo,
92-
)
93-
94-
size = humanize.naturalsize(os.lstat(tar_path).st_size)
95-
print(f"Tarball size: {size}")
96-
97-
upload_tarball(tar_path, platform, f"v{MZ_LSP_SERVER_VERSION.str_without_prefix()}")
98-
if is_latest_version():
99-
upload_latest_redirect(
100-
platform, f"v{MZ_LSP_SERVER_VERSION.str_without_prefix()}"
101-
)
102-
103-
104-
def is_latest_version() -> bool:
105-
latest_version = max(
106-
t
107-
for t in git.get_version_tags(version_type=MzLspServerVersion)
108-
if t.prerelease is None
109-
)
110-
return MZ_LSP_SERVER_VERSION == latest_version

Diff for: ‎ci/deploy_mz_lsp_server/linux.py

+6-1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import os
1111
from pathlib import Path
1212

13+
from ci import tarball_uploader
1314
from ci.deploy.deploy_util import rust_version
1415
from materialize import mzbuild, spawn, ui
1516
from materialize.mz_version import MzLspServerVersion
@@ -51,7 +52,11 @@ def main() -> None:
5152
mzbuild.chmod_x(path)
5253

5354
print(f"--- Uploading {target} binary tarball")
54-
deploy_util.deploy_tarball(target, path)
55+
uploader = tarball_uploader.TarballUploader(
56+
package_name="mz-lsp-server",
57+
version=deploy_util.MZ_LSP_SERVER_VERSION,
58+
)
59+
uploader.deploy_tarball(target, path)
5560

5661

5762
if __name__ == "__main__":

Diff for: ‎ci/deploy_mz_lsp_server/macos.py

+6-1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import os
1111
from pathlib import Path
1212

13+
from ci import tarball_uploader
1314
from materialize import spawn
1415
from materialize.xcompile import Arch
1516

@@ -27,7 +28,11 @@ def main() -> None:
2728
)
2829

2930
print(f"--- Uploading {target} binary tarball")
30-
deploy_util.deploy_tarball(target, Path("target") / "release" / "mz-lsp-server")
31+
uploader = tarball_uploader.TarballUploader(
32+
package_name="mz-lsp-server",
33+
version=deploy_util.MZ_LSP_SERVER_VERSION,
34+
)
35+
uploader.deploy_tarball(target, Path("target") / "release" / "mz-lsp-server")
3136

3237

3338
if __name__ == "__main__":

Diff for: ‎ci/deploy_mz_lsp_server/version.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@
99

1010
import boto3
1111

12-
from .deploy_util import BINARIES_BUCKET, MZ_LSP_SERVER_VERSION
12+
from ci.tarball_uploader import BINARIES_BUCKET
13+
14+
from .deploy_util import MZ_LSP_SERVER_VERSION
1315

1416

1517
def main() -> None:

Diff for: ‎ci/tarball_uploader.py

+1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323

2424
BINARIES_BUCKET = "materialize-binaries"
2525

26+
2627
class TarballUploader:
2728
def __init__(self, package_name: str, version: TypedVersionBase):
2829
self.package_name = package_name

0 commit comments

Comments
 (0)