-
Notifications
You must be signed in to change notification settings - Fork 36
Add support for S3 bucket as source for anod #669
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,6 +3,7 @@ | |
import hashlib | ||
import json | ||
import os | ||
import re | ||
import tempfile | ||
from contextlib import closing | ||
from typing import TYPE_CHECKING | ||
|
@@ -20,6 +21,8 @@ | |
from collections.abc import Callable | ||
from e3.mypy import assert_never | ||
|
||
SupportedVCS = Literal["git"] | Literal["svn"] | Literal["external"] | Literal["s3"] | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You must use |
||
|
||
logger = e3.log.getLogger("e3.anod.checkout") | ||
|
||
|
||
|
@@ -57,7 +60,7 @@ def __init__(self, name: str, working_dir: str, compute_changelog: bool = True): | |
|
||
def update( | ||
self, | ||
vcs: Literal["git"] | Literal["svn"] | Literal["external"], | ||
vcs: SupportedVCS, | ||
url: str, | ||
revision: str | None = None, | ||
) -> ReturnValue: | ||
|
@@ -84,6 +87,8 @@ def update( | |
update = self.update_svn | ||
elif vcs == "external": | ||
update = self.update_external | ||
elif vcs == "s3": | ||
update = self.update_s3 | ||
else: | ||
assert_never() | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This function takes one argument as parameter. |
||
|
||
|
@@ -102,6 +107,50 @@ def update( | |
) | ||
return result | ||
|
||
def update_s3( | ||
self, url: str, revision: str | None | ||
) -> tuple[ReturnValue, str, str]: # all: no cover | ||
"""Update working dir using a S3 bucket as source. | ||
|
||
:param url: s3 url to a s3 directory. s3://[profile@]bucket/suffix | ||
:param revision: ignored | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why create an ignored parameter? |
||
""" | ||
if os.path.isdir(self.working_dir): | ||
old_commit = get_filetree_state(self.working_dir) | ||
else: | ||
old_commit = "" | ||
|
||
# Extract profile from the url and perform basic | ||
# url validation | ||
m = re.match("s3://([a-zA-Z0-9_-]+@)?(.+)$", url) | ||
grouigrokon marked this conversation as resolved.
Show resolved
Hide resolved
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Maybe should we compile this regex? |
||
if m is None: | ||
raise e3.error.E3Error("invalid s3 url: {url}") | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. missing |
||
|
||
if m.group(1) is not None: | ||
profile = m.group(1)[:-1] | ||
else: | ||
profile = None | ||
|
||
s3_url = f"s3://{m.group(2)}" | ||
|
||
if not which("aws"): | ||
raise e3.error.E3Error("aws cli is needed") | ||
|
||
s3_command = ["aws", "s3", "sync", "--delete"] | ||
if profile is not None: | ||
s3_command += ["--profile", profile] | ||
s3_command += [s3_url, self.working_dir] | ||
|
||
p = Run(s3_command, output=None) | ||
if p.status != 0: | ||
raise e3.error.E3Error(f"aws sync failed with status {p.status}") | ||
|
||
new_commit = get_filetree_state(self.working_dir) | ||
if new_commit == old_commit: | ||
return ReturnValue.unchanged, old_commit, new_commit | ||
else: | ||
return ReturnValue.success, old_commit, new_commit | ||
|
||
def update_external( | ||
self, url: str, revision: str | None | ||
) -> tuple[ReturnValue, str, str]: | ||
|
Uh oh!
There was an error while loading. Please reload this page.