Skip to content

Commit b366b6d

Browse files
authored
fix(cache-cli): allow custom S3 endpoint to be used (#467)
1 parent c1f0a99 commit b366b6d

File tree

7 files changed

+43
-26
lines changed

7 files changed

+43
-26
lines changed

.github/workflows/test.yml

+3-1
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,16 @@ jobs:
77
- name: Install Go
88
uses: actions/setup-go@v2
99
with:
10-
go-version: 1.20.x
10+
go-version: 1.22.x
1111
- name: Check out repository code
1212
uses: actions/checkout@v2
1313
- name: Install gotestsum
1414
run: go install gotest.tools/gotestsum@latest
1515
- name: Run tests
1616
env:
1717
SEMAPHORE_CACHE_S3_URL: "http://127.0.0.1:9000"
18+
SEMAPHORE_CACHE_S3_KEY: minioadmin
19+
SEMAPHORE_CACHE_S3_SECRET: minioadmin
1820
SEMAPHORE_TOOLBOX_METRICS_ENABLED: "true"
1921
run: |
2022
New-Item C:\minio -ItemType Directory > $null

.semaphore/semaphore.yml

+6-6
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ blocks:
1515
jobs:
1616
- name: Build cache CLI
1717
commands:
18-
- sem-version go 1.20
18+
- sem-version go 1.22
1919
- "export GOPATH=~/go"
2020
- "export PATH=/home/semaphore/go/bin:$PATH"
2121
- checkout
@@ -32,7 +32,7 @@ blocks:
3232
- artifact push workflow bin/windows/cache.exe -d bin/windows/cache.exe
3333
- name: Build sem-context CLI
3434
commands:
35-
- sem-version go 1.20
35+
- sem-version go 1.22
3636
- "export GOPATH=~/go"
3737
- "export PATH=/home/semaphore/go/bin:$PATH"
3838
- checkout
@@ -360,14 +360,14 @@ blocks:
360360
commands:
361361
- bash tests/sem_service/$TEST
362362

363-
- name: "Bats: xcode14"
363+
- name: "Bats: xcode15"
364364
dependencies:
365365
- "Build local CLIs"
366366
task:
367367
agent:
368368
machine:
369369
type: a1-standard-4
370-
os_image: macos-xcode14
370+
os_image: macos-xcode15
371371

372372
prologue:
373373
commands:
@@ -612,7 +612,7 @@ blocks:
612612
- source tests/sftp_server/start_on_linux.sh
613613
- sudo apt-get install -y python3.8-dev
614614
- sem-version python 3.8
615-
- sem-version go 1.13
615+
- sem-version go 1.22
616616
- sem-version php 7.3.23
617617
jobs:
618618
- name: "Cache tests"
@@ -677,7 +677,7 @@ blocks:
677677
value: "on"
678678
prologue:
679679
commands:
680-
- sem-version go 1.20
680+
- sem-version go 1.22
681681
- checkout && cd cache-cli
682682
jobs:
683683
- name: Lint

cache-cli/Dockerfile.dev

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM golang:1.20
1+
FROM golang:1.22
22

33
RUN go install gotest.tools/gotestsum@latest
44
RUN mkdir /root/.ssh

cache-cli/docker-compose.yml

+3-1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ services:
1616
- .:/app
1717
environment:
1818
SEMAPHORE_CACHE_S3_URL: "http://s3:9000"
19+
SEMAPHORE_CACHE_S3_KEY: minioadmin
20+
SEMAPHORE_CACHE_S3_SECRET: minioadmin
1921
STORAGE_EMULATOR_HOST: "http://gcs:4443"
2022
SEMAPHORE_TOOLBOX_METRICS_ENABLED: "true"
2123
gcs:
@@ -27,7 +29,7 @@ services:
2729
- 4443:4443
2830
command: -backend memory -scheme http -port 4443 -public-host gcs:4443 -external-url http://gcs:4443
2931
s3:
30-
image: quay.io/minio/minio:RELEASE.2021-09-15T04-54-25Z
32+
image: quay.io/minio/minio:RELEASE.2024-09-22T00-33-43Z
3133
container_name: 's3'
3234
ports:
3335
- 9000:9000

cache-cli/go.mod

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
module github.com/semaphoreci/toolbox/cache-cli
22

3-
go 1.20
3+
go 1.22
44

55
require (
66
cloud.google.com/go/storage v1.30.1
@@ -9,7 +9,6 @@ require (
99
github.com/aws/aws-sdk-go-v2/credentials v1.13.24
1010
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.67
1111
github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1
12-
github.com/aws/smithy-go v1.13.5
1312
github.com/klauspost/pgzip v1.2.6
1413
github.com/pkg/sftp v1.13.5
1514
github.com/sirupsen/logrus v1.9.2
@@ -37,6 +36,7 @@ require (
3736
github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 // indirect
3837
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 // indirect
3938
github.com/aws/aws-sdk-go-v2/service/sts v1.19.0 // indirect
39+
github.com/aws/smithy-go v1.13.5 // indirect
4040
github.com/davecgh/go-spew v1.1.1 // indirect
4141
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
4242
github.com/golang/protobuf v1.5.3 // indirect

cache-cli/go.sum

+3
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2Aawl
88
cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k=
99
cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0=
1010
cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM=
11+
cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
1112
cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM=
1213
cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E=
1314
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -88,6 +89,7 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
8889
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
8990
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
9091
github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
92+
github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
9193
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
9294
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
9395
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -168,6 +170,7 @@ golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
168170
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
169171
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
170172
golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8=
173+
golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
171174
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
172175
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
173176
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

cache-cli/pkg/storage/s3.go

+25-15
Original file line numberDiff line numberDiff line change
@@ -91,31 +91,41 @@ func createDefaultS3Storage(s3Bucket, project string, storageConfig StorageConfi
9191
}
9292

9393
func createS3StorageUsingEndpoint(s3Bucket, project, s3Url string, storageConfig StorageConfig) (*S3Storage, error) {
94-
resolver := aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) {
95-
return aws.Endpoint{
96-
URL: s3Url,
97-
}, nil
98-
})
94+
options := []func(*awsConfig.LoadOptions) error{
95+
awsConfig.WithRegion("auto"),
96+
awsConfig.WithEndpointResolverWithOptions(
97+
aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
98+
return aws.Endpoint{
99+
URL: s3Url,
100+
}, nil
101+
}),
102+
),
103+
}
99104

100-
creds := credentials.NewStaticCredentialsProvider("minioadmin", "minioadmin", "")
101-
cfg, err := awsConfig.LoadDefaultConfig(context.TODO(),
102-
awsConfig.WithCredentialsProvider(creds),
103-
awsConfig.WithEndpointResolver(resolver),
104-
)
105+
// If a key/secret pair is passed, we use them.
106+
// Otherwise, we just rely on the default configuration methods
107+
// used by LoadDefaultConfig(), for example,
108+
// AWS_SECRET_ACCESS_KEY and AWS_ACCESS_KEY_ID environment variables.
109+
s3Key := os.Getenv("SEMAPHORE_CACHE_S3_KEY")
110+
s3Secret := os.Getenv("SEMAPHORE_CACHE_S3_SECRET")
111+
if s3Key != "" && s3Secret != "" {
112+
options = append(options, awsConfig.WithCredentialsProvider(
113+
credentials.NewStaticCredentialsProvider(s3Key, s3Secret, ""),
114+
))
115+
}
105116

117+
cfg, err := awsConfig.LoadDefaultConfig(context.TODO(), options...)
106118
if err != nil {
107119
return nil, err
108120
}
109121

110-
svc := s3.NewFromConfig(cfg, func(o *s3.Options) {
111-
o.UsePathStyle = true
112-
})
113-
114122
return &S3Storage{
115-
Client: svc,
116123
Bucket: s3Bucket,
117124
Project: project,
118125
StorageConfig: storageConfig,
126+
Client: s3.NewFromConfig(cfg, func(o *s3.Options) {
127+
o.UsePathStyle = true
128+
}),
119129
}, nil
120130
}
121131

0 commit comments

Comments
 (0)