Skip to content

Commit fbb1f40

Browse files
zoewanggzoewangg
authored
Fixed an issue in AWS CRT-based S3 client where checksums are not calculated for operations that require checksums when RequestChecksumCalculation.WHEN_REQUIRED is configured, resulting error. (#5873)
1 parent 9d5af05 commit fbb1f40

File tree

4 files changed

+219
-2
lines changed

4 files changed

+219
-2
lines changed

‎.changes/next-release/bugfix-AWSCRTbasedS3Client-5465b9d.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"type": "bugfix",
3+
"category": "AWS CRT-based S3 Client",
4+
"contributor": "",
5+
"description": "Fixed an issue in AWS CRT-based S3 client where checksums are not calculated for operations that require checksums when RequestChecksumCalculation.WHEN_REQUIRED is configured, resulting error."
6+
}

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/CrtChecksumUtils.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,8 @@ private static ChecksumAlgorithm crtChecksumAlgorithm(HttpChecksum httpChecksum,
7878
RequestChecksumCalculation requestChecksumCalculation) {
7979

8080
if (httpChecksum.requestAlgorithm() == null) {
81-
if (requestChecksumCalculation == RequestChecksumCalculation.WHEN_REQUIRED) {
81+
if (!httpChecksum.isRequestChecksumRequired() &&
82+
requestChecksumCalculation == RequestChecksumCalculation.WHEN_REQUIRED) {
8283
return ChecksumAlgorithm.NONE;
8384
}
8485
return DEFAULT_CHECKSUM_ALGO;

‎services/s3/src/test/java/software/amazon/awssdk/services/s3/crt/S3CrtChecksumTest.java

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.services.s3.crt;
17+
18+
import static com.github.tomakehurst.wiremock.client.WireMock.anyUrl;
19+
import static com.github.tomakehurst.wiremock.client.WireMock.equalTo;
20+
import static com.github.tomakehurst.wiremock.client.WireMock.put;
21+
import static com.github.tomakehurst.wiremock.client.WireMock.putRequestedFor;
22+
import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
23+
import static com.github.tomakehurst.wiremock.client.WireMock.verify;
24+
25+
import com.github.tomakehurst.wiremock.client.WireMock;
26+
import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo;
27+
import com.github.tomakehurst.wiremock.junit5.WireMockTest;
28+
import com.github.tomakehurst.wiremock.matching.AnythingPattern;
29+
import java.io.IOException;
30+
import java.net.URI;
31+
import java.util.stream.Stream;
32+
import org.junit.jupiter.api.BeforeEach;
33+
import org.junit.jupiter.api.Timeout;
34+
import org.junit.jupiter.params.ParameterizedTest;
35+
import org.junit.jupiter.params.provider.Arguments;
36+
import org.junit.jupiter.params.provider.MethodSource;
37+
import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
38+
import software.amazon.awssdk.core.HttpChecksumConstant;
39+
import software.amazon.awssdk.core.async.AsyncRequestBody;
40+
import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
41+
import software.amazon.awssdk.core.checksums.ResponseChecksumValidation;
42+
import software.amazon.awssdk.regions.Region;
43+
import software.amazon.awssdk.services.s3.S3AsyncClient;
44+
import software.amazon.awssdk.services.s3.S3CrtAsyncClientBuilder;
45+
import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm;
46+
import software.amazon.awssdk.services.s3.model.DefaultRetention;
47+
import software.amazon.awssdk.services.s3.model.ObjectLockConfiguration;
48+
import software.amazon.awssdk.services.s3.model.ObjectLockEnabled;
49+
import software.amazon.awssdk.services.s3.model.ObjectLockRetentionMode;
50+
import software.amazon.awssdk.services.s3.model.ObjectLockRule;
51+
import software.amazon.awssdk.services.s3.model.PutObjectLockConfigurationRequest;
52+
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
53+
54+
@WireMockTest
55+
@Timeout(10)
56+
public class S3CrtChecksumTest {
57+
58+
private S3CrtAsyncClientBuilder initializeAsync(WireMockRuntimeInfo wiremock,
59+
RequestChecksumCalculation calculation) {
60+
return S3AsyncClient.crtBuilder()
61+
.credentialsProvider(AnonymousCredentialsProvider.create())
62+
.requestChecksumCalculation(calculation)
63+
.endpointOverride(URI.create("http://localhost:" + wiremock.getHttpPort()))
64+
.forcePathStyle(true)
65+
.region(Region.US_WEST_2);
66+
}
67+
68+
private S3CrtAsyncClientBuilder initializeAsync(WireMockRuntimeInfo wiremock,
69+
ResponseChecksumValidation responseChecksumValidation) {
70+
return S3AsyncClient.crtBuilder()
71+
.credentialsProvider(AnonymousCredentialsProvider.create())
72+
.responseChecksumValidation(responseChecksumValidation)
73+
.endpointOverride(URI.create("http://localhost:" + wiremock.getHttpPort()))
74+
.forcePathStyle(true)
75+
.region(Region.US_WEST_2);
76+
}
77+
78+
@BeforeEach
79+
public void setup() throws IOException {
80+
stubFor(put(anyUrl()).willReturn(WireMock.aResponse().withStatus(200)));
81+
82+
}
83+
84+
public static Stream<Arguments> streamingInputChecksumCalculationParams() {
85+
return Stream.of(Arguments.of(RequestChecksumCalculation.WHEN_SUPPORTED, null, "x-amz-checksum-crc32",
86+
"requestChecksumWhenSupported_checksumAlgorithmNotProvided_shouldAddCrc32ChecksumTrailerByDefault"),
87+
88+
Arguments.of(RequestChecksumCalculation.WHEN_SUPPORTED, ChecksumAlgorithm.SHA1,
89+
"x-amz-checksum-sha1",
90+
"requestChecksumWhenSupported_checksumAlgorithmProvided_shouldHonor"),
91+
92+
Arguments.of(RequestChecksumCalculation.WHEN_REQUIRED, null, null,
93+
"requestChecksumWhenRequired_checksumAlgorithmNotProvided_shouldNotAddChecksum"),
94+
95+
Arguments.of(RequestChecksumCalculation.WHEN_REQUIRED, ChecksumAlgorithm.CRC32_C,
96+
"x-amz-checksum-crc32c",
97+
"requestChecksumWhenRequired_checksumAlgorithmProvided_shouldAddChecksumTrailer"));
98+
}
99+
100+
public static Stream<Arguments> checksumInHeaderRequiredParams() {
101+
return Stream.of(Arguments.of(RequestChecksumCalculation.WHEN_SUPPORTED, null, "x-amz-checksum-crc32", "Rs0ofQ==",
102+
"requestChecksumWhenSupported_checksumAlgorithmNotProvided_shouldAddCrc32ChecksumTrailerByDefault"),
103+
104+
Arguments.of(RequestChecksumCalculation.WHEN_SUPPORTED, ChecksumAlgorithm.SHA1,
105+
"x-amz-checksum-sha1", "4wnI4cDeFPttl6wSYksrgmk41qk=",
106+
"requestChecksumWhenSupported_checksumAlgorithmProvided_shouldHonor"),
107+
108+
Arguments.of(RequestChecksumCalculation.WHEN_REQUIRED, null, "x-amz-checksum-crc32", "Rs0ofQ==",
109+
"requestChecksumWhenRequired_checksumAlgorithmNotProvided_shouldAddChecksum"),
110+
111+
Arguments.of(RequestChecksumCalculation.WHEN_REQUIRED, ChecksumAlgorithm.CRC32_C,
112+
"x-amz-checksum-crc32c", "Zx3Wjw==",
113+
"requestChecksumWhenRequired_checksumAlgorithmProvided_shouldAddChecksum"));
114+
}
115+
116+
117+
@ParameterizedTest(name = "{index} {3}")
118+
@MethodSource("streamingInputChecksumCalculationParams")
119+
public void streamingInput_checksumCalculation(RequestChecksumCalculation requestChecksumCalculation,
120+
ChecksumAlgorithm checksumAlgorithm,
121+
String expectedTrailer,
122+
String description,
123+
WireMockRuntimeInfo wiremock) {
124+
125+
try (S3AsyncClient client = initializeAsync(wiremock, requestChecksumCalculation).build()) {
126+
client.putObject(PutObjectRequest.builder()
127+
.bucket("bucket").key("key")
128+
.checksumAlgorithm(checksumAlgorithm)
129+
.build(),
130+
AsyncRequestBody.fromString("Hello world")).join();
131+
132+
validateChecksumTrailerHeader(expectedTrailer, wiremock);
133+
}
134+
}
135+
136+
@ParameterizedTest(name = "{index} {4}")
137+
@MethodSource("checksumInHeaderRequiredParams")
138+
public void checksumInHeaderRequired_checksumCalculation(RequestChecksumCalculation requestChecksumCalculation,
139+
ChecksumAlgorithm checksumAlgorithm,
140+
String expectedChecksumHeader,
141+
String expectedChecksumValue,
142+
String description,
143+
WireMockRuntimeInfo wiremock) {
144+
145+
try (S3AsyncClient client = initializeAsync(wiremock, requestChecksumCalculation).build()) {
146+
PutObjectLockConfigurationRequest request =
147+
PutObjectLockConfigurationRequest.builder()
148+
.bucket("bucket")
149+
.checksumAlgorithm(checksumAlgorithm)
150+
.objectLockConfiguration(
151+
ObjectLockConfiguration.builder()
152+
.objectLockEnabled(ObjectLockEnabled.ENABLED)
153+
.rule(ObjectLockRule.builder()
154+
.defaultRetention(DefaultRetention.builder().mode(ObjectLockRetentionMode.COMPLIANCE).days(Integer.valueOf(1)).build())
155+
.build())
156+
.build())
157+
.build();
158+
159+
client.putObjectLockConfiguration(request).join();
160+
validateChecksumHeader(expectedChecksumHeader, expectedChecksumValue);
161+
}
162+
}
163+
164+
private static void validateChecksumHeader(String expectedChecksumHeader,
165+
String expectedChecksumValue) {
166+
verify(putRequestedFor(anyUrl()).withoutHeader(HttpChecksumConstant.X_AMZ_TRAILER));
167+
if (expectedChecksumHeader != null) {
168+
verify(putRequestedFor(anyUrl()).withHeader(expectedChecksumHeader, equalTo(expectedChecksumValue)));
169+
verify(putRequestedFor(anyUrl()).withHeader("x-amz-sdk-checksum-algorithm", new AnythingPattern()));
170+
} else {
171+
verify(putRequestedFor(anyUrl()).withoutHeader("x-amz-sdk-checksum-algorithm"));
172+
}
173+
}
174+
175+
private static void validateChecksumTrailerHeader(String expectedTrailer,
176+
WireMockRuntimeInfo wiremock) {
177+
178+
179+
if (expectedTrailer != null) {
180+
verify(putRequestedFor(anyUrl()).withHeader(HttpChecksumConstant.X_AMZ_TRAILER, equalTo(expectedTrailer)));
181+
verify(putRequestedFor(anyUrl()).withHeader("x-amz-content-sha256", equalTo("STREAMING-UNSIGNED-PAYLOAD-TRAILER")));
182+
verify(putRequestedFor(anyUrl()).withHeader("x-amz-sdk-checksum-algorithm", new AnythingPattern()));
183+
} else {
184+
verify(putRequestedFor(anyUrl()).withoutHeader(HttpChecksumConstant.X_AMZ_TRAILER));
185+
verify(putRequestedFor(anyUrl()).withoutHeader("x-amz-sdk-checksum-algorithm"));
186+
}
187+
}
188+
}

‎services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/CrtChecksumUtilsTest.java

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,34 @@ private static Stream<Arguments> crtChecksumInput() {
3737
checksumAlgorithms.add(ChecksumAlgorithm.SHA256);
3838
checksumAlgorithms.add(ChecksumAlgorithm.SHA1);
3939
return Stream.of(
40-
// DEFAULT request, should set default algorithm CRC32 in header
40+
// DEFAULT request, operation w/o checksum required, WHEN_SUPPORTED config, should set default algorithm CRC32 in
41+
// header
4142
Arguments.of(HttpChecksum.builder().build(), S3MetaRequestOptions.MetaRequestType.DEFAULT,
4243
RequestChecksumCalculation.WHEN_SUPPORTED, ResponseChecksumValidation.WHEN_SUPPORTED,
4344
new ChecksumConfig().withChecksumAlgorithm(ChecksumAlgorithm.CRC32)
4445
.withChecksumLocation(ChecksumConfig.ChecksumLocation.HEADER)),
4546

47+
// DEFAULT request, operation w/o checksum required, WHEN_REQUIRED config, should pass empty config
48+
Arguments.of(HttpChecksum.builder().build(), S3MetaRequestOptions.MetaRequestType.DEFAULT,
49+
RequestChecksumCalculation.WHEN_REQUIRED, ResponseChecksumValidation.WHEN_SUPPORTED,
50+
new ChecksumConfig()),
51+
52+
// DEFAULT request, operation w/ checksum required, WHEN_REQUIRED config, should set default algorithm CRC32 in
53+
// header
54+
Arguments.of(HttpChecksum.builder().requestChecksumRequired(true).build(),
55+
S3MetaRequestOptions.MetaRequestType.DEFAULT,
56+
RequestChecksumCalculation.WHEN_REQUIRED, ResponseChecksumValidation.WHEN_SUPPORTED,
57+
new ChecksumConfig().withChecksumAlgorithm(ChecksumAlgorithm.CRC32)
58+
.withChecksumLocation(ChecksumConfig.ChecksumLocation.HEADER)),
59+
60+
// DEFAULT request, operation w/ checksum required, WHEN_SUPPORTED config, should set default algorithm CRC32 in
61+
// header
62+
Arguments.of(HttpChecksum.builder().requestChecksumRequired(true).build(),
63+
S3MetaRequestOptions.MetaRequestType.DEFAULT,
64+
RequestChecksumCalculation.WHEN_SUPPORTED, ResponseChecksumValidation.WHEN_SUPPORTED,
65+
new ChecksumConfig().withChecksumAlgorithm(ChecksumAlgorithm.CRC32)
66+
.withChecksumLocation(ChecksumConfig.ChecksumLocation.HEADER)),
67+
4668
// PUT request with SHA256 request algorithm, should set algorithm in trailer
4769
Arguments.of(HttpChecksum.builder()
4870
.requestAlgorithm("sha256")

0 commit comments

Comments
 (0)