From 5bd6de21e8c9d16dc5c73fcdef0ba7844ce8e88c Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Tue, 4 Mar 2025 10:19:23 -0500 Subject: [PATCH 01/13] add support for md5 checksum validation --- .../runtime/config/profile/AwsProfile.kt | 24 ++ .../sqs/ClientConfigIntegration.kt | 92 +++++ .../SqsMd5ChecksumValidationIntegration.kt | 51 +++ .../customization/sqs/SqsModelUtils.kt | 14 + ...tlin.codegen.integration.KotlinIntegration | 2 + ...SqsMd5ChecksumValidationIntegrationTest.kt | 52 +++ .../SqsMd5ChecksumValidationInterceptor.kt | 388 ++++++++++++++++++ .../internal/FinalizeSqsConfig.kt | 35 ++ .../internal/SQSSetting.kt | 49 +++ .../internal/ValidationConfig.kt | 46 +++ .../src/SqsMd5ChecksumValidationTest.kt | 241 +++++++++++ services/sqs/e2eTest/src/SqsTestUtils.kt | 110 +++++ 12 files changed, 1104 insertions(+) create mode 100644 codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/ClientConfigIntegration.kt create mode 100644 codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt create mode 100644 codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsModelUtils.kt create mode 100644 codegen/aws-sdk-codegen/src/test/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegrationTest.kt create mode 100644 services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt create mode 100644 services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/FinalizeSqsConfig.kt create mode 100644 services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SQSSetting.kt create mode 100644 services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/ValidationConfig.kt create mode 100644 services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt create mode 100644 services/sqs/e2eTest/src/SqsTestUtils.kt diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt index 1f69799c31e..8e2bebe72c5 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt @@ -221,6 +221,30 @@ public inline fun > AwsProfile.getEnumOrNull(key: String, su ) } +/** + * Parse a config value as an enum set. + */ +@InternalSdkApi +public inline fun > AwsProfile.getEnumSetOrNull(key: String, subKey: String? = null): Set? = + getOrNull(key, subKey)?.let { rawValue -> + rawValue.split(",") + .map { it.trim() } + .map { value -> + enumValues().firstOrNull { + it.name.equals(value, ignoreCase = true) + } ?: throw ConfigurationException( + buildString { + append(key) + append(" '") + append(value) + append("' is not supported, should be one of: ") + enumValues().joinTo(this) { it.name.lowercase() } + } + ) + }.toSet() + .takeIf { it.isNotEmpty() } + } + internal fun AwsProfile.getUrlOrNull(key: String, subKey: String? = null): Url? = getOrNull(key, subKey)?.let { try { diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/ClientConfigIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/ClientConfigIntegration.kt new file mode 100644 index 00000000000..8758066a46f --- /dev/null +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/ClientConfigIntegration.kt @@ -0,0 +1,92 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.codegen.customization.sqs + +import aws.sdk.kotlin.codegen.ServiceClientCompanionObjectWriter +import software.amazon.smithy.kotlin.codegen.KotlinSettings +import software.amazon.smithy.kotlin.codegen.core.CodegenContext +import software.amazon.smithy.kotlin.codegen.integration.AppendingSectionWriter +import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration +import software.amazon.smithy.kotlin.codegen.integration.SectionWriterBinding +import software.amazon.smithy.kotlin.codegen.lang.KotlinTypes +import software.amazon.smithy.kotlin.codegen.model.buildSymbol +import software.amazon.smithy.kotlin.codegen.model.expectShape +import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigProperty +import software.amazon.smithy.model.Model +import software.amazon.smithy.model.shapes.ServiceShape + +class ClientConfigIntegration : KotlinIntegration { + override fun enabledForService(model: Model, settings: KotlinSettings): Boolean = + model.expectShape(settings.service).isSqs + + companion object { + val ValidationEnabledProp: ConfigProperty = ConfigProperty { + name = "checksumValidationEnabled" + symbol = buildSymbol { + name = "ValidationEnabled" + namespace = "aws.sdk.kotlin.services.sqs.internal" + documentation = """ + Specifies when MD5 checksum validation should be performed for SQS messages. This controls the automatic + calculation and validation of checksums during message operations. + + Valid values: + - ALWAYS (default) - Checksums are calculated and validated for both sending and receiving operations + (SendMessage, SendMessageBatch, and ReceiveMessage) + - WHEN_SENDING - Checksums are only calculated and validated during send operations + (SendMessage and SendMessageBatch) + - WHEN_RECEIVING - Checksums are only calculated and validated during receive operations + (ReceiveMessage) + - NEVER - No checksum calculation or validation is performed + """.trimIndent() + } + } + + private val validationScope = buildSymbol { + name = "ValidationScope" + namespace = "aws.sdk.kotlin.services.sqs.internal" + } + + val ValidationScopeProp: ConfigProperty = ConfigProperty { + name = "checksumValidationScopes" + symbol = KotlinTypes.Collections.set(validationScope, default = "emptySet()") + documentation = """ + Specifies which parts of an SQS message should undergo MD5 checksum validation. This configuration + accepts a set of validation scopes that determine which message components to validate. + + Valid values: + - MESSAGE_ATTRIBUTES - Validates checksums for message attributes + - MESSAGE_SYSTEM_ATTRIBUTES - Validates checksums for message system attributes + (Note: Not available for ReceiveMessage operations as SQS does not calculate checksums for + system attributes during message receipt) + - MESSAGE_BODY - Validates checksums for the message body + + Default: All three scopes (MESSAGE_ATTRIBUTES, MESSAGE_SYSTEM_ATTRIBUTES, MESSAGE_BODY) + """.trimIndent() + } + } + + override fun additionalServiceConfigProps(ctx: CodegenContext): List = + listOf( + ValidationEnabledProp, + ValidationScopeProp, + ) + + override val sectionWriters: List + get() = listOf( + SectionWriterBinding( + ServiceClientCompanionObjectWriter.FinalizeEnvironmentalConfig, + finalizeSqsConfigWriter, + ), + ) + + // add Sqs-specific config finalization + private val finalizeSqsConfigWriter = AppendingSectionWriter { writer -> + val finalizeSqsConfig = buildSymbol { + name = "finalizeSqsConfig" + namespace = "aws.sdk.kotlin.services.sqs.internal" + } + writer.write("#T(builder, sharedConfig)", finalizeSqsConfig) + } +} diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt new file mode 100644 index 00000000000..f69f5bfb4ed --- /dev/null +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt @@ -0,0 +1,51 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.codegen.customization.sqs + +import software.amazon.smithy.kotlin.codegen.KotlinSettings +import software.amazon.smithy.kotlin.codegen.core.KotlinWriter +import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration +import software.amazon.smithy.kotlin.codegen.model.buildSymbol +import software.amazon.smithy.kotlin.codegen.model.expectShape +import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator +import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware +import software.amazon.smithy.model.Model +import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.model.shapes.ServiceShape + +/** + * Register interceptor to handle SQS message MD5 checksum validation. + */ +class SqsMd5ChecksumValidationIntegration : KotlinIntegration { + override fun enabledForService(model: Model, settings: KotlinSettings): Boolean = + model.expectShape(settings.service).isSqs + + override fun customizeMiddleware( + ctx: ProtocolGenerator.GenerationContext, + resolved: List, + ): List = resolved + listOf(SqsMd5ChecksumValidationMiddleware) +} + +internal object SqsMd5ChecksumValidationMiddleware : ProtocolMiddleware { + override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean { + return when (op.id.name) { + "ReceiveMessage", + "SendMessage", + "SendMessageBatch" -> true + else -> false + } + } + + override val name: String = "SqsMd5ChecksumValidationInterceptor" + + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + val symbol = buildSymbol { + name = this@SqsMd5ChecksumValidationMiddleware.name + namespace = "aws.sdk.kotlin.services.sqs" + } + + writer.write("op.interceptors.add(#T(config.checksumValidationEnabled, config.checksumValidationScopes,))", symbol) + } +} diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsModelUtils.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsModelUtils.kt new file mode 100644 index 00000000000..f79ec678161 --- /dev/null +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsModelUtils.kt @@ -0,0 +1,14 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.codegen.customization.sqs + +import aws.sdk.kotlin.codegen.sdkId +import software.amazon.smithy.model.shapes.ServiceShape + +/** + * Returns true if the service is Sqs + */ +val ServiceShape.isSqs: Boolean + get() = sdkId.lowercase() == "sqs" diff --git a/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration b/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration index 7786616c0a6..5a8d3df78a6 100644 --- a/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration +++ b/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration @@ -49,3 +49,5 @@ aws.sdk.kotlin.codegen.smoketests.SmokeTestsDenyListIntegration aws.sdk.kotlin.codegen.smoketests.testing.SmokeTestSuccessHttpEngineIntegration aws.sdk.kotlin.codegen.smoketests.testing.SmokeTestFailHttpEngineIntegration aws.sdk.kotlin.codegen.customization.AwsQueryModeCustomization +aws.sdk.kotlin.codegen.customization.sqs.ClientConfigIntegration +aws.sdk.kotlin.codegen.customization.sqs.SqsMd5ChecksumValidationIntegration \ No newline at end of file diff --git a/codegen/aws-sdk-codegen/src/test/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegrationTest.kt b/codegen/aws-sdk-codegen/src/test/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegrationTest.kt new file mode 100644 index 00000000000..aa23286fcc0 --- /dev/null +++ b/codegen/aws-sdk-codegen/src/test/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegrationTest.kt @@ -0,0 +1,52 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.codegen.customization.sqs + +import aws.sdk.kotlin.codegen.testutil.model +import org.junit.jupiter.api.Test +import software.amazon.smithy.kotlin.codegen.core.KotlinWriter +import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator +import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware +import software.amazon.smithy.kotlin.codegen.test.defaultSettings +import software.amazon.smithy.kotlin.codegen.test.newTestContext +import software.amazon.smithy.model.shapes.OperationShape +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue +import kotlin.test.fail + +class SqsMd5ChecksumValidationIntegrationTest { + @Test + fun testNotExpectedForNonSqsModel() { + val model = model("NotSqs") + val actual = SqsMd5ChecksumValidationIntegration().enabledForService(model, model.defaultSettings()) + + assertFalse(actual) + } + + @Test + fun testExpectedForSqsModel() { + val model = model("Sqs") + val actual = SqsMd5ChecksumValidationIntegration().enabledForService(model, model.defaultSettings()) + + assertTrue(actual) + } + + @Test + fun testMiddlewareAddition() { + val model = model("Sqs") + val preexistingMiddleware = listOf(FooMiddleware) + val ctx = model.newTestContext("Sqs") + val actual = SqsMd5ChecksumValidationIntegration().customizeMiddleware(ctx.generationCtx, preexistingMiddleware) + + assertEquals(listOf(FooMiddleware, SqsMd5ChecksumValidationMiddleware), actual) + } +} + +object FooMiddleware : ProtocolMiddleware { + override val name: String = "FooMiddleware" + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) = + fail("Unexpected call to `FooMiddleware.render") +} diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt new file mode 100644 index 00000000000..f0617c492e8 --- /dev/null +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt @@ -0,0 +1,388 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.services.sqs + +import aws.sdk.kotlin.services.sqs.internal.ValidationEnabled +import aws.sdk.kotlin.services.sqs.internal.ValidationScope +import aws.sdk.kotlin.services.sqs.model.* +import aws.smithy.kotlin.runtime.ClientException +import aws.smithy.kotlin.runtime.InternalApi +import aws.smithy.kotlin.runtime.client.ResponseInterceptorContext +import aws.smithy.kotlin.runtime.hashing.Md5 +import aws.smithy.kotlin.runtime.hashing.md5 +import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException +import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor +import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.http.response.HttpResponse +import aws.smithy.kotlin.runtime.telemetry.logging.Logger +import aws.smithy.kotlin.runtime.telemetry.logging.logger +import kotlin.collections.Map +import kotlin.collections.Set +import kotlin.collections.hashMapOf +import kotlin.collections.isNullOrEmpty +import kotlin.collections.set +import kotlin.collections.sorted +import kotlin.collections.sortedBy + +/** + * Interceptor that validates MD5 checksums for SQS message operations. + * + * This interceptor performs client-side validation of MD5 checksums returned by SQS to ensure + * message integrity during transmission. It validates the following components: + * - Message body + * - Message attributes + * - Message system attributes + * + * The validation behavior can be configured using: + * - [checksumValidationEnabled] - Controls when validation occurs (ALWAYS, WHEN_SENDING, WHEN_RECEIVING, NEVER) + * - [checksumValidationScopes] - Specifies which message components to validate + * + * Supported operations: + * - SendMessage + * - SendMessageBatch + * - ReceiveMessage + */ +@OptIn(InternalApi::class, ExperimentalStdlibApi::class) +public class SqsMd5ChecksumValidationInterceptor( + private val validationEnabled: ValidationEnabled?, + private val validationScopes: Set, +) : HttpInterceptor { + public companion object { + private const val STRING_TYPE_FIELD_INDEX: Byte = 1 + private const val BINARY_TYPE_FIELD_INDEX: Byte = 2 + private const val STRING_LIST_TYPE_FIELD_INDEX: Byte = 3 + private const val BINARY_LIST_TYPE_FIELD_INDEX: Byte = 4 + + private lateinit var logger: Logger + + private fun initLogger(logger: Logger) { + this.logger = logger + } + } + + override fun readAfterExecution(context: ResponseInterceptorContext) { + val request = context.request + val response = context.response.getOrNull() + + if (validationEnabled == ValidationEnabled.NEVER) return + + val logger = context.executionContext.coroutineContext.logger() + initLogger(logger) + + if (response != null) { + when (request) { + is SendMessageRequest -> { + if (validationEnabled == ValidationEnabled.WHEN_RECEIVING) return + + val sendMessageRequest = request as SendMessageRequest + val sendMessageResponse = response as SendMessageResponse + sendMessageOperationMd5Check(sendMessageRequest, sendMessageResponse) + } + + is ReceiveMessageRequest -> { + if (validationEnabled == ValidationEnabled.WHEN_SENDING) return + + val receiveMessageResponse = response as ReceiveMessageResponse + receiveMessageResultMd5Check(receiveMessageResponse) + } + + is SendMessageBatchRequest -> { + if (validationEnabled == ValidationEnabled.WHEN_RECEIVING) return + + val sendMessageBatchRequest = request as SendMessageBatchRequest + val sendMessageBatchResponse = response as SendMessageBatchResponse + sendMessageBatchOperationMd5Check(sendMessageBatchRequest, sendMessageBatchResponse) + } + } + } + } + + private fun sendMessageOperationMd5Check( + sendMessageRequest: SendMessageRequest, + sendMessageResponse: SendMessageResponse, + ) { + if(validationScopes.contains(ValidationScope.MESSAGE_BODY)) { + val messageBodySent = sendMessageRequest.messageBody + + if (!messageBodySent.isNullOrEmpty()) { + logger.debug { "Validating message body MD5 checksum for SendMessage" } + + val bodyMD5Returned = sendMessageResponse.md5OfMessageBody + val clientSideBodyMd5 = calculateMessageBodyMd5(messageBodySent) + if (clientSideBodyMd5 != bodyMD5Returned) { + throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideBodyMd5 but was $bodyMD5Returned") + } + } + } + + if(validationScopes.contains(ValidationScope.MESSAGE_ATTRIBUTES)) { + val messageAttrSent = sendMessageRequest.messageAttributes + if (!messageAttrSent.isNullOrEmpty()) { + logger.debug { "Validating message attribute MD5 checksum for SendMessage" } + + val messageAttrMD5Returned = sendMessageResponse.md5OfMessageAttributes + val clientSideAttrMd5 = calculateMessageAttributesMd5(messageAttrSent) + if (clientSideAttrMd5 != messageAttrMD5Returned) { + throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideAttrMd5 but was $messageAttrMD5Returned") + } + } + } + + if(validationScopes.contains(ValidationScope.MESSAGE_SYSTEM_ATTRIBUTES)) { + val messageSysAttrSent = sendMessageRequest.messageSystemAttributes + if (!messageSysAttrSent.isNullOrEmpty()) { + logger.debug { "Validating message system attribute MD5 checksum for SendMessage" } + + val messageSysAttrMD5Returned = sendMessageResponse.md5OfMessageSystemAttributes + val clientSideSysAttrMd5 = calculateMessageSystemAttributesMd5(messageSysAttrSent) + if (clientSideSysAttrMd5 != messageSysAttrMD5Returned) { + throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideSysAttrMd5 but was $messageSysAttrMD5Returned") + } + } + } + } + + private fun receiveMessageResultMd5Check(receiveMessageResponse: ReceiveMessageResponse) { + val messages = receiveMessageResponse.messages + if (messages != null) { + for (messageReceived in messages) { + if(validationScopes.contains(ValidationScope.MESSAGE_BODY)) { + val messageBody = messageReceived.body + if (!messageBody.isNullOrEmpty()) { + logger.debug { "Validating message body MD5 checksum for ReceiveMessage" } + + val bodyMd5Returned = messageReceived.md5OfBody + val clientSideBodyMd5 = calculateMessageBodyMd5(messageBody) + if (clientSideBodyMd5 != bodyMd5Returned) { + throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideBodyMd5 but was $bodyMd5Returned") + } + } + } + + if(validationScopes.contains(ValidationScope.MESSAGE_ATTRIBUTES)) { + val messageAttr = messageReceived.messageAttributes + + if (!messageAttr.isNullOrEmpty()) { + logger.debug { "Validating message attribute MD5 checksum for ReceiveMessage" } + + val attrMd5Returned = messageReceived.md5OfMessageAttributes + val clientSideAttrMd5 = calculateMessageAttributesMd5(messageAttr) + if (clientSideAttrMd5 != attrMd5Returned) { + throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideAttrMd5 but was $attrMd5Returned") + } + } + } + } + } + } + + private fun sendMessageBatchOperationMd5Check( + sendMessageBatchRequest: SendMessageBatchRequest, + sendMessageBatchResponse: SendMessageBatchResponse, + ) { + val idToRequestEntryMap = hashMapOf() + val entries = sendMessageBatchRequest.entries + if (entries != null) { + for (entry in entries) { + idToRequestEntryMap[entry.id] = entry + } + } + + for (entry in sendMessageBatchResponse.successful) { + if(validationScopes.contains(ValidationScope.MESSAGE_BODY)) { + val messageBody = idToRequestEntryMap[entry.id]?.messageBody + + if (!messageBody.isNullOrEmpty()) { + logger.debug { "Validating message body MD5 checksum for SendMessageBatch: ${entry.messageId}" } + + val bodyMd5Returned = entry.md5OfMessageBody + val clientSideBodyMd5 = calculateMessageBodyMd5(messageBody) + if (clientSideBodyMd5 != bodyMd5Returned) { + throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideBodyMd5 but was $bodyMd5Returned") + } + } + } + + if(validationScopes.contains(ValidationScope.MESSAGE_ATTRIBUTES)) { + val messageAttrSent = idToRequestEntryMap[entry.id]?.messageAttributes + if (!messageAttrSent.isNullOrEmpty()) { + logger.debug { "Validating message attribute MD5 checksum for SendMessageBatch: ${entry.messageId}" } + + val messageAttrMD5Returned = entry.md5OfMessageAttributes + val clientSideAttrMd5 = calculateMessageAttributesMd5(messageAttrSent) + if (clientSideAttrMd5 != messageAttrMD5Returned) { + throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideAttrMd5 but was $messageAttrMD5Returned") + } + } + } + + if(validationScopes.contains(ValidationScope.MESSAGE_SYSTEM_ATTRIBUTES)) { + val messageSysAttrSent = idToRequestEntryMap[entry.id]?.messageSystemAttributes + if (!messageSysAttrSent.isNullOrEmpty()) { + logger.debug { "Validating message system attribute MD5 checksum for SendMessageBatch: ${entry.messageId}" } + + val messageSysAttrMD5Returned = entry.md5OfMessageSystemAttributes + val clientSideSysAttrMd5 = calculateMessageSystemAttributesMd5(messageSysAttrSent) + if (clientSideSysAttrMd5 != messageSysAttrMD5Returned) { + throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideSysAttrMd5 but was $messageSysAttrMD5Returned") + } + } + } + } + } + + private fun calculateMessageBodyMd5(messageBody: String): String { + val expectedMD5 = try { + messageBody.encodeToByteArray().md5() + } catch (e: Exception) { + throw ClientException( + "Unable to calculate the MD5 hash of the message body." + + "Potential reasons include JVM configuration or FIPS compliance issues." + + "To disable message MD5 validation, you can set checksumValidationEnabled" + + "to false when instantiating the client." + e.message, + ) + } + val expectedMD5Hex = expectedMD5.toHexString() + return expectedMD5Hex + } + + /** + * Calculates the MD5 digest for message attributes according to SQS specifications. + * https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-metadata.html#sqs-attributes-md5-message-digest-calculation + */ + @OptIn(InternalApi::class, ExperimentalStdlibApi::class) + private fun calculateMessageAttributesMd5(messageAttributes: Map): String { + val sortedAttributeNames = messageAttributes.keys.sorted() + val md5Digest = Md5() + + try { + for (attributeName in sortedAttributeNames) { + val attributeValue = messageAttributes[attributeName] + updateLengthAndBytes(md5Digest, attributeName.encodeToByteArray()) + + attributeValue?.dataType?.let { dataType -> + updateLengthAndBytes(md5Digest, dataType.encodeToByteArray()) + } + + val stringValue = attributeValue?.stringValue + val binaryValue = attributeValue?.binaryValue + val stringListValues = attributeValue?.stringListValues + val binaryListValues = attributeValue?.binaryListValues + + when { + stringValue != null -> { + md5Digest.update(STRING_TYPE_FIELD_INDEX) + updateLengthAndBytes(md5Digest, stringValue.encodeToByteArray()) + } + + binaryValue != null -> { + md5Digest.update(BINARY_TYPE_FIELD_INDEX) + updateLengthAndBytes(md5Digest, binaryValue) + } + + !stringListValues.isNullOrEmpty() -> { + md5Digest.update(STRING_LIST_TYPE_FIELD_INDEX) + for (stringListValue in stringListValues) { + updateLengthAndBytes(md5Digest, stringListValue.encodeToByteArray()) + } + } + + !binaryListValues.isNullOrEmpty() -> { + md5Digest.update(BINARY_LIST_TYPE_FIELD_INDEX) + for (binaryListValue in binaryListValues) { + updateLengthAndBytes(md5Digest, binaryListValue) + } + } + } + } + } catch (e: Exception) { + throw ClientException( + "Unable to calculate the MD5 hash of the message body." + + "Potential reasons include JVM configuration or FIPS compliance issues." + + "To disable message MD5 validation, you can set checksumValidationEnabled" + + "to false when instantiating the client." + e.message, + ) + } + val expectedMD5Hex = md5Digest.digest().toHexString() + return expectedMD5Hex + } + + private fun calculateMessageSystemAttributesMd5( + messageSysAttrs: + Map, + ): String { + val sortedAttributeNames = messageSysAttrs.keys.sortedBy { it.value } + val md5Digest = Md5() + + try { + for (attributeName in sortedAttributeNames) { + val attributeValue = messageSysAttrs[attributeName] + updateLengthAndBytes(md5Digest, attributeName.value.encodeToByteArray()) + + attributeValue?.dataType?.let { dataType -> + updateLengthAndBytes(md5Digest, dataType.encodeToByteArray()) + } + + val stringValue = attributeValue?.stringValue + val binaryValue = attributeValue?.binaryValue + val stringListValues = attributeValue?.stringListValues + val binaryListValues = attributeValue?.binaryListValues + + when { + stringValue != null -> { + md5Digest.update(STRING_TYPE_FIELD_INDEX) + updateLengthAndBytes(md5Digest, stringValue.encodeToByteArray()) + } + + binaryValue != null -> { + md5Digest.update(BINARY_TYPE_FIELD_INDEX) + updateLengthAndBytes(md5Digest, binaryValue) + } + + !stringListValues.isNullOrEmpty() -> { + md5Digest.update(STRING_LIST_TYPE_FIELD_INDEX) + for (stringListValue in stringListValues) { + updateLengthAndBytes(md5Digest, stringListValue.encodeToByteArray()) + } + } + + !binaryListValues.isNullOrEmpty() -> { + md5Digest.update(BINARY_LIST_TYPE_FIELD_INDEX) + for (binaryListValue in binaryListValues) { + updateLengthAndBytes(md5Digest, binaryListValue) + } + } + } + } + } catch (e: Exception) { + throw ClientException( + "Unable to calculate the MD5 hash of the message body." + + "Potential reasons include JVM configuration or FIPS compliance issues." + + "To disable message MD5 validation, you can set checksumValidationEnabled" + + "to false when instantiating the client." + e.message, + ) + } + val expectedMD5Hex = md5Digest.digest().toHexString() + return expectedMD5Hex + } + + /** + * Update the digest using a sequence of bytes that consists of the length (in 4 bytes) of the + * input binaryValue and all the bytes it contains. + */ + private fun updateLengthAndBytes(messageDigest: Md5, binaryValue: ByteArray) { + println("updateLengthAndBytes") + val length = binaryValue.size + val lengthBytes = byteArrayOf( + (length shr 24).toByte(), + (length shr 16).toByte(), + (length shr 8).toByte(), + length.toByte(), + ) + + messageDigest.update(lengthBytes) + messageDigest.update(binaryValue) + } +} diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/FinalizeSqsConfig.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/FinalizeSqsConfig.kt new file mode 100644 index 00000000000..178e841923c --- /dev/null +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/FinalizeSqsConfig.kt @@ -0,0 +1,35 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.services.sqs.internal + +import aws.sdk.kotlin.runtime.config.profile.* +import aws.sdk.kotlin.services.sqs.SqsClient +import aws.smithy.kotlin.runtime.config.resolve +import aws.smithy.kotlin.runtime.util.LazyAsyncValue +import aws.smithy.kotlin.runtime.util.PlatformProvider + +internal suspend fun finalizeSqsConfig( + builder: SqsClient.Builder, + sharedConfig: LazyAsyncValue, + provider: PlatformProvider = PlatformProvider.System, +) { + val activeProfile = sharedConfig.get().activeProfile + builder.config.checksumValidationEnabled = builder.config.checksumValidationEnabled + ?: SQSSetting.checksumValidationEnabled.resolve(provider) + ?: activeProfile.checksumValidationEnabled + ?: ValidationEnabled.NEVER //TODO: MD5 checksum validation is temporarily disabled. Set default to ALWAYS in next minor version + + builder.config.checksumValidationScopes = builder.config.checksumValidationScopes.ifEmpty { + SQSSetting.checksumValidationScopes.resolve(provider) + ?: activeProfile.checksumValidationScopes + ?: ValidationScope.entries.toSet() + } +} + +private val AwsProfile.checksumValidationEnabled: ValidationEnabled? + get() = getEnumOrNull("sqs_checksum_validation_enabled") + +private val AwsProfile.checksumValidationScopes: Set? + get() = getEnumSetOrNull("sqs_checksum_validation_scope") diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SQSSetting.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SQSSetting.kt new file mode 100644 index 00000000000..cbaa9ea6b9d --- /dev/null +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SQSSetting.kt @@ -0,0 +1,49 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.services.sqs.internal + +import aws.smithy.kotlin.runtime.config.* + +/** + * SQS specific system settings + */ +internal object SQSSetting { + /** + * Configure when MD5 checksum validation is performed for SQS operations. + * + * Can be configured using: + * - System property: aws.SqsChecksumValidationEnabled + * - Environment variable: AWS_SQS_CHECKSUM_VALIDATION_ENABLED + * + * Valid values: + * - ALWAYS (default) - Validates checksums for both sending and receiving operations + * - WHEN_SENDING - Validates checksums only when sending messages + * - WHEN_RECEIVING - Validates checksums only when receiving messages + * - NEVER - Disables checksum validation + * + * Note: Value matching is case-insensitive when configured via environment variables. + */ + public val checksumValidationEnabled : EnvironmentSetting = + enumEnvSetting("aws.SqsChecksumValidationEnabled", "AWS_SQS_CHECKSUM_VALIDATION_ENABLED") + + /** + * Configure the scope of checksum validation for SQS operations. + * + * Can be configured using: + * - System property: aws.SqsChecksumValidationScope + * - Environment variable: AWS_SQS_CHECKSUM_VALIDATION_SCOPE + * + * Valid values are comma-separated combinations of: + * - MESSAGE_BODY: Validate message body checksums + * - MESSAGE_ATTRIBUTES: Validate message attribute checksums + * - SYSTEM_ATTRIBUTES: Validate system attribute checksums + * + * Example: "MESSAGE_BODY,MESSAGE_ATTRIBUTES" + * + * If not specified, defaults to validating all scopes. + */ + public val checksumValidationScopes: EnvironmentSetting?> = + enumSetEnvSetting("aws.SqsChecksumValidationScope", "AWS_SQS_CHECKSUM_VALIDATION_SCOPE") +} diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/ValidationConfig.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/ValidationConfig.kt new file mode 100644 index 00000000000..b3cd56ad84e --- /dev/null +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/ValidationConfig.kt @@ -0,0 +1,46 @@ +package aws.sdk.kotlin.services.sqs.internal + +/** + * Controls when MD5 checksum validation is performed for SQS operations. + * + * This configuration determines under which conditions checksums will be automatically + * calculated and validated for SQS message operations. + * + * Valid values: + * - ALWAYS - Validates checksums for both sending and receiving operations + * (SendMessage, SendMessageBatch, and ReceiveMessage) + * - WHEN_SENDING - Validates checksums only when sending messages + * (SendMessage and SendMessageBatch) + * - WHEN_RECEIVING - Validates checksums only when receiving messages + * (ReceiveMessage) + * - NEVER - Disables checksum validation completely + * + * Default: ALWAYS + */ +public enum class ValidationEnabled { + ALWAYS, + WHEN_SENDING, + WHEN_RECEIVING, + NEVER +} + +/** + * Specifies which parts of an SQS message should undergo MD5 checksum validation. + * + * This configuration determines which components of a message will be validated + * when checksum validation is enabled. + * + * Valid values: + * - MESSAGE_ATTRIBUTES - Validates checksums for message attributes + * - MESSAGE_SYSTEM_ATTRIBUTES - Validates checksums for message system attributes + * (Note: Not available for ReceiveMessage operations as SQS does not calculate + * checksums for system attributes during message receipt) + * - MESSAGE_BODY - Validates checksums for the message body + * + * Default: All scopes enabled + */ +public enum class ValidationScope { + MESSAGE_ATTRIBUTES, + MESSAGE_SYSTEM_ATTRIBUTES, + MESSAGE_BODY +} diff --git a/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt b/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt new file mode 100644 index 00000000000..9d11148c67b --- /dev/null +++ b/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt @@ -0,0 +1,241 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.e2etest + +import aws.sdk.kotlin.e2etest.SqsTestUtils.DEFAULT_REGION +import aws.sdk.kotlin.e2etest.SqsTestUtils.TEST_MESSAGE_ATTRIBUTES_NAME +import aws.sdk.kotlin.e2etest.SqsTestUtils.TEST_MESSAGE_ATTRIBUTES_VALUE +import aws.sdk.kotlin.e2etest.SqsTestUtils.TEST_MESSAGE_BODY +import aws.sdk.kotlin.e2etest.SqsTestUtils.TEST_MESSAGE_SYSTEM_ATTRIBUTES_VALUE +import aws.sdk.kotlin.e2etest.SqsTestUtils.TEST_QUEUE_CORRECT_CHECKSUM_PREFIX +import aws.sdk.kotlin.e2etest.SqsTestUtils.TEST_QUEUE_WRONG_CHECKSUM_PREFIX +import aws.sdk.kotlin.e2etest.SqsTestUtils.buildSendMessageBatchRequestEntry +import aws.sdk.kotlin.e2etest.SqsTestUtils.deleteQueueAndAllMessages +import aws.sdk.kotlin.e2etest.SqsTestUtils.getTestQueueUrl +import aws.sdk.kotlin.services.sqs.SqsClient +import aws.sdk.kotlin.services.sqs.internal.ValidationEnabled +import aws.sdk.kotlin.services.sqs.internal.ValidationScope +import aws.sdk.kotlin.services.sqs.model.* +import aws.smithy.kotlin.runtime.client.ResponseInterceptorContext +import aws.smithy.kotlin.runtime.hashing.md5 +import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException +import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor +import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.http.response.HttpResponse +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.* + +/** + * Tests for Sqs MD5 checksum validation + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class SqsMd5ChecksumValidationTest { + // An interceptor that set wrong md5 checksums in SQS response + private val wrongChecksumInterceptor = object : HttpInterceptor { + override suspend fun modifyBeforeCompletion( + context: ResponseInterceptorContext, + ): Result { + val wrongMd5ofMessageBody = "wrong message md5".encodeToByteArray().md5().toString() + val wrongMd5ofMessageAttribute = "wrong attribute md5".encodeToByteArray().md5().toString() + val wrongMd5ofMessageSystemAttribute = "wrong system attribute md5".encodeToByteArray().md5().toString() + + when (val response = context.response.getOrNull()) { + is SendMessageResponse -> { + val modifiedResponse = SendMessageResponse.invoke { + messageId = response.messageId + sequenceNumber = response.sequenceNumber + md5OfMessageBody = wrongMd5ofMessageBody + md5OfMessageAttributes = wrongMd5ofMessageAttribute + md5OfMessageSystemAttributes = wrongMd5ofMessageSystemAttribute + } + println("modify SendMessage") + return Result.success(modifiedResponse) + } + is ReceiveMessageResponse -> { + val modifiedMessages = response.messages?.map { message -> + Message { + messageId = message.messageId + receiptHandle = message.receiptHandle + body = message.body + attributes = message.attributes + messageAttributes = message.messageAttributes + md5OfBody = wrongMd5ofMessageBody + md5OfMessageAttributes = wrongMd5ofMessageAttribute + } + } + + val modifiedResponse = ReceiveMessageResponse { + messages = modifiedMessages + } + return Result.success(modifiedResponse) + } + is SendMessageBatchResponse -> { + val modifiedEntries = response.successful.map { entry -> + SendMessageBatchResultEntry { + id = entry.id + messageId = entry.messageId + md5OfMessageBody = wrongMd5ofMessageBody + md5OfMessageAttributes = wrongMd5ofMessageAttribute + md5OfMessageSystemAttributes = wrongMd5ofMessageSystemAttribute + sequenceNumber = entry.sequenceNumber + } + } + + val modifiedResponse = SendMessageBatchResponse { + successful = modifiedEntries + failed = response.failed + } + return Result.success(modifiedResponse) + } + } + return context.response + } + } + + private val correctChecksumClient = SqsClient { + region = DEFAULT_REGION + checksumValidationEnabled = ValidationEnabled.ALWAYS + checksumValidationScopes = ValidationScope.entries.toSet() + } + + // used for wrong checksum tests + private val wrongChecksumClient = SqsClient { + region = DEFAULT_REGION + checksumValidationEnabled = ValidationEnabled.ALWAYS + checksumValidationScopes = ValidationScope.entries.toSet() + interceptors += wrongChecksumInterceptor + } + + private lateinit var correctChecksumTestQueueUrl: String + private lateinit var wrongChecksumTestQueueUrl: String + + @BeforeAll + private fun setUp(): Unit = runBlocking { + correctChecksumTestQueueUrl = getTestQueueUrl(correctChecksumClient, TEST_QUEUE_CORRECT_CHECKSUM_PREFIX, DEFAULT_REGION) + wrongChecksumTestQueueUrl = getTestQueueUrl(wrongChecksumClient, TEST_QUEUE_WRONG_CHECKSUM_PREFIX, DEFAULT_REGION) + } + + @AfterAll + private fun cleanUp(): Unit = runBlocking { + deleteQueueAndAllMessages(correctChecksumClient, correctChecksumTestQueueUrl) + deleteQueueAndAllMessages(wrongChecksumClient, wrongChecksumTestQueueUrl) + correctChecksumClient.close() + wrongChecksumClient.close() + } + + @Test + fun testSendMessage(): Unit = runBlocking { + assertDoesNotThrow { + correctChecksumClient.sendMessage( + SendMessageRequest { + queueUrl = correctChecksumTestQueueUrl + messageBody = TEST_MESSAGE_BODY + messageAttributes = hashMapOf( + TEST_MESSAGE_ATTRIBUTES_NAME to MessageAttributeValue { + dataType = "String" + stringValue = TEST_MESSAGE_ATTRIBUTES_VALUE + } + ) + messageSystemAttributes = hashMapOf( + MessageSystemAttributeNameForSends.AwsTraceHeader to MessageSystemAttributeValue { + dataType = "String" + stringValue = TEST_MESSAGE_SYSTEM_ATTRIBUTES_VALUE + } + ) + } + ) + } + } + + @Test + fun testReceiveMessage(): Unit = runBlocking { + assertDoesNotThrow { + correctChecksumClient.receiveMessage( + ReceiveMessageRequest { + queueUrl = correctChecksumTestQueueUrl + maxNumberOfMessages = 1 + messageAttributeNames = listOf(TEST_MESSAGE_ATTRIBUTES_NAME) + messageSystemAttributeNames = listOf(MessageSystemAttributeName.AwsTraceHeader) + } + ) + } + } + + @Test + fun testSendMessageBatch(): Unit = runBlocking { + val entries = (1..5).map { batchId -> + buildSendMessageBatchRequestEntry(batchId) + } + + assertDoesNotThrow { + correctChecksumClient.sendMessageBatch( + SendMessageBatchRequest { + queueUrl = correctChecksumTestQueueUrl + this.entries = entries + } + ) + } + } + + @Test + fun testSendMessageWithWrongChecksum(): Unit = runBlocking { + val exception = assertThrows { + wrongChecksumClient.sendMessage ( + SendMessageRequest { + queueUrl = wrongChecksumTestQueueUrl + messageBody = TEST_MESSAGE_BODY + messageAttributes = hashMapOf( + TEST_MESSAGE_ATTRIBUTES_NAME to MessageAttributeValue { + dataType = "String" + stringValue = TEST_MESSAGE_ATTRIBUTES_VALUE + } + ) + messageSystemAttributes = hashMapOf( + MessageSystemAttributeNameForSends.AwsTraceHeader to MessageSystemAttributeValue { + dataType = "String" + stringValue = TEST_MESSAGE_SYSTEM_ATTRIBUTES_VALUE + } + ) + } + ) + } + + assert(exception.message!!.contains("Checksum mismatch")) + } + + @Test + fun testReceiveMessageWithWrongChecksum(): Unit = runBlocking { + val exception = assertThrows { + wrongChecksumClient.receiveMessage( + ReceiveMessageRequest { + queueUrl = wrongChecksumTestQueueUrl + maxNumberOfMessages = 1 + messageAttributeNames = listOf(TEST_MESSAGE_ATTRIBUTES_NAME) + messageSystemAttributeNames = listOf(MessageSystemAttributeName.AwsTraceHeader) + } + ) + } + + assert(exception.message!!.contains("Checksum mismatch")) + } + + @Test + fun testSendMessageBatchWithWrongChecksum(): Unit = runBlocking { + val entries = (1..5).map { batchId -> + buildSendMessageBatchRequestEntry(batchId) + } + + val exception = assertThrows { + wrongChecksumClient.sendMessageBatch( + SendMessageBatchRequest { + queueUrl = wrongChecksumTestQueueUrl + this.entries = entries + } + ) + } + + assert(exception.message!!.contains("Checksum mismatch")) + } +} diff --git a/services/sqs/e2eTest/src/SqsTestUtils.kt b/services/sqs/e2eTest/src/SqsTestUtils.kt new file mode 100644 index 00000000000..9ef0f43e17d --- /dev/null +++ b/services/sqs/e2eTest/src/SqsTestUtils.kt @@ -0,0 +1,110 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.e2etest + +import aws.sdk.kotlin.services.sqs.SqsClient +import aws.sdk.kotlin.services.sqs.createQueue +import aws.sdk.kotlin.services.sqs.model.* +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.withTimeout +import java.net.URI +import java.util.* +import kotlin.time.Duration.Companion.seconds + +object SqsTestUtils { + const val DEFAULT_REGION = "us-west-2" + + const val TEST_QUEUE_WRONG_CHECKSUM_PREFIX = "sqs-test-queue-" + const val TEST_QUEUE_CORRECT_CHECKSUM_PREFIX = "sqs-test-queue-" + + const val TEST_MESSAGE_BODY = "Hello World" + const val TEST_MESSAGE_ATTRIBUTES_NAME = "TestAttribute" + const val TEST_MESSAGE_ATTRIBUTES_VALUE = "TestAttributeValue" + const val TEST_MESSAGE_SYSTEM_ATTRIBUTES_VALUE = "TestSystemAttributeValue" + + suspend fun getTestQueueUrl( + client: SqsClient, + prefix: String, + region: String? = null, + ): String = getQueueUrlWithPrefix(client, prefix, region) + + suspend fun getQueueUrlWithPrefix( + client: SqsClient, + prefix: String, + region: String? = null, + ): String = withTimeout(60.seconds) { + val queueUrls = client.listQueues().queueUrls + + var matchingQueueUrl = queueUrls?.firstOrNull { url -> + val queueUrl = URI(url).toURL() + val hostParts = queueUrl.host.split(".") + + val regionMatches = if (region != null) { + hostParts.getOrNull(1)?.equals(region, ignoreCase = true) ?: false + } else { + true + } + + val queueName = queueUrl.path.split("/").last() + val prefixMatches = queueName.startsWith(prefix) + + regionMatches && prefixMatches + } + + if (matchingQueueUrl == null) { + matchingQueueUrl = prefix + UUID.randomUUID() + println("Creating Sqs queue: $matchingQueueUrl") + + client.createQueue { + queueName = matchingQueueUrl + } + } else { + println("Using existing Sqs queue: $matchingQueueUrl") + } + + matchingQueueUrl + } + + suspend fun deleteQueueAndAllMessages(client: SqsClient, queueUrl: String): Unit = coroutineScope { + try { + println("Purging Sqs queue: $queueUrl") + val purgeRequest = PurgeQueueRequest { + this.queueUrl = queueUrl + } + + client.purgeQueue(purgeRequest) + println("Queue purged successfully.") + + println("Deleting Sqs queue: $queueUrl") + val deleteRequest = DeleteQueueRequest { + this.queueUrl = queueUrl + } + + client.deleteQueue(deleteRequest) + println("Queue deleted successfully.") + } catch (e: SqsException) { + println("Error during delete SQS queue: ${e.message}") + } + } + + fun buildSendMessageBatchRequestEntry(batchId: Int): SendMessageBatchRequestEntry{ + return SendMessageBatchRequestEntry { + id = batchId.toString() + messageBody = TEST_MESSAGE_BODY + batchId + messageAttributes = hashMapOf( + TEST_MESSAGE_ATTRIBUTES_NAME to MessageAttributeValue { + dataType = "String" + stringValue = TEST_MESSAGE_ATTRIBUTES_VALUE + batchId + } + ) + messageSystemAttributes = hashMapOf( + MessageSystemAttributeNameForSends.AwsTraceHeader to MessageSystemAttributeValue { + dataType = "String" + stringValue = TEST_MESSAGE_SYSTEM_ATTRIBUTES_VALUE + batchId + } + ) + } + } +} From 44b7d4785fee9e6b6b49bdaeadfa0e930e97895c Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Tue, 4 Mar 2025 11:36:18 -0500 Subject: [PATCH 02/13] lint --- .../runtime/config/profile/AwsProfile.kt | 2 +- .../SqsMd5ChecksumValidationIntegration.kt | 13 +++---- .../SqsMd5ChecksumValidationInterceptor.kt | 37 +++++++++---------- .../internal/FinalizeSqsConfig.kt | 2 +- .../internal/SQSSetting.kt | 2 +- .../internal/ValidationConfig.kt | 4 +- .../src/SqsMd5ChecksumValidationTest.kt | 22 +++++------ services/sqs/e2eTest/src/SqsTestUtils.kt | 32 ++++++++-------- 8 files changed, 55 insertions(+), 59 deletions(-) diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt index 8e2bebe72c5..a857769a226 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt @@ -239,7 +239,7 @@ public inline fun > AwsProfile.getEnumSetOrNull(key: String, append(value) append("' is not supported, should be one of: ") enumValues().joinTo(this) { it.name.lowercase() } - } + }, ) }.toSet() .takeIf { it.isNotEmpty() } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt index f69f5bfb4ed..3d86820d647 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt @@ -29,13 +29,12 @@ class SqsMd5ChecksumValidationIntegration : KotlinIntegration { } internal object SqsMd5ChecksumValidationMiddleware : ProtocolMiddleware { - override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean { - return when (op.id.name) { - "ReceiveMessage", - "SendMessage", - "SendMessageBatch" -> true - else -> false - } + override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = when (op.id.name) { + "ReceiveMessage", + "SendMessage", + "SendMessageBatch", + -> true + else -> false } override val name: String = "SqsMd5ChecksumValidationInterceptor" diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt index f0617c492e8..8641bca643e 100644 --- a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt @@ -103,7 +103,7 @@ public class SqsMd5ChecksumValidationInterceptor( sendMessageRequest: SendMessageRequest, sendMessageResponse: SendMessageResponse, ) { - if(validationScopes.contains(ValidationScope.MESSAGE_BODY)) { + if (validationScopes.contains(ValidationScope.MESSAGE_BODY)) { val messageBodySent = sendMessageRequest.messageBody if (!messageBodySent.isNullOrEmpty()) { @@ -117,7 +117,7 @@ public class SqsMd5ChecksumValidationInterceptor( } } - if(validationScopes.contains(ValidationScope.MESSAGE_ATTRIBUTES)) { + if (validationScopes.contains(ValidationScope.MESSAGE_ATTRIBUTES)) { val messageAttrSent = sendMessageRequest.messageAttributes if (!messageAttrSent.isNullOrEmpty()) { logger.debug { "Validating message attribute MD5 checksum for SendMessage" } @@ -130,7 +130,7 @@ public class SqsMd5ChecksumValidationInterceptor( } } - if(validationScopes.contains(ValidationScope.MESSAGE_SYSTEM_ATTRIBUTES)) { + if (validationScopes.contains(ValidationScope.MESSAGE_SYSTEM_ATTRIBUTES)) { val messageSysAttrSent = sendMessageRequest.messageSystemAttributes if (!messageSysAttrSent.isNullOrEmpty()) { logger.debug { "Validating message system attribute MD5 checksum for SendMessage" } @@ -148,7 +148,7 @@ public class SqsMd5ChecksumValidationInterceptor( val messages = receiveMessageResponse.messages if (messages != null) { for (messageReceived in messages) { - if(validationScopes.contains(ValidationScope.MESSAGE_BODY)) { + if (validationScopes.contains(ValidationScope.MESSAGE_BODY)) { val messageBody = messageReceived.body if (!messageBody.isNullOrEmpty()) { logger.debug { "Validating message body MD5 checksum for ReceiveMessage" } @@ -161,7 +161,7 @@ public class SqsMd5ChecksumValidationInterceptor( } } - if(validationScopes.contains(ValidationScope.MESSAGE_ATTRIBUTES)) { + if (validationScopes.contains(ValidationScope.MESSAGE_ATTRIBUTES)) { val messageAttr = messageReceived.messageAttributes if (!messageAttr.isNullOrEmpty()) { @@ -191,7 +191,7 @@ public class SqsMd5ChecksumValidationInterceptor( } for (entry in sendMessageBatchResponse.successful) { - if(validationScopes.contains(ValidationScope.MESSAGE_BODY)) { + if (validationScopes.contains(ValidationScope.MESSAGE_BODY)) { val messageBody = idToRequestEntryMap[entry.id]?.messageBody if (!messageBody.isNullOrEmpty()) { @@ -205,7 +205,7 @@ public class SqsMd5ChecksumValidationInterceptor( } } - if(validationScopes.contains(ValidationScope.MESSAGE_ATTRIBUTES)) { + if (validationScopes.contains(ValidationScope.MESSAGE_ATTRIBUTES)) { val messageAttrSent = idToRequestEntryMap[entry.id]?.messageAttributes if (!messageAttrSent.isNullOrEmpty()) { logger.debug { "Validating message attribute MD5 checksum for SendMessageBatch: ${entry.messageId}" } @@ -218,7 +218,7 @@ public class SqsMd5ChecksumValidationInterceptor( } } - if(validationScopes.contains(ValidationScope.MESSAGE_SYSTEM_ATTRIBUTES)) { + if (validationScopes.contains(ValidationScope.MESSAGE_SYSTEM_ATTRIBUTES)) { val messageSysAttrSent = idToRequestEntryMap[entry.id]?.messageSystemAttributes if (!messageSysAttrSent.isNullOrEmpty()) { logger.debug { "Validating message system attribute MD5 checksum for SendMessageBatch: ${entry.messageId}" } @@ -239,9 +239,9 @@ public class SqsMd5ChecksumValidationInterceptor( } catch (e: Exception) { throw ClientException( "Unable to calculate the MD5 hash of the message body." + - "Potential reasons include JVM configuration or FIPS compliance issues." + - "To disable message MD5 validation, you can set checksumValidationEnabled" + - "to false when instantiating the client." + e.message, + "Potential reasons include JVM configuration or FIPS compliance issues." + + "To disable message MD5 validation, you can set checksumValidationEnabled" + + "to false when instantiating the client." + e.message, ) } val expectedMD5Hex = expectedMD5.toHexString() @@ -300,9 +300,9 @@ public class SqsMd5ChecksumValidationInterceptor( } catch (e: Exception) { throw ClientException( "Unable to calculate the MD5 hash of the message body." + - "Potential reasons include JVM configuration or FIPS compliance issues." + - "To disable message MD5 validation, you can set checksumValidationEnabled" + - "to false when instantiating the client." + e.message, + "Potential reasons include JVM configuration or FIPS compliance issues." + + "To disable message MD5 validation, you can set checksumValidationEnabled" + + "to false when instantiating the client." + e.message, ) } val expectedMD5Hex = md5Digest.digest().toHexString() @@ -310,8 +310,7 @@ public class SqsMd5ChecksumValidationInterceptor( } private fun calculateMessageSystemAttributesMd5( - messageSysAttrs: - Map, + messageSysAttrs: Map, ): String { val sortedAttributeNames = messageSysAttrs.keys.sortedBy { it.value } val md5Digest = Md5() @@ -359,9 +358,9 @@ public class SqsMd5ChecksumValidationInterceptor( } catch (e: Exception) { throw ClientException( "Unable to calculate the MD5 hash of the message body." + - "Potential reasons include JVM configuration or FIPS compliance issues." + - "To disable message MD5 validation, you can set checksumValidationEnabled" + - "to false when instantiating the client." + e.message, + "Potential reasons include JVM configuration or FIPS compliance issues." + + "To disable message MD5 validation, you can set checksumValidationEnabled" + + "to false when instantiating the client." + e.message, ) } val expectedMD5Hex = md5Digest.digest().toHexString() diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/FinalizeSqsConfig.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/FinalizeSqsConfig.kt index 178e841923c..8ed86777313 100644 --- a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/FinalizeSqsConfig.kt +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/FinalizeSqsConfig.kt @@ -19,7 +19,7 @@ internal suspend fun finalizeSqsConfig( builder.config.checksumValidationEnabled = builder.config.checksumValidationEnabled ?: SQSSetting.checksumValidationEnabled.resolve(provider) ?: activeProfile.checksumValidationEnabled - ?: ValidationEnabled.NEVER //TODO: MD5 checksum validation is temporarily disabled. Set default to ALWAYS in next minor version + ?: ValidationEnabled.NEVER // TODO: MD5 checksum validation is temporarily disabled. Set default to ALWAYS in next minor version builder.config.checksumValidationScopes = builder.config.checksumValidationScopes.ifEmpty { SQSSetting.checksumValidationScopes.resolve(provider) diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SQSSetting.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SQSSetting.kt index cbaa9ea6b9d..5a8308599be 100644 --- a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SQSSetting.kt +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SQSSetting.kt @@ -25,7 +25,7 @@ internal object SQSSetting { * * Note: Value matching is case-insensitive when configured via environment variables. */ - public val checksumValidationEnabled : EnvironmentSetting = + public val checksumValidationEnabled: EnvironmentSetting = enumEnvSetting("aws.SqsChecksumValidationEnabled", "AWS_SQS_CHECKSUM_VALIDATION_ENABLED") /** diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/ValidationConfig.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/ValidationConfig.kt index b3cd56ad84e..61ef7c15927 100644 --- a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/ValidationConfig.kt +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/ValidationConfig.kt @@ -21,7 +21,7 @@ public enum class ValidationEnabled { ALWAYS, WHEN_SENDING, WHEN_RECEIVING, - NEVER + NEVER, } /** @@ -42,5 +42,5 @@ public enum class ValidationEnabled { public enum class ValidationScope { MESSAGE_ATTRIBUTES, MESSAGE_SYSTEM_ATTRIBUTES, - MESSAGE_BODY + MESSAGE_BODY, } diff --git a/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt b/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt index 9d11148c67b..618f34c31eb 100644 --- a/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt +++ b/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt @@ -136,15 +136,15 @@ class SqsMd5ChecksumValidationTest { TEST_MESSAGE_ATTRIBUTES_NAME to MessageAttributeValue { dataType = "String" stringValue = TEST_MESSAGE_ATTRIBUTES_VALUE - } + }, ) messageSystemAttributes = hashMapOf( MessageSystemAttributeNameForSends.AwsTraceHeader to MessageSystemAttributeValue { dataType = "String" stringValue = TEST_MESSAGE_SYSTEM_ATTRIBUTES_VALUE - } + }, ) - } + }, ) } } @@ -158,7 +158,7 @@ class SqsMd5ChecksumValidationTest { maxNumberOfMessages = 1 messageAttributeNames = listOf(TEST_MESSAGE_ATTRIBUTES_NAME) messageSystemAttributeNames = listOf(MessageSystemAttributeName.AwsTraceHeader) - } + }, ) } } @@ -174,7 +174,7 @@ class SqsMd5ChecksumValidationTest { SendMessageBatchRequest { queueUrl = correctChecksumTestQueueUrl this.entries = entries - } + }, ) } } @@ -182,7 +182,7 @@ class SqsMd5ChecksumValidationTest { @Test fun testSendMessageWithWrongChecksum(): Unit = runBlocking { val exception = assertThrows { - wrongChecksumClient.sendMessage ( + wrongChecksumClient.sendMessage( SendMessageRequest { queueUrl = wrongChecksumTestQueueUrl messageBody = TEST_MESSAGE_BODY @@ -190,15 +190,15 @@ class SqsMd5ChecksumValidationTest { TEST_MESSAGE_ATTRIBUTES_NAME to MessageAttributeValue { dataType = "String" stringValue = TEST_MESSAGE_ATTRIBUTES_VALUE - } + }, ) messageSystemAttributes = hashMapOf( MessageSystemAttributeNameForSends.AwsTraceHeader to MessageSystemAttributeValue { dataType = "String" stringValue = TEST_MESSAGE_SYSTEM_ATTRIBUTES_VALUE - } + }, ) - } + }, ) } @@ -214,7 +214,7 @@ class SqsMd5ChecksumValidationTest { maxNumberOfMessages = 1 messageAttributeNames = listOf(TEST_MESSAGE_ATTRIBUTES_NAME) messageSystemAttributeNames = listOf(MessageSystemAttributeName.AwsTraceHeader) - } + }, ) } @@ -232,7 +232,7 @@ class SqsMd5ChecksumValidationTest { SendMessageBatchRequest { queueUrl = wrongChecksumTestQueueUrl this.entries = entries - } + }, ) } diff --git a/services/sqs/e2eTest/src/SqsTestUtils.kt b/services/sqs/e2eTest/src/SqsTestUtils.kt index 9ef0f43e17d..7e3c5edeafa 100644 --- a/services/sqs/e2eTest/src/SqsTestUtils.kt +++ b/services/sqs/e2eTest/src/SqsTestUtils.kt @@ -89,22 +89,20 @@ object SqsTestUtils { } } - fun buildSendMessageBatchRequestEntry(batchId: Int): SendMessageBatchRequestEntry{ - return SendMessageBatchRequestEntry { - id = batchId.toString() - messageBody = TEST_MESSAGE_BODY + batchId - messageAttributes = hashMapOf( - TEST_MESSAGE_ATTRIBUTES_NAME to MessageAttributeValue { - dataType = "String" - stringValue = TEST_MESSAGE_ATTRIBUTES_VALUE + batchId - } - ) - messageSystemAttributes = hashMapOf( - MessageSystemAttributeNameForSends.AwsTraceHeader to MessageSystemAttributeValue { - dataType = "String" - stringValue = TEST_MESSAGE_SYSTEM_ATTRIBUTES_VALUE + batchId - } - ) - } + fun buildSendMessageBatchRequestEntry(batchId: Int): SendMessageBatchRequestEntry = SendMessageBatchRequestEntry { + id = batchId.toString() + messageBody = TEST_MESSAGE_BODY + batchId + messageAttributes = hashMapOf( + TEST_MESSAGE_ATTRIBUTES_NAME to MessageAttributeValue { + dataType = "String" + stringValue = TEST_MESSAGE_ATTRIBUTES_VALUE + batchId + }, + ) + messageSystemAttributes = hashMapOf( + MessageSystemAttributeNameForSends.AwsTraceHeader to MessageSystemAttributeValue { + dataType = "String" + stringValue = TEST_MESSAGE_SYSTEM_ATTRIBUTES_VALUE + batchId + }, + ) } } From 7f1eaffcc3edfbcf560e2bf9b488571574fd0992 Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Tue, 4 Mar 2025 13:19:44 -0500 Subject: [PATCH 03/13] reduce duplication --- .../SqsMd5ChecksumValidationInterceptor.kt | 82 ++++++++----------- 1 file changed, 34 insertions(+), 48 deletions(-) diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt index 8641bca643e..a3b89990760 100644 --- a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt @@ -272,34 +272,15 @@ public class SqsMd5ChecksumValidationInterceptor( val binaryListValues = attributeValue?.binaryListValues when { - stringValue != null -> { - md5Digest.update(STRING_TYPE_FIELD_INDEX) - updateLengthAndBytes(md5Digest, stringValue.encodeToByteArray()) - } - - binaryValue != null -> { - md5Digest.update(BINARY_TYPE_FIELD_INDEX) - updateLengthAndBytes(md5Digest, binaryValue) - } - - !stringListValues.isNullOrEmpty() -> { - md5Digest.update(STRING_LIST_TYPE_FIELD_INDEX) - for (stringListValue in stringListValues) { - updateLengthAndBytes(md5Digest, stringListValue.encodeToByteArray()) - } - } - - !binaryListValues.isNullOrEmpty() -> { - md5Digest.update(BINARY_LIST_TYPE_FIELD_INDEX) - for (binaryListValue in binaryListValues) { - updateLengthAndBytes(md5Digest, binaryListValue) - } - } + stringValue != null -> updateForStringType(md5Digest, stringValue) + binaryValue != null -> updateForBinaryType(md5Digest, binaryValue) + !stringListValues.isNullOrEmpty() -> updateForStringListType(md5Digest, stringListValues) + !binaryListValues.isNullOrEmpty() -> updateForBinaryListType(md5Digest, binaryListValues) } } } catch (e: Exception) { throw ClientException( - "Unable to calculate the MD5 hash of the message body." + + "Unable to calculate the MD5 hash of the message attributes." + "Potential reasons include JVM configuration or FIPS compliance issues." + "To disable message MD5 validation, you can set checksumValidationEnabled" + "to false when instantiating the client." + e.message, @@ -330,34 +311,15 @@ public class SqsMd5ChecksumValidationInterceptor( val binaryListValues = attributeValue?.binaryListValues when { - stringValue != null -> { - md5Digest.update(STRING_TYPE_FIELD_INDEX) - updateLengthAndBytes(md5Digest, stringValue.encodeToByteArray()) - } - - binaryValue != null -> { - md5Digest.update(BINARY_TYPE_FIELD_INDEX) - updateLengthAndBytes(md5Digest, binaryValue) - } - - !stringListValues.isNullOrEmpty() -> { - md5Digest.update(STRING_LIST_TYPE_FIELD_INDEX) - for (stringListValue in stringListValues) { - updateLengthAndBytes(md5Digest, stringListValue.encodeToByteArray()) - } - } - - !binaryListValues.isNullOrEmpty() -> { - md5Digest.update(BINARY_LIST_TYPE_FIELD_INDEX) - for (binaryListValue in binaryListValues) { - updateLengthAndBytes(md5Digest, binaryListValue) - } - } + stringValue != null -> updateForStringType(md5Digest, stringValue) + binaryValue != null -> updateForBinaryType(md5Digest, binaryValue) + !stringListValues.isNullOrEmpty() -> updateForStringListType(md5Digest, stringListValues) + !binaryListValues.isNullOrEmpty() -> updateForBinaryListType(md5Digest, binaryListValues) } } } catch (e: Exception) { throw ClientException( - "Unable to calculate the MD5 hash of the message body." + + "Unable to calculate the MD5 hash of the message system attributes." + "Potential reasons include JVM configuration or FIPS compliance issues." + "To disable message MD5 validation, you can set checksumValidationEnabled" + "to false when instantiating the client." + e.message, @@ -367,6 +329,30 @@ public class SqsMd5ChecksumValidationInterceptor( return expectedMD5Hex } + private fun updateForStringType(md5Digest: Md5, value: String) { + md5Digest.update(STRING_TYPE_FIELD_INDEX) + updateLengthAndBytes(md5Digest, value.encodeToByteArray()) + } + + private fun updateForBinaryType(md5Digest: Md5, value: ByteArray) { + md5Digest.update(BINARY_TYPE_FIELD_INDEX) + updateLengthAndBytes(md5Digest, value) + } + + private fun updateForStringListType(md5Digest: Md5, values: List) { + md5Digest.update(STRING_LIST_TYPE_FIELD_INDEX) + values.forEach { value -> + updateLengthAndBytes(md5Digest, value.encodeToByteArray()) + } + } + + private fun updateForBinaryListType(md5Digest: Md5, values: List) { + md5Digest.update(BINARY_LIST_TYPE_FIELD_INDEX) + values.forEach { value -> + updateLengthAndBytes(md5Digest, value) + } + } + /** * Update the digest using a sequence of bytes that consists of the length (in 4 bytes) of the * input binaryValue and all the bytes it contains. From dc370a63c234c5445be9df6d34f8c2289acc9271 Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Tue, 4 Mar 2025 18:07:00 -0500 Subject: [PATCH 04/13] address pr feedbacks --- .../sqs/ClientConfigIntegration.kt | 92 ------ .../SqsMd5ChecksumValidationIntegration.kt | 77 ++++- ...tlin.codegen.integration.KotlinIntegration | 1 - ...SqsMd5ChecksumValidationIntegrationTest.kt | 2 +- .../SqsMd5ChecksumValidationInterceptor.kt | 266 ++++++++---------- .../internal/FinalizeSqsConfig.kt | 4 +- .../{SQSSetting.kt => SqsSettings.kt} | 2 +- .../src/SqsMd5ChecksumValidationTest.kt | 24 +- services/sqs/e2eTest/src/SqsTestUtils.kt | 57 ++-- 9 files changed, 232 insertions(+), 293 deletions(-) delete mode 100644 codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/ClientConfigIntegration.kt rename services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/{SQSSetting.kt => SqsSettings.kt} (98%) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/ClientConfigIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/ClientConfigIntegration.kt deleted file mode 100644 index 8758066a46f..00000000000 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/ClientConfigIntegration.kt +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package aws.sdk.kotlin.codegen.customization.sqs - -import aws.sdk.kotlin.codegen.ServiceClientCompanionObjectWriter -import software.amazon.smithy.kotlin.codegen.KotlinSettings -import software.amazon.smithy.kotlin.codegen.core.CodegenContext -import software.amazon.smithy.kotlin.codegen.integration.AppendingSectionWriter -import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration -import software.amazon.smithy.kotlin.codegen.integration.SectionWriterBinding -import software.amazon.smithy.kotlin.codegen.lang.KotlinTypes -import software.amazon.smithy.kotlin.codegen.model.buildSymbol -import software.amazon.smithy.kotlin.codegen.model.expectShape -import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigProperty -import software.amazon.smithy.model.Model -import software.amazon.smithy.model.shapes.ServiceShape - -class ClientConfigIntegration : KotlinIntegration { - override fun enabledForService(model: Model, settings: KotlinSettings): Boolean = - model.expectShape(settings.service).isSqs - - companion object { - val ValidationEnabledProp: ConfigProperty = ConfigProperty { - name = "checksumValidationEnabled" - symbol = buildSymbol { - name = "ValidationEnabled" - namespace = "aws.sdk.kotlin.services.sqs.internal" - documentation = """ - Specifies when MD5 checksum validation should be performed for SQS messages. This controls the automatic - calculation and validation of checksums during message operations. - - Valid values: - - ALWAYS (default) - Checksums are calculated and validated for both sending and receiving operations - (SendMessage, SendMessageBatch, and ReceiveMessage) - - WHEN_SENDING - Checksums are only calculated and validated during send operations - (SendMessage and SendMessageBatch) - - WHEN_RECEIVING - Checksums are only calculated and validated during receive operations - (ReceiveMessage) - - NEVER - No checksum calculation or validation is performed - """.trimIndent() - } - } - - private val validationScope = buildSymbol { - name = "ValidationScope" - namespace = "aws.sdk.kotlin.services.sqs.internal" - } - - val ValidationScopeProp: ConfigProperty = ConfigProperty { - name = "checksumValidationScopes" - symbol = KotlinTypes.Collections.set(validationScope, default = "emptySet()") - documentation = """ - Specifies which parts of an SQS message should undergo MD5 checksum validation. This configuration - accepts a set of validation scopes that determine which message components to validate. - - Valid values: - - MESSAGE_ATTRIBUTES - Validates checksums for message attributes - - MESSAGE_SYSTEM_ATTRIBUTES - Validates checksums for message system attributes - (Note: Not available for ReceiveMessage operations as SQS does not calculate checksums for - system attributes during message receipt) - - MESSAGE_BODY - Validates checksums for the message body - - Default: All three scopes (MESSAGE_ATTRIBUTES, MESSAGE_SYSTEM_ATTRIBUTES, MESSAGE_BODY) - """.trimIndent() - } - } - - override fun additionalServiceConfigProps(ctx: CodegenContext): List = - listOf( - ValidationEnabledProp, - ValidationScopeProp, - ) - - override val sectionWriters: List - get() = listOf( - SectionWriterBinding( - ServiceClientCompanionObjectWriter.FinalizeEnvironmentalConfig, - finalizeSqsConfigWriter, - ), - ) - - // add Sqs-specific config finalization - private val finalizeSqsConfigWriter = AppendingSectionWriter { writer -> - val finalizeSqsConfig = buildSymbol { - name = "finalizeSqsConfig" - namespace = "aws.sdk.kotlin.services.sqs.internal" - } - writer.write("#T(builder, sharedConfig)", finalizeSqsConfig) - } -} diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt index 3d86820d647..36efc644ccf 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt @@ -4,13 +4,19 @@ */ package aws.sdk.kotlin.codegen.customization.sqs +import aws.sdk.kotlin.codegen.ServiceClientCompanionObjectWriter import software.amazon.smithy.kotlin.codegen.KotlinSettings +import software.amazon.smithy.kotlin.codegen.core.CodegenContext import software.amazon.smithy.kotlin.codegen.core.KotlinWriter +import software.amazon.smithy.kotlin.codegen.integration.AppendingSectionWriter import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration +import software.amazon.smithy.kotlin.codegen.integration.SectionWriterBinding +import software.amazon.smithy.kotlin.codegen.lang.KotlinTypes import software.amazon.smithy.kotlin.codegen.model.buildSymbol import software.amazon.smithy.kotlin.codegen.model.expectShape import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware +import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigProperty import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.ServiceShape @@ -22,6 +28,75 @@ class SqsMd5ChecksumValidationIntegration : KotlinIntegration { override fun enabledForService(model: Model, settings: KotlinSettings): Boolean = model.expectShape(settings.service).isSqs + companion object { + val ValidationEnabledProp: ConfigProperty = ConfigProperty { + name = "checksumValidationEnabled" + symbol = buildSymbol { + name = "ValidationEnabled" + namespace = "aws.sdk.kotlin.services.sqs.internal" + } + documentation = """ + Specifies when MD5 checksum validation should be performed for SQS messages. This controls the automatic + calculation and validation of checksums during message operations. + + Valid values: + - `ALWAYS` (default) - Checksums are calculated and validated for both sending and receiving operations + (SendMessage, SendMessageBatch, and ReceiveMessage) + - `WHEN_SENDING` - Checksums are only calculated and validated during send operations + (SendMessage and SendMessageBatch) + - `WHEN_RECEIVING` - Checksums are only calculated and validated during receive operations + (ReceiveMessage) + - `NEVER` - No checksum calculation or validation is performed + """.trimIndent() + } + + private val validationScope = buildSymbol { + name = "ValidationScope" + namespace = "aws.sdk.kotlin.services.sqs.internal" + } + + val ValidationScopeProp: ConfigProperty = ConfigProperty { + name = "checksumValidationScopes" + symbol = KotlinTypes.Collections.set(validationScope, default = "emptySet()") + documentation = """ + Specifies which parts of an SQS message should undergo MD5 checksum validation. This configuration + accepts a set of validation scopes that determine which message components to validate. + + Valid values: + - `MESSAGE_ATTRIBUTES` - Validates checksums for message attributes + - `MESSAGE_SYSTEM_ATTRIBUTES` - Validates checksums for message system attributes + (Note: Not available for ReceiveMessage operations as SQS does not calculate checksums for + system attributes during message receipt) + - `MESSAGE_BODY` - Validates checksums for the message body + + Default: All three scopes (MESSAGE_ATTRIBUTES, MESSAGE_SYSTEM_ATTRIBUTES, MESSAGE_BODY) + """.trimIndent() + } + } + + override fun additionalServiceConfigProps(ctx: CodegenContext): List = + listOf( + ValidationEnabledProp, + ValidationScopeProp, + ) + + override val sectionWriters: List + get() = listOf( + SectionWriterBinding( + ServiceClientCompanionObjectWriter.FinalizeEnvironmentalConfig, + finalizeSqsConfigWriter, + ), + ) + + // add Sqs-specific config finalization + private val finalizeSqsConfigWriter = AppendingSectionWriter { writer -> + val finalizeSqsConfig = buildSymbol { + name = "finalizeSqsConfig" + namespace = "aws.sdk.kotlin.services.sqs.internal" + } + writer.write("#T(builder, sharedConfig)", finalizeSqsConfig) + } + override fun customizeMiddleware( ctx: ProtocolGenerator.GenerationContext, resolved: List, @@ -45,6 +120,6 @@ internal object SqsMd5ChecksumValidationMiddleware : ProtocolMiddleware { namespace = "aws.sdk.kotlin.services.sqs" } - writer.write("op.interceptors.add(#T(config.checksumValidationEnabled, config.checksumValidationScopes,))", symbol) + writer.write("op.interceptors.add(#T(config.checksumValidationEnabled, config.checksumValidationScopes))", symbol) } } diff --git a/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration b/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration index 5a8d3df78a6..369fe2ccfc1 100644 --- a/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration +++ b/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration @@ -49,5 +49,4 @@ aws.sdk.kotlin.codegen.smoketests.SmokeTestsDenyListIntegration aws.sdk.kotlin.codegen.smoketests.testing.SmokeTestSuccessHttpEngineIntegration aws.sdk.kotlin.codegen.smoketests.testing.SmokeTestFailHttpEngineIntegration aws.sdk.kotlin.codegen.customization.AwsQueryModeCustomization -aws.sdk.kotlin.codegen.customization.sqs.ClientConfigIntegration aws.sdk.kotlin.codegen.customization.sqs.SqsMd5ChecksumValidationIntegration \ No newline at end of file diff --git a/codegen/aws-sdk-codegen/src/test/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegrationTest.kt b/codegen/aws-sdk-codegen/src/test/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegrationTest.kt index aa23286fcc0..a5175e7d069 100644 --- a/codegen/aws-sdk-codegen/src/test/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegrationTest.kt +++ b/codegen/aws-sdk-codegen/src/test/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegrationTest.kt @@ -48,5 +48,5 @@ class SqsMd5ChecksumValidationIntegrationTest { object FooMiddleware : ProtocolMiddleware { override val name: String = "FooMiddleware" override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) = - fail("Unexpected call to `FooMiddleware.render") + fail("Unexpected call to `FooMiddleware.render`") } diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt index a3b89990760..b3d903dfc24 100644 --- a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt @@ -8,23 +8,15 @@ import aws.sdk.kotlin.services.sqs.internal.ValidationEnabled import aws.sdk.kotlin.services.sqs.internal.ValidationScope import aws.sdk.kotlin.services.sqs.model.* import aws.smithy.kotlin.runtime.ClientException -import aws.smithy.kotlin.runtime.InternalApi import aws.smithy.kotlin.runtime.client.ResponseInterceptorContext -import aws.smithy.kotlin.runtime.hashing.Md5 import aws.smithy.kotlin.runtime.hashing.md5 import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor import aws.smithy.kotlin.runtime.http.request.HttpRequest import aws.smithy.kotlin.runtime.http.response.HttpResponse +import aws.smithy.kotlin.runtime.io.SdkBuffer import aws.smithy.kotlin.runtime.telemetry.logging.Logger import aws.smithy.kotlin.runtime.telemetry.logging.logger -import kotlin.collections.Map -import kotlin.collections.Set -import kotlin.collections.hashMapOf -import kotlin.collections.isNullOrEmpty -import kotlin.collections.set -import kotlin.collections.sorted -import kotlin.collections.sortedBy /** * Interceptor that validates MD5 checksums for SQS message operations. @@ -44,7 +36,7 @@ import kotlin.collections.sortedBy * - SendMessageBatch * - ReceiveMessage */ -@OptIn(InternalApi::class, ExperimentalStdlibApi::class) +@OptIn(ExperimentalStdlibApi::class) public class SqsMd5ChecksumValidationInterceptor( private val validationEnabled: ValidationEnabled?, private val validationScopes: Set, @@ -54,22 +46,23 @@ public class SqsMd5ChecksumValidationInterceptor( private const val BINARY_TYPE_FIELD_INDEX: Byte = 2 private const val STRING_LIST_TYPE_FIELD_INDEX: Byte = 3 private const val BINARY_LIST_TYPE_FIELD_INDEX: Byte = 4 - - private lateinit var logger: Logger - - private fun initLogger(logger: Logger) { - this.logger = logger - } } override fun readAfterExecution(context: ResponseInterceptorContext) { - val request = context.request - val response = context.response.getOrNull() - if (validationEnabled == ValidationEnabled.NEVER) return val logger = context.executionContext.coroutineContext.logger() - initLogger(logger) + + // Test MD5 availability + try { + "MD5".encodeToByteArray().md5() + } catch (e: Exception) { + logger.error { "MD5 checksums are not available (likely due to FIPS mode). Checksum validation will be disabled." } + return + } + + val request = context.request + val response = context.response.getOrNull() if (response != null) { when (request) { @@ -78,14 +71,14 @@ public class SqsMd5ChecksumValidationInterceptor( val sendMessageRequest = request as SendMessageRequest val sendMessageResponse = response as SendMessageResponse - sendMessageOperationMd5Check(sendMessageRequest, sendMessageResponse) + sendMessageOperationMd5Check(sendMessageRequest, sendMessageResponse, logger) } is ReceiveMessageRequest -> { if (validationEnabled == ValidationEnabled.WHEN_SENDING) return val receiveMessageResponse = response as ReceiveMessageResponse - receiveMessageResultMd5Check(receiveMessageResponse) + receiveMessageResultMd5Check(receiveMessageResponse, logger) } is SendMessageBatchRequest -> { @@ -93,7 +86,7 @@ public class SqsMd5ChecksumValidationInterceptor( val sendMessageBatchRequest = request as SendMessageBatchRequest val sendMessageBatchResponse = response as SendMessageBatchResponse - sendMessageBatchOperationMd5Check(sendMessageBatchRequest, sendMessageBatchResponse) + sendMessageBatchOperationMd5Check(sendMessageBatchRequest, sendMessageBatchResponse, logger) } } } @@ -102,6 +95,7 @@ public class SqsMd5ChecksumValidationInterceptor( private fun sendMessageOperationMd5Check( sendMessageRequest: SendMessageRequest, sendMessageResponse: SendMessageResponse, + logger: Logger ) { if (validationScopes.contains(ValidationScope.MESSAGE_BODY)) { val messageBodySent = sendMessageRequest.messageBody @@ -114,6 +108,8 @@ public class SqsMd5ChecksumValidationInterceptor( if (clientSideBodyMd5 != bodyMD5Returned) { throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideBodyMd5 but was $bodyMD5Returned") } + + logger.debug { "Message body MD5 checksum for SendMessage validated" } } } @@ -127,6 +123,8 @@ public class SqsMd5ChecksumValidationInterceptor( if (clientSideAttrMd5 != messageAttrMD5Returned) { throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideAttrMd5 but was $messageAttrMD5Returned") } + + logger.debug { "Message attribute MD5 checksum for SendMessage validated" } } } @@ -140,39 +138,42 @@ public class SqsMd5ChecksumValidationInterceptor( if (clientSideSysAttrMd5 != messageSysAttrMD5Returned) { throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideSysAttrMd5 but was $messageSysAttrMD5Returned") } + + logger.debug { "Message system attribute MD5 checksum for SendMessage validated" } } } } - private fun receiveMessageResultMd5Check(receiveMessageResponse: ReceiveMessageResponse) { - val messages = receiveMessageResponse.messages - if (messages != null) { - for (messageReceived in messages) { - if (validationScopes.contains(ValidationScope.MESSAGE_BODY)) { - val messageBody = messageReceived.body - if (!messageBody.isNullOrEmpty()) { - logger.debug { "Validating message body MD5 checksum for ReceiveMessage" } - - val bodyMd5Returned = messageReceived.md5OfBody - val clientSideBodyMd5 = calculateMessageBodyMd5(messageBody) - if (clientSideBodyMd5 != bodyMd5Returned) { - throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideBodyMd5 but was $bodyMd5Returned") - } + private fun receiveMessageResultMd5Check(receiveMessageResponse: ReceiveMessageResponse, logger: Logger) { + receiveMessageResponse.messages?.forEach { messageReceived -> + if (validationScopes.contains(ValidationScope.MESSAGE_BODY)) { + val messageBody = messageReceived.body + if (!messageBody.isNullOrEmpty()) { + logger.debug { "Validating message body MD5 checksum for ReceiveMessage" } + + val bodyMd5Returned = messageReceived.md5OfBody + val clientSideBodyMd5 = calculateMessageBodyMd5(messageBody) + if (clientSideBodyMd5 != bodyMd5Returned) { + throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideBodyMd5 but was $bodyMd5Returned") } + + logger.debug { "Message body MD5 checksum for ReceiveMessage validated " } } + } - if (validationScopes.contains(ValidationScope.MESSAGE_ATTRIBUTES)) { - val messageAttr = messageReceived.messageAttributes + if (validationScopes.contains(ValidationScope.MESSAGE_ATTRIBUTES)) { + val messageAttr = messageReceived.messageAttributes - if (!messageAttr.isNullOrEmpty()) { - logger.debug { "Validating message attribute MD5 checksum for ReceiveMessage" } + if (!messageAttr.isNullOrEmpty()) { + logger.debug { "Validating message attribute MD5 checksum for ReceiveMessage" } - val attrMd5Returned = messageReceived.md5OfMessageAttributes - val clientSideAttrMd5 = calculateMessageAttributesMd5(messageAttr) - if (clientSideAttrMd5 != attrMd5Returned) { - throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideAttrMd5 but was $attrMd5Returned") - } + val attrMd5Returned = messageReceived.md5OfMessageAttributes + val clientSideAttrMd5 = calculateMessageAttributesMd5(messageAttr) + if (clientSideAttrMd5 != attrMd5Returned) { + throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideAttrMd5 but was $attrMd5Returned") } + + logger.debug { "Message attribute MD5 checksum for ReceiveMessage validated " } } } } @@ -181,14 +182,12 @@ public class SqsMd5ChecksumValidationInterceptor( private fun sendMessageBatchOperationMd5Check( sendMessageBatchRequest: SendMessageBatchRequest, sendMessageBatchResponse: SendMessageBatchResponse, + logger: Logger ) { - val idToRequestEntryMap = hashMapOf() - val entries = sendMessageBatchRequest.entries - if (entries != null) { - for (entry in entries) { - idToRequestEntryMap[entry.id] = entry - } - } + val idToRequestEntryMap = sendMessageBatchRequest + .entries + .orEmpty() + .associateBy { it.id } for (entry in sendMessageBatchResponse.successful) { if (validationScopes.contains(ValidationScope.MESSAGE_BODY)) { @@ -202,6 +201,8 @@ public class SqsMd5ChecksumValidationInterceptor( if (clientSideBodyMd5 != bodyMd5Returned) { throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideBodyMd5 but was $bodyMd5Returned") } + + logger.debug { "Message body MD5 checksum for SendMessageBatch: ${entry.messageId} validated" } } } @@ -215,6 +216,8 @@ public class SqsMd5ChecksumValidationInterceptor( if (clientSideAttrMd5 != messageAttrMD5Returned) { throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideAttrMd5 but was $messageAttrMD5Returned") } + + logger.debug { "Message attribute MD5 checksum for SendMessageBatch: ${entry.messageId} validated" } } } @@ -228,23 +231,17 @@ public class SqsMd5ChecksumValidationInterceptor( if (clientSideSysAttrMd5 != messageSysAttrMD5Returned) { throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideSysAttrMd5 but was $messageSysAttrMD5Returned") } + + logger.debug { "Message system attribute MD5 checksum for SendMessageBatch: ${entry.messageId} validated" } } } } } private fun calculateMessageBodyMd5(messageBody: String): String { - val expectedMD5 = try { - messageBody.encodeToByteArray().md5() - } catch (e: Exception) { - throw ClientException( - "Unable to calculate the MD5 hash of the message body." + - "Potential reasons include JVM configuration or FIPS compliance issues." + - "To disable message MD5 validation, you can set checksumValidationEnabled" + - "to false when instantiating the client." + e.message, - ) - } + val expectedMD5 = messageBody.encodeToByteArray().md5() val expectedMD5Hex = expectedMD5.toHexString() + return expectedMD5Hex } @@ -252,122 +249,97 @@ public class SqsMd5ChecksumValidationInterceptor( * Calculates the MD5 digest for message attributes according to SQS specifications. * https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-metadata.html#sqs-attributes-md5-message-digest-calculation */ - @OptIn(InternalApi::class, ExperimentalStdlibApi::class) private fun calculateMessageAttributesMd5(messageAttributes: Map): String { - val sortedAttributeNames = messageAttributes.keys.sorted() - val md5Digest = Md5() + val buffer = SdkBuffer() - try { - for (attributeName in sortedAttributeNames) { - val attributeValue = messageAttributes[attributeName] - updateLengthAndBytes(md5Digest, attributeName.encodeToByteArray()) + messageAttributes + .entries + .sortedBy { (name, _) -> name } + .forEach { (attributeName, attributeValue) -> + updateLengthAndBytes(buffer, attributeName) - attributeValue?.dataType?.let { dataType -> - updateLengthAndBytes(md5Digest, dataType.encodeToByteArray()) - } + updateLengthAndBytes(buffer, attributeValue.dataType) - val stringValue = attributeValue?.stringValue - val binaryValue = attributeValue?.binaryValue - val stringListValues = attributeValue?.stringListValues - val binaryListValues = attributeValue?.binaryListValues + val stringValue = attributeValue.stringValue + val binaryValue = attributeValue.binaryValue + val stringListValues = attributeValue.stringListValues + val binaryListValues = attributeValue.binaryListValues when { - stringValue != null -> updateForStringType(md5Digest, stringValue) - binaryValue != null -> updateForBinaryType(md5Digest, binaryValue) - !stringListValues.isNullOrEmpty() -> updateForStringListType(md5Digest, stringListValues) - !binaryListValues.isNullOrEmpty() -> updateForBinaryListType(md5Digest, binaryListValues) + stringValue != null -> updateForStringType(buffer, stringValue) + binaryValue != null -> updateForBinaryType(buffer, binaryValue) + !stringListValues.isNullOrEmpty() -> updateForStringListType(buffer, stringListValues) + !binaryListValues.isNullOrEmpty() -> updateForBinaryListType(buffer, binaryListValues) } - } - } catch (e: Exception) { - throw ClientException( - "Unable to calculate the MD5 hash of the message attributes." + - "Potential reasons include JVM configuration or FIPS compliance issues." + - "To disable message MD5 validation, you can set checksumValidationEnabled" + - "to false when instantiating the client." + e.message, - ) } - val expectedMD5Hex = md5Digest.digest().toHexString() - return expectedMD5Hex + + val payload = buffer.readByteArray() + return payload.md5().toHexString() } private fun calculateMessageSystemAttributesMd5( messageSysAttrs: Map, ): String { - val sortedAttributeNames = messageSysAttrs.keys.sortedBy { it.value } - val md5Digest = Md5() - - try { - for (attributeName in sortedAttributeNames) { - val attributeValue = messageSysAttrs[attributeName] - updateLengthAndBytes(md5Digest, attributeName.value.encodeToByteArray()) - - attributeValue?.dataType?.let { dataType -> - updateLengthAndBytes(md5Digest, dataType.encodeToByteArray()) - } - - val stringValue = attributeValue?.stringValue - val binaryValue = attributeValue?.binaryValue - val stringListValues = attributeValue?.stringListValues - val binaryListValues = attributeValue?.binaryListValues - - when { - stringValue != null -> updateForStringType(md5Digest, stringValue) - binaryValue != null -> updateForBinaryType(md5Digest, binaryValue) - !stringListValues.isNullOrEmpty() -> updateForStringListType(md5Digest, stringListValues) - !binaryListValues.isNullOrEmpty() -> updateForBinaryListType(md5Digest, binaryListValues) - } + val buffer = SdkBuffer() + + messageSysAttrs + .entries + .sortedBy { (name, _) -> name.value } + .forEach { (attributeName, attributeValue) -> + updateLengthAndBytes(buffer, attributeName.value) + + updateLengthAndBytes(buffer, attributeValue.dataType) + + val stringValue = attributeValue.stringValue + val binaryValue = attributeValue.binaryValue + val stringListValues = attributeValue.stringListValues + val binaryListValues = attributeValue.binaryListValues + + when { + stringValue != null -> updateForStringType(buffer, stringValue) + binaryValue != null -> updateForBinaryType(buffer, binaryValue) + !stringListValues.isNullOrEmpty() -> updateForStringListType(buffer, stringListValues) + !binaryListValues.isNullOrEmpty() -> updateForBinaryListType(buffer, binaryListValues) } - } catch (e: Exception) { - throw ClientException( - "Unable to calculate the MD5 hash of the message system attributes." + - "Potential reasons include JVM configuration or FIPS compliance issues." + - "To disable message MD5 validation, you can set checksumValidationEnabled" + - "to false when instantiating the client." + e.message, - ) } - val expectedMD5Hex = md5Digest.digest().toHexString() - return expectedMD5Hex + + val payload = buffer.readByteArray() + return payload.md5().toHexString() } - private fun updateForStringType(md5Digest: Md5, value: String) { - md5Digest.update(STRING_TYPE_FIELD_INDEX) - updateLengthAndBytes(md5Digest, value.encodeToByteArray()) + private fun updateForStringType(buffer: SdkBuffer, value: String) { + buffer.writeByte(STRING_TYPE_FIELD_INDEX) + updateLengthAndBytes(buffer, value) } - private fun updateForBinaryType(md5Digest: Md5, value: ByteArray) { - md5Digest.update(BINARY_TYPE_FIELD_INDEX) - updateLengthAndBytes(md5Digest, value) + private fun updateForBinaryType(buffer: SdkBuffer, value: ByteArray) { + buffer.writeByte(BINARY_TYPE_FIELD_INDEX) + updateLengthAndBytes(buffer, value) } - private fun updateForStringListType(md5Digest: Md5, values: List) { - md5Digest.update(STRING_LIST_TYPE_FIELD_INDEX) + private fun updateForStringListType(buffer: SdkBuffer, values: List) { + buffer.writeByte(STRING_LIST_TYPE_FIELD_INDEX) values.forEach { value -> - updateLengthAndBytes(md5Digest, value.encodeToByteArray()) + updateLengthAndBytes(buffer, value) } } - private fun updateForBinaryListType(md5Digest: Md5, values: List) { - md5Digest.update(BINARY_LIST_TYPE_FIELD_INDEX) + private fun updateForBinaryListType(buffer: SdkBuffer, values: List) { + buffer.writeByte(BINARY_LIST_TYPE_FIELD_INDEX) values.forEach { value -> - updateLengthAndBytes(md5Digest, value) + updateLengthAndBytes(buffer, value) } } + private fun updateLengthAndBytes(buffer: SdkBuffer, stringValue: String) = + updateLengthAndBytes(buffer, stringValue.encodeToByteArray()) + /** * Update the digest using a sequence of bytes that consists of the length (in 4 bytes) of the * input binaryValue and all the bytes it contains. */ - private fun updateLengthAndBytes(messageDigest: Md5, binaryValue: ByteArray) { - println("updateLengthAndBytes") - val length = binaryValue.size - val lengthBytes = byteArrayOf( - (length shr 24).toByte(), - (length shr 16).toByte(), - (length shr 8).toByte(), - length.toByte(), - ) - - messageDigest.update(lengthBytes) - messageDigest.update(binaryValue) + private fun updateLengthAndBytes(buffer: SdkBuffer, binaryValue: ByteArray) { + buffer.writeInt(binaryValue.size) + buffer.write(binaryValue) } } diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/FinalizeSqsConfig.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/FinalizeSqsConfig.kt index 8ed86777313..6b81e1657db 100644 --- a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/FinalizeSqsConfig.kt +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/FinalizeSqsConfig.kt @@ -17,12 +17,12 @@ internal suspend fun finalizeSqsConfig( ) { val activeProfile = sharedConfig.get().activeProfile builder.config.checksumValidationEnabled = builder.config.checksumValidationEnabled - ?: SQSSetting.checksumValidationEnabled.resolve(provider) + ?: SqsSettings.checksumValidationEnabled.resolve(provider) ?: activeProfile.checksumValidationEnabled ?: ValidationEnabled.NEVER // TODO: MD5 checksum validation is temporarily disabled. Set default to ALWAYS in next minor version builder.config.checksumValidationScopes = builder.config.checksumValidationScopes.ifEmpty { - SQSSetting.checksumValidationScopes.resolve(provider) + SqsSettings.checksumValidationScopes.resolve(provider) ?: activeProfile.checksumValidationScopes ?: ValidationScope.entries.toSet() } diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SQSSetting.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SqsSettings.kt similarity index 98% rename from services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SQSSetting.kt rename to services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SqsSettings.kt index 5a8308599be..71dcb0865bb 100644 --- a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SQSSetting.kt +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SqsSettings.kt @@ -9,7 +9,7 @@ import aws.smithy.kotlin.runtime.config.* /** * SQS specific system settings */ -internal object SQSSetting { +internal object SqsSettings { /** * Configure when MD5 checksum validation is performed for SQS operations. * diff --git a/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt b/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt index 618f34c31eb..723551bbce7 100644 --- a/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt +++ b/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt @@ -33,34 +33,27 @@ import org.junit.jupiter.api.* @TestInstance(TestInstance.Lifecycle.PER_CLASS) class SqsMd5ChecksumValidationTest { // An interceptor that set wrong md5 checksums in SQS response + @OptIn(ExperimentalStdlibApi::class) private val wrongChecksumInterceptor = object : HttpInterceptor { override suspend fun modifyBeforeCompletion( context: ResponseInterceptorContext, ): Result { - val wrongMd5ofMessageBody = "wrong message md5".encodeToByteArray().md5().toString() - val wrongMd5ofMessageAttribute = "wrong attribute md5".encodeToByteArray().md5().toString() - val wrongMd5ofMessageSystemAttribute = "wrong system attribute md5".encodeToByteArray().md5().toString() + val wrongMd5ofMessageBody = "wrong message md5".encodeToByteArray().md5().toHexString() + val wrongMd5ofMessageAttribute = "wrong attribute md5".encodeToByteArray().md5().toHexString() + val wrongMd5ofMessageSystemAttribute = "wrong system attribute md5".encodeToByteArray().md5().toHexString() when (val response = context.response.getOrNull()) { is SendMessageResponse -> { - val modifiedResponse = SendMessageResponse.invoke { - messageId = response.messageId - sequenceNumber = response.sequenceNumber + val modifiedResponse = response.copy { md5OfMessageBody = wrongMd5ofMessageBody md5OfMessageAttributes = wrongMd5ofMessageAttribute md5OfMessageSystemAttributes = wrongMd5ofMessageSystemAttribute } - println("modify SendMessage") return Result.success(modifiedResponse) } is ReceiveMessageResponse -> { val modifiedMessages = response.messages?.map { message -> - Message { - messageId = message.messageId - receiptHandle = message.receiptHandle - body = message.body - attributes = message.attributes - messageAttributes = message.messageAttributes + message.copy { md5OfBody = wrongMd5ofMessageBody md5OfMessageAttributes = wrongMd5ofMessageAttribute } @@ -73,13 +66,10 @@ class SqsMd5ChecksumValidationTest { } is SendMessageBatchResponse -> { val modifiedEntries = response.successful.map { entry -> - SendMessageBatchResultEntry { - id = entry.id - messageId = entry.messageId + entry.copy { md5OfMessageBody = wrongMd5ofMessageBody md5OfMessageAttributes = wrongMd5ofMessageAttribute md5OfMessageSystemAttributes = wrongMd5ofMessageSystemAttribute - sequenceNumber = entry.sequenceNumber } } diff --git a/services/sqs/e2eTest/src/SqsTestUtils.kt b/services/sqs/e2eTest/src/SqsTestUtils.kt index 7e3c5edeafa..f32cda50075 100644 --- a/services/sqs/e2eTest/src/SqsTestUtils.kt +++ b/services/sqs/e2eTest/src/SqsTestUtils.kt @@ -7,7 +7,9 @@ package aws.sdk.kotlin.e2etest import aws.sdk.kotlin.services.sqs.SqsClient import aws.sdk.kotlin.services.sqs.createQueue import aws.sdk.kotlin.services.sqs.model.* -import kotlinx.coroutines.coroutineScope +import aws.sdk.kotlin.services.sqs.paginators.listQueuesPaginated +import aws.sdk.kotlin.services.sqs.paginators.queueUrls +import kotlinx.coroutines.flow.firstOrNull import kotlinx.coroutines.withTimeout import java.net.URI import java.util.* @@ -16,8 +18,8 @@ import kotlin.time.Duration.Companion.seconds object SqsTestUtils { const val DEFAULT_REGION = "us-west-2" - const val TEST_QUEUE_WRONG_CHECKSUM_PREFIX = "sqs-test-queue-" - const val TEST_QUEUE_CORRECT_CHECKSUM_PREFIX = "sqs-test-queue-" + const val TEST_QUEUE_WRONG_CHECKSUM_PREFIX = "sqs-test-queue-wrong-checksum" + const val TEST_QUEUE_CORRECT_CHECKSUM_PREFIX = "sqs-test-queue-correct-checksum" const val TEST_MESSAGE_BODY = "Hello World" const val TEST_MESSAGE_ATTRIBUTES_NAME = "TestAttribute" @@ -30,28 +32,17 @@ object SqsTestUtils { region: String? = null, ): String = getQueueUrlWithPrefix(client, prefix, region) - suspend fun getQueueUrlWithPrefix( + private suspend fun getQueueUrlWithPrefix( client: SqsClient, prefix: String, region: String? = null, ): String = withTimeout(60.seconds) { - val queueUrls = client.listQueues().queueUrls + //val queueUrls = client.listQueues().queueUrls - var matchingQueueUrl = queueUrls?.firstOrNull { url -> - val queueUrl = URI(url).toURL() - val hostParts = queueUrl.host.split(".") - - val regionMatches = if (region != null) { - hostParts.getOrNull(1)?.equals(region, ignoreCase = true) ?: false - } else { - true - } - - val queueName = queueUrl.path.split("/").last() - val prefixMatches = queueName.startsWith(prefix) - - regionMatches && prefixMatches - } + var matchingQueueUrl = client + .listQueuesPaginated { queueNamePrefix = prefix } + .queueUrls() + .firstOrNull() if (matchingQueueUrl == null) { matchingQueueUrl = prefix + UUID.randomUUID() @@ -67,22 +58,26 @@ object SqsTestUtils { matchingQueueUrl } - suspend fun deleteQueueAndAllMessages(client: SqsClient, queueUrl: String): Unit = coroutineScope { + suspend fun deleteQueueAndAllMessages(client: SqsClient, queueUrl: String) { try { println("Purging Sqs queue: $queueUrl") - val purgeRequest = PurgeQueueRequest { - this.queueUrl = queueUrl - } - client.purgeQueue(purgeRequest) + client.purgeQueue ( + PurgeQueueRequest { + this.queueUrl = queueUrl + } + ) + println("Queue purged successfully.") println("Deleting Sqs queue: $queueUrl") - val deleteRequest = DeleteQueueRequest { - this.queueUrl = queueUrl - } - client.deleteQueue(deleteRequest) + client.deleteQueue( + DeleteQueueRequest { + this.queueUrl = queueUrl + } + ) + println("Queue deleted successfully.") } catch (e: SqsException) { println("Error during delete SQS queue: ${e.message}") @@ -92,13 +87,13 @@ object SqsTestUtils { fun buildSendMessageBatchRequestEntry(batchId: Int): SendMessageBatchRequestEntry = SendMessageBatchRequestEntry { id = batchId.toString() messageBody = TEST_MESSAGE_BODY + batchId - messageAttributes = hashMapOf( + messageAttributes = mapOf( TEST_MESSAGE_ATTRIBUTES_NAME to MessageAttributeValue { dataType = "String" stringValue = TEST_MESSAGE_ATTRIBUTES_VALUE + batchId }, ) - messageSystemAttributes = hashMapOf( + messageSystemAttributes = mapOf( MessageSystemAttributeNameForSends.AwsTraceHeader to MessageSystemAttributeValue { dataType = "String" stringValue = TEST_MESSAGE_SYSTEM_ATTRIBUTES_VALUE + batchId From 70ed7299e5b2e795e4ab3b2ca6791968e74cea8b Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Tue, 4 Mar 2025 18:09:10 -0500 Subject: [PATCH 05/13] address pr feedbacks --- .../src/SqsMd5ChecksumValidationTest.kt | 70 +++++++++---------- services/sqs/e2eTest/src/SqsTestUtils.kt | 4 +- 2 files changed, 34 insertions(+), 40 deletions(-) diff --git a/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt b/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt index 723551bbce7..af7e2148fb0 100644 --- a/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt +++ b/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt @@ -117,40 +117,36 @@ class SqsMd5ChecksumValidationTest { @Test fun testSendMessage(): Unit = runBlocking { - assertDoesNotThrow { - correctChecksumClient.sendMessage( - SendMessageRequest { - queueUrl = correctChecksumTestQueueUrl - messageBody = TEST_MESSAGE_BODY - messageAttributes = hashMapOf( - TEST_MESSAGE_ATTRIBUTES_NAME to MessageAttributeValue { - dataType = "String" - stringValue = TEST_MESSAGE_ATTRIBUTES_VALUE - }, - ) - messageSystemAttributes = hashMapOf( - MessageSystemAttributeNameForSends.AwsTraceHeader to MessageSystemAttributeValue { - dataType = "String" - stringValue = TEST_MESSAGE_SYSTEM_ATTRIBUTES_VALUE - }, - ) - }, - ) - } + correctChecksumClient.sendMessage( + SendMessageRequest { + queueUrl = correctChecksumTestQueueUrl + messageBody = TEST_MESSAGE_BODY + messageAttributes = hashMapOf( + TEST_MESSAGE_ATTRIBUTES_NAME to MessageAttributeValue { + dataType = "String" + stringValue = TEST_MESSAGE_ATTRIBUTES_VALUE + }, + ) + messageSystemAttributes = hashMapOf( + MessageSystemAttributeNameForSends.AwsTraceHeader to MessageSystemAttributeValue { + dataType = "String" + stringValue = TEST_MESSAGE_SYSTEM_ATTRIBUTES_VALUE + }, + ) + }, + ) } @Test fun testReceiveMessage(): Unit = runBlocking { - assertDoesNotThrow { - correctChecksumClient.receiveMessage( - ReceiveMessageRequest { - queueUrl = correctChecksumTestQueueUrl - maxNumberOfMessages = 1 - messageAttributeNames = listOf(TEST_MESSAGE_ATTRIBUTES_NAME) - messageSystemAttributeNames = listOf(MessageSystemAttributeName.AwsTraceHeader) - }, - ) - } + correctChecksumClient.receiveMessage( + ReceiveMessageRequest { + queueUrl = correctChecksumTestQueueUrl + maxNumberOfMessages = 1 + messageAttributeNames = listOf(TEST_MESSAGE_ATTRIBUTES_NAME) + messageSystemAttributeNames = listOf(MessageSystemAttributeName.AwsTraceHeader) + }, + ) } @Test @@ -159,14 +155,12 @@ class SqsMd5ChecksumValidationTest { buildSendMessageBatchRequestEntry(batchId) } - assertDoesNotThrow { - correctChecksumClient.sendMessageBatch( - SendMessageBatchRequest { - queueUrl = correctChecksumTestQueueUrl - this.entries = entries - }, - ) - } + correctChecksumClient.sendMessageBatch( + SendMessageBatchRequest { + queueUrl = correctChecksumTestQueueUrl + this.entries = entries + }, + ) } @Test diff --git a/services/sqs/e2eTest/src/SqsTestUtils.kt b/services/sqs/e2eTest/src/SqsTestUtils.kt index f32cda50075..81a665b0e95 100644 --- a/services/sqs/e2eTest/src/SqsTestUtils.kt +++ b/services/sqs/e2eTest/src/SqsTestUtils.kt @@ -18,8 +18,8 @@ import kotlin.time.Duration.Companion.seconds object SqsTestUtils { const val DEFAULT_REGION = "us-west-2" - const val TEST_QUEUE_WRONG_CHECKSUM_PREFIX = "sqs-test-queue-wrong-checksum" - const val TEST_QUEUE_CORRECT_CHECKSUM_PREFIX = "sqs-test-queue-correct-checksum" + const val TEST_QUEUE_WRONG_CHECKSUM_PREFIX = "sqs-test-queue-wrong-checksum-" + const val TEST_QUEUE_CORRECT_CHECKSUM_PREFIX = "sqs-test-queue-correct-checksum-" const val TEST_MESSAGE_BODY = "Hello World" const val TEST_MESSAGE_ATTRIBUTES_NAME = "TestAttribute" From f84e6e4d36613aa6afc08892fa5cf996e7b3fbae Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Wed, 5 Mar 2025 10:23:10 -0500 Subject: [PATCH 06/13] lint --- .../SqsMd5ChecksumValidationIntegration.kt | 24 +++++++------- .../SqsMd5ChecksumValidationInterceptor.kt | 31 +++++++++---------- .../internal/FinalizeSqsConfig.kt | 4 +-- .../{SqsSettings.kt => SqsSetting.kt} | 2 +- services/sqs/e2eTest/src/SqsTestUtils.kt | 9 ++---- 5 files changed, 33 insertions(+), 37 deletions(-) rename services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/{SqsSettings.kt => SqsSetting.kt} (98%) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt index 36efc644ccf..9228e03a61a 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt @@ -36,18 +36,18 @@ class SqsMd5ChecksumValidationIntegration : KotlinIntegration { namespace = "aws.sdk.kotlin.services.sqs.internal" } documentation = """ - Specifies when MD5 checksum validation should be performed for SQS messages. This controls the automatic - calculation and validation of checksums during message operations. - - Valid values: - - `ALWAYS` (default) - Checksums are calculated and validated for both sending and receiving operations - (SendMessage, SendMessageBatch, and ReceiveMessage) - - `WHEN_SENDING` - Checksums are only calculated and validated during send operations - (SendMessage and SendMessageBatch) - - `WHEN_RECEIVING` - Checksums are only calculated and validated during receive operations - (ReceiveMessage) - - `NEVER` - No checksum calculation or validation is performed - """.trimIndent() + Specifies when MD5 checksum validation should be performed for SQS messages. This controls the automatic + calculation and validation of checksums during message operations. + + Valid values: + - `ALWAYS` (default) - Checksums are calculated and validated for both sending and receiving operations + (SendMessage, SendMessageBatch, and ReceiveMessage) + - `WHEN_SENDING` - Checksums are only calculated and validated during send operations + (SendMessage and SendMessageBatch) + - `WHEN_RECEIVING` - Checksums are only calculated and validated during receive operations + (ReceiveMessage) + - `NEVER` - No checksum calculation or validation is performed + """.trimIndent() } private val validationScope = buildSymbol { diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt index b3d903dfc24..b4f137e2b54 100644 --- a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt @@ -7,7 +7,6 @@ package aws.sdk.kotlin.services.sqs import aws.sdk.kotlin.services.sqs.internal.ValidationEnabled import aws.sdk.kotlin.services.sqs.internal.ValidationScope import aws.sdk.kotlin.services.sqs.model.* -import aws.smithy.kotlin.runtime.ClientException import aws.smithy.kotlin.runtime.client.ResponseInterceptorContext import aws.smithy.kotlin.runtime.hashing.md5 import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException @@ -95,7 +94,7 @@ public class SqsMd5ChecksumValidationInterceptor( private fun sendMessageOperationMd5Check( sendMessageRequest: SendMessageRequest, sendMessageResponse: SendMessageResponse, - logger: Logger + logger: Logger, ) { if (validationScopes.contains(ValidationScope.MESSAGE_BODY)) { val messageBodySent = sendMessageRequest.messageBody @@ -182,7 +181,7 @@ public class SqsMd5ChecksumValidationInterceptor( private fun sendMessageBatchOperationMd5Check( sendMessageBatchRequest: SendMessageBatchRequest, sendMessageBatchResponse: SendMessageBatchResponse, - logger: Logger + logger: Logger, ) { val idToRequestEntryMap = sendMessageBatchRequest .entries @@ -271,7 +270,7 @@ public class SqsMd5ChecksumValidationInterceptor( !stringListValues.isNullOrEmpty() -> updateForStringListType(buffer, stringListValues) !binaryListValues.isNullOrEmpty() -> updateForBinaryListType(buffer, binaryListValues) } - } + } val payload = buffer.readByteArray() return payload.md5().toHexString() @@ -286,22 +285,22 @@ public class SqsMd5ChecksumValidationInterceptor( .entries .sortedBy { (name, _) -> name.value } .forEach { (attributeName, attributeValue) -> - updateLengthAndBytes(buffer, attributeName.value) + updateLengthAndBytes(buffer, attributeName.value) - updateLengthAndBytes(buffer, attributeValue.dataType) + updateLengthAndBytes(buffer, attributeValue.dataType) - val stringValue = attributeValue.stringValue - val binaryValue = attributeValue.binaryValue - val stringListValues = attributeValue.stringListValues - val binaryListValues = attributeValue.binaryListValues + val stringValue = attributeValue.stringValue + val binaryValue = attributeValue.binaryValue + val stringListValues = attributeValue.stringListValues + val binaryListValues = attributeValue.binaryListValues - when { - stringValue != null -> updateForStringType(buffer, stringValue) - binaryValue != null -> updateForBinaryType(buffer, binaryValue) - !stringListValues.isNullOrEmpty() -> updateForStringListType(buffer, stringListValues) - !binaryListValues.isNullOrEmpty() -> updateForBinaryListType(buffer, binaryListValues) + when { + stringValue != null -> updateForStringType(buffer, stringValue) + binaryValue != null -> updateForBinaryType(buffer, binaryValue) + !stringListValues.isNullOrEmpty() -> updateForStringListType(buffer, stringListValues) + !binaryListValues.isNullOrEmpty() -> updateForBinaryListType(buffer, binaryListValues) + } } - } val payload = buffer.readByteArray() return payload.md5().toHexString() diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/FinalizeSqsConfig.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/FinalizeSqsConfig.kt index 6b81e1657db..f38fbb476e7 100644 --- a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/FinalizeSqsConfig.kt +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/FinalizeSqsConfig.kt @@ -17,12 +17,12 @@ internal suspend fun finalizeSqsConfig( ) { val activeProfile = sharedConfig.get().activeProfile builder.config.checksumValidationEnabled = builder.config.checksumValidationEnabled - ?: SqsSettings.checksumValidationEnabled.resolve(provider) + ?: SqsSetting.checksumValidationEnabled.resolve(provider) ?: activeProfile.checksumValidationEnabled ?: ValidationEnabled.NEVER // TODO: MD5 checksum validation is temporarily disabled. Set default to ALWAYS in next minor version builder.config.checksumValidationScopes = builder.config.checksumValidationScopes.ifEmpty { - SqsSettings.checksumValidationScopes.resolve(provider) + SqsSetting.checksumValidationScopes.resolve(provider) ?: activeProfile.checksumValidationScopes ?: ValidationScope.entries.toSet() } diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SqsSettings.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SqsSetting.kt similarity index 98% rename from services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SqsSettings.kt rename to services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SqsSetting.kt index 71dcb0865bb..cc6c2c78791 100644 --- a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SqsSettings.kt +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SqsSetting.kt @@ -9,7 +9,7 @@ import aws.smithy.kotlin.runtime.config.* /** * SQS specific system settings */ -internal object SqsSettings { +internal object SqsSetting { /** * Configure when MD5 checksum validation is performed for SQS operations. * diff --git a/services/sqs/e2eTest/src/SqsTestUtils.kt b/services/sqs/e2eTest/src/SqsTestUtils.kt index 81a665b0e95..fbd67085aad 100644 --- a/services/sqs/e2eTest/src/SqsTestUtils.kt +++ b/services/sqs/e2eTest/src/SqsTestUtils.kt @@ -11,7 +11,6 @@ import aws.sdk.kotlin.services.sqs.paginators.listQueuesPaginated import aws.sdk.kotlin.services.sqs.paginators.queueUrls import kotlinx.coroutines.flow.firstOrNull import kotlinx.coroutines.withTimeout -import java.net.URI import java.util.* import kotlin.time.Duration.Companion.seconds @@ -37,8 +36,6 @@ object SqsTestUtils { prefix: String, region: String? = null, ): String = withTimeout(60.seconds) { - //val queueUrls = client.listQueues().queueUrls - var matchingQueueUrl = client .listQueuesPaginated { queueNamePrefix = prefix } .queueUrls() @@ -62,10 +59,10 @@ object SqsTestUtils { try { println("Purging Sqs queue: $queueUrl") - client.purgeQueue ( + client.purgeQueue( PurgeQueueRequest { this.queueUrl = queueUrl - } + }, ) println("Queue purged successfully.") @@ -75,7 +72,7 @@ object SqsTestUtils { client.deleteQueue( DeleteQueueRequest { this.queueUrl = queueUrl - } + }, ) println("Queue deleted successfully.") From 3b8c02d4a9d6593742228656e62d1f2904fef310 Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Wed, 5 Mar 2025 11:04:47 -0500 Subject: [PATCH 07/13] deduplication --- .../SqsMd5ChecksumValidationInterceptor.kt | 30 +++++-------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt index b4f137e2b54..abb09008174 100644 --- a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt @@ -256,19 +256,12 @@ public class SqsMd5ChecksumValidationInterceptor( .sortedBy { (name, _) -> name } .forEach { (attributeName, attributeValue) -> updateLengthAndBytes(buffer, attributeName) - updateLengthAndBytes(buffer, attributeValue.dataType) - - val stringValue = attributeValue.stringValue - val binaryValue = attributeValue.binaryValue - val stringListValues = attributeValue.stringListValues - val binaryListValues = attributeValue.binaryListValues - when { - stringValue != null -> updateForStringType(buffer, stringValue) - binaryValue != null -> updateForBinaryType(buffer, binaryValue) - !stringListValues.isNullOrEmpty() -> updateForStringListType(buffer, stringListValues) - !binaryListValues.isNullOrEmpty() -> updateForBinaryListType(buffer, binaryListValues) + attributeValue.stringValue != null -> updateForStringType(buffer, attributeValue.stringValue) + attributeValue.binaryValue != null -> updateForBinaryType(buffer, attributeValue.binaryValue) + !attributeValue.stringListValues.isNullOrEmpty() -> updateForStringListType(buffer, attributeValue.stringListValues) + !attributeValue.binaryListValues.isNullOrEmpty() -> updateForBinaryListType(buffer, attributeValue.binaryListValues) } } @@ -286,19 +279,12 @@ public class SqsMd5ChecksumValidationInterceptor( .sortedBy { (name, _) -> name.value } .forEach { (attributeName, attributeValue) -> updateLengthAndBytes(buffer, attributeName.value) - updateLengthAndBytes(buffer, attributeValue.dataType) - - val stringValue = attributeValue.stringValue - val binaryValue = attributeValue.binaryValue - val stringListValues = attributeValue.stringListValues - val binaryListValues = attributeValue.binaryListValues - when { - stringValue != null -> updateForStringType(buffer, stringValue) - binaryValue != null -> updateForBinaryType(buffer, binaryValue) - !stringListValues.isNullOrEmpty() -> updateForStringListType(buffer, stringListValues) - !binaryListValues.isNullOrEmpty() -> updateForBinaryListType(buffer, binaryListValues) + attributeValue.stringValue != null -> updateForStringType(buffer, attributeValue.stringValue) + attributeValue.binaryValue != null -> updateForBinaryType(buffer, attributeValue.binaryValue) + !attributeValue.stringListValues.isNullOrEmpty() -> updateForStringListType(buffer, attributeValue.stringListValues) + !attributeValue.binaryListValues.isNullOrEmpty() -> updateForBinaryListType(buffer, attributeValue.binaryListValues) } } From 4c669771104f3b6cb4f9815be3ca855e003ff539 Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Tue, 11 Mar 2025 15:46:39 -0400 Subject: [PATCH 08/13] address feedback --- .../runtime/config/profile/AwsProfile.kt | 9 +- .../SqsMd5ChecksumValidationIntegration.kt | 35 +++- .../customization/sqs/SqsModelUtils.kt | 14 -- ...SqsMd5ChecksumValidationIntegrationTest.kt | 12 +- .../SqsMd5ChecksumValidationInterceptor.kt | 184 ++++++++++-------- .../internal/FinalizeSqsConfig.kt | 11 +- .../internal/SqsSetting.kt | 4 +- .../internal/ValidationConfig.kt | 17 +- .../src/SqsMd5ChecksumValidationTest.kt | 93 +++++---- services/sqs/e2eTest/src/SqsTestUtils.kt | 24 +-- 10 files changed, 216 insertions(+), 187 deletions(-) delete mode 100644 codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsModelUtils.kt diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt index a857769a226..2699202e1aa 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt @@ -228,10 +228,10 @@ public inline fun > AwsProfile.getEnumOrNull(key: String, su public inline fun > AwsProfile.getEnumSetOrNull(key: String, subKey: String? = null): Set? = getOrNull(key, subKey)?.let { rawValue -> rawValue.split(",") - .map { it.trim() } - .map { value -> - enumValues().firstOrNull { - it.name.equals(value, ignoreCase = true) + .map { it -> + val value = it.trim() + enumValues().firstOrNull { enumValue -> + enumValue.name.equals(value, ignoreCase = true) } ?: throw ConfigurationException( buildString { append(key) @@ -242,7 +242,6 @@ public inline fun > AwsProfile.getEnumSetOrNull(key: String, }, ) }.toSet() - .takeIf { it.isNotEmpty() } } internal fun AwsProfile.getUrlOrNull(key: String, subKey: String? = null): Url? = diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt index 9228e03a61a..9de776332df 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt @@ -5,6 +5,7 @@ package aws.sdk.kotlin.codegen.customization.sqs import aws.sdk.kotlin.codegen.ServiceClientCompanionObjectWriter +import aws.sdk.kotlin.codegen.sdkId import software.amazon.smithy.kotlin.codegen.KotlinSettings import software.amazon.smithy.kotlin.codegen.core.CodegenContext import software.amazon.smithy.kotlin.codegen.core.KotlinWriter @@ -17,6 +18,7 @@ import software.amazon.smithy.kotlin.codegen.model.expectShape import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigProperty +import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigPropertyType import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.ServiceShape @@ -26,7 +28,7 @@ import software.amazon.smithy.model.shapes.ServiceShape */ class SqsMd5ChecksumValidationIntegration : KotlinIntegration { override fun enabledForService(model: Model, settings: KotlinSettings): Boolean = - model.expectShape(settings.service).isSqs + model.expectShape(settings.service).sdkId.lowercase() == "sqs" companion object { val ValidationEnabledProp: ConfigProperty = ConfigProperty { @@ -34,30 +36,53 @@ class SqsMd5ChecksumValidationIntegration : KotlinIntegration { symbol = buildSymbol { name = "ValidationEnabled" namespace = "aws.sdk.kotlin.services.sqs.internal" + nullable = false } + propertyType = ConfigPropertyType.Custom( + render = { prop, writer -> + writer.write("public val #1L: #2T = builder.#1L ?: #2T.NEVER", prop.propertyName, prop.symbol) + }, + renderBuilder = { prop, writer -> + prop.documentation?.let(writer::dokka) + writer.write("public var #L: #T? = null", prop.propertyName, prop.symbol) + writer.write("") + }, + ) documentation = """ Specifies when MD5 checksum validation should be performed for SQS messages. This controls the automatic calculation and validation of checksums during message operations. Valid values: - - `ALWAYS` (default) - Checksums are calculated and validated for both sending and receiving operations + - `ALWAYS` - Checksums are calculated and validated for both sending and receiving operations (SendMessage, SendMessageBatch, and ReceiveMessage) - `WHEN_SENDING` - Checksums are only calculated and validated during send operations (SendMessage and SendMessageBatch) - `WHEN_RECEIVING` - Checksums are only calculated and validated during receive operations (ReceiveMessage) - - `NEVER` - No checksum calculation or validation is performed + - `NEVER` (default) - No checksum calculation or validation is performed """.trimIndent() + // TODO: MD5 checksum validation is temporarily disabled. Change default to ALWAYS in v1.5 } private val validationScope = buildSymbol { name = "ValidationScope" namespace = "aws.sdk.kotlin.services.sqs.internal" + nullable = false } val ValidationScopeProp: ConfigProperty = ConfigProperty { name = "checksumValidationScopes" - symbol = KotlinTypes.Collections.set(validationScope, default = "emptySet()") + symbol = KotlinTypes.Collections.set(validationScope) + propertyType = ConfigPropertyType.Custom( + render = { prop, writer -> + writer.write("public val #1L: #2T = builder.#1L ?: #3T.entries.toSet()", prop.propertyName, prop.symbol, validationScope) + }, + renderBuilder = { prop, writer -> + prop.documentation?.let(writer::dokka) + writer.write("public var #L: #T? = null", prop.propertyName, prop.symbol) + writer.write("") + }, + ) documentation = """ Specifies which parts of an SQS message should undergo MD5 checksum validation. This configuration accepts a set of validation scopes that determine which message components to validate. @@ -69,7 +94,7 @@ class SqsMd5ChecksumValidationIntegration : KotlinIntegration { system attributes during message receipt) - `MESSAGE_BODY` - Validates checksums for the message body - Default: All three scopes (MESSAGE_ATTRIBUTES, MESSAGE_SYSTEM_ATTRIBUTES, MESSAGE_BODY) + Default: All three scopes (`MESSAGE_ATTRIBUTES`, `MESSAGE_SYSTEM_ATTRIBUTES`, `MESSAGE_BODY`) """.trimIndent() } } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsModelUtils.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsModelUtils.kt deleted file mode 100644 index f79ec678161..00000000000 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsModelUtils.kt +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package aws.sdk.kotlin.codegen.customization.sqs - -import aws.sdk.kotlin.codegen.sdkId -import software.amazon.smithy.model.shapes.ServiceShape - -/** - * Returns true if the service is Sqs - */ -val ServiceShape.isSqs: Boolean - get() = sdkId.lowercase() == "sqs" diff --git a/codegen/aws-sdk-codegen/src/test/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegrationTest.kt b/codegen/aws-sdk-codegen/src/test/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegrationTest.kt index a5175e7d069..0c0e32eced5 100644 --- a/codegen/aws-sdk-codegen/src/test/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegrationTest.kt +++ b/codegen/aws-sdk-codegen/src/test/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegrationTest.kt @@ -18,6 +18,12 @@ import kotlin.test.assertTrue import kotlin.test.fail class SqsMd5ChecksumValidationIntegrationTest { + object FooMiddleware : ProtocolMiddleware { + override val name: String = "FooMiddleware" + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) = + fail("Unexpected call to `FooMiddleware.render`") + } + @Test fun testNotExpectedForNonSqsModel() { val model = model("NotSqs") @@ -44,9 +50,3 @@ class SqsMd5ChecksumValidationIntegrationTest { assertEquals(listOf(FooMiddleware, SqsMd5ChecksumValidationMiddleware), actual) } } - -object FooMiddleware : ProtocolMiddleware { - override val name: String = "FooMiddleware" - override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) = - fail("Unexpected call to `FooMiddleware.render`") -} diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt index abb09008174..3e38423a5a3 100644 --- a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt @@ -4,10 +4,12 @@ */ package aws.sdk.kotlin.services.sqs +import aws.sdk.kotlin.runtime.ClientException import aws.sdk.kotlin.services.sqs.internal.ValidationEnabled import aws.sdk.kotlin.services.sqs.internal.ValidationScope import aws.sdk.kotlin.services.sqs.model.* import aws.smithy.kotlin.runtime.client.ResponseInterceptorContext +import aws.smithy.kotlin.runtime.collections.AttributeKey import aws.smithy.kotlin.runtime.hashing.md5 import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor @@ -15,7 +17,16 @@ import aws.smithy.kotlin.runtime.http.request.HttpRequest import aws.smithy.kotlin.runtime.http.response.HttpResponse import aws.smithy.kotlin.runtime.io.SdkBuffer import aws.smithy.kotlin.runtime.telemetry.logging.Logger +import aws.smithy.kotlin.runtime.telemetry.logging.error import aws.smithy.kotlin.runtime.telemetry.logging.logger +import aws.smithy.kotlin.runtime.util.asyncLazy +import kotlinx.coroutines.runBlocking +import kotlin.coroutines.coroutineContext + +private const val STRING_TYPE_FIELD_INDEX: Byte = 1 +private const val BINARY_TYPE_FIELD_INDEX: Byte = 2 +private const val STRING_LIST_TYPE_FIELD_INDEX: Byte = 3 +private const val BINARY_LIST_TYPE_FIELD_INDEX: Byte = 4 /** * Interceptor that validates MD5 checksums for SQS message operations. @@ -27,7 +38,7 @@ import aws.smithy.kotlin.runtime.telemetry.logging.logger * - Message system attributes * * The validation behavior can be configured using: - * - [checksumValidationEnabled] - Controls when validation occurs (ALWAYS, WHEN_SENDING, WHEN_RECEIVING, NEVER) + * - [checksumValidationEnabled] - Controls when validation occurs (`ALWAYS`, `WHEN_SENDING`, `WHEN_RECEIVING`, `NEVER`) * - [checksumValidationScopes] - Specifies which message components to validate * * Supported operations: @@ -37,40 +48,37 @@ import aws.smithy.kotlin.runtime.telemetry.logging.logger */ @OptIn(ExperimentalStdlibApi::class) public class SqsMd5ChecksumValidationInterceptor( - private val validationEnabled: ValidationEnabled?, + private val validationEnabled: ValidationEnabled, private val validationScopes: Set, ) : HttpInterceptor { public companion object { - private const val STRING_TYPE_FIELD_INDEX: Byte = 1 - private const val BINARY_TYPE_FIELD_INDEX: Byte = 2 - private const val STRING_LIST_TYPE_FIELD_INDEX: Byte = 3 - private const val BINARY_LIST_TYPE_FIELD_INDEX: Byte = 4 + private val isMd5Available = asyncLazy { + try { + "MD5".encodeToByteArray().md5() + true + } catch (e: Exception) { + coroutineContext.error(e) { + "MD5 checksums are not available (likely due to FIPS mode). Checksum validation will be disabled." + } + false + } + } } override fun readAfterExecution(context: ResponseInterceptorContext) { - if (validationEnabled == ValidationEnabled.NEVER) return + if (validationEnabled == ValidationEnabled.NEVER || runBlocking { !isMd5Available.get() }) return val logger = context.executionContext.coroutineContext.logger() - // Test MD5 availability - try { - "MD5".encodeToByteArray().md5() - } catch (e: Exception) { - logger.error { "MD5 checksums are not available (likely due to FIPS mode). Checksum validation will be disabled." } - return - } - val request = context.request - val response = context.response.getOrNull() - if (response != null) { + context.response.getOrNull()?.let { response -> when (request) { is SendMessageRequest -> { if (validationEnabled == ValidationEnabled.WHEN_RECEIVING) return - val sendMessageRequest = request as SendMessageRequest val sendMessageResponse = response as SendMessageResponse - sendMessageOperationMd5Check(sendMessageRequest, sendMessageResponse, logger) + sendMessageOperationMd5Check(request, sendMessageResponse, logger) } is ReceiveMessageRequest -> { @@ -83,12 +91,15 @@ public class SqsMd5ChecksumValidationInterceptor( is SendMessageBatchRequest -> { if (validationEnabled == ValidationEnabled.WHEN_RECEIVING) return - val sendMessageBatchRequest = request as SendMessageBatchRequest val sendMessageBatchResponse = response as SendMessageBatchResponse - sendMessageBatchOperationMd5Check(sendMessageBatchRequest, sendMessageBatchResponse, logger) + sendMessageBatchOperationMd5Check(request, sendMessageBatchResponse, logger) } } } + + // Record MD5 checksum validation was performed for this request + val checksumValidated: AttributeKey = AttributeKey("checksumValidated") + context.executionContext[checksumValidated] = true } private fun sendMessageOperationMd5Check( @@ -97,46 +108,45 @@ public class SqsMd5ChecksumValidationInterceptor( logger: Logger, ) { if (validationScopes.contains(ValidationScope.MESSAGE_BODY)) { + val messageBodyMd5Returned = sendMessageResponse.md5OfMessageBody val messageBodySent = sendMessageRequest.messageBody - if (!messageBodySent.isNullOrEmpty()) { + if (!messageBodyMd5Returned.isNullOrEmpty() && !messageBodySent.isNullOrEmpty()) { logger.debug { "Validating message body MD5 checksum for SendMessage" } - val bodyMD5Returned = sendMessageResponse.md5OfMessageBody val clientSideBodyMd5 = calculateMessageBodyMd5(messageBodySent) - if (clientSideBodyMd5 != bodyMD5Returned) { - throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideBodyMd5 but was $bodyMD5Returned") - } + + validateMd5(clientSideBodyMd5, messageBodyMd5Returned) logger.debug { "Message body MD5 checksum for SendMessage validated" } } } if (validationScopes.contains(ValidationScope.MESSAGE_ATTRIBUTES)) { + val messageAttrMd5Returned = sendMessageResponse.md5OfMessageAttributes val messageAttrSent = sendMessageRequest.messageAttributes - if (!messageAttrSent.isNullOrEmpty()) { + + if (!messageAttrMd5Returned.isNullOrEmpty() && !messageAttrSent.isNullOrEmpty()) { logger.debug { "Validating message attribute MD5 checksum for SendMessage" } - val messageAttrMD5Returned = sendMessageResponse.md5OfMessageAttributes val clientSideAttrMd5 = calculateMessageAttributesMd5(messageAttrSent) - if (clientSideAttrMd5 != messageAttrMD5Returned) { - throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideAttrMd5 but was $messageAttrMD5Returned") - } + + validateMd5(clientSideAttrMd5, messageAttrMd5Returned) logger.debug { "Message attribute MD5 checksum for SendMessage validated" } } } if (validationScopes.contains(ValidationScope.MESSAGE_SYSTEM_ATTRIBUTES)) { + val messageSysAttrMD5Returned = sendMessageResponse.md5OfMessageSystemAttributes val messageSysAttrSent = sendMessageRequest.messageSystemAttributes - if (!messageSysAttrSent.isNullOrEmpty()) { + + if (!messageSysAttrMD5Returned.isNullOrEmpty() && !messageSysAttrSent.isNullOrEmpty()) { logger.debug { "Validating message system attribute MD5 checksum for SendMessage" } - val messageSysAttrMD5Returned = sendMessageResponse.md5OfMessageSystemAttributes val clientSideSysAttrMd5 = calculateMessageSystemAttributesMd5(messageSysAttrSent) - if (clientSideSysAttrMd5 != messageSysAttrMD5Returned) { - throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideSysAttrMd5 but was $messageSysAttrMD5Returned") - } + + validateMd5(clientSideSysAttrMd5, messageSysAttrMD5Returned) logger.debug { "Message system attribute MD5 checksum for SendMessage validated" } } @@ -146,31 +156,30 @@ public class SqsMd5ChecksumValidationInterceptor( private fun receiveMessageResultMd5Check(receiveMessageResponse: ReceiveMessageResponse, logger: Logger) { receiveMessageResponse.messages?.forEach { messageReceived -> if (validationScopes.contains(ValidationScope.MESSAGE_BODY)) { - val messageBody = messageReceived.body - if (!messageBody.isNullOrEmpty()) { + val messageBodyMd5Returned = messageReceived.md5OfBody + val messageBodyReturned = messageReceived.body + + if (!messageBodyMd5Returned.isNullOrEmpty() && !messageBodyReturned.isNullOrEmpty()) { logger.debug { "Validating message body MD5 checksum for ReceiveMessage" } - val bodyMd5Returned = messageReceived.md5OfBody - val clientSideBodyMd5 = calculateMessageBodyMd5(messageBody) - if (clientSideBodyMd5 != bodyMd5Returned) { - throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideBodyMd5 but was $bodyMd5Returned") - } + val clientSideBodyMd5 = calculateMessageBodyMd5(messageBodyReturned) + + validateMd5(clientSideBodyMd5, messageBodyMd5Returned) logger.debug { "Message body MD5 checksum for ReceiveMessage validated " } } } if (validationScopes.contains(ValidationScope.MESSAGE_ATTRIBUTES)) { - val messageAttr = messageReceived.messageAttributes + val messageAttrMd5Returned = messageReceived.md5OfMessageAttributes + val messageAttrReturned = messageReceived.messageAttributes - if (!messageAttr.isNullOrEmpty()) { + if (!messageAttrMd5Returned.isNullOrEmpty() && !messageAttrReturned.isNullOrEmpty()) { logger.debug { "Validating message attribute MD5 checksum for ReceiveMessage" } - val attrMd5Returned = messageReceived.md5OfMessageAttributes - val clientSideAttrMd5 = calculateMessageAttributesMd5(messageAttr) - if (clientSideAttrMd5 != attrMd5Returned) { - throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideAttrMd5 but was $attrMd5Returned") - } + val clientSideAttrMd5 = calculateMessageAttributesMd5(messageAttrReturned) + + validateMd5(clientSideAttrMd5, messageAttrMd5Returned) logger.debug { "Message attribute MD5 checksum for ReceiveMessage validated " } } @@ -183,53 +192,52 @@ public class SqsMd5ChecksumValidationInterceptor( sendMessageBatchResponse: SendMessageBatchResponse, logger: Logger, ) { - val idToRequestEntryMap = sendMessageBatchRequest + val idToRequestEntry = sendMessageBatchRequest .entries .orEmpty() .associateBy { it.id } for (entry in sendMessageBatchResponse.successful) { if (validationScopes.contains(ValidationScope.MESSAGE_BODY)) { - val messageBody = idToRequestEntryMap[entry.id]?.messageBody + val messageBodyMd5Returned = entry.md5OfMessageBody + val messageBodySent = idToRequestEntry[entry.id]?.messageBody - if (!messageBody.isNullOrEmpty()) { + if (!messageBodyMd5Returned.isNullOrEmpty() && !messageBodySent.isNullOrEmpty()) { logger.debug { "Validating message body MD5 checksum for SendMessageBatch: ${entry.messageId}" } - val bodyMd5Returned = entry.md5OfMessageBody - val clientSideBodyMd5 = calculateMessageBodyMd5(messageBody) - if (clientSideBodyMd5 != bodyMd5Returned) { - throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideBodyMd5 but was $bodyMd5Returned") - } + val clientSideBodyMd5 = calculateMessageBodyMd5(messageBodySent) + + validateMd5(clientSideBodyMd5, messageBodyMd5Returned) logger.debug { "Message body MD5 checksum for SendMessageBatch: ${entry.messageId} validated" } } } if (validationScopes.contains(ValidationScope.MESSAGE_ATTRIBUTES)) { - val messageAttrSent = idToRequestEntryMap[entry.id]?.messageAttributes - if (!messageAttrSent.isNullOrEmpty()) { + val messageAttrMD5Returned = entry.md5OfMessageAttributes + val messageAttrSent = idToRequestEntry[entry.id]?.messageAttributes + + if (!messageAttrMD5Returned.isNullOrEmpty() && !messageAttrSent.isNullOrEmpty()) { logger.debug { "Validating message attribute MD5 checksum for SendMessageBatch: ${entry.messageId}" } - val messageAttrMD5Returned = entry.md5OfMessageAttributes val clientSideAttrMd5 = calculateMessageAttributesMd5(messageAttrSent) - if (clientSideAttrMd5 != messageAttrMD5Returned) { - throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideAttrMd5 but was $messageAttrMD5Returned") - } + + validateMd5(clientSideAttrMd5, messageAttrMD5Returned) logger.debug { "Message attribute MD5 checksum for SendMessageBatch: ${entry.messageId} validated" } } } if (validationScopes.contains(ValidationScope.MESSAGE_SYSTEM_ATTRIBUTES)) { - val messageSysAttrSent = idToRequestEntryMap[entry.id]?.messageSystemAttributes - if (!messageSysAttrSent.isNullOrEmpty()) { + val messageSysAttrMD5Returned = entry.md5OfMessageSystemAttributes + val messageSysAttrSent = idToRequestEntry[entry.id]?.messageSystemAttributes + + if (!messageSysAttrMD5Returned.isNullOrEmpty() && !messageSysAttrSent.isNullOrEmpty()) { logger.debug { "Validating message system attribute MD5 checksum for SendMessageBatch: ${entry.messageId}" } - val messageSysAttrMD5Returned = entry.md5OfMessageSystemAttributes val clientSideSysAttrMd5 = calculateMessageSystemAttributesMd5(messageSysAttrSent) - if (clientSideSysAttrMd5 != messageSysAttrMD5Returned) { - throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideSysAttrMd5 but was $messageSysAttrMD5Returned") - } + + validateMd5(clientSideSysAttrMd5, messageSysAttrMD5Returned) logger.debug { "Message system attribute MD5 checksum for SendMessageBatch: ${entry.messageId} validated" } } @@ -237,13 +245,15 @@ public class SqsMd5ChecksumValidationInterceptor( } } - private fun calculateMessageBodyMd5(messageBody: String): String { - val expectedMD5 = messageBody.encodeToByteArray().md5() - val expectedMD5Hex = expectedMD5.toHexString() - - return expectedMD5Hex + private fun validateMd5(clientSideMd5: String, md5Returned: String) { + if (clientSideMd5 != md5Returned) { + throw ChecksumMismatchException("Checksum mismatch. Expected $clientSideMd5 but was $md5Returned") + } } + private fun calculateMessageBodyMd5(messageBody: String) = + messageBody.encodeToByteArray().md5().toHexString() + /** * Calculates the MD5 digest for message attributes according to SQS specifications. * https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-metadata.html#sqs-attributes-md5-message-digest-calculation @@ -253,7 +263,7 @@ public class SqsMd5ChecksumValidationInterceptor( messageAttributes .entries - .sortedBy { (name, _) -> name } + .sortedBy { (attributeName, _) -> attributeName } .forEach { (attributeName, attributeValue) -> updateLengthAndBytes(buffer, attributeName) updateLengthAndBytes(buffer, attributeValue.dataType) @@ -262,6 +272,7 @@ public class SqsMd5ChecksumValidationInterceptor( attributeValue.binaryValue != null -> updateForBinaryType(buffer, attributeValue.binaryValue) !attributeValue.stringListValues.isNullOrEmpty() -> updateForStringListType(buffer, attributeValue.stringListValues) !attributeValue.binaryListValues.isNullOrEmpty() -> updateForBinaryListType(buffer, attributeValue.binaryListValues) + else -> throw ClientException("No value type found for attribute $attributeName") } } @@ -270,21 +281,22 @@ public class SqsMd5ChecksumValidationInterceptor( } private fun calculateMessageSystemAttributesMd5( - messageSysAttrs: Map, + messageSystemAttributes: Map, ): String { val buffer = SdkBuffer() - messageSysAttrs + messageSystemAttributes .entries - .sortedBy { (name, _) -> name.value } - .forEach { (attributeName, attributeValue) -> - updateLengthAndBytes(buffer, attributeName.value) - updateLengthAndBytes(buffer, attributeValue.dataType) + .sortedBy { (systemAttributeName, _) -> systemAttributeName.value } + .forEach { (systemAttributeName, systemAttributeValue) -> + updateLengthAndBytes(buffer, systemAttributeName.value) + updateLengthAndBytes(buffer, systemAttributeValue.dataType) when { - attributeValue.stringValue != null -> updateForStringType(buffer, attributeValue.stringValue) - attributeValue.binaryValue != null -> updateForBinaryType(buffer, attributeValue.binaryValue) - !attributeValue.stringListValues.isNullOrEmpty() -> updateForStringListType(buffer, attributeValue.stringListValues) - !attributeValue.binaryListValues.isNullOrEmpty() -> updateForBinaryListType(buffer, attributeValue.binaryListValues) + systemAttributeValue.stringValue != null -> updateForStringType(buffer, systemAttributeValue.stringValue) + systemAttributeValue.binaryValue != null -> updateForBinaryType(buffer, systemAttributeValue.binaryValue) + !systemAttributeValue.stringListValues.isNullOrEmpty() -> updateForStringListType(buffer, systemAttributeValue.stringListValues) + !systemAttributeValue.binaryListValues.isNullOrEmpty() -> updateForBinaryListType(buffer, systemAttributeValue.binaryListValues) + else -> throw ClientException("No value type found for system attribute $systemAttributeName") } } diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/FinalizeSqsConfig.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/FinalizeSqsConfig.kt index f38fbb476e7..a176cc8bb48 100644 --- a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/FinalizeSqsConfig.kt +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/FinalizeSqsConfig.kt @@ -19,13 +19,12 @@ internal suspend fun finalizeSqsConfig( builder.config.checksumValidationEnabled = builder.config.checksumValidationEnabled ?: SqsSetting.checksumValidationEnabled.resolve(provider) ?: activeProfile.checksumValidationEnabled - ?: ValidationEnabled.NEVER // TODO: MD5 checksum validation is temporarily disabled. Set default to ALWAYS in next minor version + ?: ValidationEnabled.NEVER // TODO: MD5 checksum validation is temporarily disabled. Set default to ALWAYS in v1.5 - builder.config.checksumValidationScopes = builder.config.checksumValidationScopes.ifEmpty { - SqsSetting.checksumValidationScopes.resolve(provider) - ?: activeProfile.checksumValidationScopes - ?: ValidationScope.entries.toSet() - } + builder.config.checksumValidationScopes = builder.config.checksumValidationScopes + ?: SqsSetting.checksumValidationScopes.resolve(provider) + ?: activeProfile.checksumValidationScopes + ?: ValidationScope.entries.toSet() } private val AwsProfile.checksumValidationEnabled: ValidationEnabled? diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SqsSetting.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SqsSetting.kt index cc6c2c78791..bb97697b11b 100644 --- a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SqsSetting.kt +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/SqsSetting.kt @@ -18,10 +18,10 @@ internal object SqsSetting { * - Environment variable: AWS_SQS_CHECKSUM_VALIDATION_ENABLED * * Valid values: - * - ALWAYS (default) - Validates checksums for both sending and receiving operations + * - ALWAYS - Validates checksums for both sending and receiving operations * - WHEN_SENDING - Validates checksums only when sending messages * - WHEN_RECEIVING - Validates checksums only when receiving messages - * - NEVER - Disables checksum validation + * - NEVER (default) - Disables checksum validation * * Note: Value matching is case-insensitive when configured via environment variables. */ diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/ValidationConfig.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/ValidationConfig.kt index 61ef7c15927..bc03181bab1 100644 --- a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/ValidationConfig.kt +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/internal/ValidationConfig.kt @@ -7,16 +7,17 @@ package aws.sdk.kotlin.services.sqs.internal * calculated and validated for SQS message operations. * * Valid values: - * - ALWAYS - Validates checksums for both sending and receiving operations + * - `ALWAYS` - Validates checksums for both sending and receiving operations * (SendMessage, SendMessageBatch, and ReceiveMessage) - * - WHEN_SENDING - Validates checksums only when sending messages + * - `WHEN_SENDING` - Validates checksums only when sending messages * (SendMessage and SendMessageBatch) - * - WHEN_RECEIVING - Validates checksums only when receiving messages + * - `WHEN_RECEIVING` - Validates checksums only when receiving messages * (ReceiveMessage) - * - NEVER - Disables checksum validation completely + * - `NEVER` - Disables checksum validation completely * - * Default: ALWAYS + * Default: `NEVER` */ +// TODO: MD5 checksum validation is temporarily disabled. Change default to ALWAYS in v1.5 public enum class ValidationEnabled { ALWAYS, WHEN_SENDING, @@ -31,11 +32,11 @@ public enum class ValidationEnabled { * when checksum validation is enabled. * * Valid values: - * - MESSAGE_ATTRIBUTES - Validates checksums for message attributes - * - MESSAGE_SYSTEM_ATTRIBUTES - Validates checksums for message system attributes + * - `MESSAGE_ATTRIBUTES` - Validates checksums for message attributes + * - `MESSAGE_SYSTEM_ATTRIBUTES` - Validates checksums for message system attributes * (Note: Not available for ReceiveMessage operations as SQS does not calculate * checksums for system attributes during message receipt) - * - MESSAGE_BODY - Validates checksums for the message body + * - `MESSAGE_BODY` - Validates checksums for the message body * * Default: All scopes enabled */ diff --git a/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt b/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt index af7e2148fb0..399878e24b5 100644 --- a/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt +++ b/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt @@ -9,16 +9,16 @@ import aws.sdk.kotlin.e2etest.SqsTestUtils.TEST_MESSAGE_ATTRIBUTES_NAME import aws.sdk.kotlin.e2etest.SqsTestUtils.TEST_MESSAGE_ATTRIBUTES_VALUE import aws.sdk.kotlin.e2etest.SqsTestUtils.TEST_MESSAGE_BODY import aws.sdk.kotlin.e2etest.SqsTestUtils.TEST_MESSAGE_SYSTEM_ATTRIBUTES_VALUE -import aws.sdk.kotlin.e2etest.SqsTestUtils.TEST_QUEUE_CORRECT_CHECKSUM_PREFIX -import aws.sdk.kotlin.e2etest.SqsTestUtils.TEST_QUEUE_WRONG_CHECKSUM_PREFIX +import aws.sdk.kotlin.e2etest.SqsTestUtils.TEST_QUEUE_PREFIX import aws.sdk.kotlin.e2etest.SqsTestUtils.buildSendMessageBatchRequestEntry import aws.sdk.kotlin.e2etest.SqsTestUtils.deleteQueueAndAllMessages import aws.sdk.kotlin.e2etest.SqsTestUtils.getTestQueueUrl import aws.sdk.kotlin.services.sqs.SqsClient import aws.sdk.kotlin.services.sqs.internal.ValidationEnabled -import aws.sdk.kotlin.services.sqs.internal.ValidationScope import aws.sdk.kotlin.services.sqs.model.* import aws.smithy.kotlin.runtime.client.ResponseInterceptorContext +import aws.smithy.kotlin.runtime.collections.AttributeKey +import aws.smithy.kotlin.runtime.collections.get import aws.smithy.kotlin.runtime.hashing.md5 import aws.smithy.kotlin.runtime.http.interceptors.ChecksumMismatchException import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor @@ -26,6 +26,7 @@ import aws.smithy.kotlin.runtime.http.request.HttpRequest import aws.smithy.kotlin.runtime.http.response.HttpResponse import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.* +import org.junit.jupiter.api.Assertions.assertNotNull /** * Tests for Sqs MD5 checksum validation @@ -84,33 +85,53 @@ class SqsMd5ChecksumValidationTest { } } - private val correctChecksumClient = SqsClient { - region = DEFAULT_REGION - checksumValidationEnabled = ValidationEnabled.ALWAYS - checksumValidationScopes = ValidationScope.entries.toSet() - } + // An interceptor that checks if the SQS md5 checksum was validated + private val checksumValidationAssertionInterceptor = object : HttpInterceptor { + private val supportedOperations = setOf( + "SendMessage", + "SendMessageBatch", + "ReceiveMessage", + ) + + override fun readAfterExecution(context: ResponseInterceptorContext) { + val operationName = context.executionContext.attributes[AttributeKey("aws.smithy.kotlin#OperationName")] as String + + if (operationName !in supportedOperations) { + return + } + + assertNotNull(context.executionContext.attributes[AttributeKey("checksumValidated")]) - // used for wrong checksum tests - private val wrongChecksumClient = SqsClient { - region = DEFAULT_REGION - checksumValidationEnabled = ValidationEnabled.ALWAYS - checksumValidationScopes = ValidationScope.entries.toSet() - interceptors += wrongChecksumInterceptor + val isChecksumValidated = context.executionContext.attributes[AttributeKey("checksumValidated")] as Boolean + + assert(isChecksumValidated) + } } - private lateinit var correctChecksumTestQueueUrl: String - private lateinit var wrongChecksumTestQueueUrl: String + private lateinit var correctChecksumClient: SqsClient + + private lateinit var wrongChecksumClient: SqsClient + + private lateinit var testQueueUrl: String @BeforeAll private fun setUp(): Unit = runBlocking { - correctChecksumTestQueueUrl = getTestQueueUrl(correctChecksumClient, TEST_QUEUE_CORRECT_CHECKSUM_PREFIX, DEFAULT_REGION) - wrongChecksumTestQueueUrl = getTestQueueUrl(wrongChecksumClient, TEST_QUEUE_WRONG_CHECKSUM_PREFIX, DEFAULT_REGION) + correctChecksumClient = SqsClient.fromEnvironment { + region = DEFAULT_REGION + checksumValidationEnabled = ValidationEnabled.ALWAYS + interceptors += checksumValidationAssertionInterceptor + } + wrongChecksumClient = SqsClient.fromEnvironment { + region = DEFAULT_REGION + checksumValidationEnabled = ValidationEnabled.ALWAYS + interceptors += wrongChecksumInterceptor + } + testQueueUrl = getTestQueueUrl(correctChecksumClient, TEST_QUEUE_PREFIX) } @AfterAll private fun cleanUp(): Unit = runBlocking { - deleteQueueAndAllMessages(correctChecksumClient, correctChecksumTestQueueUrl) - deleteQueueAndAllMessages(wrongChecksumClient, wrongChecksumTestQueueUrl) + deleteQueueAndAllMessages(correctChecksumClient, testQueueUrl) correctChecksumClient.close() wrongChecksumClient.close() } @@ -119,15 +140,15 @@ class SqsMd5ChecksumValidationTest { fun testSendMessage(): Unit = runBlocking { correctChecksumClient.sendMessage( SendMessageRequest { - queueUrl = correctChecksumTestQueueUrl + queueUrl = testQueueUrl messageBody = TEST_MESSAGE_BODY - messageAttributes = hashMapOf( + messageAttributes = mapOf( TEST_MESSAGE_ATTRIBUTES_NAME to MessageAttributeValue { dataType = "String" stringValue = TEST_MESSAGE_ATTRIBUTES_VALUE }, ) - messageSystemAttributes = hashMapOf( + messageSystemAttributes = mapOf( MessageSystemAttributeNameForSends.AwsTraceHeader to MessageSystemAttributeValue { dataType = "String" stringValue = TEST_MESSAGE_SYSTEM_ATTRIBUTES_VALUE @@ -141,7 +162,7 @@ class SqsMd5ChecksumValidationTest { fun testReceiveMessage(): Unit = runBlocking { correctChecksumClient.receiveMessage( ReceiveMessageRequest { - queueUrl = correctChecksumTestQueueUrl + queueUrl = testQueueUrl maxNumberOfMessages = 1 messageAttributeNames = listOf(TEST_MESSAGE_ATTRIBUTES_NAME) messageSystemAttributeNames = listOf(MessageSystemAttributeName.AwsTraceHeader) @@ -157,7 +178,7 @@ class SqsMd5ChecksumValidationTest { correctChecksumClient.sendMessageBatch( SendMessageBatchRequest { - queueUrl = correctChecksumTestQueueUrl + queueUrl = testQueueUrl this.entries = entries }, ) @@ -165,18 +186,18 @@ class SqsMd5ChecksumValidationTest { @Test fun testSendMessageWithWrongChecksum(): Unit = runBlocking { - val exception = assertThrows { + assertThrows { wrongChecksumClient.sendMessage( SendMessageRequest { - queueUrl = wrongChecksumTestQueueUrl + queueUrl = testQueueUrl messageBody = TEST_MESSAGE_BODY - messageAttributes = hashMapOf( + messageAttributes = mapOf( TEST_MESSAGE_ATTRIBUTES_NAME to MessageAttributeValue { dataType = "String" stringValue = TEST_MESSAGE_ATTRIBUTES_VALUE }, ) - messageSystemAttributes = hashMapOf( + messageSystemAttributes = mapOf( MessageSystemAttributeNameForSends.AwsTraceHeader to MessageSystemAttributeValue { dataType = "String" stringValue = TEST_MESSAGE_SYSTEM_ATTRIBUTES_VALUE @@ -185,24 +206,20 @@ class SqsMd5ChecksumValidationTest { }, ) } - - assert(exception.message!!.contains("Checksum mismatch")) } @Test fun testReceiveMessageWithWrongChecksum(): Unit = runBlocking { - val exception = assertThrows { + assertThrows { wrongChecksumClient.receiveMessage( ReceiveMessageRequest { - queueUrl = wrongChecksumTestQueueUrl + queueUrl = testQueueUrl maxNumberOfMessages = 1 messageAttributeNames = listOf(TEST_MESSAGE_ATTRIBUTES_NAME) messageSystemAttributeNames = listOf(MessageSystemAttributeName.AwsTraceHeader) }, ) } - - assert(exception.message!!.contains("Checksum mismatch")) } @Test @@ -211,15 +228,13 @@ class SqsMd5ChecksumValidationTest { buildSendMessageBatchRequestEntry(batchId) } - val exception = assertThrows { + assertThrows { wrongChecksumClient.sendMessageBatch( SendMessageBatchRequest { - queueUrl = wrongChecksumTestQueueUrl + queueUrl = testQueueUrl this.entries = entries }, ) } - - assert(exception.message!!.contains("Checksum mismatch")) } } diff --git a/services/sqs/e2eTest/src/SqsTestUtils.kt b/services/sqs/e2eTest/src/SqsTestUtils.kt index fbd67085aad..07128e89f76 100644 --- a/services/sqs/e2eTest/src/SqsTestUtils.kt +++ b/services/sqs/e2eTest/src/SqsTestUtils.kt @@ -17,25 +17,17 @@ import kotlin.time.Duration.Companion.seconds object SqsTestUtils { const val DEFAULT_REGION = "us-west-2" - const val TEST_QUEUE_WRONG_CHECKSUM_PREFIX = "sqs-test-queue-wrong-checksum-" - const val TEST_QUEUE_CORRECT_CHECKSUM_PREFIX = "sqs-test-queue-correct-checksum-" + const val TEST_QUEUE_PREFIX = "sqs-test-queue-" const val TEST_MESSAGE_BODY = "Hello World" const val TEST_MESSAGE_ATTRIBUTES_NAME = "TestAttribute" const val TEST_MESSAGE_ATTRIBUTES_VALUE = "TestAttributeValue" const val TEST_MESSAGE_SYSTEM_ATTRIBUTES_VALUE = "TestSystemAttributeValue" - suspend fun getTestQueueUrl( - client: SqsClient, - prefix: String, - region: String? = null, - ): String = getQueueUrlWithPrefix(client, prefix, region) - - private suspend fun getQueueUrlWithPrefix( - client: SqsClient, - prefix: String, - region: String? = null, - ): String = withTimeout(60.seconds) { + suspend fun getTestQueueUrl(client: SqsClient, prefix: String): String = + getQueueUrlWithPrefix(client, prefix) + + private suspend fun getQueueUrlWithPrefix(client: SqsClient, prefix: String): String = withTimeout(60.seconds) { var matchingQueueUrl = client .listQueuesPaginated { queueNamePrefix = prefix } .queueUrls() @@ -43,13 +35,13 @@ object SqsTestUtils { if (matchingQueueUrl == null) { matchingQueueUrl = prefix + UUID.randomUUID() - println("Creating Sqs queue: $matchingQueueUrl") + println("Creating SQS queue: $matchingQueueUrl") client.createQueue { queueName = matchingQueueUrl } } else { - println("Using existing Sqs queue: $matchingQueueUrl") + println("Using existing SQS queue: $matchingQueueUrl") } matchingQueueUrl @@ -57,7 +49,7 @@ object SqsTestUtils { suspend fun deleteQueueAndAllMessages(client: SqsClient, queueUrl: String) { try { - println("Purging Sqs queue: $queueUrl") + println("Purging SQS queue: $queueUrl") client.purgeQueue( PurgeQueueRequest { From 027ee3dc3894f0ac709412b53b2661597ff1fadb Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Tue, 11 Mar 2025 15:56:20 -0400 Subject: [PATCH 09/13] add changelog --- .changes/cc154f8b-62ba-4ab5-8da5-84d1b5fe0d9f.json | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .changes/cc154f8b-62ba-4ab5-8da5-84d1b5fe0d9f.json diff --git a/.changes/cc154f8b-62ba-4ab5-8da5-84d1b5fe0d9f.json b/.changes/cc154f8b-62ba-4ab5-8da5-84d1b5fe0d9f.json new file mode 100644 index 00000000000..1f6faf97e9a --- /dev/null +++ b/.changes/cc154f8b-62ba-4ab5-8da5-84d1b5fe0d9f.json @@ -0,0 +1,8 @@ +{ + "id": "cc154f8b-62ba-4ab5-8da5-84d1b5fe0d9f", + "type": "feature", + "description": "Added MD5 checksum validation for SQS message operations", + "issues": [ + "awslabs/aws-sdk-kotlin#222" + ] +} \ No newline at end of file From a37a4f26954a6571de717295e4ea386779e3a1db Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Tue, 11 Mar 2025 16:04:47 -0400 Subject: [PATCH 10/13] better comment wording --- .../SqsMd5ChecksumValidationInterceptor.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt index 3e38423a5a3..bb565a3cb07 100644 --- a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt @@ -97,7 +97,7 @@ public class SqsMd5ChecksumValidationInterceptor( } } - // Record MD5 checksum validation was performed for this request + // Sets validation flag in execution context for e2e test assertions val checksumValidated: AttributeKey = AttributeKey("checksumValidated") context.executionContext[checksumValidated] = true } From 30ca40da592b0b40fb5787bcc41cfeb73f8c9951 Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Tue, 11 Mar 2025 16:49:23 -0400 Subject: [PATCH 11/13] pr feedback --- .../runtime/config/profile/AwsProfile.kt | 33 +++++++------------ 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt index 2699202e1aa..439b2bab712 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt @@ -212,10 +212,7 @@ public inline fun > AwsProfile.getEnumOrNull(key: String, su it.name.equals(value, ignoreCase = true) } ?: throw ConfigurationException( buildString { - append(key) - append(" '") - append(value) - append("' is not supported, should be one of: ") + append("$key '$value' is not supported, should be one of: ") enumValues().joinTo(this) { it.name.lowercase() } }, ) @@ -226,23 +223,17 @@ public inline fun > AwsProfile.getEnumOrNull(key: String, su */ @InternalSdkApi public inline fun > AwsProfile.getEnumSetOrNull(key: String, subKey: String? = null): Set? = - getOrNull(key, subKey)?.let { rawValue -> - rawValue.split(",") - .map { it -> - val value = it.trim() - enumValues().firstOrNull { enumValue -> - enumValue.name.equals(value, ignoreCase = true) - } ?: throw ConfigurationException( - buildString { - append(key) - append(" '") - append(value) - append("' is not supported, should be one of: ") - enumValues().joinTo(this) { it.name.lowercase() } - }, - ) - }.toSet() - } + getOrNull(key, subKey)?.split(",")?.map { it -> + val value = it.trim() + enumValues().firstOrNull { enumValue -> + enumValue.name.equals(value, ignoreCase = true) + } ?: throw ConfigurationException( + buildString { + append("$key '$value' is not supported, should be one of: ") + enumValues().joinTo(this) { it.name.lowercase() } + }, + ) + }?.toSet() internal fun AwsProfile.getUrlOrNull(key: String, subKey: String? = null): Url? = getOrNull(key, subKey)?.let { From d86b08ddbc2e81d69fe7bf0f01c979b77ec9428c Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Tue, 11 Mar 2025 17:23:46 -0400 Subject: [PATCH 12/13] increase test coverage --- services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt b/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt index 399878e24b5..0ca8719db13 100644 --- a/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt +++ b/services/sqs/e2eTest/src/SqsMd5ChecksumValidationTest.kt @@ -46,9 +46,7 @@ class SqsMd5ChecksumValidationTest { when (val response = context.response.getOrNull()) { is SendMessageResponse -> { val modifiedResponse = response.copy { - md5OfMessageBody = wrongMd5ofMessageBody md5OfMessageAttributes = wrongMd5ofMessageAttribute - md5OfMessageSystemAttributes = wrongMd5ofMessageSystemAttribute } return Result.success(modifiedResponse) } @@ -56,7 +54,6 @@ class SqsMd5ChecksumValidationTest { val modifiedMessages = response.messages?.map { message -> message.copy { md5OfBody = wrongMd5ofMessageBody - md5OfMessageAttributes = wrongMd5ofMessageAttribute } } @@ -68,8 +65,6 @@ class SqsMd5ChecksumValidationTest { is SendMessageBatchResponse -> { val modifiedEntries = response.successful.map { entry -> entry.copy { - md5OfMessageBody = wrongMd5ofMessageBody - md5OfMessageAttributes = wrongMd5ofMessageAttribute md5OfMessageSystemAttributes = wrongMd5ofMessageSystemAttribute } } @@ -147,6 +142,10 @@ class SqsMd5ChecksumValidationTest { dataType = "String" stringValue = TEST_MESSAGE_ATTRIBUTES_VALUE }, + TEST_MESSAGE_ATTRIBUTES_NAME to MessageAttributeValue { + dataType = "Binary" + binaryValue = TEST_MESSAGE_ATTRIBUTES_VALUE.toByteArray() + }, ) messageSystemAttributes = mapOf( MessageSystemAttributeNameForSends.AwsTraceHeader to MessageSystemAttributeValue { From 893148c08c7a0d3df27dba5aefd7d448bf827601 Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Wed, 12 Mar 2025 13:36:53 -0400 Subject: [PATCH 13/13] address pr feedback --- .../customization/sqs/SqsMd5ChecksumValidationIntegration.kt | 2 +- .../SqsMd5ChecksumValidationInterceptor.kt | 4 ++-- services/sqs/e2eTest/src/SqsTestUtils.kt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt index 9de776332df..9c74f6cb81e 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/sqs/SqsMd5ChecksumValidationIntegration.kt @@ -113,7 +113,7 @@ class SqsMd5ChecksumValidationIntegration : KotlinIntegration { ), ) - // add Sqs-specific config finalization + // add SQS-specific config finalization private val finalizeSqsConfigWriter = AppendingSectionWriter { writer -> val finalizeSqsConfig = buildSymbol { name = "finalizeSqsConfig" diff --git a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt index bb565a3cb07..4adbe3aa5c2 100644 --- a/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt +++ b/services/sqs/common/src/aws.sdk.kotlin.services.sqs/SqsMd5ChecksumValidationInterceptor.kt @@ -166,7 +166,7 @@ public class SqsMd5ChecksumValidationInterceptor( validateMd5(clientSideBodyMd5, messageBodyMd5Returned) - logger.debug { "Message body MD5 checksum for ReceiveMessage validated " } + logger.debug { "Message body MD5 checksum for ReceiveMessage validated" } } } @@ -181,7 +181,7 @@ public class SqsMd5ChecksumValidationInterceptor( validateMd5(clientSideAttrMd5, messageAttrMd5Returned) - logger.debug { "Message attribute MD5 checksum for ReceiveMessage validated " } + logger.debug { "Message attribute MD5 checksum for ReceiveMessage validated" } } } } diff --git a/services/sqs/e2eTest/src/SqsTestUtils.kt b/services/sqs/e2eTest/src/SqsTestUtils.kt index 07128e89f76..3f8b0a35236 100644 --- a/services/sqs/e2eTest/src/SqsTestUtils.kt +++ b/services/sqs/e2eTest/src/SqsTestUtils.kt @@ -59,7 +59,7 @@ object SqsTestUtils { println("Queue purged successfully.") - println("Deleting Sqs queue: $queueUrl") + println("Deleting SQS queue: $queueUrl") client.deleteQueue( DeleteQueueRequest {