From 0cc81eec59c4d4901da3db43c55cf3d4557897c7 Mon Sep 17 00:00:00 2001 From: Tim Van Laer Date: Tue, 4 Feb 2025 11:16:12 +0100 Subject: [PATCH 1/4] Use amazonlinux:2023 docker base image with python 3.9 --- consumer/Dockerfile | 5 +++-- consumer/run.sh | 2 +- consumer/sample.properties | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/consumer/Dockerfile b/consumer/Dockerfile index af1b447..5334ac4 100644 --- a/consumer/Dockerfile +++ b/consumer/Dockerfile @@ -1,6 +1,7 @@ -FROM public.ecr.aws/amazonlinux/amazonlinux:2 -RUN yum install python3 git java-11-amazon-corretto-headless -y +FROM public.ecr.aws/amazonlinux/amazonlinux:2023 +RUN yum install python3-pip git java-11-amazon-corretto-headless -y RUN git clone --depth 1 --branch v2.1.1 https://github.com/awslabs/amazon-kinesis-client-python +RUN pip3 install setuptools RUN cd amazon-kinesis-client-python &&\ python3 setup.py download_jars &&\ python3 setup.py install &&\ diff --git a/consumer/run.sh b/consumer/run.sh index 4039b42..8176658 100755 --- a/consumer/run.sh +++ b/consumer/run.sh @@ -1,7 +1,7 @@ #!/bin/sh python3 /usr/bin/set_properties.py chmod 777 /usr/bin/record_processor.py -`python3 /usr/local/lib/python3.7/site-packages/samples/amazon_kclpy_helper.py --print_command --java /usr/bin/java --properties /usr/bin/record_processor.properties --log-configuration /usr/bin/logback.xml` & +`python3 /usr/local/lib/python3.9/site-packages/samples/amazon_kclpy_helper.py --print_command --java /usr/bin/java --properties /usr/bin/record_processor.properties --log-configuration /usr/bin/logback.xml` & mkdir -p /app/logs touch /app/logs/record_processor.log exec tail -F /app/logs/record_processor.log diff --git a/consumer/sample.properties b/consumer/sample.properties index d8c1dd9..b013556 100644 --- a/consumer/sample.properties +++ b/consumer/sample.properties @@ -19,7 +19,7 @@ AWSCredentialsProvider = DefaultAWSCredentialsProviderChain # Appended to the user agent of the KCL. Does not impact the functionality of the # KCL in any other way. -processingLanguage = python/3.6 +processingLanguage = python/3.9 # Valid options at TRIM_HORIZON or LATEST. # See http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html#API_GetShardIterator_RequestSyntax From 4036afe3f08fc0273d970772c22ee042b5e17802 Mon Sep 17 00:00:00 2001 From: Tim Van Laer Date: Tue, 4 Feb 2025 11:16:33 +0100 Subject: [PATCH 2/4] Upgrade amazon-kinesis-client-python to v3.0.1 --- consumer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consumer/Dockerfile b/consumer/Dockerfile index 5334ac4..2a826d0 100644 --- a/consumer/Dockerfile +++ b/consumer/Dockerfile @@ -1,6 +1,6 @@ FROM public.ecr.aws/amazonlinux/amazonlinux:2023 RUN yum install python3-pip git java-11-amazon-corretto-headless -y -RUN git clone --depth 1 --branch v2.1.1 https://github.com/awslabs/amazon-kinesis-client-python +RUN git clone --depth 1 --branch v3.0.1 https://github.com/awslabs/amazon-kinesis-client-python RUN pip3 install setuptools RUN cd amazon-kinesis-client-python &&\ python3 setup.py download_jars &&\ From 315f5df2c6aa69c03e817e2d650d23ae3366af41 Mon Sep 17 00:00:00 2001 From: Tim Van Laer Date: Tue, 4 Feb 2025 11:17:48 +0100 Subject: [PATCH 3/4] Use DefaultCredentialsProvider from the Java SDK v2 This allows us to use a task role when running this container on Fargate. --- consumer/sample.properties | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/consumer/sample.properties b/consumer/sample.properties index b013556..9f1ef8a 100644 --- a/consumer/sample.properties +++ b/consumer/sample.properties @@ -12,10 +12,12 @@ streamName = STREAM_NAME applicationName = APPLICATION_NAME # Users can change the credentials provider the KCL will use to retrieve credentials. -# The DefaultAWSCredentialsProviderChain checks several other providers, which is +# Expected key name (case-sensitive): +# AwsCredentialsProvider / AwsCredentialsProviderDynamoDB / AwsCredentialsProviderCloudWatch +# The DefaultCredentialsProvider checks several other providers, which is # described here: -# http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html -AWSCredentialsProvider = DefaultAWSCredentialsProviderChain +# https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/DefaultCredentialsProvider.html +AwsCredentialsProvider = DefaultCredentialsProvider # Appended to the user agent of the KCL. Does not impact the functionality of the # KCL in any other way. From febccfa0ac5ce39511602e02af41f4915f2aebbb Mon Sep 17 00:00:00 2001 From: Tim Van Laer Date: Tue, 4 Feb 2025 11:28:46 +0100 Subject: [PATCH 4/4] Make it clear that amazon_kclpy_helper generates a command and execute that command in the background. Use modern bash syntax to do so (`$()` instead of backticks `````) --- consumer/run.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/consumer/run.sh b/consumer/run.sh index 8176658..b2ebe7e 100755 --- a/consumer/run.sh +++ b/consumer/run.sh @@ -1,7 +1,12 @@ #!/bin/sh +# transform sample.properties into record_processor.properties python3 /usr/bin/set_properties.py + chmod 777 /usr/bin/record_processor.py -`python3 /usr/local/lib/python3.9/site-packages/samples/amazon_kclpy_helper.py --print_command --java /usr/bin/java --properties /usr/bin/record_processor.properties --log-configuration /usr/bin/logback.xml` & +command=$(python3 /usr/local/lib/python3.9/site-packages/samples/amazon_kclpy_helper.py --print_command --java /usr/bin/java --properties /usr/bin/record_processor.properties --log-configuration /usr/bin/logback.xml) +# execute the generated command in the background (&) +eval "${command}" & + mkdir -p /app/logs touch /app/logs/record_processor.log exec tail -F /app/logs/record_processor.log