Sprint 15

.gitignore

@@ -1,6 +1,9 @@
 # Idea
 .idea
 
+# VS Code
+.vscode
+
 #OS
 .DS_Store
 .tmp

backend/bin/compile_requirements.sh

@@ -1,21 +1,23 @@
 set -e
 
-pip-compile --no-emit-index-url --upgrade multi-account/requirements-dev.in
-pip-compile --no-emit-index-url --upgrade multi-account/requirements.in
-pip-compile --no-emit-index-url --upgrade compact-connect/requirements-dev.in
-pip-compile --no-emit-index-url --upgrade compact-connect/requirements.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/common/requirements-dev.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/common/requirements.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/custom-resources/requirements-dev.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/custom-resources/requirements.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/data-events/requirements-dev.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/data-events/requirements.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/provider-data-v1/requirements-dev.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/provider-data-v1/requirements.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/purchases/requirements-dev.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/purchases/requirements.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/staff-user-pre-token/requirements.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/staff-users/requirements-dev.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/staff-users/requirements.in
+pip-compile --no-emit-index-url --upgrade --no-strip-extras multi-account/requirements-dev.in
+pip-compile --no-emit-index-url --upgrade --no-strip-extras multi-account/requirements.in
+pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/requirements-dev.in
+pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/requirements.in
+pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/attestations/requirements-dev.in
+pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/attestations/requirements.in
+pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/common/requirements-dev.in
+pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/common/requirements.in
+pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/custom-resources/requirements-dev.in
+pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/custom-resources/requirements.in
+pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/data-events/requirements-dev.in
+pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/data-events/requirements.in
+pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/provider-data-v1/requirements-dev.in
+pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/provider-data-v1/requirements.in
+pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/purchases/requirements-dev.in
+pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/purchases/requirements.in
+pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.in
+pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/staff-user-pre-token/requirements.in
+pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/staff-users/requirements-dev.in
+pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/staff-users/requirements.in
 bin/sync_deps.sh

backend/bin/run_tests.sh

@@ -25,13 +25,14 @@ if [[ "$LANGUAGE" == 'python' || "$LANGUAGE" == 'all' ]]; then
     pytest --cov=. --cov-config=.coveragerc tests
   ) || exit "$?"
   for dir in \
-    compact-connect/lambdas/python/common \
+    compact-connect/lambdas/python/attestations \
     compact-connect/lambdas/python/custom-resources \
     compact-connect/lambdas/python/data-events \
     compact-connect/lambdas/python/provider-data-v1 \
     compact-connect/lambdas/python/purchases \
     compact-connect/lambdas/python/staff-user-pre-token \
     compact-connect/lambdas/python/staff-users \
+    compact-connect/lambdas/python/common \
     multi-account
     do
     (

backend/bin/sync_deps.sh

@@ -8,6 +8,8 @@ pip-sync \
   multi-account/requirements.txt \
   compact-connect/requirements-dev.txt \
   compact-connect/requirements.txt \
+  compact-connect/lambdas/python/attestations/requirements-dev.txt \
+  compact-connect/lambdas/python/attestations/requirements.txt \
   compact-connect/lambdas/python/common/requirements-dev.txt \
   compact-connect/lambdas/python/common/requirements.txt \
   compact-connect/lambdas/python/custom-resources/requirements-dev.txt \

backend/compact-connect/README.md

@@ -63,7 +63,7 @@ $ cdk synth
 ```
 
 For development work there are additional requirements in `requirements-dev.txt` to install with
-`pip install -r requirements.txt`.
+`pip install -r requirements-dev.txt`.
 
 To add additional dependencies, for example other CDK libraries, just add them to the `requirements.in` file and rerun
 `pip-compile requirements.in`, then `pip install -r requirements.txt` command.
@@ -118,7 +118,7 @@ its environment:
    your app. See [About Route53 Hosted Zones](#about-route53-hosted-zones) for more. Note: Without this step, you will
    not be able to log in to the UI hosted in CloudFront. The Oauth2 authentication process requires a predictable
    callback url to be pre-configured, which the domain name provides. You can still run a local UI against this app,
-   so long as you leave the `allow_local_ui` context value set to `true` in your environment's context. 
+   so long as you leave the `allow_local_ui` context value set to `true` and remove the `domain_name` param in your environment's context.
 2) *Optional if testing SES email notifications with custom domain:* By default, AWS does not allow sending emails to unverified email
    addresses. If you need to test SES email notifications and do not want to request AWS to remove your account from
    the SES sandbox, you will need to set up a verified SES email identity for each address you want to send emails to.

backend/compact-connect/bin/create_provider_user.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""Provider user generation helper script. Run from `backend/compact-connect`.
+
+Note: This script requires the boto3 library and the environment variable:
+# The provider user pool id
+USER_POOL_ID=us-east-1_7zzexample
+
+The CLI must also be configured with AWS credentials that have appropriate access to Cognito
+"""
+
+import json
+import os
+import sys
+
+import boto3
+from botocore.exceptions import ClientError
+
+with open('cdk.json') as context_file:
+    _context = json.load(context_file)['context']
+
+COMPACTS = _context['compacts']
+USER_POOL_ID = os.environ['USER_POOL_ID']
+
+
+cognito_client = boto3.client('cognito-idp')
+
+
+def create_cognito_user(*, email: str, compact: str, provider_id: str):
+    sys.stdout.write(f"Creating new provider user, '{email}', in {compact}")
+
+    def get_sub_from_attributes(user_attributes: list):
+        for attribute in user_attributes:
+            if attribute['Name'] == 'sub':
+                return attribute['Value']
+        raise ValueError('Failed to find user sub!')
+
+    try:
+        user_data = cognito_client.admin_create_user(
+            UserPoolId=USER_POOL_ID,
+            Username=email,
+            UserAttributes=[
+                {'Name': 'email', 'Value': email},
+                {'Name': 'custom:compact', 'Value': compact},
+                {'Name': 'custom:providerId', 'Value': provider_id},
+            ],
+            DesiredDeliveryMediums=['EMAIL'],
+        )
+        return get_sub_from_attributes(user_data['User']['Attributes'])
+
+    except ClientError as e:
+        if e.response['Error']['Code'] == 'UsernameExistsException':
+            user_data = cognito_client.admin_get_user(UserPoolId=USER_POOL_ID, Username=email)
+            return get_sub_from_attributes(user_data['UserAttributes'])
+        raise
+
+
+if __name__ == '__main__':
+    from argparse import ArgumentParser
+
+    parser = ArgumentParser(
+        description='Create a provider user',
+        epilog='example: bin/create_provider_user.py -e [email protected] -c octp -p ac5f9901-e4e6-4a2e-8982-27d2517a3ab8',  # noqa: E501 line-too-long
+    )
+    parser.add_argument('-e', '--email', help="The new user's email address", required=True)
+    parser.add_argument('-c', '--compact', help="The new user's compact", required=True, choices=COMPACTS)
+    parser.add_argument('-p', '--provider-id', help="The new user's associated provider id", required=True)
+
+    args = parser.parse_args()
+
+    create_cognito_user(email=args.email, compact=args.compact, provider_id=args.provider_id)

backend/compact-connect/bin/create_staff_user.py

@@ -17,6 +17,7 @@
 
 provider_data_path = os.path.join('lambdas', 'python', 'staff-users')
 common_lib_path = os.path.join('lambdas', 'python', 'common')
+
 sys.path.append(provider_data_path)
 sys.path.append(common_lib_path)
 

backend/compact-connect/bin/generate_mock_data.py

@@ -23,16 +23,17 @@
 COMPACTS = _context['compacts']
 LICENSE_TYPES = _context['license_types']
 
+
 os.environ['COMPACTS'] = json.dumps(COMPACTS)
 os.environ['JURISDICTIONS'] = json.dumps(JURISDICTIONS)
 
-from cc_common.data_model.schema.license import LicensePostSchema  # noqa: E402
+from cc_common.data_model.schema.license.api import LicensePostRequestSchema  # noqa: E402
 
 # We'll grab three different localizations to provide a variety of names/characters
 name_faker = Faker(['en_US', 'ja_JP', 'es_MX'])
 faker = Faker(['en_US'])
 
-schema = LicensePostSchema()
+schema = LicensePostRequestSchema()
 
 FIELDS = (
     'ssn',
@@ -59,7 +60,7 @@
 
 
 def generate_mock_csv_file(count, *, compact: str, jurisdiction: str = None):
-    with open('mock-data.csv', 'w', encoding='utf-8') as data_file:
+    with open(f'{compact}-{jurisdiction}-mock-data.csv', 'w', encoding='utf-8') as data_file:
         writer = DictWriter(data_file, fieldnames=FIELDS)
         writer.writeheader()
         for row in generate_csv_rows(count, compact=compact, jurisdiction=jurisdiction):

backend/compact-connect/common_constructs/nodejs_function.py

@@ -71,7 +71,7 @@ def __init__(
             suppressions=[
                 {
                     'id': 'AwsSolutions-IAM4',
-                    'applies_to': [
+                    'appliesTo': [
                         'Policy::arn:<AWS::Partition>:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole',
                     ],
                     'reason': 'The BasicExecutionRole policy is appropriate for these lambdas',
@@ -84,7 +84,9 @@ def __init__(
             suppressions=[
                 {
                     'id': 'AwsSolutions-IAM4',
-                    'applies_to': 'Policy::arn:<AWS::Partition>:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole',  # noqa: E501 line-too-long
+                    'appliesTo': [
+                        'Policy::arn:<AWS::Partition>:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole'
+                    ],  # noqa: E501 line-too-long
                     'reason': 'This policy is appropriate for the log retention lambda',
                 },
             ],
@@ -95,7 +97,7 @@ def __init__(
             suppressions=[
                 {
                     'id': 'AwsSolutions-IAM5',
-                    'applies_to': ['Resource::*'],
+                    'appliesTo': ['Resource::*'],
                     'reason': 'This lambda needs to be able to configure log groups across the account, though the'
                     ' actions it is allowed are scoped specifically for this task.',
                 },

backend/compact-connect/common_constructs/python_function.py

@@ -2,10 +2,10 @@
 
 import os
 
-import stacks.persistent_stack as ps
 from aws_cdk import Duration, Stack
 from aws_cdk.aws_cloudwatch import Alarm, ComparisonOperator, Stats, TreatMissingData
 from aws_cdk.aws_cloudwatch_actions import SnsAction
+from aws_cdk.aws_iam import IRole
 from aws_cdk.aws_lambda import ILayerVersion, Runtime
 from aws_cdk.aws_lambda_python_alpha import PythonFunction as CdkPythonFunction
 from aws_cdk.aws_lambda_python_alpha import PythonLayerVersion
@@ -14,6 +14,7 @@
 from aws_cdk.aws_ssm import StringParameter
 from cdk_nag import NagSuppressions
 from constructs import Construct
+from stacks import persistent_stack as ps
 
 COMMON_PYTHON_LAMBDA_LAYER_SSM_PARAMETER_NAME = '/deployment/lambda/layers/common-python-layer-arn'
 
@@ -31,8 +32,9 @@ def __init__(
         construct_id: str,
         *,
         lambda_dir: str,
-        log_retention: RetentionDays = RetentionDays.ONE_MONTH,
+        log_retention: RetentionDays = RetentionDays.INFINITE,
         alarm_topic: ITopic = None,
+        role: IRole = None,
         **kwargs,
     ):
         defaults = {
@@ -46,6 +48,7 @@ def __init__(
             entry=os.path.join('lambdas', 'python', lambda_dir),
             runtime=Runtime.PYTHON_3_12,
             log_retention=log_retention,
+            role=role,
             **defaults,
         )
         self.add_layers(self._get_common_layer())
@@ -72,26 +75,32 @@ def __init__(
                 },
             ],
         )
-        NagSuppressions.add_resource_suppressions_by_path(
-            stack,
-            path=f'{self.node.path}/ServiceRole/Resource',
-            suppressions=[
-                {
-                    'id': 'AwsSolutions-IAM4',
-                    'applies_to': [
-                        'Policy::arn:<AWS::Partition>:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole',
-                    ],
-                    'reason': 'The BasicExecutionRole policy is appropriate for these lambdas',
-                },
-            ],
-        )
+
+        # If a role is provided from elsewhere for this lambda (role is not None), we don't need to run suppressions for
+        # the role that this construct normally creates.
+        if role is None:
+            NagSuppressions.add_resource_suppressions_by_path(
+                stack,
+                path=f'{self.node.path}/ServiceRole/Resource',
+                suppressions=[
+                    {
+                        'id': 'AwsSolutions-IAM4',
+                        'appliesTo': [
+                            'Policy::arn:<AWS::Partition>:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole',
+                        ],
+                        'reason': 'The BasicExecutionRole policy is appropriate for these lambdas',
+                    },
+                ],
+            )
         NagSuppressions.add_resource_suppressions_by_path(
             stack,
             path=f'{stack.node.path}/LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8a/ServiceRole/Resource',
             suppressions=[
                 {
                     'id': 'AwsSolutions-IAM4',
-                    'applies_to': 'Policy::arn:<AWS::Partition>:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole',  # noqa: E501 line-too-long
+                    'appliesTo': [
+                        'Policy::arn:<AWS::Partition>:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole'
+                    ],  # noqa: E501 line-too-long
                     'reason': 'This policy is appropriate for the log retention lambda',
                 },
             ],
@@ -102,7 +111,7 @@ def __init__(
             suppressions=[
                 {
                     'id': 'AwsSolutions-IAM5',
-                    'applies_to': ['Resource::*'],
+                    'appliesTo': ['Resource::*'],
                     'reason': 'This lambda needs to be able to configure log groups across the account, though the'
                     ' actions it is allowed are scoped specifically for this task.',
                 },

backend/compact-connect/common_constructs/queued_lambda_processor.py

@@ -1,9 +1,9 @@
-from aws_cdk import Duration
+from aws_cdk import Duration, Names
 from aws_cdk.aws_cloudwatch import Alarm, ComparisonOperator, TreatMissingData
 from aws_cdk.aws_cloudwatch_actions import SnsAction
+from aws_cdk.aws_iam import Effect, PolicyStatement
 from aws_cdk.aws_kms import IKey
 from aws_cdk.aws_lambda import IFunction
-from aws_cdk.aws_lambda_event_sources import SqsEventSource
 from aws_cdk.aws_logs import QueryDefinition, QueryString
 from aws_cdk.aws_sns import ITopic
 from aws_cdk.aws_sqs import DeadLetterQueue, IQueue, Queue, QueueEncryption
@@ -51,14 +51,46 @@ def __init__(
             dead_letter_queue=DeadLetterQueue(max_receive_count=max_receive_count, queue=self.dlq),
         )
 
-        process_function.add_event_source(
-            SqsEventSource(
-                self.queue,
-                batch_size=batch_size,
-                max_batching_window=max_batching_window,
-                report_batch_item_failures=True,
-            ),
+        # The following section of code is equivalent to:
+        # process_function.add_event_source(
+        #     SqsEventSource(
+        #         self.queue,
+        #         batch_size=batch_size,
+        #         max_batching_window=max_batching_window,
+        #         report_batch_item_failures=True,
+        #     ),
+        # )
+        #
+        # Except that we are granting the lambda permission to consume SQS messages via resource policy
+        # on the queue, rather than the more conventional approach of principal policy on the IAM role.
+        #
+        # We use a lower-level add_event_source_mapping method here so that we can control how those
+        # permissions are granted. In this case, we need to grant permissions via resource policy on
+        # the Queue rather than principal policy on the role to avoid creating a dependency from the
+        # role on the queue. In some cases, adding the dependency on the role can cause a circular
+        # dependency.
+        process_function.add_event_source_mapping(
+            f'SqsEventSource:{Names.node_unique_id(self.queue.node)}',
+            batch_size=batch_size,
+            max_batching_window=max_batching_window,
+            report_batch_item_failures=True,
+            event_source_arn=self.queue.queue_arn,
+        )
+        self.queue.add_to_resource_policy(
+            PolicyStatement(
+                effect=Effect.ALLOW,
+                principals=[process_function.role],
+                actions=[
+                    'sqs:ReceiveMessage',
+                    'sqs:ChangeMessageVisibility',
+                    'sqs:GetQueueUrl',
+                    'sqs:DeleteMessage',
+                    'sqs:GetQueueAttributes',
+                ],
+                resources=[self.queue.queue_arn],
+            )
         )
+
         self._add_queue_alarms(
             retention_period=retention_period, queue=self.queue, dlq=self.dlq, alarm_topic=alarm_topic
         )