Skip to content

Commit 412c341

Browse files
isabeleliassenisabeleliassen
authored
Merge pull request #459 from csg-org/development
Sprint 15
2 parents d86bb1c + 1a3a88e commit 412c341

File tree

408 files changed

+16932
-4137
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

408 files changed

+16932
-4137
lines changed

‎.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
# Idea
22
.idea
33

4+
# VS Code
5+
.vscode
6+
47
#OS
58
.DS_Store
69
.tmp

‎backend/bin/compile_requirements.sh

Lines changed: 20 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,23 @@
11
set -e
22

3-
pip-compile --no-emit-index-url --upgrade multi-account/requirements-dev.in
4-
pip-compile --no-emit-index-url --upgrade multi-account/requirements.in
5-
pip-compile --no-emit-index-url --upgrade compact-connect/requirements-dev.in
6-
pip-compile --no-emit-index-url --upgrade compact-connect/requirements.in
7-
pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/common/requirements-dev.in
8-
pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/common/requirements.in
9-
pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/custom-resources/requirements-dev.in
10-
pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/custom-resources/requirements.in
11-
pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/data-events/requirements-dev.in
12-
pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/data-events/requirements.in
13-
pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/provider-data-v1/requirements-dev.in
14-
pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/provider-data-v1/requirements.in
15-
pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/purchases/requirements-dev.in
16-
pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/purchases/requirements.in
17-
pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.in
18-
pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/staff-user-pre-token/requirements.in
19-
pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/staff-users/requirements-dev.in
20-
pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/staff-users/requirements.in
3+
pip-compile --no-emit-index-url --upgrade --no-strip-extras multi-account/requirements-dev.in
4+
pip-compile --no-emit-index-url --upgrade --no-strip-extras multi-account/requirements.in
5+
pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/requirements-dev.in
6+
pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/requirements.in
7+
pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/attestations/requirements-dev.in
8+
pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/attestations/requirements.in
9+
pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/common/requirements-dev.in
10+
pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/common/requirements.in
11+
pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/custom-resources/requirements-dev.in
12+
pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/custom-resources/requirements.in
13+
pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/data-events/requirements-dev.in
14+
pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/data-events/requirements.in
15+
pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/provider-data-v1/requirements-dev.in
16+
pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/provider-data-v1/requirements.in
17+
pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/purchases/requirements-dev.in
18+
pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/purchases/requirements.in
19+
pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.in
20+
pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/staff-user-pre-token/requirements.in
21+
pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/staff-users/requirements-dev.in
22+
pip-compile --no-emit-index-url --upgrade --no-strip-extras compact-connect/lambdas/python/staff-users/requirements.in
2123
bin/sync_deps.sh

‎backend/bin/run_tests.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,14 @@ if [[ "$LANGUAGE" == 'python' || "$LANGUAGE" == 'all' ]]; then
2525
pytest --cov=. --cov-config=.coveragerc tests
2626
) || exit "$?"
2727
for dir in \
28-
compact-connect/lambdas/python/common \
28+
compact-connect/lambdas/python/attestations \
2929
compact-connect/lambdas/python/custom-resources \
3030
compact-connect/lambdas/python/data-events \
3131
compact-connect/lambdas/python/provider-data-v1 \
3232
compact-connect/lambdas/python/purchases \
3333
compact-connect/lambdas/python/staff-user-pre-token \
3434
compact-connect/lambdas/python/staff-users \
35+
compact-connect/lambdas/python/common \
3536
multi-account
3637
do
3738
(

‎backend/bin/sync_deps.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ pip-sync \
88
multi-account/requirements.txt \
99
compact-connect/requirements-dev.txt \
1010
compact-connect/requirements.txt \
11+
compact-connect/lambdas/python/attestations/requirements-dev.txt \
12+
compact-connect/lambdas/python/attestations/requirements.txt \
1113
compact-connect/lambdas/python/common/requirements-dev.txt \
1214
compact-connect/lambdas/python/common/requirements.txt \
1315
compact-connect/lambdas/python/custom-resources/requirements-dev.txt \

‎backend/compact-connect/README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ $ cdk synth
6363
```
6464

6565
For development work there are additional requirements in `requirements-dev.txt` to install with
66-
`pip install -r requirements.txt`.
66+
`pip install -r requirements-dev.txt`.
6767

6868
To add additional dependencies, for example other CDK libraries, just add them to the `requirements.in` file and rerun
6969
`pip-compile requirements.in`, then `pip install -r requirements.txt` command.
@@ -118,7 +118,7 @@ its environment:
118118
your app. See [About Route53 Hosted Zones](#about-route53-hosted-zones) for more. Note: Without this step, you will
119119
not be able to log in to the UI hosted in CloudFront. The Oauth2 authentication process requires a predictable
120120
callback url to be pre-configured, which the domain name provides. You can still run a local UI against this app,
121-
so long as you leave the `allow_local_ui` context value set to `true` in your environment's context.
121+
so long as you leave the `allow_local_ui` context value set to `true` and remove the `domain_name` param in your environment's context.
122122
2) *Optional if testing SES email notifications with custom domain:* By default, AWS does not allow sending emails to unverified email
123123
addresses. If you need to test SES email notifications and do not want to request AWS to remove your account from
124124
the SES sandbox, you will need to set up a verified SES email identity for each address you want to send emails to.

‎backend/compact-connect/bin/create_provider_user.py

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
#!/usr/bin/env python3
2+
"""Provider user generation helper script. Run from `backend/compact-connect`.
3+
4+
Note: This script requires the boto3 library and the environment variable:
5+
# The provider user pool id
6+
USER_POOL_ID=us-east-1_7zzexample
7+
8+
The CLI must also be configured with AWS credentials that have appropriate access to Cognito
9+
"""
10+
11+
import json
12+
import os
13+
import sys
14+
15+
import boto3
16+
from botocore.exceptions import ClientError
17+
18+
with open('cdk.json') as context_file:
19+
_context = json.load(context_file)['context']
20+
21+
COMPACTS = _context['compacts']
22+
USER_POOL_ID = os.environ['USER_POOL_ID']
23+
24+
25+
cognito_client = boto3.client('cognito-idp')
26+
27+
28+
def create_cognito_user(*, email: str, compact: str, provider_id: str):
29+
sys.stdout.write(f"Creating new provider user, '{email}', in {compact}")
30+
31+
def get_sub_from_attributes(user_attributes: list):
32+
for attribute in user_attributes:
33+
if attribute['Name'] == 'sub':
34+
return attribute['Value']
35+
raise ValueError('Failed to find user sub!')
36+
37+
try:
38+
user_data = cognito_client.admin_create_user(
39+
UserPoolId=USER_POOL_ID,
40+
Username=email,
41+
UserAttributes=[
42+
{'Name': 'email', 'Value': email},
43+
{'Name': 'custom:compact', 'Value': compact},
44+
{'Name': 'custom:providerId', 'Value': provider_id},
45+
],
46+
DesiredDeliveryMediums=['EMAIL'],
47+
)
48+
return get_sub_from_attributes(user_data['User']['Attributes'])
49+
50+
except ClientError as e:
51+
if e.response['Error']['Code'] == 'UsernameExistsException':
52+
user_data = cognito_client.admin_get_user(UserPoolId=USER_POOL_ID, Username=email)
53+
return get_sub_from_attributes(user_data['UserAttributes'])
54+
raise
55+
56+
57+
if __name__ == '__main__':
58+
from argparse import ArgumentParser
59+
60+
parser = ArgumentParser(
61+
description='Create a provider user',
62+
epilog='example: bin/create_provider_user.py -e [email protected] -c octp -p ac5f9901-e4e6-4a2e-8982-27d2517a3ab8', # noqa: E501 line-too-long
63+
)
64+
parser.add_argument('-e', '--email', help="The new user's email address", required=True)
65+
parser.add_argument('-c', '--compact', help="The new user's compact", required=True, choices=COMPACTS)
66+
parser.add_argument('-p', '--provider-id', help="The new user's associated provider id", required=True)
67+
68+
args = parser.parse_args()
69+
70+
create_cognito_user(email=args.email, compact=args.compact, provider_id=args.provider_id)

‎backend/compact-connect/bin/create_staff_user.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
provider_data_path = os.path.join('lambdas', 'python', 'staff-users')
1919
common_lib_path = os.path.join('lambdas', 'python', 'common')
20+
2021
sys.path.append(provider_data_path)
2122
sys.path.append(common_lib_path)
2223

‎backend/compact-connect/bin/generate_mock_data.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,16 +23,17 @@
2323
COMPACTS = _context['compacts']
2424
LICENSE_TYPES = _context['license_types']
2525

26+
2627
os.environ['COMPACTS'] = json.dumps(COMPACTS)
2728
os.environ['JURISDICTIONS'] = json.dumps(JURISDICTIONS)
2829

29-
from cc_common.data_model.schema.license import LicensePostSchema # noqa: E402
30+
from cc_common.data_model.schema.license.api import LicensePostRequestSchema # noqa: E402
3031

3132
# We'll grab three different localizations to provide a variety of names/characters
3233
name_faker = Faker(['en_US', 'ja_JP', 'es_MX'])
3334
faker = Faker(['en_US'])
3435

35-
schema = LicensePostSchema()
36+
schema = LicensePostRequestSchema()
3637

3738
FIELDS = (
3839
'ssn',
@@ -59,7 +60,7 @@
5960

6061

6162
def generate_mock_csv_file(count, *, compact: str, jurisdiction: str = None):
62-
with open('mock-data.csv', 'w', encoding='utf-8') as data_file:
63+
with open(f'{compact}-{jurisdiction}-mock-data.csv', 'w', encoding='utf-8') as data_file:
6364
writer = DictWriter(data_file, fieldnames=FIELDS)
6465
writer.writeheader()
6566
for row in generate_csv_rows(count, compact=compact, jurisdiction=jurisdiction):

‎backend/compact-connect/common_constructs/nodejs_function.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ def __init__(
7171
suppressions=[
7272
{
7373
'id': 'AwsSolutions-IAM4',
74-
'applies_to': [
74+
'appliesTo': [
7575
'Policy::arn:<AWS::Partition>:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole',
7676
],
7777
'reason': 'The BasicExecutionRole policy is appropriate for these lambdas',
@@ -84,7 +84,9 @@ def __init__(
8484
suppressions=[
8585
{
8686
'id': 'AwsSolutions-IAM4',
87-
'applies_to': 'Policy::arn:<AWS::Partition>:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole', # noqa: E501 line-too-long
87+
'appliesTo': [
88+
'Policy::arn:<AWS::Partition>:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole'
89+
], # noqa: E501 line-too-long
8890
'reason': 'This policy is appropriate for the log retention lambda',
8991
},
9092
],
@@ -95,7 +97,7 @@ def __init__(
9597
suppressions=[
9698
{
9799
'id': 'AwsSolutions-IAM5',
98-
'applies_to': ['Resource::*'],
100+
'appliesTo': ['Resource::*'],
99101
'reason': 'This lambda needs to be able to configure log groups across the account, though the'
100102
' actions it is allowed are scoped specifically for this task.',
101103
},

‎backend/compact-connect/common_constructs/python_function.py

Lines changed: 26 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@
22

33
import os
44

5-
import stacks.persistent_stack as ps
65
from aws_cdk import Duration, Stack
76
from aws_cdk.aws_cloudwatch import Alarm, ComparisonOperator, Stats, TreatMissingData
87
from aws_cdk.aws_cloudwatch_actions import SnsAction
8+
from aws_cdk.aws_iam import IRole
99
from aws_cdk.aws_lambda import ILayerVersion, Runtime
1010
from aws_cdk.aws_lambda_python_alpha import PythonFunction as CdkPythonFunction
1111
from aws_cdk.aws_lambda_python_alpha import PythonLayerVersion
@@ -14,6 +14,7 @@
1414
from aws_cdk.aws_ssm import StringParameter
1515
from cdk_nag import NagSuppressions
1616
from constructs import Construct
17+
from stacks import persistent_stack as ps
1718

1819
COMMON_PYTHON_LAMBDA_LAYER_SSM_PARAMETER_NAME = '/deployment/lambda/layers/common-python-layer-arn'
1920

@@ -31,8 +32,9 @@ def __init__(
3132
construct_id: str,
3233
*,
3334
lambda_dir: str,
34-
log_retention: RetentionDays = RetentionDays.ONE_MONTH,
35+
log_retention: RetentionDays = RetentionDays.INFINITE,
3536
alarm_topic: ITopic = None,
37+
role: IRole = None,
3638
**kwargs,
3739
):
3840
defaults = {
@@ -46,6 +48,7 @@ def __init__(
4648
entry=os.path.join('lambdas', 'python', lambda_dir),
4749
runtime=Runtime.PYTHON_3_12,
4850
log_retention=log_retention,
51+
role=role,
4952
**defaults,
5053
)
5154
self.add_layers(self._get_common_layer())
@@ -72,26 +75,32 @@ def __init__(
7275
},
7376
],
7477
)
75-
NagSuppressions.add_resource_suppressions_by_path(
76-
stack,
77-
path=f'{self.node.path}/ServiceRole/Resource',
78-
suppressions=[
79-
{
80-
'id': 'AwsSolutions-IAM4',
81-
'applies_to': [
82-
'Policy::arn:<AWS::Partition>:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole',
83-
],
84-
'reason': 'The BasicExecutionRole policy is appropriate for these lambdas',
85-
},
86-
],
87-
)
78+
79+
# If a role is provided from elsewhere for this lambda (role is not None), we don't need to run suppressions for
80+
# the role that this construct normally creates.
81+
if role is None:
82+
NagSuppressions.add_resource_suppressions_by_path(
83+
stack,
84+
path=f'{self.node.path}/ServiceRole/Resource',
85+
suppressions=[
86+
{
87+
'id': 'AwsSolutions-IAM4',
88+
'appliesTo': [
89+
'Policy::arn:<AWS::Partition>:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole',
90+
],
91+
'reason': 'The BasicExecutionRole policy is appropriate for these lambdas',
92+
},
93+
],
94+
)
8895
NagSuppressions.add_resource_suppressions_by_path(
8996
stack,
9097
path=f'{stack.node.path}/LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8a/ServiceRole/Resource',
9198
suppressions=[
9299
{
93100
'id': 'AwsSolutions-IAM4',
94-
'applies_to': 'Policy::arn:<AWS::Partition>:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole', # noqa: E501 line-too-long
101+
'appliesTo': [
102+
'Policy::arn:<AWS::Partition>:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole'
103+
], # noqa: E501 line-too-long
95104
'reason': 'This policy is appropriate for the log retention lambda',
96105
},
97106
],
@@ -102,7 +111,7 @@ def __init__(
102111
suppressions=[
103112
{
104113
'id': 'AwsSolutions-IAM5',
105-
'applies_to': ['Resource::*'],
114+
'appliesTo': ['Resource::*'],
106115
'reason': 'This lambda needs to be able to configure log groups across the account, though the'
107116
' actions it is allowed are scoped specifically for this task.',
108117
},

‎backend/compact-connect/common_constructs/queued_lambda_processor.py

Lines changed: 41 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
from aws_cdk import Duration
1+
from aws_cdk import Duration, Names
22
from aws_cdk.aws_cloudwatch import Alarm, ComparisonOperator, TreatMissingData
33
from aws_cdk.aws_cloudwatch_actions import SnsAction
4+
from aws_cdk.aws_iam import Effect, PolicyStatement
45
from aws_cdk.aws_kms import IKey
56
from aws_cdk.aws_lambda import IFunction
6-
from aws_cdk.aws_lambda_event_sources import SqsEventSource
77
from aws_cdk.aws_logs import QueryDefinition, QueryString
88
from aws_cdk.aws_sns import ITopic
99
from aws_cdk.aws_sqs import DeadLetterQueue, IQueue, Queue, QueueEncryption
@@ -51,14 +51,46 @@ def __init__(
5151
dead_letter_queue=DeadLetterQueue(max_receive_count=max_receive_count, queue=self.dlq),
5252
)
5353

54-
process_function.add_event_source(
55-
SqsEventSource(
56-
self.queue,
57-
batch_size=batch_size,
58-
max_batching_window=max_batching_window,
59-
report_batch_item_failures=True,
60-
),
54+
# The following section of code is equivalent to:
55+
# process_function.add_event_source(
56+
# SqsEventSource(
57+
# self.queue,
58+
# batch_size=batch_size,
59+
# max_batching_window=max_batching_window,
60+
# report_batch_item_failures=True,
61+
# ),
62+
# )
63+
#
64+
# Except that we are granting the lambda permission to consume SQS messages via resource policy
65+
# on the queue, rather than the more conventional approach of principal policy on the IAM role.
66+
#
67+
# We use a lower-level add_event_source_mapping method here so that we can control how those
68+
# permissions are granted. In this case, we need to grant permissions via resource policy on
69+
# the Queue rather than principal policy on the role to avoid creating a dependency from the
70+
# role on the queue. In some cases, adding the dependency on the role can cause a circular
71+
# dependency.
72+
process_function.add_event_source_mapping(
73+
f'SqsEventSource:{Names.node_unique_id(self.queue.node)}',
74+
batch_size=batch_size,
75+
max_batching_window=max_batching_window,
76+
report_batch_item_failures=True,
77+
event_source_arn=self.queue.queue_arn,
78+
)
79+
self.queue.add_to_resource_policy(
80+
PolicyStatement(
81+
effect=Effect.ALLOW,
82+
principals=[process_function.role],
83+
actions=[
84+
'sqs:ReceiveMessage',
85+
'sqs:ChangeMessageVisibility',
86+
'sqs:GetQueueUrl',
87+
'sqs:DeleteMessage',
88+
'sqs:GetQueueAttributes',
89+
],
90+
resources=[self.queue.queue_arn],
91+
)
6192
)
93+
6294
self._add_queue_alarms(
6395
retention_period=retention_period, queue=self.queue, dlq=self.dlq, alarm_topic=alarm_topic
6496
)

0 commit comments

Comments
 (0)