Merge pull request #401 from csg-org/development

Sprint 13

Author: isabeleliassen

.github/workflows/check-lambda-js.yml

@@ -10,15 +10,16 @@ on:
       - main
       - development
       - ia-web-development
+      - ia-development
     paths:
-      - backend/compact-connect/lambdas/cloudfront-csp/**
+      - backend/compact-connect/lambdas/nodejs/**
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
-  CheckLambdaJS:
+  CheckLambdas:
     runs-on: ubuntu-latest
 
     steps:
@@ -36,28 +37,30 @@ jobs:
       - name: Setup Node
         uses: actions/setup-node@v1
         with:
-          node-version: '20.15.1'
+          node-version: '22.1.0'
 
       # Use any cached yarn dependencies (saves build time)
       - uses: actions/cache@v2
         with:
           path: '**/node_modules'
           key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
 
-      # =======================================================================
-      # =                      Cloudfront CSP Lambda                          =
-      # =======================================================================
       # Install Yarn Dependencies
       - name: Install JS dependencies (Cloudfront CSP Lambda)
         run: yarn install
-        working-directory: ./backend/compact-connect/lambdas/cloudfront-csp
+        working-directory: ./backend/compact-connect/lambdas/nodejs
 
       # Run Linter Checks
       - name: Run linter (Cloudfront CSP Lambda)
         run: yarn run lint
-        working-directory: ./backend/compact-connect/lambdas/cloudfront-csp
+        working-directory: ./backend/compact-connect/lambdas/nodejs
 
       # Run Unit Tests
       - name: Run unit tests (Cloudfront CSP Lambda)
         run: yarn test
-        working-directory: ./backend/compact-connect/lambdas/cloudfront-csp
+        working-directory: ./backend/compact-connect/lambdas/nodejs
+
+      # Audit dependencies for vulnerabilities
+      - name: Audit dependencies (Cloudfront CSP Lambda)
+        run: yarn run audit:dependencies
+        working-directory: ./backend/compact-connect/lambdas/nodejs

.github/workflows/check-python.yml

@@ -33,7 +33,8 @@ jobs:
   TestPython:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-python@v5
+      - name: Setup Python
+        uses: actions/setup-python@v5
         with:
           python-version: '3.12'
 
@@ -47,7 +48,7 @@ jobs:
         run: "cd backend; bin/sync_deps.sh"
 
       - name: Test backend
-        run: "cd backend; bin/run_tests.sh no-report"
+        run: "cd backend; bin/run_tests.sh python no-report"
 
       - name: Check Dependencies
         run: "pip-audit"

backend/.gitignore

@@ -12,3 +12,5 @@ cdk.out
 # Save for local config
 cdk.context.json
 *-cdk.context.json
+# smoke test env
+smoke_tests_env.json

backend/bin/compile_requirements.sh

@@ -1,23 +1,21 @@
 set -e
 
-pip-compile --no-emit-index-url --upgrade multi-account/requirements.in
 pip-compile --no-emit-index-url --upgrade multi-account/requirements-dev.in
-pip-compile --no-emit-index-url --upgrade compact-connect/requirements.in
+pip-compile --no-emit-index-url --upgrade multi-account/requirements.in
 pip-compile --no-emit-index-url --upgrade compact-connect/requirements-dev.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/common-python/requirements.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/common-python/requirements-dev.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/custom-resources/requirements.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/custom-resources/requirements-dev.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/license-data/requirements.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/license-data/requirements-dev.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/provider-data-v1/requirements.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/provider-data-v1/requirements-dev.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/purchases/requirements.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/purchases/requirements-dev.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/staff-user-pre-token/requirements.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/staff-user-pre-token/requirements-dev.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/staff-users/requirements.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/staff-users/requirements-dev.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/delete-objects/requirements.in
-pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/delete-objects/requirements-dev.in
+pip-compile --no-emit-index-url --upgrade compact-connect/requirements.in
+pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/common/requirements-dev.in
+pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/common/requirements.in
+pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/custom-resources/requirements-dev.in
+pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/custom-resources/requirements.in
+pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/data-events/requirements-dev.in
+pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/data-events/requirements.in
+pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/provider-data-v1/requirements-dev.in
+pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/provider-data-v1/requirements.in
+pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/purchases/requirements-dev.in
+pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/purchases/requirements.in
+pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.in
+pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/staff-user-pre-token/requirements.in
+pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/staff-users/requirements-dev.in
+pip-compile --no-emit-index-url --upgrade compact-connect/lambdas/python/staff-users/requirements.in
 bin/sync_deps.sh

backend/bin/pre-commit

@@ -95,7 +95,7 @@ if [ -n "$files" ]; then
 fi
 
 # Run the back-end tests
-bin/run_tests.sh no-report || exit "$?"
+bin/run_tests.sh all no-report || exit "$?"
 
 # Check dependencies for known vulnerabilities
 pip-audit || exit "$?"

backend/bin/run_tests.sh

@@ -1,37 +1,55 @@
 # To disable the report, provide basically anything as a first argument
-REPORT="$1"
+set -e
+LANGUAGE="${1:-all}"
+REPORT="$2"
 
-# Run CDK tests, tracking code coverage in a new data file
-(
-  cd compact-connect
-  pytest --cov=. --cov-config=.coveragerc tests
-) || exit "$?"
-for dir in \
-  compact-connect/lambdas/common-python \
-  compact-connect/lambdas/license-data \
-  compact-connect/lambdas/provider-data-v1 \
-  compact-connect/lambdas/staff-user-pre-token \
-  compact-connect/lambdas/staff-users \
-  compact-connect/lambdas/delete-objects \
-  compact-connect/lambdas/custom-resources \
-  compact-connect/lambdas/purchases \
-  multi-account
-  do
+# Set this to 1 ahead of running tests, so this script will fail if neither node or python tests ran
+EXIT=1
+if [[ "$LANGUAGE" == 'nodejs' || "$LANGUAGE" == 'all' ]]; then
+  # Run NodeJS tests first
   (
-    cd "$dir"
-    echo "Running tests in $dir"
-    # update the PYTHONPATH to include the shared code if not common-python
-    [ "$dir" = "compact-connect/lambdas/common-python" ] || export PYTHONPATH=../common-python
-    # Run lambda tests, appending data to the same data file
-    pytest --cov=. --cov-config=.coveragerc --cov-append tests
+    cd compact-connect/lambdas/nodejs
+    yarn test || exit "$?"
+    if [ -z "$REPORT" ]; then
+      open 'coverage/lcov-report/index.html'
+    fi
   ) || exit "$?"
-done
+  # If this didn't exit already, we'll set our exit status to success for now
+  EXIT=0
+fi
+
+if [[ "$LANGUAGE" == 'python' || "$LANGUAGE" == 'all' ]]; then
+  # Run CDK tests, tracking code coverage in a new, combined, data file
+  (
+    cd compact-connect
+    pytest --cov=. --cov-config=.coveragerc tests
+  ) || exit "$?"
+  for dir in \
+    compact-connect/lambdas/python/common \
+    compact-connect/lambdas/python/custom-resources \
+    compact-connect/lambdas/python/data-events \
+    compact-connect/lambdas/python/provider-data-v1 \
+    compact-connect/lambdas/python/purchases \
+    compact-connect/lambdas/python/staff-user-pre-token \
+    compact-connect/lambdas/python/staff-users \
+    multi-account
+    do
+    (
+      cd "$dir"
+      echo "Running tests in $dir"
+      # update the PYTHONPATH to include the shared code if not python/common
+      [ "$dir" = "compact-connect/lambdas/python/common" ] || export PYTHONPATH=../common
+      # Run lambda tests, appending data to the same data file
+      pytest --cov=. --cov-config=.coveragerc --cov-append tests
+    ) || exit "$?"
+  done
 
-# Run a coverage report with the combined data
-coverage html --fail-under=90
-EXIT=$?
-if [ -z "$REPORT" ]; then
-  open 'coverage/index.html'
+  # Run a coverage report with the combined data
+  coverage html --fail-under=90
+  EXIT=$?
+  if [ -z "$REPORT" ]; then
+    open 'coverage/index.html'
+  fi
 fi
 
 exit "$EXIT"

backend/bin/sync_deps.sh

@@ -1,21 +1,24 @@
+(
+  cd compact-connect/lambdas/nodejs
+  yarn install
+)
+
 pip-sync \
-  multi-account/requirements.txt \
   multi-account/requirements-dev.txt \
-  compact-connect/requirements.txt \
+  multi-account/requirements.txt \
   compact-connect/requirements-dev.txt \
-  compact-connect/lambdas/license-data/requirements.txt \
-  compact-connect/lambdas/license-data/requirements-dev.txt \
-  compact-connect/lambdas/provider-data-v1/requirements.txt \
-  compact-connect/lambdas/provider-data-v1/requirements-dev.txt \
-  compact-connect/lambdas/staff-user-pre-token/requirements.txt \
-  compact-connect/lambdas/staff-user-pre-token/requirements-dev.txt \
-  compact-connect/lambdas/staff-users/requirements.txt \
-  compact-connect/lambdas/staff-users/requirements-dev.txt \
-  compact-connect/lambdas/delete-objects/requirements.txt \
-  compact-connect/lambdas/delete-objects/requirements-dev.txt \
-  compact-connect/lambdas/custom-resources/requirements.txt \
-  compact-connect/lambdas/custom-resources/requirements-dev.txt \
-  compact-connect/lambdas/purchases/requirements.txt \
-  compact-connect/lambdas/purchases/requirements-dev.txt \
-  compact-connect/lambdas/common-python/requirements.txt \
-  compact-connect/lambdas/common-python/requirements-dev.txt
+  compact-connect/requirements.txt \
+  compact-connect/lambdas/python/common/requirements-dev.txt \
+  compact-connect/lambdas/python/common/requirements.txt \
+  compact-connect/lambdas/python/custom-resources/requirements-dev.txt \
+  compact-connect/lambdas/python/custom-resources/requirements.txt \
+  compact-connect/lambdas/python/data-events/requirements-dev.txt \
+  compact-connect/lambdas/python/data-events/requirements.txt \
+  compact-connect/lambdas/python/provider-data-v1/requirements-dev.txt \
+  compact-connect/lambdas/python/provider-data-v1/requirements.txt \
+  compact-connect/lambdas/python/purchases/requirements-dev.txt \
+  compact-connect/lambdas/python/purchases/requirements.txt \
+  compact-connect/lambdas/python/staff-user-pre-token/requirements-dev.txt \
+  compact-connect/lambdas/python/staff-user-pre-token/requirements.txt \
+  compact-connect/lambdas/python/staff-users/requirements-dev.txt \
+  compact-connect/lambdas/python/staff-users/requirements.txt

backend/compact-connect/README.md

@@ -49,6 +49,13 @@ Python requirements are pinned in [`requirements.txt`](requirements.txt). Instal
 $ pip install -r requirements.txt
 ```
 
+Node.js requirements (for some selected Lambda runtimes) are defined in [`package.json`](./lambdas/nodejs). Install them using `yarn`.
+
+```shell
+$ cd lambdas/nodejs
+$ yarn install
+```
+
 At this point you can now synthesize the CloudFormation template(s) for this code.
 
 ```
@@ -89,7 +96,17 @@ tests are defined under the [tests](./tests) directory. Runtime code tests shoul
 lambda folders. Code that is common across all lambdas should be tested in the `common-python` directory, to reduce
 duplication and ensure consistency across the app.
 
-To execute the tests, simply run `bin/run_tests.sh` from the `backend` directory.
+To execute the tests, simply run `bin/sync_deps.sh` then `bin/run_tests.sh` from the `backend` directory.
+
+## Documentation
+[Back to top](#compact-connect---backend-developer-documentation)
+
+Keeping documentation current is an important part of feature development in this project. If the feature involves a non-trivial amount of architecture or other technical design, be sure that the design and considerations are captured in the [design documentation](./docs/design). If any updates are made to the API, be sure to follow these steps to keep the documentation current:
+1) Export a fresh api specification (OAS 3.0) is exported from API Gateway and used to update [the Open API Specification JSON file](./docs/api-specification/latest-oas30.json).
+2) Run `bin/trim_oas30.py` to organize and trim the API to include only supported API endpoints (and update the script itself, if needed).
+3) If you exported the api specification from somewhere other than the CSG Test environment, be sure to set the `servers[0].url` entry back to the correct base URL for the CSG Test environment.
+4) Update the change summary at the bottom of the [Technical User Docs](./docs/README.md).
+5) Update the [Postman Collection and Environment](./docs/postman) as appropriate.
 
 ## Deployment
 [Back to top](#compact-connect---backend-developer-documentation)
@@ -101,7 +118,7 @@ its environment:
    your app. See [About Route53 Hosted Zones](#about-route53-hosted-zones) for more. Note: Without this step, you will
    not be able to log in to the UI hosted in CloudFront. The Oauth2 authentication process requires a predictable
    callback url to be pre-configured, which the domain name provides. You can still run a local UI against this app,
-   so long as you leave the `allow_local_ui` context value set to `true` in your environment's context.
+   so long as you leave the `allow_local_ui` context value set to `true` in your environment's context. 
 2) *Optional if testing SES email notifications with custom domain:* By default, AWS does not allow sending emails to unverified email
    addresses. If you need to test SES email notifications and do not want to request AWS to remove your account from
    the SES sandbox, you will need to set up a verified SES email identity for each address you want to send emails to.
@@ -111,10 +128,11 @@ its environment:
    See [Default User Pool Email Settings](https://docs.aws.amazon.com/cognito/latest/developerguide/user-pool-email.html#user-pool-email-default)
 3) Copy [cdk.context.sandbox-example.json](./cdk.context.sandbox-example.json) to `cdk.context.json`.
 4) At the top level of the JSON structure update the `"environment_name"` field to your own name.
-5) Update the environment entry under `ssm_context.environments` to your own name and your own AWS sandbox account id,
-   and domain name, if you set one up. If you opted not to create a HostedZone, just remove the `domain_name` field.
+5) Update the environment entry under `ssm_context.environments` to your own name and your own AWS sandbox account id (which you can find by following these [these instructions](https://docs.aws.amazon.com/accounts/latest/reference/manage-acct-identifiers.html#FindAccountId)),
+   and domain name, if you set one up. **If you opted not to create a HostedZone, remove the `domain_name` field.**
    The key under `environments` must match the value you put under `environment_name`.
-6) Configure your aws cli to authenticate against your own account.
+6) Configure your aws cli to authenticate against your own account. There are several ways to do this based on the
+   type of authentication you use to login to your account. See the [AWS CLI Configuration Guide](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-quickstart.html).
 7) Run `cdk bootstrap` to add some base CDK support infrastructure to your AWS account.
 8) Run `cdk deploy 'Sandbox/*'` to get the initial stack resources deployed.
 

backend/compact-connect/bin/create_staff_user.py

@@ -15,8 +15,10 @@
 import boto3
 from botocore.exceptions import ClientError
 
-provider_data_path = os.path.join('lambdas', 'staff-users')
+provider_data_path = os.path.join('lambdas', 'python', 'staff-users')
+common_lib_path = os.path.join('lambdas', 'python', 'common')
 sys.path.append(provider_data_path)
+sys.path.append(common_lib_path)
 
 with open('cdk.json') as context_file:
     _context = json.load(context_file)['context']
@@ -27,7 +29,7 @@
 os.environ['JURISDICTIONS'] = json.dumps(JURISDICTIONS)
 
 # We have to import this after we've mucked with our path and environment
-from data_model.schema.user import UserRecordSchema  # noqa: E402
+from cc_common.data_model.schema.user import UserRecordSchema  # noqa: E402
 
 USER_POOL_ID = os.environ['USER_POOL_ID']
 USER_TABLE_NAME = os.environ['USER_TABLE_NAME']

backend/compact-connect/bin/generate_mock_data.py

@@ -14,7 +14,7 @@
 
 # We have to do some set-up before we can import everything we need
 # Add the provider data lambda runtime to our pythonpath
-provider_data_path = os.path.join('lambdas', 'provider-data-v1')
+provider_data_path = os.path.join('lambdas', 'python', 'common')
 sys.path.append(provider_data_path)
 
 with open('cdk.json') as context_file:
@@ -26,7 +26,7 @@
 os.environ['COMPACTS'] = json.dumps(COMPACTS)
 os.environ['JURISDICTIONS'] = json.dumps(JURISDICTIONS)
 
-from data_model.schema.license import LicensePostSchema  # noqa: E402
+from cc_common.data_model.schema.license import LicensePostSchema  # noqa: E402
 
 # We'll grab three different localizations to provide a variety of names/characters
 name_faker = Faker(['en_US', 'ja_JP', 'es_MX'])

backend/compact-connect/cdk.context.sandbox-example.json

@@ -16,6 +16,11 @@
           "email": [
             "[email protected]"
           ]
+        },
+        "jurisdiction_configuration_overrides": {
+          "jurisdictionOperationsTeamEmails": [
+            "[email protected]"
+          ]
         }
       }
     }