pipeline.yaml

AWSTemplateFormatVersion: "2010-09-09"
Description: "CloudFormation Template to deploy an Event Driven Application with AWS Lambda and Amazon Redshift Data API"
Parameters:
  RedshiftClusterIdentifier:
    Description: Cluster Identifier for your redshift cluster
    Type: String
    Default: 'redshift-cluster-1'
  DbUsername:
    Description: Redshift database user name which has access to run SQL Script.
    Type: String
    AllowedPattern: "([a-z])([a-z]|[0-9])*"
    Default: 'awsuser'
  DatabaseName:
    Description: Name of the Redshift database where SQL Script would be run.
    Type: String
    Default: 'dev'
  RedshiftIAMRoleName:
    Description: AWS IAM Role Name associated with the Redshift cluster
    Type: String
    Default: 'myRedshiftRole'
  ExecutionSchedule:
    Type: String
    Description: Cron expression to schedule extract-load-transform (ELT) process through EventBridge rule
    Default: 'cron(0 11 ? * MON-FRI *)'
  Statements:
    Type: List #TODO: figure out what this needs to be
    Description: SQL statements to be run as part of the extract-load-transform (ELT) process
    Default: 'call run_elt_process();'
Metadata:
  AWS::CloudFormation::Interface:
    ParameterGroups:
      -
        Label:
          default: "Input Parameters"
        Parameters:
        - RedshiftClusterIdentifier
        - DbUsername
        - DatabaseName
        - RedshiftIAMRoleName
        - ExecutionSchedule
        - SqlText
Resources:
  LambdaRedshiftDataApiETLRole:
    Type: AWS::IAM::Role
    Properties:
      Description : IAM Role for lambda to access Redshift and SNS topic
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole
      AssumeRolePolicyDocument:
          Version: 2012-10-17
          Statement:
            -
              Effect: Allow
              Principal:
                Service:
                  - lambda.amazonaws.com
              Action:
                - sts:AssumeRole
      Path: /
      Policies:
          -
            PolicyName: RedshiftAccessPolicy
            PolicyDocument :
              Version: 2012-10-17
              Statement:
                -
                  Effect: Allow
                  Action: redshift:GetClusterCredentials
                  Resource:
                    - !Sub arn:aws:redshift:${AWS::Region}:${AWS::AccountId}:cluster:${RedshiftClusterIdentifier}
                    - !Sub arn:aws:redshift:${AWS::Region}:${AWS::AccountId}:dbname:${RedshiftClusterIdentifier}/${DatabaseName}
                    - !Sub arn:aws:redshift:${AWS::Region}:${AWS::AccountId}:dbuser:${RedshiftClusterIdentifier}/${DbUsername}
                -
                  Effect: "Allow"
                  Action:
                  - redshift-data:ExecuteStatement
                  - redshift-data:ListStatements
                  - redshift-data:GetStatementResult
                  - redshift-data:DescribeStatement
                  Resource: "*"
  EventBridgeScheduledEventRule:
     Type: "AWS::Events::Rule"
     Properties:
       Description: Scheduled Event Rule to be triggered periodically based on cron expression.
       ScheduleExpression: !Ref ExecutionSchedule
       State: "ENABLED"
       Targets:
          -
            Arn:
              Fn::GetAtt:
                - "LambdaRedshiftDataApiETL"
                - "Arn"
            Id: ScheduledEventRedshiftELT
            #TODO: Adjust this based on the actual structure of an incoming event
            Input: !Sub "{\"Input\":{\"redshift_cluster_id\":\"${RedshiftClusterIdentifier}\",\"redshift_database\":\"${DatabaseName}\",\"redshift_user\":\"${DbUsername}\",\"statements\":\"${SqlText}\",\"secret_arn\":\"${RedshiftArn}\"}}"
  PermissionForScheduledEventToInvokeLambda:
    Type: AWS::Lambda::Permission
    Properties:
      FunctionName:
        Ref: "LambdaRedshiftDataApiETL"
      Action: "lambda:InvokeFunction"
      Principal: "events.amazonaws.com"
      SourceArn:
        Fn::GetAtt:
          - "EventBridgeScheduledEventRule"
          - "Arn"
  LambdaRedshiftDataApiETL:
    DependsOn:
      - LambdaRedshiftDataApiETLRole
    Type: AWS::Lambda::Function
    Properties:
      Description: Lambda function to asynchronously execute defined commands with Amazon Redshift Data API
      Handler: index.handler
      Runtime: go1.13.8
      Role: !GetAtt 'LambdaRedshiftDataApiETLRole.Arn'
      Timeout: 30
      Code: # TODO: Add finished go handler to zip file
        ZipFile: |
            # handler