fix: change BatchReplayer props to solve JSII conversion issue in Python with IS3Sink (#654)

* fix: fix JSII conversion issue in Python with IS3Sink

---------

Co-authored-by: Automation <[email protected]>

Author: vgkowski

core/API.md

@@ -3,7 +3,7 @@ import { ITable } from 'aws-cdk-lib/aws-dynamodb';
 import { Bucket } from 'aws-cdk-lib/aws-s3';
 
 
-export interface IS3Sink {
+export interface S3Sink {
   /**
    * The S3 Bucket sink where the BatchReplayer writes data.
    * :warning: **If the Bucket is encrypted with KMS, the Key must be managed by this stack.
@@ -13,12 +13,12 @@ export interface IS3Sink {
    * The S3 object key sink where the BatchReplayer writes data.
    * @default - No object key is used and the BatchReplayer writes the dataset in s3://<BUCKET_NAME>/<TABLE_NAME>
    */
-  sinkObjectKey?: string;
+  readonly sinkObjectKey?: string;
   /**
    * The maximum file size in Bytes written by the BatchReplayer
    * @default - The BatchReplayer writes 100MB files maximum
    */
-  outputFileMaxSizeInBytes?: number;
+  readonly outputFileMaxSizeInBytes?: number;
 }
 
 export interface DbSink {
@@ -54,7 +54,7 @@ export interface DynamoDbSink {
  * @param dataBucketName
  * @param dataObjectKey
  */
-export function prepareS3Target(S3Props: IS3Sink, dataBucketName: string, dataObjectKey: string)
+export function prepareS3Target(S3Props: S3Sink, dataBucketName: string, dataObjectKey: string)
   : {policy:PolicyStatement; taskInputParams:Object} {
   // Add policy to allow access to bucket
   const policy = new PolicyStatement({

core/src/data-generator/batch-replayer-helpers.ts

@@ -20,7 +20,7 @@ import {
   prepareRdsTarget,
   prepareRedshiftTarget,
   prepareS3Target,
-  IS3Sink,
+  S3Sink,
   DbSink,
   DynamoDbSink,
 } from './batch-replayer-helpers';
@@ -42,7 +42,7 @@ export interface BatchReplayerProps {
   /**
    * Parameters to write to S3 target
    */
-  readonly s3Props?: IS3Sink;
+  readonly s3Props?: S3Sink;
   /**
    * Parameters to write to DynamoDB target
    */
@@ -115,7 +115,7 @@ export interface BatchReplayerProps {
  *
  * const myBucket = new Bucket(stack, "MyBucket")
  *
- * let myProps: IS3Sink = {
+ * let myProps: S3Sink = {
  *  sinkBucket: myBucket,
  *  sinkObjectKey: 'some-prefix',
  *  outputFileMaxSizeInBytes: 10000000,
@@ -145,7 +145,7 @@ export class BatchReplayer extends Construct {
   /**
    * Parameters to write to S3 target
    */
-  public readonly s3Props?: IS3Sink;
+  public readonly s3Props?: S3Sink;
   /**
    * Parameters to write to DynamoDB target
    */
@@ -191,13 +191,6 @@ export class BatchReplayer extends Construct {
     this.frequency = props.frequency?.toSeconds() || 60;
     this.additionalStepFunctionTasks = props.additionalStepFunctionTasks;
 
-    // Properties for S3 target
-    if (props.s3Props) {
-      this.s3Props = props.s3Props;
-      if (!this.s3Props.outputFileMaxSizeInBytes) {
-        this.s3Props.outputFileMaxSizeInBytes = 100 * 1024 * 1024; //Default to 100 MB
-      }
-    }
     const manifestBucketName = this.dataset.manifestLocation.bucketName;
     const manifestObjectKey = this.dataset.manifestLocation.objectKey;
     const dataBucketName = this.dataset.location.bucketName;
@@ -274,12 +267,18 @@ export class BatchReplayer extends Construct {
     };
 
     // S3 target is selected
-    if (this.s3Props) {
+    if (props.s3Props) {
       // Used to force S3 bucket auto cleaning after deletion of this
-      this.node.addDependency(this.s3Props.sinkBucket);
+      this.node.addDependency(props.s3Props.sinkBucket);
+      var modS3Props = { ...props.s3Props };
+      if (!props.s3Props.outputFileMaxSizeInBytes) {
+        modS3Props.outputFileMaxSizeInBytes = 100 * 1024 * 1024; //Default to 100 MB
+      }
 
-      this.s3Props.sinkObjectKey = this.s3Props.sinkObjectKey ?
-        `${this.s3Props.sinkObjectKey}/${this.dataset.tableName}` : this.dataset.tableName;
+      modS3Props.sinkObjectKey = props.s3Props.sinkObjectKey ?
+        `${props.s3Props.sinkObjectKey}/${this.dataset.tableName}` : this.dataset.tableName;
+        
+      this.s3Props = modS3Props;
 
       const { policy, taskInputParams } = prepareS3Target(this.s3Props, dataBucketName, dataObjectKey);
       writeInBatchFnPolicy.push(policy);

core/src/data-generator/batch-replayer.ts

@@ -1,4 +1,4 @@
 export { BatchReplayerProps, BatchReplayer } from './batch-replayer';
-export { IS3Sink, DbSink, DynamoDbSink } from './batch-replayer-helpers';
+export { S3Sink, DbSink, DynamoDbSink } from './batch-replayer-helpers';
 export { PreparedDataset, PreparedDatasetProps } from './prepared-dataset';
 export { CustomDataset, CustomDatasetProps, CustomDatasetInputFormat } from './custom-dataset';

core/src/data-generator/index.ts

@@ -3,7 +3,7 @@
 
 export { DataLakeStorageProps, DataLakeStorage } from './data-lake-storage';
 export { SynchronousCrawlerProps, SynchronousCrawler } from './synchronous-crawler';
-export { BatchReplayerProps, BatchReplayer, PreparedDatasetProps, PreparedDataset, CustomDataset, CustomDatasetProps, CustomDatasetInputFormat, IS3Sink, DbSink, DynamoDbSink } from './data-generator';
+export { BatchReplayerProps, BatchReplayer, PreparedDatasetProps, PreparedDataset, CustomDataset, CustomDatasetProps, CustomDatasetInputFormat, S3Sink, DbSink, DynamoDbSink } from './data-generator';
 export { SynchronousAthenaQueryProps, SynchronousAthenaQuery } from './synchronous-athena-query';
 export { AraBucket, AraBucketProps } from './ara-bucket';
 export { Ec2SsmRole } from './ec2-ssm-role';

core/src/index.ts

@@ -14,7 +14,7 @@ import { Cluster } from '@aws-cdk/aws-redshift-alpha';
 import { deployStack, destroyStack } from './utils';
 
 import { BatchReplayer } from '../../src/data-generator/batch-replayer';
-import { IS3Sink, DynamoDbSink, DbSink } from '../../src/data-generator/batch-replayer-helpers';
+import { S3Sink, DynamoDbSink, DbSink } from '../../src/data-generator/batch-replayer-helpers';
 import { PreparedDataset } from '../../src/data-generator/prepared-dataset';
 import { PreBundledFunction } from '../../src/common/pre-bundled-function';
 import { Runtime } from 'aws-cdk-lib/aws-lambda';
@@ -31,7 +31,7 @@ const sinkBucket = new Bucket(stack, 'SinkBucket', {
   removalPolicy: RemovalPolicy.DESTROY,
   autoDeleteObjects: true,
 });
-let s3Props: IS3Sink = { sinkBucket: sinkBucket };
+let s3Props: S3Sink = { sinkBucket: sinkBucket };
 
 const vpc = new aws_ec2.Vpc(stack, 'Vpc');
 

core/test/e2e/batch-replayer.test.ts

@@ -13,14 +13,14 @@ import { App, Aspects, Duration, Stack } from 'aws-cdk-lib';
 // eslint-disable-next-line import/no-extraneous-dependencies
 import { AwsSolutionsChecks, NagSuppressions } from 'cdk-nag';
 import { BatchReplayer, PreparedDataset } from '../../../src';
-import { IS3Sink } from '../../../lib';
+import { S3Sink } from '../../../lib';
 
 const mockApp = new App();
 
 const batchReplayerStack = new Stack(mockApp, 'BatchReplayer');
 const sinkBucket = new Bucket(batchReplayerStack, 'SinkBucket');
 // Instantiate a DataGenerator
-const s3Props: IS3Sink = { sinkBucket: sinkBucket, sinkObjectKey: 'test' };
+const s3Props: S3Sink = { sinkBucket: sinkBucket, sinkObjectKey: 'test' };
 const batchReplayer = new BatchReplayer(batchReplayerStack, 'TestBatchReplayer', {
   dataset: PreparedDataset.RETAIL_1_GB_WEB_SALE,
   frequency: Duration.seconds(120),

core/test/unit/cdk-nag/nag-batch-replayer.test.ts

@@ -18,14 +18,14 @@ import {
 
 import { Template } from 'aws-cdk-lib/assertions';
 import { Bucket } from 'aws-cdk-lib/aws-s3';
-import { BatchReplayer, DbSink, DynamoDbSink, IS3Sink, PreparedDataset } from '../../../src';
+import { BatchReplayer, DbSink, DynamoDbSink, S3Sink, PreparedDataset } from '../../../src';
 import { IVpc, SecurityGroup } from 'aws-cdk-lib/aws-ec2';
 import { PreBundledFunction } from '../../../src/common/pre-bundled-function';
 import { Runtime } from 'aws-cdk-lib/aws-lambda';
 import { RetentionDays } from 'aws-cdk-lib/aws-logs';
 
 let testStack: Stack;
-let s3Props: IS3Sink;
+let s3Props: S3Sink;
 let batchReplayer: BatchReplayer;
 let template: Template;
 let ddbProps: DynamoDbSink;
@@ -105,7 +105,7 @@ test("BatchReplayer should use default frequency", () => {
 });
 
 test("BatchReplayer should use given max output file size", () => {
-  const s3MaxOutputFileSizeSet: IS3Sink = {
+  const s3MaxOutputFileSizeSet: S3Sink = {
     sinkBucket: new Bucket(testStack, 'filesizeBucket'),
     outputFileMaxSizeInBytes: 20480,
   };
@@ -119,7 +119,7 @@ test("BatchReplayer should use given max output file size", () => {
 });
 
 test("BatchReplayer should use default max output file size 100MB", () => {
-  const s3MaxOutputFileSizeDefault: IS3Sink = {
+  const s3MaxOutputFileSizeDefault: S3Sink = {
     sinkBucket: new Bucket(testStack, 'noFilesizeBucket'),
   };
   const batchReplayerWithNoFilesizeProp = new BatchReplayer(testStack, "TestBatchReplayerWithNoFreqProp", {