Fix DynamoDB streams

Resolve deferred outputs before uploading the template

Modified logging to show resolutions

Use non-locking deferred outputs for event references.

Author: Frank Schmid

.eslintrc

@@ -0,0 +1,47 @@
+---
+env:
+  es6: true
+  node: true
+  mocha: true
+
+extends:
+  - eslint:recommended
+  - plugin:lodash/recommended
+  - plugin:import/errors
+  - plugin:import/warnings
+
+plugins:
+  - promise
+  - lodash
+  - import
+
+rules:
+  indent:
+    - error
+    - tab
+    -
+      MemberExpression: off
+  linebreak-style:
+    - error
+    - unix
+  semi:
+    - error
+    - always
+  promise/always-return: error
+  promise/no-return-wrap: error
+  promise/param-names: error
+  promise/catch-or-return: error
+  promise/no-native: off
+  promise/no-nesting: off
+  promise/no-promise-in-callback: warn
+  promise/no-callback-in-promise: warn
+  promise/avoid-new: warn
+  import/no-named-as-default: off
+  lodash/import-scope: off
+  lodash/preferred-alias: off
+  lodash/prop-shorthand: off
+  lodash/prefer-lodash-method:
+    - error
+    -
+      ignoreObjects:
+        - BbPromise

.eslintrc.json

@@ -5,20 +5,21 @@
  */
 
 const BbPromise = require('bluebird')
-		, _ = require('lodash')
-		, Path = require('path')
-		, validate = require('./lib/validate')
-		, configureAliasStack = require('./lib/configureAliasStack')
-		, createAliasStack = require('./lib/createAliasStack')
-		, updateAliasStack = require('./lib/updateAliasStack')
-		, aliasRestructureStack = require('./lib/aliasRestructureStack')
-		, stackInformation = require('./lib/stackInformation')
-		, listAliases = require('./lib/listAliases')
-		, removeAlias = require('./lib/removeAlias')
-		, logs = require('./lib/logs')
-		, collectUserResources = require('./lib/collectUserResources')
-		, uploadAliasArtifacts = require('./lib/uploadAliasArtifacts')
-		, updateFunctionAlias = require('./lib/updateFunctionAlias');
+	, _ = require('lodash')
+	, Path = require('path')
+	, validate = require('./lib/validate')
+	, configureAliasStack = require('./lib/configureAliasStack')
+	, createAliasStack = require('./lib/createAliasStack')
+	, updateAliasStack = require('./lib/updateAliasStack')
+	, aliasRestructureStack = require('./lib/aliasRestructureStack')
+	, stackInformation = require('./lib/stackInformation')
+	, listAliases = require('./lib/listAliases')
+	, removeAlias = require('./lib/removeAlias')
+	, logs = require('./lib/logs')
+	, collectUserResources = require('./lib/collectUserResources')
+	, uploadAliasArtifacts = require('./lib/uploadAliasArtifacts')
+	, updateFunctionAlias = require('./lib/updateFunctionAlias')
+	, deferredOutputs = require('./lib/deferredOutputs');
 
 class AwsAlias {
 
@@ -40,18 +41,18 @@ class AwsAlias {
 		 * Load stack helpers from Serverless installation.
 		 */
 		const monitorStack = require(
-		Path.join(this._serverless.config.serverlessPath,
-			'plugins',
-			'aws',
-			'lib',
-			'monitorStack')
+			Path.join(this._serverless.config.serverlessPath,
+				'plugins',
+				'aws',
+				'lib',
+				'monitorStack')
 		);
 		const setBucketName = require(
-		Path.join(this._serverless.config.serverlessPath,
-			'plugins',
-			'aws',
-			'lib',
-			'setBucketName')
+			Path.join(this._serverless.config.serverlessPath,
+				'plugins',
+				'aws',
+				'lib',
+				'setBucketName')
 		);
 
 		_.assign(
@@ -69,7 +70,8 @@ class AwsAlias {
 			uploadAliasArtifacts,
 			updateFunctionAlias,
 			setBucketName,
-			monitorStack
+			monitorStack,
+			deferredOutputs
 		);
 
 		this._commands = {
@@ -116,10 +118,11 @@ class AwsAlias {
 				.then(this.createAliasStack),
 
 			'after:aws:deploy:deploy:uploadArtifacts': () => BbPromise.bind(this)
-				.then(this.setBucketName)
-				.then(this.uploadAliasArtifacts),
+				.then(() => BbPromise.resolve()),
 
 			'after:aws:deploy:deploy:updateStack': () => BbPromise.bind(this)
+				.then(this.setBucketName)
+				.then(this.uploadAliasArtifacts)
 				.then(this.updateAliasStack),
 
 			'before:deploy:function:initialize': () => BbPromise.bind(this)

index.js

@@ -0,0 +1,59 @@
+'use strict';
+
+/**
+ * Handle deferred output resolution.
+ * Some references to outputs of the base stack cannot be done
+ * by Fn::ImportValue because they will change from deployment to deployment.
+ * So we resolve them after the base stack has been deployed and set their
+ * values accordingly.
+ */
+
+const _ = require('lodash');
+const BbPromise = require('bluebird');
+
+const deferredOutputs = {};
+
+module.exports = {
+
+	/**
+	 * Register a deferred output
+	 * @param {string} sourceOutput 
+	 * @param {Object} targetObject 
+	 * @param {string} targetPropertyName 
+	 */
+	addDeferredOutput(sourceOutput, targetObject, targetPropertyName) {
+		this.options.verbose && this.serverless.cli.log(`Register deferred output ${sourceOutput} -> ${targetPropertyName}`);
+
+		deferredOutputs[sourceOutput] = deferredOutputs[sourceOutput] || [];
+		deferredOutputs[sourceOutput].push({
+			target: targetObject,
+			property: targetPropertyName
+		});
+	},
+
+	resolveDeferredOutputs() {
+		this.options.verbose && this.serverless.cli.log('Resolving deferred outputs');
+
+		if (_.isEmpty(deferredOutputs)) {
+			return BbPromise.resolve();
+		}
+
+		return this.aliasGetExports()
+		.then(cfExports => {
+			_.forOwn(deferredOutputs, (references, output) => {
+				if (_.has(cfExports, output)) {
+					const value = cfExports[output];
+					this.options.verbose && this.serverless.cli.log(`  ${output} -> ${value}`);
+					_.forEach(references, reference => {
+						_.set(reference.target, reference.property, value);
+					});
+				}
+				else {
+					this.serverless.cli.log(`ERROR: Output ${output} not found.`);
+				}
+			});
+			return null;
+		});
+	}
+
+};

lib/deferredOutputs.js

@@ -141,6 +141,30 @@ module.exports = {
 		});
 	},
 
+	aliasGetExports() {
+		const fetchExports = (result, token) => {
+			const params = {};
+			if (token) {
+				params.NextToken = token;
+			}
+			return this._provider.request('CloudFormation', 'listExports', params)
+			.then(cfData => {
+				const newResult = _.reduce(cfData.Exports, (__, cfExport) => {
+					__[cfExport.Name] = cfExport.Value;
+					return __;
+				}, result);
+
+				if (cfData.NextToken) {
+					return fetchExports(newResult, cfData.NextToken);
+				}
+
+				return newResult;
+			});
+		};
+
+		return fetchExports({});
+	},
+
 	aliasStackLoadCurrentCFStackAndDependencies() {
 		return BbPromise.join(
 			BbPromise.bind(this).then(this.aliasStackLoadCurrentTemplate),

lib/stackInformation.js

@@ -15,6 +15,8 @@ module.exports = function(currentTemplate, aliasStackTemplates, currentAliasStac
 
 	const subscriptions = _.assign({}, _.pickBy(_.get(stageStack, 'Resources', {}), [ 'Type', 'AWS::Lambda::EventSourceMapping' ]));
 
+	this.options.verbose && this._serverless.cli.log('Processing event source subscriptions');
+
 	_.forOwn(subscriptions, (subscription, name) => {
 		// Reference alias as FunctionName
 		const functionNameRef = utils.findAllReferences(_.get(subscription, 'Properties.FunctionName'));
@@ -44,14 +46,17 @@ module.exports = function(currentTemplate, aliasStackTemplates, currentAliasStac
 				Name: `${stackName}-${resourceRefName}`
 			}
 		};
-		// Add the outpur to the referenced alias outputs
+		// Add the output to the referenced alias outputs
 		const aliasOutputs = JSON.parse(aliasStack.Outputs.AliasOutputs.Value);
 		aliasOutputs.push(resourceRefName);
 		aliasStack.Outputs.AliasOutputs.Value = JSON.stringify(aliasOutputs);
 		// Replace the reference with the cross stack reference
-		subscription.Properties.EventSourceArn = {
-			'Fn::ImportValue': `${stackName}-${resourceRefName}`
-		};
+		subscription.Properties.EventSourceArn = {};
+
+		// Event source ARNs can be volatile - e.g. DynamoDB and must not be referenced
+		// with Fn::ImportValue which does not allow for changes. So we have to register
+		// them for delayed lookup until the base stack has been updated.
+		this.addDeferredOutput(`${stackName}-${resourceRefName}`, subscription.Properties, 'EventSourceArn');
 
 		// Remove mapping from stage stack
 		delete stageStack.Resources[name];

lib/stackops/events.js

@@ -36,6 +36,7 @@ module.exports = {
 		}
 
 		return BbPromise.bind(this)
+			.then(this.resolveDeferredOutputs)
 			.then(this.uploadAliasCloudFormationFile);
 	},