working aggregator

Author: jamiebrown201

backend/aggregator.js

@@ -1,21 +1,46 @@
 "use strict";
+const request = require("request-promise");
+const Bottleneck = require("bottleneck");
+
+const limiter = new Bottleneck({
+  maxConcurrent: 1,
+  minTime: 1000
+});
+
 const fetchArticles = require("./lib/fetchArticles");
 
+const SENTIMENT_URL = process.env.SENTIMENT_URL;
+
 module.exports.main = async event => {
   try {
     const searchTerm = event.queryStringParameters.searchTerm;
 
     const rawSapiResult = await fetchArticles.search(searchTerm);
     const uuids = rawSapiResult.sapiObj.results[0].results.map(obj => obj.id);
+    console.log(SENTIMENT_URL);
+
+    const result = await Promise.all(
+      uuids.map(uuid =>
+        limiter.schedule(async () => {
+          const result = await request(createParams(uuid));
+          if (result !== "not article") {
+            return JSON.parse(result);
+          }
+          return result;
+        })
+      )
+    );
     return {
       statusCode: 200,
-      body: JSON.stringify(uuids),
+      body: JSON.stringify(result),
       headers: {
         "Access-Control-Allow-Origin": "*"
       }
     };
   } catch (err) {
-    console.error(err);
+    console.log(err);
+    console.log("GETTING IN ERROR");
+    console.error(JSON.stringify(err));
     return {
       statusCode: 401,
       body: "Something went wrong",
@@ -25,3 +50,11 @@ module.exports.main = async event => {
     };
   }
 };
+
+function createParams(uuid) {
+  return {
+    method: "GET",
+    url: SENTIMENT_URL + uuid + "?provider=meaningCloud,aws",
+    headers: { "content-type": "application/json" }
+  };
+}

backend/handler.js

@@ -39,10 +39,25 @@ module.exports.main = async event => {
     }
     const uuid = event.pathParameters.uuid;
     const result = await fetchContent.getArticle(uuid);
+    if (!result.bodyXML || !result.title || !result.standfirst) {
+      console.log("getting in catch");
+      return {
+        statusCode: 200,
+        body: "not article",
+        headers: {
+          "Access-Control-Allow-Origin": "*"
+        }
+      };
+    }
+    console.log(JSON.stringify(result));
     const articleContent = extractText(result.bodyXML);
     const title = result.title;
     const standfirst = result.standfirst;
 
+    console.log("articleContent", articleContent);
+    console.log("title", title);
+    console.log("standfirst", standfirst);
+
     const sentimentResult = await Promise.all(
       finalProviderArray.map(async provider => {
         const result = await provider.functionality.getSentiment({
@@ -68,10 +83,10 @@ module.exports.main = async event => {
       }
     };
   } catch (err) {
-    console.error(err);
+    console.error(JSON.stringify(err));
     return {
       statusCode: 401,
-      body: "Something went wrong",
+      body: "Something went wrong in handler",
       headers: {
         "Access-Control-Allow-Origin": "*"
       }

backend/lib/fetchArticles.js

@@ -11,7 +11,7 @@ const searchBody = {
     curations: ["ARTICLES"]
   },
   resultContext: {
-    maxResults: "10",
+    maxResults: "5",
     offset: "0",
     aspects: [
       "audioVisual",
@@ -48,7 +48,7 @@ const searchBody = {
 function constructSAPIQuery(params) {
   const defaults = {
     queryString: "",
-    maxResults: 10,
+    maxResults: 5,
     offset: 0,
     aspects: ["title", "lifecycle", "location"], // [ "title", "location", "summary", "lifecycle", "metadata"],
     constraints: [],

backend/package-lock.json

@@ -21,6 +21,7 @@
   "homepage": "https://github.com/ftlabs/sentiment#readme",
   "dependencies": {
     "aws-sdk": "^2.551.0",
+    "bottleneck": "^2.19.5",
     "ibm-watson": "^5.1.0",
     "node-fetch": "^2.6.0",
     "request": "^2.88.0",

backend/package.json

@@ -3,13 +3,16 @@ service: sentiment
 provider:
   name: aws
   runtime: nodejs10.x
+  timeout: 30
+  memorySize: 3008
   environment:
     CAPI_KEY: "${file(./serverless.env.yml):CAPI_API_KEY}"
     TONE_ANALYZER_IAM_APIKEY: "${file(./serverless.env.yml):TONE_ANALYZER_IAM_APIKEY}"
     TONE_ANALYZER_URL: "${file(./serverless.env.yml):TONE_ANALYZER_URL}"
     TEST: "${file(./serverless.env.yml):TEST}"
     MEANING_CLOUD_URL: "${file(./serverless.env.yml):MEANING_CLOUD_URL}"
     MEANING_CLOUD_APIKEY: "${file(./serverless.env.yml):MEANING_CLOUD_APIKEY}"
+    SENTIMENT_URL: "https://umwi7bvika.execute-api.us-east-1.amazonaws.com/${opt:stage, self:provider.stage}/ftlabs-sentiment/"
   stackTags:
     environment: p
     teamDL: [email protected]

backend/serverless.yml

@@ -3,6 +3,7 @@ const truncate = require("truncate-utf8-bytes");
 const comprehend = new AWS.Comprehend();
 
 async function getSentiment({ articleContent, title, standfirst }) {
+  console.log("aws", articleContent, title, standfirst);
   const result = await Promise.all([
     comprehend
       .detectSentiment({

backend/services/aws.js

@@ -11,6 +11,8 @@ const toneAnalyzer = new ToneAnalyzerV3({
 });
 
 async function getSentiment({ articleContent, standfirst, title }) {
+  console.log("ibm", articleContent, title, standfirst);
+
   const result = await Promise.all([
     new Promise(function(resolve, reject) {
       toneAnalyzer.tone(

backend/services/ibm.js

@@ -3,6 +3,7 @@ const URL = process.env.MEANING_CLOUD_URL;
 const API_KEY = process.env.MEANING_CLOUD_APIKEY;
 
 async function getSentiment({ articleContent, title, standfirst }) {
+  console.log("meaningCloud", articleContent, title, standfirst);
   var request = require("request-promise");
 
   const result = await Promise.all([