Merge pull request #360 from topcoder-platform/develop

ES to OS migration to aid with OpenSearch consolidation

Author: jmgasper

.circleci/config.yml

@@ -9,8 +9,8 @@ install_dependency: &install_dependency
         sudo apt update
         sudo apt install python3-pip
         sudo pip3 install awscli --upgrade
-        sudo pip3 install docker==6.1.3
-        sudo pip3 install docker-compose   
+        # sudo pip3 install docker==6.1.3
+        # sudo pip3 install docker-compose   
 
 install_deploysuite: &install_deploysuite
   name: Installation of install_deploysuite.

README.md

@@ -20,7 +20,7 @@ Dev: [![CircleCI](https://circleci.com/gh/topcoder-platform/submissions-api/tree
 
 ## Related repos
 
-- [ES Processor](https://github.com/topcoder-platform/submission-processor-es) - Updates data in ElasticSearch
+- [ES Processor](https://github.com/topcoder-platform/submission-processor-es) - Updates data in Opensearch
 
 ## Pre-requisites
 
@@ -179,8 +179,8 @@ npm run start
    ```
 
    This command will do 2 things:
-   - Import the data to the database and index it to ElasticSearch
-   - Note, to migrate the existing data from DynamoDB to ES, run the following script
+   - Import the data to the database and index it to OpenSearch
+   - Note, to migrate the existing data from DynamoDB to OS, run the following script
       ```
       npm run db-to-es
       ```

build.sh

@@ -10,5 +10,20 @@ set -eo pipefail
 #sed -i='' "s|submissions-api:latest|$TAG|" docker/docker-compose.yml
 echo "" > docker/api.env
 docker-compose -f docker/docker-compose.yml build submissions-api
-docker images
+#docker images
+docker create --name app submissions-api:latest
+if [ -d node_modules ]
+then
+  mv package-lock.json old-package-lock.json
+  docker cp app:/submissions-api/package-lock.json package-lock.json
+  set +eo pipefail
+  UPDATE_CACHE=$(cmp package-lock.json old-package-lock.json)
+  set -eo pipefail
+else
+  UPDATE_CACHE=1
+fi
 
+if [ "$UPDATE_CACHE" == 1 ]
+then
+  docker cp app:/submissions-api/node_modules .
+fi

config/default.js

@@ -29,11 +29,10 @@ module.exports = {
   TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME,
   AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID,
   AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET,
-  esConfig: {
-    HOST: process.env.ES_HOST || 'localhost:9200',
-    API_VERSION: process.env.ES_API_VERSION || '6.3',
-    ES_INDEX: process.env.ES_INDEX || 'submission',
-    ES_TYPE: process.env.ES_TYPE || '_doc' // ES 6.x accepts only 1 Type per index and it's mandatory to define it
+  osConfig: {
+    HOST: process.env.OS_HOST || 'localhost:9200',
+    OS_INDEX: process.env.OS_INDEX || 'submission-api',
+    OS_TYPE: process.env.OS_TYPE || '_doc' // ES 6.x accepts only 1 Type per index and it's mandatory to define it
   },
   PAGE_SIZE: process.env.PAGE_SIZE || 20,
   MAX_PAGE_SIZE: parseInt(process.env.MAX_PAGE_SIZE) || 100,

config/test.js

@@ -19,9 +19,9 @@ module.exports = {
   BUSAPI_EVENTS_URL: 'https://api.topcoder-dev.com/v5/bus/events',
   BUSAPI_URL: 'https://api.topcoder-dev.com/v5',
   CHALLENGEAPI_V5_URL: 'https://api.topcoder-dev.com/v5/challenges',
-  esConfig: {
-    ES_INDEX: process.env.ES_INDEX_TEST || 'submission-test',
-    ES_TYPE: process.env.ES_TYPE_TEST || '_doc' // ES 6.x accepts only 1 Type per index and it's mandatory to define it
+  osConfig: {
+    OS_INDEX: process.env.OS_INDEX_TEST || 'submission-test',
+    OS_TYPE: process.env.OS_TYPE_TEST || '_doc' // ES 6.x accepts only 1 Type per index and it's mandatory to define it
   },
   AUTH0_URL: process.env.AUTH0_URL, // Auth0 credentials for Submission Service
   AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE,

package-lock.json

@@ -26,7 +26,7 @@
     "local:init": "npm run init-db && npm run init-es"
   },
   "dependencies": {
-    "@elastic/elasticsearch": "^6.8.8",
+    "@opensearch-project/opensearch": "^2.11.0",
     "amazon-s3-uri": "0.1.1",
     "aws-sdk": "^2.265.1",
     "axios": "^1.4.0",

package.json

@@ -12,107 +12,102 @@ const submissions = require('./data/Submissions.json')
 const reviews = require('./data/Reviews.json')
 const reviewSummations = require('./data/ReviewSummations.json')
 
-const esClient = helper.getEsClient()
+const osClient = helper.getOsClient()
 
 /*
- * Delete all data from ES
+ * Delete all data from OS
  */
-function deleteDatafromES () {
-  logger.info('Clear data from ES if any')
+function deleteDatafromOS () {
+  logger.info('Clear data from OS if any')
   const filter = {
-    index: config.get('esConfig.ES_INDEX'),
-    type: config.get('esConfig.ES_TYPE'),
+    index: config.get('osConfig.OS_INDEX'),
     q: '*'
   }
-  return esClient.deleteByQuery(filter)
+  return osClient.deleteByQuery(filter)
 }
 
 /*
- * Load Review Types from JSON into ES
+ * Load Review Types from JSON into OS
  */
 async function loadReviewTypes () {
   const promises = []
   reviewTypes.forEach((reviewType) => {
     const record = {
-      index: config.get('esConfig.ES_INDEX'),
-      type: config.get('esConfig.ES_TYPE'),
+      index: config.get('osConfig.OS_INDEX'),
       id: reviewType.id,
       body: _.extend({ resource: 'reviewType' }, reviewType)
     }
-    promises.push(esClient.create(record))
+    promises.push(osClient.index(record))
   })
   await Promise.all(promises)
 }
 
 /*
- * Load Submissions from JSON into ES
+ * Load Submissions from JSON into OS
  */
 async function loadSubmissions () {
   const promises = []
   submissions.forEach((submission) => {
     const record = {
-      index: config.get('esConfig.ES_INDEX'),
-      type: config.get('esConfig.ES_TYPE'),
+      index: config.get('osConfig.OS_INDEX'),
       id: submission.id,
       body: _.extend({ resource: 'submission' }, submission)
     }
-    promises.push(esClient.create(record))
+    promises.push(osClient.index(record))
   })
   await Promise.all(promises)
 }
 
 /*
- * Load Reviews from JSON into ES
+ * Load Reviews from JSON into OS
  */
 async function loadReviews () {
   const promises = []
   reviews.forEach((review) => {
     const record = {
-      index: config.get('esConfig.ES_INDEX'),
-      type: config.get('esConfig.ES_TYPE'),
+      index: config.get('osConfig.OS_INDEX'),
       id: review.id,
       body: _.extend({ resource: 'review' }, review)
     }
-    promises.push(esClient.create(record))
+    promises.push(osClient.index(record))
   })
   await Promise.all(promises)
 }
 
 /*
- * Load Review Summations from JSON into ES
+ * Load Review Summations from JSON into OS
  */
 async function loadReviewSummations () {
   const promises = []
   reviewSummations.forEach((reviewSummation) => {
     const record = {
-      index: config.get('esConfig.ES_INDEX'),
-      type: config.get('esConfig.ES_TYPE'),
+      index: config.get('osConfig.OS_INDEX'),
       id: reviewSummation.id,
       body: _.extend({ resource: 'reviewSummation' }, reviewSummation)
     }
-    promises.push(esClient.create(record))
+    promises.push(osClient.index(record))
   })
   await Promise.all(promises)
 }
 
 /*
- * Load data into ES after removing existing data
+ * Load data into OS after removing existing data
  */
-async function loadES () {
-  await deleteDatafromES()
-  logger.info('ES Loading started!')
+async function loadOS () {
+  await deleteDatafromOS()
+  logger.info('OS Loading started!')
   await loadReviewTypes()
   await loadSubmissions()
   await loadReviews()
   await loadReviewSummations()
-  logger.info('ES Loading succeeded!')
+  logger.info('OS Loading succeeded!')
 }
 
 module.exports = {
-  deleteDatafromES,
+  deleteDatafromOS,
   loadReviewTypes,
   loadSubmissions,
   loadReviews,
   loadReviewSummations,
-  loadES
+  loadOS
 }