refactor: upgrade to opensearch & build fix

Signed-off-by: Rakib Ansary <[email protected]>

Author: rakibansary

.circleci/config.yml

@@ -18,10 +18,10 @@ install_deploysuite: &install_deploysuite
     cp ./../buildscript/buildenv.sh .
     cp ./../buildscript/awsconfiguration.sh .
 restore_cache_settings_for_build: &restore_cache_settings_for_build
-  key: docker-node-modules-{{ checksum "package-lock.json" }}
+  key: docker-node-modules-{{ checksum "yarn.lock" }}
 
 save_cache_settings: &save_cache_settings
-  key: docker-node-modules-{{ checksum "package-lock.json" }}
+  key: docker-node-modules-{{ checksum "yarn.lock" }}
   paths:
     - node_modules
 

README.md

@@ -58,7 +58,6 @@ The following parameters can be set in config files or in env variables:
 - ES.HOST: Elasticsearch host
 - ES.API_VERSION: Elasticsearch API version
 - ES.ES_INDEX: Elasticsearch index name
-- ES.ES_TYPE: Elasticsearch index type
 - ES.ES_REFRESH: Elasticsearch refresh method. Default to string `true`(i.e. refresh immediately)
 - FILE_UPLOAD_SIZE_LIMIT: the file upload size limit in bytes
 - RESOURCES_API_URL: TC resources API base URL

config/default.js

@@ -43,7 +43,6 @@ module.exports = {
     HOST: process.env.ES_HOST || "localhost:9200",
     API_VERSION: process.env.ES_API_VERSION || "6.8",
     ES_INDEX: process.env.ES_INDEX || "challenge",
-    ES_TYPE: process.env.ES_TYPE || "_doc", // ES 6.x accepts only 1 Type per index and it's mandatory to define it
     ES_REFRESH: process.env.ES_REFRESH || "true",
     TEMP_REINDEXING: process.env.TEMP_REINDEXING || true, // if true, it won't delete the existing index when reindexing data
   },

docker/Dockerfile

@@ -1,5 +1,5 @@
 # Use the base image with Node.js
-FROM node:14.21.2-bullseye
+FROM node:18.14.1-alpine3.17
 
 # Copy the current directory into the Docker image
 COPY . /challenge-api

docs/prod.env

@@ -14,7 +14,6 @@ S3_API_VERSION=
 ES_HOST=
 ES_API_VERSION=
 ES_INDEX=
-ES_TYPE=
 ES_REFRESH=true
 RESOURCES_API_URL=
 GROUPS_API_URL=

package.json

@@ -39,6 +39,7 @@
     "prettier": "^2.8.1"
   },
   "dependencies": {
+    "@opensearch-project/opensearch": "^2.2.0",
     "@topcoder-framework/domain-challenge": "^0.4.3",
     "@topcoder-framework/lib-common": "^0.4.3",
     "aws-sdk": "^2.1145.0",
@@ -50,7 +51,6 @@
     "cors": "^2.7.1",
     "dotenv": "^8.2.0",
     "dynamoose": "^1.11.1",
-    "elasticsearch": "^16.1.1",
     "express": "^4.15.4",
     "express-fileupload": "^1.1.6",
     "express-interceptor": "^1.2.0",

src/common/helper.js

@@ -13,12 +13,13 @@ const config = require("config");
 const axios = require("axios");
 const axiosRetry = require("axios-retry");
 const busApi = require("topcoder-bus-api-wrapper");
-const elasticsearch = require("elasticsearch");
 const NodeCache = require("node-cache");
 const HttpStatus = require("http-status-codes");
 const xss = require("xss");
 const logger = require("./logger");
 
+const { Client: ESClient } = require("@opensearch-project/opensearch");
+
 const projectHelper = require("./project-helper");
 const m2mHelper = require("./m2m-helper");
 
@@ -850,23 +851,14 @@ function getESClient() {
     return esClient;
   }
   const esHost = config.get("ES.HOST");
-  // AWS ES configuration is different from other providers
-  if (/.*amazonaws.*/.test(esHost)) {
-    esClient = elasticsearch.Client({
-      apiVersion: config.get("ES.API_VERSION"),
-      hosts: esHost,
-      connectionClass: require("http-aws-es"), // eslint-disable-line global-require
-      amazonES: {
-        region: config.get("AMAZON.AWS_REGION"),
-        credentials: new AWS.EnvironmentCredentials("AWS"),
-      },
-    });
-  } else {
-    esClient = new elasticsearch.Client({
-      apiVersion: config.get("ES.API_VERSION"),
-      hosts: esHost,
-    });
-  }
+
+  esClient = new ESClient({
+    node: esHost,
+    ssl: {
+      rejectUnauthorized: false,
+    },
+  });
+
   return esClient;
 }
 

src/init-es.js

@@ -7,81 +7,85 @@
  * node src/init-es force
  */
 const config = require("config");
-// const logger = require('./common/logger')
+const logger = require("./common/logger");
 const helper = require("./common/helper");
 
 const client = helper.getESClient();
 
 const initES = async () => {
   if (process.argv.length === 3 && process.argv[2] === "force") {
-    console.log(`Delete index ${config.ES.ES_INDEX} if any.`);
+    logger.info(`Delete index ${config.ES.ES_INDEX} if any.`);
     try {
       await client.indices.delete({ index: config.ES.ES_INDEX });
     } catch (err) {
       // ignore
     }
   }
 
-  const exists = await client.indices.exists({ index: config.ES.ES_INDEX });
+  const { body: exists } = await client.indices.exists({ index: config.ES.ES_INDEX });
   if (exists) {
-    console.log(`The index ${config.ES.ES_INDEX} exists.`);
+    logger.info(`The index ${config.ES.ES_INDEX} exists.`);
   } else {
-    console.log(`The index ${config.ES.ES_INDEX} will be created.`);
+    logger.info(`The index ${config.ES.ES_INDEX} will be created.`);
 
-    const body = { mappings: {} };
-    body.mappings[config.get("ES.ES_TYPE")] = {
-      properties: {
-        id: { type: "keyword" },
-        name: {
-          type: "keyword",
-          fields: {
-            text: {
-              type: "text",
+    const body = {
+      mappings: {
+        // properties: {
+        properties: {
+          id: { type: "keyword" },
+          name: {
+            type: "keyword",
+            fields: {
+              text: {
+                type: "text",
+              },
             },
+            normalizer: "custom_sort_normalizer",
           },
-          normalizer: "custom_sort_normalizer",
-        },
-        status: {
-          type: 'keyword',
-          fields: {
-            text: {
-              type: 'text'
-            }
+          status: {
+            type: "keyword",
+            fields: {
+              text: {
+                type: "text",
+              },
+            },
+            normalizer: "custom_sort_normalizer",
           },
-          normalizer: 'custom_sort_normalizer'
-        },
-        type: {
-          type: 'keyword',
-          fields: {
-            text: {
-              type: 'text'
-            }
+          type: {
+            type: "keyword",
+            fields: {
+              text: {
+                type: "text",
+              },
+            },
+            normalizer: "custom_sort_normalizer",
           },
-          normalizer: 'custom_sort_normalizer'
-        },
-        prizeSets: {
-          properties: {
-            type: { type: "text" },
-            prizes: {
-              properties: {
-                type: { type: "text" },
-                value: { type: "float" },
+          prizeSets: {
+            properties: {
+              type: { type: "text" },
+              prizes: {
+                properties: {
+                  type: { type: "text" },
+                  value: { type: "float" },
+                },
               },
             },
           },
         },
-      },
-      dynamic_templates: [
-        {
-          metadata: {
-            path_match: "metadata.*",
-            mapping: {
-              type: "text",
+        dynamic_templates: [
+          {
+            metadata: {
+              path_match: "metadata.*",
+              mapping: {
+                type: "text",
+              },
             },
           },
-        },
-      ],
+        ],
+        //}
+      },
     };
+
     body.settings = {
       analysis: {
         normalizer: {
@@ -103,11 +107,10 @@ const initES = async () => {
 
 initES()
   .then(() => {
-    console.log("Done!");
+    logger.info("Done!");
     process.exit();
   })
   .catch((e) => {
-    // logger.logFullError(e);
-    e.printStackTrace();
+    logger.logFullError(e);
     process.exit();
   });

src/scripts/sync-es.js

@@ -16,7 +16,6 @@ async function migrateRecords() {
   for (const challenge of result) {
     await esClient.update({
       index: config.get("ES.ES_INDEX"),
-      type: config.get("ES.ES_TYPE"),
       id: challenge.id,
       body: { doc: challenge, doc_as_upsert: true },
     });

src/scripts/view-es-data.js

@@ -11,7 +11,6 @@ const esClient = helper.getESClient();
 async function showESData() {
   const result = await esClient.search({
     index: config.get("ES.ES_INDEX"),
-    type: config.get("ES.ES_TYPE"),
   });
   return result;
 }

src/services/ChallengeService.js

@@ -794,7 +794,6 @@ async function searchChallenges(currentUser, criteria) {
 
   const esQuery = {
     index: config.get("ES.ES_INDEX"),
-    type: config.get("ES.ES_TYPE"),
     size: perPage,
     from: (page - 1) * perPage, // Es Index starts from 0
     body: {
@@ -817,7 +816,7 @@ async function searchChallenges(currentUser, criteria) {
   // Search with constructed query
   let docs;
   try {
-    docs = await esClient.search(esQuery);
+    docs = (await esClient.search(esQuery)).body;
   } catch (e) {
     // Catch error when the ES is fresh and has no data
     logger.error(`Query Error from ES ${JSON.stringify(e, null, 4)}`);
@@ -1139,7 +1138,6 @@ async function createChallenge(currentUser, challenge, userToken) {
   // Create in ES
   await esClient.create({
     index: config.get("ES.ES_INDEX"),
-    type: config.get("ES.ES_TYPE"),
     refresh: config.get("ES.ES_REFRESH"),
     id: ret.id,
     body: ret,
@@ -1304,15 +1302,15 @@ async function getChallenge(currentUser, id, checkIfExists) {
   let challenge;
   // logger.warn(JSON.stringify({
   //   index: config.get('ES.ES_INDEX'),
-  //   type: config.get('ES.ES_TYPE'),
   //   _id: id
   // }))
   try {
-    challenge = await esClient.getSource({
-      index: config.get("ES.ES_INDEX"),
-      type: config.get("ES.ES_TYPE"),
-      id,
-    });
+    challenge = (
+      await esClient.getSource({
+        index: config.get("ES.ES_INDEX"),
+        id,
+      })
+    ).body;
   } catch (e) {
     if (e.statusCode === HttpStatus.NOT_FOUND) {
       throw new errors.NotFoundError(`Challenge of id ${id} is not found.`);
@@ -2195,7 +2193,6 @@ async function update(currentUser, challengeId, data, isFull) {
   // Update ES
   await esClient.update({
     index: config.get("ES.ES_INDEX"),
-    type: config.get("ES.ES_TYPE"),
     refresh: config.get("ES.ES_REFRESH"),
     id: challengeId,
     body: {
@@ -2701,7 +2698,6 @@ async function deleteChallenge(currentUser, challengeId) {
   // delete ES document
   await esClient.delete({
     index: config.get("ES.ES_INDEX"),
-    type: config.get("ES.ES_TYPE"),
     refresh: config.get("ES.ES_REFRESH"),
     id: challengeId,
   });