notebook/CovidFAQ/07.Generating_BERT_Prediction_Results.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "07.Generating_BERT_Prediction_Results.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GWIhLk6uIODF",
        "outputId": "981eb50a-6fec-4d60-f972-3a5e45d2fed5"
      },
      "source": [
        "# install required libraries\n",
        "!pip3 install sentence-transformers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting sentence-transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/35/aa/f672ce489063c4ee7a566ebac1b723c53ac0cea19d9e36599cc241d8ed56/sentence-transformers-1.0.4.tar.gz (74kB)\n",
            "\u001b[K     |████████████████████████████████| 81kB 7.9MB/s \n",
            "\u001b[?25hCollecting transformers<5.0.0,>=3.1.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ed/d5/f4157a376b8a79489a76ce6cfe147f4f3be1e029b7144fa7b8432e8acb26/transformers-4.4.2-py3-none-any.whl (2.0MB)\n",
            "\u001b[K     |████████████████████████████████| 2.0MB 24.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from sentence-transformers) (4.41.1)\n",
            "Requirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from sentence-transformers) (1.8.1+cu101)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from sentence-transformers) (1.19.5)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from sentence-transformers) (0.22.2.post1)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from sentence-transformers) (1.4.1)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.7/dist-packages (from sentence-transformers) (3.2.5)\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f5/99/e0808cb947ba10f575839c43e8fafc9cc44e4a7a2c8f79c60db48220a577/sentencepiece-0.1.95-cp37-cp37m-manylinux2014_x86_64.whl (1.2MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2MB 41.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=3.1.0->sentence-transformers) (2.23.0)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=3.1.0->sentence-transformers) (3.8.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=3.1.0->sentence-transformers) (20.9)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=3.1.0->sentence-transformers) (3.0.12)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/23/2ddc317b2121117bf34dd00f5b0de194158f2a44ee2bf5e47c7166878a97/tokenizers-0.10.1-cp37-cp37m-manylinux2010_x86_64.whl (3.2MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2MB 53.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=3.1.0->sentence-transformers) (2019.12.20)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/08/cd/342e584ee544d044fb573ae697404ce22ede086c9e87ce5960772084cad0/sacremoses-0.0.44.tar.gz (862kB)\n",
            "\u001b[K     |████████████████████████████████| 870kB 52.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch>=1.6.0->sentence-transformers) (3.7.4.3)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->sentence-transformers) (1.0.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from nltk->sentence-transformers) (1.15.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers<5.0.0,>=3.1.0->sentence-transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers<5.0.0,>=3.1.0->sentence-transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers<5.0.0,>=3.1.0->sentence-transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers<5.0.0,>=3.1.0->sentence-transformers) (2020.12.5)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers<5.0.0,>=3.1.0->sentence-transformers) (3.4.1)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers<5.0.0,>=3.1.0->sentence-transformers) (2.4.7)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers<5.0.0,>=3.1.0->sentence-transformers) (7.1.2)\n",
            "Building wheels for collected packages: sentence-transformers, sacremoses\n",
            "  Building wheel for sentence-transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sentence-transformers: filename=sentence_transformers-1.0.4-cp37-none-any.whl size=114307 sha256=2f444a7eac4bbc0253f7c8a0d046960542d44279601dded700077788291a7591\n",
            "  Stored in directory: /root/.cache/pip/wheels/26/ea/89/d0d2e013d951b6d23270aa9ca4018b82632ab7cd933c331316\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.44-cp37-none-any.whl size=886084 sha256=384237782ba656f2fa1e4e1d71909115660f0ead952e141ebd9eaba880a3b10b\n",
            "  Stored in directory: /root/.cache/pip/wheels/3e/fb/c0/13ab4d63d537658f448366744654323077c4d90069b6512f3c\n",
            "Successfully built sentence-transformers sacremoses\n",
            "Installing collected packages: tokenizers, sacremoses, transformers, sentencepiece, sentence-transformers\n",
            "Successfully installed sacremoses-0.0.44 sentence-transformers-1.0.4 sentencepiece-0.1.95 tokenizers-0.10.1 transformers-4.4.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wHrXaJsSLhrJ",
        "outputId": "bffc2663-f897-44d6-b4dc-e5d80c4f1f12"
      },
      "source": [
        "!pip3 install elasticsearch"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting elasticsearch\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/09/93/461a042becf2a35a666fb7dbb2fa31f0f766dfd1b01e7d971f4ad51f0d69/elasticsearch-7.12.0-py2.py3-none-any.whl (334kB)\n",
            "\r\u001b[K     |█                               | 10kB 26.8MB/s eta 0:00:01\r\u001b[K     |██                              | 20kB 31.3MB/s eta 0:00:01\r\u001b[K     |███                             | 30kB 19.2MB/s eta 0:00:01\r\u001b[K     |████                            | 40kB 22.6MB/s eta 0:00:01\r\u001b[K     |█████                           | 51kB 25.3MB/s eta 0:00:01\r\u001b[K     |█████▉                          | 61kB 28.1MB/s eta 0:00:01\r\u001b[K     |██████▉                         | 71kB 18.4MB/s eta 0:00:01\r\u001b[K     |███████▉                        | 81kB 19.4MB/s eta 0:00:01\r\u001b[K     |████████▉                       | 92kB 18.3MB/s eta 0:00:01\r\u001b[K     |█████████▉                      | 102kB 18.5MB/s eta 0:00:01\r\u001b[K     |██████████▊                     | 112kB 18.5MB/s eta 0:00:01\r\u001b[K     |███████████▊                    | 122kB 18.5MB/s eta 0:00:01\r\u001b[K     |████████████▊                   | 133kB 18.5MB/s eta 0:00:01\r\u001b[K     |█████████████▊                  | 143kB 18.5MB/s eta 0:00:01\r\u001b[K     |██████████████▊                 | 153kB 18.5MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 163kB 18.5MB/s eta 0:00:01\r\u001b[K     |████████████████▋               | 174kB 18.5MB/s eta 0:00:01\r\u001b[K     |█████████████████▋              | 184kB 18.5MB/s eta 0:00:01\r\u001b[K     |██████████████████▋             | 194kB 18.5MB/s eta 0:00:01\r\u001b[K     |███████████████████▋            | 204kB 18.5MB/s eta 0:00:01\r\u001b[K     |████████████████████▋           | 215kB 18.5MB/s eta 0:00:01\r\u001b[K     |█████████████████████▌          | 225kB 18.5MB/s eta 0:00:01\r\u001b[K     |██████████████████████▌         | 235kB 18.5MB/s eta 0:00:01\r\u001b[K     |███████████████████████▌        | 245kB 18.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████▌       | 256kB 18.5MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▌      | 266kB 18.5MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▍     | 276kB 18.5MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▍    | 286kB 18.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 296kB 18.5MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▍  | 307kB 18.5MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▍ | 317kB 18.5MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▍| 327kB 18.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 337kB 18.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: certifi in /usr/local/lib/python3.7/dist-packages (from elasticsearch) (2020.12.5)\n",
            "Requirement already satisfied: urllib3<2,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from elasticsearch) (1.24.3)\n",
            "Installing collected packages: elasticsearch\n",
            "Successfully installed elasticsearch-7.12.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Nx6PkXGeh5s3"
      },
      "source": [
        "from google.colab import drive"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L-v9U9E5h5gb",
        "outputId": "8d0c5e0c-9d9f-4762-f076-26aceac1dbe8"
      },
      "source": [
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dMP1Wg7Bh5RT",
        "outputId": "ff8a9744-480a-422e-8630-47dd8e6f0b6d"
      },
      "source": [
        "%cd /content/drive/MyDrive/BERT-FAQ"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/BERT-FAQ\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-ESSxl0XiXAD",
        "outputId": "96a70ca6-5e40-41a1-966b-73e983909149"
      },
      "source": [
        "!ls"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "data\t\t\t     __init__.py  reranker.py\n",
            "evaluation.py\t\t     metric.py\t  searcher.py\n",
            "faq_bert_finetuning.py\t     notebook\t  shared\n",
            "faq_bert.py\t\t     output\t  training_data_generator.py\n",
            "faq_bert_ranker.py\t     parser\t  webserver.py\n",
            "hard_negatives_generator.py  __pycache__\n",
            "indexer.py\t\t     README.md\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fu6mIC_NKTRR"
      },
      "source": [
        "# import required dependencies\n",
        "from evaluation import get_relevance_label_df\n",
        "from shared.utils import load_from_json\n",
        "from shared.utils import dump_to_json\n",
        "from shared.utils import make_dirs\n",
        "from reranker import ReRanker"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_O3F8NYk19Go"
      },
      "source": [
        "output_path=\"data/CovidFAQ/rank_results\"\n",
        "\n",
        "# load user_query ES results from json files\n",
        "es_output_path = output_path + \"/unsupervised\"\n",
        "es_query_by_question = load_from_json(es_output_path + '/es_query_by_question.json')\n",
        "es_query_by_answer = load_from_json(es_output_path + '/es_query_by_answer.json')\n",
        "es_query_by_question_answer = load_from_json(es_output_path + '/es_query_by_question_answer.json')\n",
        "es_query_by_question_answer_concat = load_from_json(es_output_path + '/es_query_by_question_answer_concat.json')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pyA4nPfi1885"
      },
      "source": [
        "# load test_queries, relevance_label_df for ReRanker\n",
        "query_answer_pair_filepath = 'data/CovidFAQ/query_answer_pairs.json'\n",
        "relevance_label_df = get_relevance_label_df(query_answer_pair_filepath)\n",
        "test_queries = relevance_label_df[relevance_label_df['query_type'] == 'user_query'].question.unique()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MhodYHua180S",
        "outputId": "a16473d4-472b-4471-f420-49860bb2260f"
      },
      "source": [
        "test_queries[:10]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['What does Corona and Covid mean?',\n",
              "       'when will the social distancing end? and what is the economic consequent of the pandemic?',\n",
              "       'How do I go grocery shopping? ', 'what is covid 19',\n",
              "       \"What is COVID-19's definition?\", 'Should I stay home ?',\n",
              "       'Should I practice social distancing? ', 'How did it started',\n",
              "       'what are the 3 symptoms of covid-19', 'Should I wear a mask?'],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "o2S73_R42DCF",
        "outputId": "79e86c1e-2dd6-411a-d000-0f5fd4a2604e"
      },
      "source": [
        "# total number of test queries\n",
        "len(test_queries)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1078"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CDsNe63Nix7W"
      },
      "source": [
        "**1. Generating BERT prediction results from Answer (BERT-Q-a)\"**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dvplanC2jY-y"
      },
      "source": [
        "# define rank_field parameter\n",
        "rank_field=\"BERT-Q-a\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SxcqS2Dj2liG"
      },
      "source": [
        "**query_type=\"user_query\"; neg_type=\"hard\"; loss_type='triplet'**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "__jcFHSRBZSY"
      },
      "source": [
        "# define variables\n",
        "query_type=\"user_query\"; neg_type=\"hard\"; version=\"1.1\"; loss_type='triplet'\n",
        "bert_model_path='output/CovidFAQ/models/' + loss_type + '_' + neg_type + '_' + query_type + '_' + version"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n9ycYBipNnP9"
      },
      "source": [
        "# create instance of ReRanker class\n",
        "r = ReRanker(\n",
        "    bert_model_path=bert_model_path, \n",
        "    test_queries=test_queries, relevance_label_df=relevance_label_df,\n",
        "    rank_field=rank_field\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xJMgG4aZKTN5",
        "outputId": "7b1c2e6c-761c-4503-c081-af8466b18f28"
      },
      "source": [
        "# generate directory structure\n",
        "pred_output_path = output_path + \"/supervised/\" + rank_field + \"/\" + loss_type + \"/\" + query_type + \"/\" + neg_type\n",
        "make_dirs(pred_output_path)\n",
        "\n",
        "# next, generate BERT, Re-ranked top-k results and dump to files\n",
        "bert_query_by_question = r.get_bert_topk_preds(es_query_by_question)\n",
        "dump_to_json(bert_query_by_question, pred_output_path + '/bert_query_by_question.json')\n",
        "\n",
        "bert_query_by_answer = r.get_bert_topk_preds(es_query_by_answer)\n",
        "dump_to_json(bert_query_by_answer, pred_output_path + '/bert_query_by_answer.json')\n",
        "\n",
        "bert_query_by_question_answer = r.get_bert_topk_preds(es_query_by_question_answer)\n",
        "dump_to_json(bert_query_by_question_answer, pred_output_path + '/bert_query_by_question_answer.json')\n",
        "\n",
        "bert_query_by_question_answer_concat = r.get_bert_topk_preds(es_query_by_question_answer_concat)\n",
        "dump_to_json(bert_query_by_question_answer_concat, pred_output_path + '/bert_query_by_question_answer_concat.json')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-03 09:09:49 - Generating BERT top-k results ...\n",
            "2021-04-03 09:09:50 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_hard_user_query_1.1\n",
            "2021-04-03 09:09:50 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_hard_user_query_1.1\n",
            "2021-04-03 09:10:02 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [04:47<00:00,  3.75it/s]\n",
            "2021-04-03 09:14:50 - Generating BERT top-k results ...\n",
            "2021-04-03 09:14:50 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_hard_user_query_1.1\n",
            "2021-04-03 09:14:50 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_hard_user_query_1.1\n",
            "2021-04-03 09:14:51 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [08:20<00:00,  2.15it/s]\n",
            "2021-04-03 09:23:13 - Generating BERT top-k results ...\n",
            "2021-04-03 09:23:13 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_hard_user_query_1.1\n",
            "2021-04-03 09:23:13 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_hard_user_query_1.1\n",
            "2021-04-03 09:23:15 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [08:53<00:00,  2.02it/s]\n",
            "2021-04-03 09:32:09 - Generating BERT top-k results ...\n",
            "2021-04-03 09:32:09 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_hard_user_query_1.1\n",
            "2021-04-03 09:32:09 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_hard_user_query_1.1\n",
            "2021-04-03 09:32:11 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [08:53<00:00,  2.02it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ni5fBXXCM8oo"
      },
      "source": [
        "**query_type=\"user_query\"; neg_type=\"simple\"; loss_type='triplet'**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YahPWqlrM4eU"
      },
      "source": [
        "# define variables\n",
        "query_type=\"user_query\"; neg_type=\"simple\"; version=\"1.1\"; loss_type='triplet'\n",
        "bert_model_path='output/CovidFAQ/models/' + loss_type + '_' + neg_type + '_' + query_type + '_' + version"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vfz-PNgZM4FL"
      },
      "source": [
        "# create instance of ReRanker class\n",
        "r = ReRanker(\n",
        "    bert_model_path=bert_model_path, \n",
        "    test_queries=test_queries, relevance_label_df=relevance_label_df,\n",
        "    rank_field=rank_field\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "90jQn9shzcsj",
        "outputId": "f241c875-02db-433d-ed10-9258a8909017"
      },
      "source": [
        "# generate directory structure\n",
        "pred_output_path = output_path + \"/supervised/\" + rank_field + \"/\" + loss_type + \"/\" + query_type + \"/\" + neg_type\n",
        "make_dirs(pred_output_path)\n",
        "\n",
        "# next, generate BERT, Re-ranked top-k results and dump to files\n",
        "bert_query_by_question = r.get_bert_topk_preds(es_query_by_question)\n",
        "dump_to_json(bert_query_by_question, pred_output_path + '/bert_query_by_question.json')\n",
        "\n",
        "bert_query_by_answer = r.get_bert_topk_preds(es_query_by_answer)\n",
        "dump_to_json(bert_query_by_answer, pred_output_path + '/bert_query_by_answer.json')\n",
        "\n",
        "bert_query_by_question_answer = r.get_bert_topk_preds(es_query_by_question_answer)\n",
        "dump_to_json(bert_query_by_question_answer, pred_output_path + '/bert_query_by_question_answer.json')\n",
        "\n",
        "bert_query_by_question_answer_concat = r.get_bert_topk_preds(es_query_by_question_answer_concat)\n",
        "dump_to_json(bert_query_by_question_answer_concat, pred_output_path + '/bert_query_by_question_answer_concat.json')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-03 09:41:06 - Generating BERT top-k results ...\n",
            "2021-04-03 09:41:06 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_simple_user_query_1.1\n",
            "2021-04-03 09:41:06 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_simple_user_query_1.1\n",
            "2021-04-03 09:41:16 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [04:43<00:00,  3.80it/s]\n",
            "2021-04-03 09:46:00 - Generating BERT top-k results ...\n",
            "2021-04-03 09:46:00 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_simple_user_query_1.1\n",
            "2021-04-03 09:46:00 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_simple_user_query_1.1\n",
            "2021-04-03 09:46:01 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [08:21<00:00,  2.15it/s]\n",
            "2021-04-03 09:54:24 - Generating BERT top-k results ...\n",
            "2021-04-03 09:54:24 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_simple_user_query_1.1\n",
            "2021-04-03 09:54:24 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_simple_user_query_1.1\n",
            "2021-04-03 09:54:25 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [08:56<00:00,  2.01it/s]\n",
            "2021-04-03 10:03:22 - Generating BERT top-k results ...\n",
            "2021-04-03 10:03:22 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_simple_user_query_1.1\n",
            "2021-04-03 10:03:22 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_simple_user_query_1.1\n",
            "2021-04-03 10:03:24 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [08:51<00:00,  2.03it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h0-VLGZ-Np04"
      },
      "source": [
        "**query_type=\"faq\"; neg_type=\"hard\"; loss_type='triplet'**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sGUqO4kSNcGJ"
      },
      "source": [
        "# define variables\n",
        "query_type=\"faq\"; neg_type=\"hard\"; version=\"1.1\"; loss_type='triplet'\n",
        "bert_model_path='output/CovidFAQ/models/' + loss_type + '_' + neg_type + '_' + query_type + '_' + version"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bkdb3DQON4A9"
      },
      "source": [
        "# create instance of ReRanker class\n",
        "r = ReRanker(\n",
        "    bert_model_path=bert_model_path, \n",
        "    test_queries=test_queries, relevance_label_df=relevance_label_df,\n",
        "    rank_field=rank_field\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9whF6pZTzfHK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0d7b2711-31b2-4b6b-d11b-e36af65d2101"
      },
      "source": [
        "# generate directory structure\n",
        "pred_output_path = output_path + \"/supervised/\" + rank_field + \"/\" + loss_type + \"/\" + query_type + \"/\" + neg_type\n",
        "make_dirs(pred_output_path)\n",
        "\n",
        "# next, generate BERT, Re-ranked top-k results and dump to files\n",
        "bert_query_by_question = r.get_bert_topk_preds(es_query_by_question)\n",
        "dump_to_json(bert_query_by_question, pred_output_path + '/bert_query_by_question.json')\n",
        "\n",
        "bert_query_by_answer = r.get_bert_topk_preds(es_query_by_answer)\n",
        "dump_to_json(bert_query_by_answer, pred_output_path + '/bert_query_by_answer.json')\n",
        "\n",
        "bert_query_by_question_answer = r.get_bert_topk_preds(es_query_by_question_answer)\n",
        "dump_to_json(bert_query_by_question_answer, pred_output_path + '/bert_query_by_question_answer.json')\n",
        "\n",
        "bert_query_by_question_answer_concat = r.get_bert_topk_preds(es_query_by_question_answer_concat)\n",
        "dump_to_json(bert_query_by_question_answer_concat, pred_output_path + '/bert_query_by_question_answer_concat.json')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-03 10:12:17 - Generating BERT top-k results ...\n",
            "2021-04-03 10:12:17 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_hard_faq_1.1\n",
            "2021-04-03 10:12:17 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_hard_faq_1.1\n",
            "2021-04-03 10:12:29 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [04:40<00:00,  3.84it/s]\n",
            "2021-04-03 10:17:10 - Generating BERT top-k results ...\n",
            "2021-04-03 10:17:10 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_hard_faq_1.1\n",
            "2021-04-03 10:17:10 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_hard_faq_1.1\n",
            "2021-04-03 10:17:11 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [08:18<00:00,  2.16it/s]\n",
            "2021-04-03 10:25:31 - Generating BERT top-k results ...\n",
            "2021-04-03 10:25:31 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_hard_faq_1.1\n",
            "2021-04-03 10:25:31 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_hard_faq_1.1\n",
            "2021-04-03 10:25:33 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [08:50<00:00,  2.03it/s]\n",
            "2021-04-03 10:34:25 - Generating BERT top-k results ...\n",
            "2021-04-03 10:34:25 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_hard_faq_1.1\n",
            "2021-04-03 10:34:25 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_hard_faq_1.1\n",
            "2021-04-03 10:34:26 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [08:51<00:00,  2.03it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LW6WhP1POh4R"
      },
      "source": [
        "**query_type=\"faq\"; neg_type=\"simple\"; loss_type='triplet'**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "td1x8prBOXIo"
      },
      "source": [
        "# define variables\n",
        "query_type=\"faq\"; neg_type=\"simple\"; version=\"1.1\"; loss_type='triplet'\n",
        "bert_model_path='output/CovidFAQ/models/' + loss_type + '_' + neg_type + '_' + query_type + '_' + version"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z8ZPxcmWOny9"
      },
      "source": [
        "# create instance of ReRanker class\n",
        "r = ReRanker(\n",
        "    bert_model_path=bert_model_path, \n",
        "    test_queries=test_queries, relevance_label_df=relevance_label_df,\n",
        "    rank_field=rank_field\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eWO9iOd4zjrX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "036e921d-8615-4739-da66-84bbc020b268"
      },
      "source": [
        "# generate directory structure\n",
        "pred_output_path = output_path + \"/supervised/\" + rank_field + \"/\" + loss_type + \"/\" + query_type + \"/\" + neg_type\n",
        "make_dirs(pred_output_path)\n",
        "\n",
        "# next, generate BERT, Re-ranked top-k results and dump to files\n",
        "bert_query_by_question = r.get_bert_topk_preds(es_query_by_question)\n",
        "dump_to_json(bert_query_by_question, pred_output_path + '/bert_query_by_question.json')\n",
        "\n",
        "bert_query_by_answer = r.get_bert_topk_preds(es_query_by_answer)\n",
        "dump_to_json(bert_query_by_answer, pred_output_path + '/bert_query_by_answer.json')\n",
        "\n",
        "bert_query_by_question_answer = r.get_bert_topk_preds(es_query_by_question_answer)\n",
        "dump_to_json(bert_query_by_question_answer, pred_output_path + '/bert_query_by_question_answer.json')\n",
        "\n",
        "bert_query_by_question_answer_concat = r.get_bert_topk_preds(es_query_by_question_answer_concat)\n",
        "dump_to_json(bert_query_by_question_answer_concat, pred_output_path + '/bert_query_by_question_answer_concat.json')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-03 10:43:19 - Generating BERT top-k results ...\n",
            "2021-04-03 10:43:19 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_simple_faq_1.1\n",
            "2021-04-03 10:43:19 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_simple_faq_1.1\n",
            "2021-04-03 10:43:29 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [04:40<00:00,  3.84it/s]\n",
            "2021-04-03 10:48:10 - Generating BERT top-k results ...\n",
            "2021-04-03 10:48:10 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_simple_faq_1.1\n",
            "2021-04-03 10:48:10 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_simple_faq_1.1\n",
            "2021-04-03 10:48:12 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [08:18<00:00,  2.16it/s]\n",
            "2021-04-03 10:56:32 - Generating BERT top-k results ...\n",
            "2021-04-03 10:56:32 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_simple_faq_1.1\n",
            "2021-04-03 10:56:32 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_simple_faq_1.1\n",
            "2021-04-03 10:56:33 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [08:51<00:00,  2.03it/s]\n",
            "2021-04-03 11:05:26 - Generating BERT top-k results ...\n",
            "2021-04-03 11:05:26 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_simple_faq_1.1\n",
            "2021-04-03 11:05:26 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_simple_faq_1.1\n",
            "2021-04-03 11:05:27 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [08:54<00:00,  2.02it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TQWBMnXNuN1r"
      },
      "source": [
        "**query_type=\"user_query\"; neg_type=\"hard\"; loss_type='softmax'**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qWbql3bypSdF"
      },
      "source": [
        "# define variables\n",
        "query_type=\"user_query\"; neg_type=\"hard\"; version=\"1.1\"; loss_type='softmax'\n",
        "bert_model_path='output/CovidFAQ/models/' + loss_type + '_' + neg_type + '_' + query_type + '_' + version"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nXbORhXguIpu"
      },
      "source": [
        "# create instance of ReRanker class\n",
        "r = ReRanker(\n",
        "    bert_model_path=bert_model_path, \n",
        "    test_queries=test_queries, relevance_label_df=relevance_label_df,\n",
        "    rank_field=rank_field\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JxveUtZbzlnb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3e8b16c2-82a0-454a-ae32-dbb6b1a1905c"
      },
      "source": [
        "# generate directory structure\n",
        "pred_output_path = output_path + \"/supervised/\" + rank_field + \"/\" + loss_type + \"/\" + query_type + \"/\" + neg_type\n",
        "make_dirs(pred_output_path)\n",
        "\n",
        "# next, generate BERT, Re-ranked top-k results and dump to files\n",
        "bert_query_by_question = r.get_bert_topk_preds(es_query_by_question)\n",
        "dump_to_json(bert_query_by_question, pred_output_path + '/bert_query_by_question.json')\n",
        "\n",
        "bert_query_by_answer = r.get_bert_topk_preds(es_query_by_answer)\n",
        "dump_to_json(bert_query_by_answer, pred_output_path + '/bert_query_by_answer.json')\n",
        "\n",
        "bert_query_by_question_answer = r.get_bert_topk_preds(es_query_by_question_answer)\n",
        "dump_to_json(bert_query_by_question_answer, pred_output_path + '/bert_query_by_question_answer.json')\n",
        "\n",
        "bert_query_by_question_answer_concat = r.get_bert_topk_preds(es_query_by_question_answer_concat)\n",
        "dump_to_json(bert_query_by_question_answer_concat, pred_output_path + '/bert_query_by_question_answer_concat.json')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-03 11:14:23 - Generating BERT top-k results ...\n",
            "2021-04-03 11:14:33 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [04:08<00:00,  4.34it/s]\n",
            "2021-04-03 11:18:42 - Generating BERT top-k results ...\n",
            "2021-04-03 11:18:44 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [07:20<00:00,  2.45it/s]\n",
            "2021-04-03 11:26:05 - Generating BERT top-k results ...\n",
            "2021-04-03 11:26:07 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [07:49<00:00,  2.30it/s]\n",
            "2021-04-03 11:33:58 - Generating BERT top-k results ...\n",
            "2021-04-03 11:34:00 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [07:49<00:00,  2.30it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "toLHtw4zugOk"
      },
      "source": [
        "**query_type=\"user_query\"; neg_type=\"simple\"; loss_type='softmax'**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SBPKGt7ktRLQ"
      },
      "source": [
        "# define variables\n",
        "query_type=\"user_query\"; neg_type=\"simple\"; version=\"1.1\"; loss_type='softmax'\n",
        "bert_model_path='output/CovidFAQ/models/' + loss_type + '_' + neg_type + '_' + query_type + '_' + version"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8zMO_kPHtRBO"
      },
      "source": [
        "# create instance of ReRanker class\n",
        "r = ReRanker(\n",
        "    bert_model_path=bert_model_path, \n",
        "    test_queries=test_queries, relevance_label_df=relevance_label_df,\n",
        "    rank_field=rank_field\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8vUWF6aUzmvS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d6fe0a73-1af3-47e7-d648-55a5114854d5"
      },
      "source": [
        "# generate directory structure\n",
        "pred_output_path = output_path + \"/supervised/\" + rank_field + \"/\" + loss_type + \"/\" + query_type + \"/\" + neg_type\n",
        "make_dirs(pred_output_path)\n",
        "\n",
        "# next, generate BERT, Re-ranked top-k results and dump to files\n",
        "bert_query_by_question = r.get_bert_topk_preds(es_query_by_question)\n",
        "dump_to_json(bert_query_by_question, pred_output_path + '/bert_query_by_question.json')\n",
        "\n",
        "bert_query_by_answer = r.get_bert_topk_preds(es_query_by_answer)\n",
        "dump_to_json(bert_query_by_answer, pred_output_path + '/bert_query_by_answer.json')\n",
        "\n",
        "bert_query_by_question_answer = r.get_bert_topk_preds(es_query_by_question_answer)\n",
        "dump_to_json(bert_query_by_question_answer, pred_output_path + '/bert_query_by_question_answer.json')\n",
        "\n",
        "bert_query_by_question_answer_concat = r.get_bert_topk_preds(es_query_by_question_answer_concat)\n",
        "dump_to_json(bert_query_by_question_answer_concat, pred_output_path + '/bert_query_by_question_answer_concat.json')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-03 11:41:51 - Generating BERT top-k results ...\n",
            "2021-04-03 11:42:04 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [04:08<00:00,  4.33it/s]\n",
            "2021-04-03 11:46:13 - Generating BERT top-k results ...\n",
            "2021-04-03 11:46:15 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [07:21<00:00,  2.44it/s]\n",
            "2021-04-03 11:53:38 - Generating BERT top-k results ...\n",
            "2021-04-03 11:53:40 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [07:49<00:00,  2.30it/s]\n",
            "2021-04-03 12:01:31 - Generating BERT top-k results ...\n",
            "2021-04-03 12:01:33 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [07:49<00:00,  2.30it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lNDZBk_GvM1z"
      },
      "source": [
        "**query_type=\"faq\"; neg_type=\"hard\"; loss_type='softmax'**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VmtGmXzrtQda"
      },
      "source": [
        "# define variables\n",
        "query_type=\"faq\"; neg_type=\"hard\"; version=\"1.1\"; loss_type='softmax'\n",
        "bert_model_path='output/CovidFAQ/models/' + loss_type + '_' + neg_type + '_' + query_type + '_' + version"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pzg0l8rEtQZh"
      },
      "source": [
        "# create instance of ReRanker class\n",
        "r = ReRanker(\n",
        "    bert_model_path=bert_model_path, \n",
        "    test_queries=test_queries, relevance_label_df=relevance_label_df,\n",
        "    rank_field=rank_field\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Q_F0a8Qzon2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5307fd6e-c7b7-448d-ddb9-766ed5236789"
      },
      "source": [
        "# generate directory structure\n",
        "pred_output_path = output_path + \"/supervised/\" + rank_field + \"/\" + loss_type + \"/\" + query_type + \"/\" + neg_type\n",
        "make_dirs(pred_output_path)\n",
        "\n",
        "# next, generate BERT, Re-ranked top-k results and dump to files\n",
        "bert_query_by_question = r.get_bert_topk_preds(es_query_by_question)\n",
        "dump_to_json(bert_query_by_question, pred_output_path + '/bert_query_by_question.json')\n",
        "\n",
        "bert_query_by_answer = r.get_bert_topk_preds(es_query_by_answer)\n",
        "dump_to_json(bert_query_by_answer, pred_output_path + '/bert_query_by_answer.json')\n",
        "\n",
        "bert_query_by_question_answer = r.get_bert_topk_preds(es_query_by_question_answer)\n",
        "dump_to_json(bert_query_by_question_answer, pred_output_path + '/bert_query_by_question_answer.json')\n",
        "\n",
        "bert_query_by_question_answer_concat = r.get_bert_topk_preds(es_query_by_question_answer_concat)\n",
        "dump_to_json(bert_query_by_question_answer_concat, pred_output_path + '/bert_query_by_question_answer_concat.json')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-03 12:09:24 - Generating BERT top-k results ...\n",
            "2021-04-03 12:09:35 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [04:07<00:00,  4.36it/s]\n",
            "2021-04-03 12:13:42 - Generating BERT top-k results ...\n",
            "2021-04-03 12:13:44 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [07:20<00:00,  2.45it/s]\n",
            "2021-04-03 12:21:06 - Generating BERT top-k results ...\n",
            "2021-04-03 12:21:08 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [07:49<00:00,  2.29it/s]\n",
            "2021-04-03 12:28:59 - Generating BERT top-k results ...\n",
            "2021-04-03 12:29:01 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [07:50<00:00,  2.29it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o2EgbhAkvgUc"
      },
      "source": [
        "**query_type=\"faq\"; neg_type=\"simple\"; loss_type='softmax'**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4tUfy5fOtP2M"
      },
      "source": [
        "# define variables\n",
        "query_type=\"faq\"; neg_type=\"simple\"; version=\"1.1\"; loss_type='softmax'\n",
        "bert_model_path='output/CovidFAQ/models/' + loss_type + '_' + neg_type + '_' + query_type + '_' + version"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x94VfKuitPva"
      },
      "source": [
        "# create instance of ReRanker class\n",
        "r = ReRanker(\n",
        "    bert_model_path=bert_model_path, \n",
        "    test_queries=test_queries, relevance_label_df=relevance_label_df,\n",
        "    rank_field=rank_field\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PzcVcYUwzqLm",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "545a3cf6-ef94-4d9a-a667-96b393e07e3c"
      },
      "source": [
        "# generate directory structure\n",
        "pred_output_path = output_path + \"/supervised/\" + rank_field + \"/\" + loss_type + \"/\" + query_type + \"/\" + neg_type\n",
        "make_dirs(pred_output_path)\n",
        "\n",
        "# next, generate BERT, Re-ranked top-k results and dump to files\n",
        "bert_query_by_question = r.get_bert_topk_preds(es_query_by_question)\n",
        "dump_to_json(bert_query_by_question, pred_output_path + '/bert_query_by_question.json')\n",
        "\n",
        "bert_query_by_answer = r.get_bert_topk_preds(es_query_by_answer)\n",
        "dump_to_json(bert_query_by_answer, pred_output_path + '/bert_query_by_answer.json')\n",
        "\n",
        "bert_query_by_question_answer = r.get_bert_topk_preds(es_query_by_question_answer)\n",
        "dump_to_json(bert_query_by_question_answer, pred_output_path + '/bert_query_by_question_answer.json')\n",
        "\n",
        "bert_query_by_question_answer_concat = r.get_bert_topk_preds(es_query_by_question_answer_concat)\n",
        "dump_to_json(bert_query_by_question_answer_concat, pred_output_path + '/bert_query_by_question_answer_concat.json')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-04 04:47:10 - Generating BERT top-k results ...\n",
            "2021-04-04 04:47:23 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [04:14<00:00,  4.23it/s]\n",
            "2021-04-04 04:51:38 - Generating BERT top-k results ...\n",
            "2021-04-04 04:51:40 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [07:16<00:00,  2.47it/s]\n",
            "2021-04-04 04:58:58 - Generating BERT top-k results ...\n",
            "2021-04-04 04:59:00 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [07:44<00:00,  2.32it/s]\n",
            "2021-04-04 05:06:46 - Generating BERT top-k results ...\n",
            "2021-04-04 05:06:47 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [07:50<00:00,  2.29it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IRyZyUa9kTKP"
      },
      "source": [
        "**2. Generating BERT prediction results from Question (BERT-Q-q)\"**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JYQ_Ixk4kZp6"
      },
      "source": [
        "# define rank_field parameter\n",
        "rank_field=\"BERT-Q-q\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EZXbr22mlcPA"
      },
      "source": [
        "**query_type=\"user_query\"; neg_type=\"hard\"; loss_type='triplet'**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GkAAGDUrlWKr"
      },
      "source": [
        "# define variables\n",
        "query_type=\"user_query\"; neg_type=\"hard\"; version=\"1.1\"; loss_type='triplet'\n",
        "bert_model_path='output/CovidFAQ/models/' + loss_type + '_' + neg_type + '_' + query_type + '_' + version"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xXmZkX1mlWDG"
      },
      "source": [
        "# create instance of ReRanker class\n",
        "r = ReRanker(\n",
        "    bert_model_path=bert_model_path, \n",
        "    test_queries=test_queries, relevance_label_df=relevance_label_df,\n",
        "    rank_field=rank_field\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "47GyLe1BlV7l",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1fce8c14-d3a0-4b1d-a072-4ff1515c43e0"
      },
      "source": [
        "# generate directory structure\n",
        "pred_output_path = output_path + \"/supervised/\" + rank_field + \"/\" + loss_type + \"/\" + query_type + \"/\" + neg_type\n",
        "make_dirs(pred_output_path)\n",
        "\n",
        "# next, generate BERT, Re-ranked top-k results and dump to files\n",
        "bert_query_by_question = r.get_bert_topk_preds(es_query_by_question)\n",
        "dump_to_json(bert_query_by_question, pred_output_path + '/bert_query_by_question.json')\n",
        "\n",
        "bert_query_by_answer = r.get_bert_topk_preds(es_query_by_answer)\n",
        "dump_to_json(bert_query_by_answer, pred_output_path + '/bert_query_by_answer.json')\n",
        "\n",
        "bert_query_by_question_answer = r.get_bert_topk_preds(es_query_by_question_answer)\n",
        "dump_to_json(bert_query_by_question_answer, pred_output_path + '/bert_query_by_question_answer.json')\n",
        "\n",
        "bert_query_by_question_answer_concat = r.get_bert_topk_preds(es_query_by_question_answer_concat)\n",
        "dump_to_json(bert_query_by_question_answer_concat, pred_output_path + '/bert_query_by_question_answer_concat.json')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-04 05:14:40 - Generating BERT top-k results ...\n",
            "2021-04-04 05:14:40 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_hard_user_query_1.1\n",
            "2021-04-04 05:14:40 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_hard_user_query_1.1\n",
            "2021-04-04 05:14:52 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [03:54<00:00,  4.59it/s]\n",
            "2021-04-04 05:18:47 - Generating BERT top-k results ...\n",
            "2021-04-04 05:18:47 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_hard_user_query_1.1\n",
            "2021-04-04 05:18:47 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_hard_user_query_1.1\n",
            "2021-04-04 05:18:49 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [06:48<00:00,  2.64it/s]\n",
            "2021-04-04 05:25:39 - Generating BERT top-k results ...\n",
            "2021-04-04 05:25:39 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_hard_user_query_1.1\n",
            "2021-04-04 05:25:39 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_hard_user_query_1.1\n",
            "2021-04-04 05:25:40 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [07:18<00:00,  2.46it/s]\n",
            "2021-04-04 05:33:00 - Generating BERT top-k results ...\n",
            "2021-04-04 05:33:00 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_hard_user_query_1.1\n",
            "2021-04-04 05:33:00 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_hard_user_query_1.1\n",
            "2021-04-04 05:33:01 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [07:21<00:00,  2.44it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tM5qc6jrl1DE"
      },
      "source": [
        "**query_type=\"user_query\"; neg_type=\"simple\"; loss_type='triplet'**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uW6EmCdAlVzK"
      },
      "source": [
        "# define variables\n",
        "query_type=\"user_query\"; neg_type=\"simple\"; version=\"1.1\"; loss_type='triplet'\n",
        "bert_model_path='output/CovidFAQ/models/' + loss_type + '_' + neg_type + '_' + query_type + '_' + version"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z6poNdx0lVrh"
      },
      "source": [
        "# create instance of ReRanker class\n",
        "r = ReRanker(\n",
        "    bert_model_path=bert_model_path, \n",
        "    test_queries=test_queries, relevance_label_df=relevance_label_df,\n",
        "    rank_field=rank_field\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kOJ8nTMslVkJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9dd38f36-e1f7-46d6-d63f-bf78045e3a3b"
      },
      "source": [
        "# generate directory structure\n",
        "pred_output_path = output_path + \"/supervised/\" + rank_field + \"/\" + loss_type + \"/\" + query_type + \"/\" + neg_type\n",
        "make_dirs(pred_output_path)\n",
        "\n",
        "# next, generate BERT, Re-ranked top-k results and dump to files\n",
        "bert_query_by_question = r.get_bert_topk_preds(es_query_by_question)\n",
        "dump_to_json(bert_query_by_question, pred_output_path + '/bert_query_by_question.json')\n",
        "\n",
        "bert_query_by_answer = r.get_bert_topk_preds(es_query_by_answer)\n",
        "dump_to_json(bert_query_by_answer, pred_output_path + '/bert_query_by_answer.json')\n",
        "\n",
        "bert_query_by_question_answer = r.get_bert_topk_preds(es_query_by_question_answer)\n",
        "dump_to_json(bert_query_by_question_answer, pred_output_path + '/bert_query_by_question_answer.json')\n",
        "\n",
        "bert_query_by_question_answer_concat = r.get_bert_topk_preds(es_query_by_question_answer_concat)\n",
        "dump_to_json(bert_query_by_question_answer_concat, pred_output_path + '/bert_query_by_question_answer_concat.json')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-04 05:40:24 - Generating BERT top-k results ...\n",
            "2021-04-04 05:40:24 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_simple_user_query_1.1\n",
            "2021-04-04 05:40:24 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_simple_user_query_1.1\n",
            "2021-04-04 05:40:37 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [03:57<00:00,  4.54it/s]\n",
            "2021-04-04 05:44:35 - Generating BERT top-k results ...\n",
            "2021-04-04 05:44:35 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_simple_user_query_1.1\n",
            "2021-04-04 05:44:35 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_simple_user_query_1.1\n",
            "2021-04-04 05:44:36 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [06:49<00:00,  2.63it/s]\n",
            "2021-04-04 05:51:26 - Generating BERT top-k results ...\n",
            "2021-04-04 05:51:26 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_simple_user_query_1.1\n",
            "2021-04-04 05:51:26 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_simple_user_query_1.1\n",
            "2021-04-04 05:51:28 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [07:20<00:00,  2.44it/s]\n",
            "2021-04-04 05:58:50 - Generating BERT top-k results ...\n",
            "2021-04-04 05:58:50 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_simple_user_query_1.1\n",
            "2021-04-04 05:58:50 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_simple_user_query_1.1\n",
            "2021-04-04 05:58:51 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [07:25<00:00,  2.42it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YycMuPkZoHSJ"
      },
      "source": [
        "**query_type=\"faq\"; neg_type=\"hard\"; loss_type='triplet'**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1V5_byZblVT0"
      },
      "source": [
        "# define variables\n",
        "query_type=\"faq\"; neg_type=\"hard\"; version=\"1.1\"; loss_type='triplet'\n",
        "bert_model_path='output/CovidFAQ/models/' + loss_type + '_' + neg_type + '_' + query_type + '_' + version"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "leu15hKDlVMo"
      },
      "source": [
        "# create instance of ReRanker class\n",
        "r = ReRanker(\n",
        "    bert_model_path=bert_model_path, \n",
        "    test_queries=test_queries, relevance_label_df=relevance_label_df,\n",
        "    rank_field=rank_field\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LDdzleOUlVFB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "96693252-50bc-48d2-f02f-a60228b33a05"
      },
      "source": [
        "# generate directory structure\n",
        "pred_output_path = output_path + \"/supervised/\" + rank_field + \"/\" + loss_type + \"/\" + query_type + \"/\" + neg_type\n",
        "make_dirs(pred_output_path)\n",
        "\n",
        "# next, generate BERT, Re-ranked top-k results and dump to files\n",
        "bert_query_by_question = r.get_bert_topk_preds(es_query_by_question)\n",
        "dump_to_json(bert_query_by_question, pred_output_path + '/bert_query_by_question.json')\n",
        "\n",
        "bert_query_by_answer = r.get_bert_topk_preds(es_query_by_answer)\n",
        "dump_to_json(bert_query_by_answer, pred_output_path + '/bert_query_by_answer.json')\n",
        "\n",
        "bert_query_by_question_answer = r.get_bert_topk_preds(es_query_by_question_answer)\n",
        "dump_to_json(bert_query_by_question_answer, pred_output_path + '/bert_query_by_question_answer.json')\n",
        "\n",
        "bert_query_by_question_answer_concat = r.get_bert_topk_preds(es_query_by_question_answer_concat)\n",
        "dump_to_json(bert_query_by_question_answer_concat, pred_output_path + '/bert_query_by_question_answer_concat.json')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-04 06:06:18 - Generating BERT top-k results ...\n",
            "2021-04-04 06:06:18 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_hard_faq_1.1\n",
            "2021-04-04 06:06:18 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_hard_faq_1.1\n",
            "2021-04-04 06:06:30 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [03:56<00:00,  4.55it/s]\n",
            "2021-04-04 06:10:27 - Generating BERT top-k results ...\n",
            "2021-04-04 06:10:27 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_hard_faq_1.1\n",
            "2021-04-04 06:10:27 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_hard_faq_1.1\n",
            "2021-04-04 06:10:28 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [06:52<00:00,  2.61it/s]\n",
            "2021-04-04 06:17:22 - Generating BERT top-k results ...\n",
            "2021-04-04 06:17:22 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_hard_faq_1.1\n",
            "2021-04-04 06:17:22 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_hard_faq_1.1\n",
            "2021-04-04 06:17:23 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [07:24<00:00,  2.43it/s]\n",
            "2021-04-04 06:24:49 - Generating BERT top-k results ...\n",
            "2021-04-04 06:24:49 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_hard_faq_1.1\n",
            "2021-04-04 06:24:49 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_hard_faq_1.1\n",
            "2021-04-04 06:24:50 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [07:22<00:00,  2.44it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ugHFLDp3oLKm"
      },
      "source": [
        "**query_type=\"faq\"; neg_type=\"simple\"; loss_type='triplet'**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0eEmYNyYlU5h"
      },
      "source": [
        "# define variables\n",
        "query_type=\"faq\"; neg_type=\"simple\"; version=\"1.1\"; loss_type='triplet'\n",
        "bert_model_path='output/CovidFAQ/models/' + loss_type + '_' + neg_type + '_' + query_type + '_' + version"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eZhB3zuSlUyb"
      },
      "source": [
        "# create instance of ReRanker class\n",
        "r = ReRanker(\n",
        "    bert_model_path=bert_model_path, \n",
        "    test_queries=test_queries, relevance_label_df=relevance_label_df,\n",
        "    rank_field=rank_field\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qee6WH2blUqi",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6617923b-f7a4-4879-b4e5-0c76cda5d4c5"
      },
      "source": [
        "# generate directory structure\n",
        "pred_output_path = output_path + \"/supervised/\" + rank_field + \"/\" + loss_type + \"/\" + query_type + \"/\" + neg_type\n",
        "make_dirs(pred_output_path)\n",
        "\n",
        "# next, generate BERT, Re-ranked top-k results and dump to files\n",
        "bert_query_by_question = r.get_bert_topk_preds(es_query_by_question)\n",
        "dump_to_json(bert_query_by_question, pred_output_path + '/bert_query_by_question.json')\n",
        "\n",
        "bert_query_by_answer = r.get_bert_topk_preds(es_query_by_answer)\n",
        "dump_to_json(bert_query_by_answer, pred_output_path + '/bert_query_by_answer.json')\n",
        "\n",
        "bert_query_by_question_answer = r.get_bert_topk_preds(es_query_by_question_answer)\n",
        "dump_to_json(bert_query_by_question_answer, pred_output_path + '/bert_query_by_question_answer.json')\n",
        "\n",
        "bert_query_by_question_answer_concat = r.get_bert_topk_preds(es_query_by_question_answer_concat)\n",
        "dump_to_json(bert_query_by_question_answer_concat, pred_output_path + '/bert_query_by_question_answer_concat.json')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-04 06:32:14 - Generating BERT top-k results ...\n",
            "2021-04-04 06:32:14 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_simple_faq_1.1\n",
            "2021-04-04 06:32:14 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_simple_faq_1.1\n",
            "2021-04-04 06:32:25 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [03:56<00:00,  4.55it/s]\n",
            "2021-04-04 06:36:22 - Generating BERT top-k results ...\n",
            "2021-04-04 06:36:22 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_simple_faq_1.1\n",
            "2021-04-04 06:36:22 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_simple_faq_1.1\n",
            "2021-04-04 06:36:24 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [06:50<00:00,  2.63it/s]\n",
            "2021-04-04 06:43:15 - Generating BERT top-k results ...\n",
            "2021-04-04 06:43:15 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_simple_faq_1.1\n",
            "2021-04-04 06:43:15 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_simple_faq_1.1\n",
            "2021-04-04 06:43:17 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [07:23<00:00,  2.43it/s]\n",
            "2021-04-04 06:50:41 - Generating BERT top-k results ...\n",
            "2021-04-04 06:50:41 - Load pretrained SentenceTransformer: output/CovidFAQ/models/triplet_simple_faq_1.1\n",
            "2021-04-04 06:50:41 - Load SentenceTransformer from folder: output/CovidFAQ/models/triplet_simple_faq_1.1\n",
            "2021-04-04 06:50:43 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [07:17<00:00,  2.46it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DqE2Blb3oPw1"
      },
      "source": [
        "**query_type=\"user_query\"; neg_type=\"hard\"; loss_type='softmax'**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C8noZJecmltk"
      },
      "source": [
        "# define variables\n",
        "query_type=\"user_query\"; neg_type=\"hard\"; version=\"1.1\"; loss_type='softmax'\n",
        "bert_model_path='output/CovidFAQ/models/' + loss_type + '_' + neg_type + '_' + query_type + '_' + version"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U_iwsKK6mljZ"
      },
      "source": [
        "# create instance of ReRanker class\n",
        "r = ReRanker(\n",
        "    bert_model_path=bert_model_path, \n",
        "    test_queries=test_queries, relevance_label_df=relevance_label_df,\n",
        "    rank_field=rank_field\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kkfY7SAAmlZy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "be62e021-714c-4e59-cd61-e012ae530713"
      },
      "source": [
        "# generate directory structure\n",
        "pred_output_path = output_path + \"/supervised/\" + rank_field + \"/\" + loss_type + \"/\" + query_type + \"/\" + neg_type\n",
        "make_dirs(pred_output_path)\n",
        "\n",
        "# next, generate BERT, Re-ranked top-k results and dump to files\n",
        "bert_query_by_question = r.get_bert_topk_preds(es_query_by_question)\n",
        "dump_to_json(bert_query_by_question, pred_output_path + '/bert_query_by_question.json')\n",
        "\n",
        "bert_query_by_answer = r.get_bert_topk_preds(es_query_by_answer)\n",
        "dump_to_json(bert_query_by_answer, pred_output_path + '/bert_query_by_answer.json')\n",
        "\n",
        "bert_query_by_question_answer = r.get_bert_topk_preds(es_query_by_question_answer)\n",
        "dump_to_json(bert_query_by_question_answer, pred_output_path + '/bert_query_by_question_answer.json')\n",
        "\n",
        "bert_query_by_question_answer_concat = r.get_bert_topk_preds(es_query_by_question_answer_concat)\n",
        "dump_to_json(bert_query_by_question_answer_concat, pred_output_path + '/bert_query_by_question_answer_concat.json')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-04 06:58:02 - Generating BERT top-k results ...\n",
            "2021-04-04 06:58:12 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [03:43<00:00,  4.83it/s]\n",
            "2021-04-04 07:01:56 - Generating BERT top-k results ...\n",
            "2021-04-04 07:01:58 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [06:26<00:00,  2.79it/s]\n",
            "2021-04-04 07:08:26 - Generating BERT top-k results ...\n",
            "2021-04-04 07:08:27 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [06:54<00:00,  2.60it/s]\n",
            "2021-04-04 07:15:23 - Generating BERT top-k results ...\n",
            "2021-04-04 07:15:25 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [06:53<00:00,  2.61it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0lO2b2wxoUWS"
      },
      "source": [
        "**query_type=\"user_query\"; neg_type=\"simple\"; loss_type='softmax'**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lctMrtS0mlHO"
      },
      "source": [
        "# define variables\n",
        "query_type=\"user_query\"; neg_type=\"simple\"; version=\"1.1\"; loss_type='softmax'\n",
        "bert_model_path='output/CovidFAQ/models/' + loss_type + '_' + neg_type + '_' + query_type + '_' + version"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H-NvD-Sgmk9l"
      },
      "source": [
        "# create instance of ReRanker class\n",
        "r = ReRanker(\n",
        "    bert_model_path=bert_model_path, \n",
        "    test_queries=test_queries, relevance_label_df=relevance_label_df,\n",
        "    rank_field=rank_field\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8sIIkgKhmk4C",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c22f1435-da4a-4661-8f81-e5160a197408"
      },
      "source": [
        "# generate directory structure\n",
        "pred_output_path = output_path + \"/supervised/\" + rank_field + \"/\" + loss_type + \"/\" + query_type + \"/\" + neg_type\n",
        "make_dirs(pred_output_path)\n",
        "\n",
        "# next, generate BERT, Re-ranked top-k results and dump to files\n",
        "bert_query_by_question = r.get_bert_topk_preds(es_query_by_question)\n",
        "dump_to_json(bert_query_by_question, pred_output_path + '/bert_query_by_question.json')\n",
        "\n",
        "bert_query_by_answer = r.get_bert_topk_preds(es_query_by_answer)\n",
        "dump_to_json(bert_query_by_answer, pred_output_path + '/bert_query_by_answer.json')\n",
        "\n",
        "bert_query_by_question_answer = r.get_bert_topk_preds(es_query_by_question_answer)\n",
        "dump_to_json(bert_query_by_question_answer, pred_output_path + '/bert_query_by_question_answer.json')\n",
        "\n",
        "bert_query_by_question_answer_concat = r.get_bert_topk_preds(es_query_by_question_answer_concat)\n",
        "dump_to_json(bert_query_by_question_answer_concat, pred_output_path + '/bert_query_by_question_answer_concat.json')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-04 07:22:20 - Generating BERT top-k results ...\n",
            "2021-04-04 07:22:31 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [03:43<00:00,  4.83it/s]\n",
            "2021-04-04 07:26:15 - Generating BERT top-k results ...\n",
            "2021-04-04 07:26:17 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [06:21<00:00,  2.83it/s]\n",
            "2021-04-04 07:32:39 - Generating BERT top-k results ...\n",
            "2021-04-04 07:32:41 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [06:55<00:00,  2.60it/s]\n",
            "2021-04-04 07:39:38 - Generating BERT top-k results ...\n",
            "2021-04-04 07:39:40 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [06:53<00:00,  2.61it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hcXgJnWboXMy"
      },
      "source": [
        "**query_type=\"faq\"; neg_type=\"hard\"; loss_type='softmax'**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uU_9X36LnNNi"
      },
      "source": [
        "# define variables\n",
        "query_type=\"faq\"; neg_type=\"hard\"; version=\"1.1\"; loss_type='softmax'\n",
        "bert_model_path='output/CovidFAQ/models/' + loss_type + '_' + neg_type + '_' + query_type + '_' + version"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KdUQo5RznNCE"
      },
      "source": [
        "# create instance of ReRanker class\n",
        "r = ReRanker(\n",
        "    bert_model_path=bert_model_path, \n",
        "    test_queries=test_queries, relevance_label_df=relevance_label_df,\n",
        "    rank_field=rank_field\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vjDLglyJnM28",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "00175dce-5db9-44b1-9677-ede302ceeea7"
      },
      "source": [
        "# generate directory structure\n",
        "pred_output_path = output_path + \"/supervised/\" + rank_field + \"/\" + loss_type + \"/\" + query_type + \"/\" + neg_type\n",
        "make_dirs(pred_output_path)\n",
        "\n",
        "# next, generate BERT, Re-ranked top-k results and dump to files\n",
        "bert_query_by_question = r.get_bert_topk_preds(es_query_by_question)\n",
        "dump_to_json(bert_query_by_question, pred_output_path + '/bert_query_by_question.json')\n",
        "\n",
        "bert_query_by_answer = r.get_bert_topk_preds(es_query_by_answer)\n",
        "dump_to_json(bert_query_by_answer, pred_output_path + '/bert_query_by_answer.json')\n",
        "\n",
        "bert_query_by_question_answer = r.get_bert_topk_preds(es_query_by_question_answer)\n",
        "dump_to_json(bert_query_by_question_answer, pred_output_path + '/bert_query_by_question_answer.json')\n",
        "\n",
        "bert_query_by_question_answer_concat = r.get_bert_topk_preds(es_query_by_question_answer_concat)\n",
        "dump_to_json(bert_query_by_question_answer_concat, pred_output_path + '/bert_query_by_question_answer_concat.json')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-04 07:46:34 - Generating BERT top-k results ...\n",
            "2021-04-04 07:46:46 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [03:42<00:00,  4.84it/s]\n",
            "2021-04-04 07:50:29 - Generating BERT top-k results ...\n",
            "2021-04-04 07:50:31 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [06:25<00:00,  2.80it/s]\n",
            "2021-04-04 07:56:57 - Generating BERT top-k results ...\n",
            "2021-04-04 07:56:59 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [06:53<00:00,  2.60it/s]\n",
            "2021-04-04 08:03:54 - Generating BERT top-k results ...\n",
            "2021-04-04 08:03:56 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [06:58<00:00,  2.58it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZubZiiwqobdT"
      },
      "source": [
        "**query_type=\"faq\"; neg_type=\"simple\"; loss_type='softmax'**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aEoVIgvQnMf-"
      },
      "source": [
        "# define variables\n",
        "query_type=\"faq\"; neg_type=\"simple\"; version=\"1.1\"; loss_type='softmax'\n",
        "bert_model_path='output/CovidFAQ/models/' + loss_type + '_' + neg_type + '_' + query_type + '_' + version"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZaVyBLM9nMUn"
      },
      "source": [
        "# create instance of ReRanker class\n",
        "r = ReRanker(\n",
        "    bert_model_path=bert_model_path, \n",
        "    test_queries=test_queries, relevance_label_df=relevance_label_df,\n",
        "    rank_field=rank_field\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ydEelLV-nMIn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "74376544-158c-46ca-8fba-6cef8f0214c4"
      },
      "source": [
        "# generate directory structure\n",
        "pred_output_path = output_path + \"/supervised/\" + rank_field + \"/\" + loss_type + \"/\" + query_type + \"/\" + neg_type\n",
        "make_dirs(pred_output_path)\n",
        "\n",
        "# next, generate BERT, Re-ranked top-k results and dump to files\n",
        "bert_query_by_question = r.get_bert_topk_preds(es_query_by_question)\n",
        "dump_to_json(bert_query_by_question, pred_output_path + '/bert_query_by_question.json')\n",
        "\n",
        "bert_query_by_answer = r.get_bert_topk_preds(es_query_by_answer)\n",
        "dump_to_json(bert_query_by_answer, pred_output_path + '/bert_query_by_answer.json')\n",
        "\n",
        "bert_query_by_question_answer = r.get_bert_topk_preds(es_query_by_question_answer)\n",
        "dump_to_json(bert_query_by_question_answer, pred_output_path + '/bert_query_by_question_answer.json')\n",
        "\n",
        "bert_query_by_question_answer_concat = r.get_bert_topk_preds(es_query_by_question_answer_concat)\n",
        "dump_to_json(bert_query_by_question_answer_concat, pred_output_path + '/bert_query_by_question_answer_concat.json')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-04 08:10:56 - Generating BERT top-k results ...\n",
            "2021-04-04 08:10:58 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [03:42<00:00,  4.84it/s]\n",
            "2021-04-04 08:14:41 - Generating BERT top-k results ...\n",
            "2021-04-04 08:14:43 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [06:25<00:00,  2.79it/s]\n",
            "2021-04-04 08:21:10 - Generating BERT top-k results ...\n",
            "2021-04-04 08:21:12 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [06:54<00:00,  2.60it/s]\n",
            "2021-04-04 08:28:08 - Generating BERT top-k results ...\n",
            "2021-04-04 08:28:10 - Use pytorch device: cuda\n",
            "100%|██████████| 1078/1078 [06:52<00:00,  2.61it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HbmSw6KqVuEH"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}