notebook/StackFAQ/02.Ingesting_Data_to_Elasticsearch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: elasticsearch in /Users/nattiya/miniconda3/envs/dl/lib/python3.8/site-packages (7.9.1)\n",
      "Requirement already satisfied: urllib3>=1.21.1 in /Users/nattiya/miniconda3/envs/dl/lib/python3.8/site-packages (from elasticsearch) (1.25.10)\n",
      "Requirement already satisfied: certifi in /Users/nattiya/miniconda3/envs/dl/lib/python3.8/site-packages (from elasticsearch) (2020.12.5)\n"
     ]
    }
   ],
   "source": [
    "!pip3 install elasticsearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: elasticsearch-dsl in /Users/nattiya/miniconda3/envs/dl/lib/python3.8/site-packages (7.3.0)\n",
      "Requirement already satisfied: elasticsearch<8.0.0,>=7.0.0 in /Users/nattiya/miniconda3/envs/dl/lib/python3.8/site-packages (from elasticsearch-dsl) (7.9.1)\n",
      "Requirement already satisfied: six in /Users/nattiya/miniconda3/envs/dl/lib/python3.8/site-packages (from elasticsearch-dsl) (1.15.0)\n",
      "Requirement already satisfied: python-dateutil in /Users/nattiya/miniconda3/envs/dl/lib/python3.8/site-packages (from elasticsearch-dsl) (2.8.1)\n",
      "Requirement already satisfied: urllib3>=1.21.1 in /Users/nattiya/miniconda3/envs/dl/lib/python3.8/site-packages (from elasticsearch<8.0.0,>=7.0.0->elasticsearch-dsl) (1.25.10)\n",
      "Requirement already satisfied: certifi in /Users/nattiya/miniconda3/envs/dl/lib/python3.8/site-packages (from elasticsearch<8.0.0,>=7.0.0->elasticsearch-dsl) (2020.12.5)\n"
     ]
    }
   ],
   "source": [
    "!pip3 install elasticsearch-dsl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, '../../../BERT-FAQ/')\n",
    "\n",
    "# import required libraries\n",
    "from elasticsearch_dsl import Index, Document, Integer, Text, analyzer, Keyword, Double\n",
    "from elasticsearch_dsl.connections import connections\n",
    "from elasticsearch import Elasticsearch, helpers\n",
    "\n",
    "from evaluation import get_relevance_label_df\n",
    "from shared.utils import load_from_json\n",
    "from shared.utils import dump_to_json\n",
    "from shared.utils import make_dirs\n",
    "from indexer import ingest_data\n",
    "from indexer import QA\n",
    "import logging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get list of query answer pairs\n",
    "\n",
    "query_answer_pairs_filepath = '../../../BERT-FAQ/data/StackFAQ/query_answer_pairs.json'\n",
    "relevance_label_df = get_relevance_label_df(query_answer_pairs_filepath)\n",
    "faq_qa_pair_df = relevance_label_df[relevance_label_df['query_type'] == 'faq']\n",
    "faq_qa_pairs = faq_qa_pair_df.T.to_dict().values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>query_type</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>How to make font strikethrough on github.</td>\n",
       "      <td>Use ~~tildes around the words~~.</td>\n",
       "      <td>user_query</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Is it possible to get  strikethrough letter fo...</td>\n",
       "      <td>Use ~~tildes around the words~~.</td>\n",
       "      <td>user_query</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Making the text on github crossed out.</td>\n",
       "      <td>Use ~~tildes around the words~~.</td>\n",
       "      <td>user_query</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Introducing stikethrough formatting on markdow...</td>\n",
       "      <td>Use ~~tildes around the words~~.</td>\n",
       "      <td>user_query</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>The &lt;s&gt; tag for font on github markdown doesnt...</td>\n",
       "      <td>Use ~~tildes around the words~~.</td>\n",
       "      <td>user_query</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7898</th>\n",
       "      <td>7899</td>\n",
       "      <td>How do I get an m dash in google docs?</td>\n",
       "      <td>On Mac, just hold fn + option + -</td>\n",
       "      <td>user_query</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7899</th>\n",
       "      <td>7900</td>\n",
       "      <td>Do google docs support the longer style dash (...</td>\n",
       "      <td>On Mac, just hold fn + option + -</td>\n",
       "      <td>user_query</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7900</th>\n",
       "      <td>7901</td>\n",
       "      <td>I desperately need the em dash in google docs ...</td>\n",
       "      <td>On Mac, just hold fn + option + -</td>\n",
       "      <td>user_query</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7901</th>\n",
       "      <td>7902</td>\n",
       "      <td>I require the longer (em) dash in google docs,...</td>\n",
       "      <td>On Mac, just hold fn + option + -</td>\n",
       "      <td>user_query</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7902</th>\n",
       "      <td>7903</td>\n",
       "      <td>How to create an em dash in Google Drive (Goog...</td>\n",
       "      <td>On Mac, just hold fn + option + -</td>\n",
       "      <td>faq</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7903 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                           question  \\\n",
       "0        1          How to make font strikethrough on github.   \n",
       "1        2  Is it possible to get  strikethrough letter fo...   \n",
       "2        3             Making the text on github crossed out.   \n",
       "3        4  Introducing stikethrough formatting on markdow...   \n",
       "4        5  The <s> tag for font on github markdown doesnt...   \n",
       "...    ...                                                ...   \n",
       "7898  7899             How do I get an m dash in google docs?   \n",
       "7899  7900  Do google docs support the longer style dash (...   \n",
       "7900  7901  I desperately need the em dash in google docs ...   \n",
       "7901  7902  I require the longer (em) dash in google docs,...   \n",
       "7902  7903  How to create an em dash in Google Drive (Goog...   \n",
       "\n",
       "                                 answer  query_type  label  \n",
       "0      Use ~~tildes around the words~~.  user_query      1  \n",
       "1      Use ~~tildes around the words~~.  user_query      1  \n",
       "2      Use ~~tildes around the words~~.  user_query      1  \n",
       "3      Use ~~tildes around the words~~.  user_query      1  \n",
       "4      Use ~~tildes around the words~~.  user_query      1  \n",
       "...                                 ...         ...    ...  \n",
       "7898  On Mac, just hold fn + option + -  user_query      1  \n",
       "7899  On Mac, just hold fn + option + -  user_query      1  \n",
       "7900  On Mac, just hold fn + option + -  user_query      1  \n",
       "7901  On Mac, just hold fn + option + -  user_query      1  \n",
       "7902  On Mac, just hold fn + option + -         faq      1  \n",
       "\n",
       "[7903 rows x 5 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check relevance_label_df\n",
    "relevance_label_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>query_type</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11</td>\n",
       "      <td>Strikethrough with GitHub Markdown</td>\n",
       "      <td>Use ~~tildes around the words~~.</td>\n",
       "      <td>faq</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>22</td>\n",
       "      <td>Strikethrough with GitHub Markdown</td>\n",
       "      <td>I just used the following syntax  &lt;del&gt; ... &lt;/...</td>\n",
       "      <td>faq</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>33</td>\n",
       "      <td>How do I delete my Facebook account?</td>\n",
       "      <td>from here :   Go here: http://www.facebook.com...</td>\n",
       "      <td>faq</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>44</td>\n",
       "      <td>How do I delete my Facebook account?</td>\n",
       "      <td>Go to \"Your Account\".  Go to \"Deactivate Accou...</td>\n",
       "      <td>faq</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>55</td>\n",
       "      <td>How do I delete my Facebook account?</td>\n",
       "      <td>It is not neccessary to go through the deactiv...</td>\n",
       "      <td>faq</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7858</th>\n",
       "      <td>7859</td>\n",
       "      <td>Can I trust dropbox and similar services? [clo...</td>\n",
       "      <td>Not sure about Dropbox. But from the Wuala ser...</td>\n",
       "      <td>faq</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7869</th>\n",
       "      <td>7870</td>\n",
       "      <td>How to create an em dash in Google Drive (Goog...</td>\n",
       "      <td>Actually, in Google Docs you can have it autom...</td>\n",
       "      <td>faq</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7880</th>\n",
       "      <td>7881</td>\n",
       "      <td>How to create an em dash in Google Drive (Goog...</td>\n",
       "      <td>Answer by @SamtheBrand is correct, but there a...</td>\n",
       "      <td>faq</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7891</th>\n",
       "      <td>7892</td>\n",
       "      <td>How to create an em dash in Google Drive (Goog...</td>\n",
       "      <td>Insert &gt; Special Characters...   Make sure you...</td>\n",
       "      <td>faq</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7902</th>\n",
       "      <td>7903</td>\n",
       "      <td>How to create an em dash in Google Drive (Goog...</td>\n",
       "      <td>On Mac, just hold fn + option + -</td>\n",
       "      <td>faq</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>719 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                           question  \\\n",
       "10      11                 Strikethrough with GitHub Markdown   \n",
       "21      22                 Strikethrough with GitHub Markdown   \n",
       "32      33               How do I delete my Facebook account?   \n",
       "43      44               How do I delete my Facebook account?   \n",
       "54      55               How do I delete my Facebook account?   \n",
       "...    ...                                                ...   \n",
       "7858  7859  Can I trust dropbox and similar services? [clo...   \n",
       "7869  7870  How to create an em dash in Google Drive (Goog...   \n",
       "7880  7881  How to create an em dash in Google Drive (Goog...   \n",
       "7891  7892  How to create an em dash in Google Drive (Goog...   \n",
       "7902  7903  How to create an em dash in Google Drive (Goog...   \n",
       "\n",
       "                                                 answer query_type  label  \n",
       "10                     Use ~~tildes around the words~~.        faq      1  \n",
       "21    I just used the following syntax  <del> ... </...        faq      1  \n",
       "32    from here :   Go here: http://www.facebook.com...        faq      1  \n",
       "43    Go to \"Your Account\".  Go to \"Deactivate Accou...        faq      1  \n",
       "54    It is not neccessary to go through the deactiv...        faq      1  \n",
       "...                                                 ...        ...    ...  \n",
       "7858  Not sure about Dropbox. But from the Wuala ser...        faq      1  \n",
       "7869  Actually, in Google Docs you can have it autom...        faq      1  \n",
       "7880  Answer by @SamtheBrand is correct, but there a...        faq      1  \n",
       "7891  Insert > Special Characters...   Make sure you...        faq      1  \n",
       "7902                  On Mac, just hold fn + option + -        faq      1  \n",
       "\n",
       "[719 rows x 5 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check faq_qa_pair_df\n",
    "faq_qa_pair_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert to list\n",
    "faq_qa_pairs = list(faq_qa_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'id': '11',\n",
       "  'question': 'Strikethrough with GitHub Markdown',\n",
       "  'answer': 'Use ~~tildes around the words~~.',\n",
       "  'query_type': 'faq',\n",
       "  'label': 1},\n",
       " {'id': '22',\n",
       "  'question': 'Strikethrough with GitHub Markdown',\n",
       "  'answer': 'I just used the following syntax  <del> ... </del>   successfully on GitHub in an issue description.',\n",
       "  'query_type': 'faq',\n",
       "  'label': 1},\n",
       " {'id': '33',\n",
       "  'question': 'How do I delete my Facebook account?',\n",
       "  'answer': 'from here :   Go here: http://www.facebook.com/help/delete_account  Click \"Submit\" and follow the instructions.  Your account will be deactivated for two weeks, and if you DO NOT USE FACEBOOK IN ANY WAY during that period, your account is permanently deleted.',\n",
       "  'query_type': 'faq',\n",
       "  'label': 1},\n",
       " {'id': '44',\n",
       "  'question': 'How do I delete my Facebook account?',\n",
       "  'answer': 'Go to \"Your Account\".  Go to \"Deactivate Account\".  Facebook is a megalomaniac.  It will never ever ever EVER delete your profile.  If you decide ten years later to sign in, then it will still be there.  I\\'d erase my pictures from the site before deleting my account.  Actually, I just did last week.',\n",
       "  'query_type': 'faq',\n",
       "  'label': 1},\n",
       " {'id': '55',\n",
       "  'question': 'How do I delete my Facebook account?',\n",
       "  'answer': 'It is not neccessary to go through the deactivation 2 week wait, you can delete your account directly at:  www.facebook.com/help/delete_account   If you do not think you will use Facebook again and would like your account deleted, we can take care of this for you. Keep in mind that you will not be able to reactivate your account or retrieve any of the content or information you have added.',\n",
       "  'query_type': 'faq',\n",
       "  'label': 1}]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check first 5 entries in faq_qa_pairs\n",
    "faq_qa_pairs[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 719/719 [00:00<00:00, 10197.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished indexing 719 records to stackfaq index\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/nattiya/miniconda3/envs/dl/lib/python3.8/site-packages/elasticsearch/connection/base.py:190: ElasticsearchDeprecationWarning: [types removal] Specifying types in bulk requests is deprecated.\n",
      "  warnings.warn(message, category=ElasticsearchDeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    \n",
    "    es = connections.create_connection(hosts=['localhost'])\n",
    "    \n",
    "    # Index data to Elasticsearch \n",
    "    index_name = \"stackfaq\"\n",
    "    \n",
    "    # Initialize index (only perform once)\n",
    "    index = Index(index_name)\n",
    "\n",
    "    # Define custom settings\n",
    "    index.settings(\n",
    "        number_of_shards=1,\n",
    "        number_of_replicas=0\n",
    "    )\n",
    "\n",
    "    # Delete the index, ignore if it doesn't exist\n",
    "    index.delete(ignore=404)\n",
    "\n",
    "    # Create the index in Elasticsearch\n",
    "    index.create()\n",
    "\n",
    "    # Register a document with the index\n",
    "    index.document(QA)\n",
    "\n",
    "    # Ingest data to Elasticsearch\n",
    "    ingest_data(faq_qa_pairs, es=es, index=index_name)\n",
    "\n",
    "    print(\"Finished indexing {} records to {} index\".format(len(faq_qa_pairs), index_name))\n",
    "\n",
    "except Exception:\n",
    "    logging.error('exception occured', exc_info=True)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}