notebook/CovidFAQ/01.Parsing_aligned_question_question_answer.csv_file.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, \"../../../BERT-FAQ\")\n",
    "\n",
    "from parser.covidfaq import CovidFAQ_Parser\n",
    "from shared.utils import dump_to_json\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>answer</th>\n",
       "      <th>source</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>How quickly can the global economy recover fro...</td>\n",
       "      <td>Can I get COVID-19 from animals when travellin...</td>\n",
       "      <td>Although the current spread and growth of the ...</td>\n",
       "      <td>JHU Public Health Canda Public Health Services</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>How quickly can the global economy recover fro...</td>\n",
       "      <td>How can I protect myself and others?</td>\n",
       "      <td>The best way to prevent illness from COVID-19 ...</td>\n",
       "      <td>JHU Public Health CDC</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How quickly can the global economy recover fro...</td>\n",
       "      <td>Where did COVID-19 come from?</td>\n",
       "      <td>It was first found in Wuhan City, Hubei Provin...</td>\n",
       "      <td>JHU Public Health CDC</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>How quickly can the global economy recover fro...</td>\n",
       "      <td>Can my pet or other animals get sick from COVI...</td>\n",
       "      <td>There is currently no evidence to suggest that...</td>\n",
       "      <td>JHU Public Health Canda Public Health Services</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>How quickly can the global economy recover fro...</td>\n",
       "      <td>How can I protect my child from COVID-19?</td>\n",
       "      <td>By having them practice the same things you ha...</td>\n",
       "      <td>JHU Public Health</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24235</th>\n",
       "      <td>austrian coronavirus</td>\n",
       "      <td>What is novel (new) coronavirus?</td>\n",
       "      <td>A novel, or new, coronavirus is one that has n...</td>\n",
       "      <td>JHU Public Health CDC</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24236</th>\n",
       "      <td>bbc coronavirus espanol</td>\n",
       "      <td>What is a coronavirus?</td>\n",
       "      <td>Coronavirus are a type of virus - there are ma...</td>\n",
       "      <td>JHU Public Health WHO</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24237</th>\n",
       "      <td>bbc coronavirus espanol</td>\n",
       "      <td>What is novel (new) coronavirus?</td>\n",
       "      <td>A novel, or new, coronavirus is one that has n...</td>\n",
       "      <td>JHU Public Health CDC</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24238</th>\n",
       "      <td>bbc coronavirus espanol</td>\n",
       "      <td>Do viruses get weaker over time? Will coronavi...</td>\n",
       "      <td>They can, but it is not guaranteed; it depends...</td>\n",
       "      <td>JHU Public Health ScienceMag</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24239</th>\n",
       "      <td>bbc coronavirus espanol</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24240 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               question1  \\\n",
       "0      How quickly can the global economy recover fro...   \n",
       "1      How quickly can the global economy recover fro...   \n",
       "2      How quickly can the global economy recover fro...   \n",
       "3      How quickly can the global economy recover fro...   \n",
       "4      How quickly can the global economy recover fro...   \n",
       "...                                                  ...   \n",
       "24235                               austrian coronavirus   \n",
       "24236                            bbc coronavirus espanol   \n",
       "24237                            bbc coronavirus espanol   \n",
       "24238                            bbc coronavirus espanol   \n",
       "24239                            bbc coronavirus espanol   \n",
       "\n",
       "                                               question2  \\\n",
       "0      Can I get COVID-19 from animals when travellin...   \n",
       "1                   How can I protect myself and others?   \n",
       "2                          Where did COVID-19 come from?   \n",
       "3      Can my pet or other animals get sick from COVI...   \n",
       "4              How can I protect my child from COVID-19?   \n",
       "...                                                  ...   \n",
       "24235                   What is novel (new) coronavirus?   \n",
       "24236                             What is a coronavirus?   \n",
       "24237                   What is novel (new) coronavirus?   \n",
       "24238  Do viruses get weaker over time? Will coronavi...   \n",
       "24239                                                NaN   \n",
       "\n",
       "                                                  answer  \\\n",
       "0      Although the current spread and growth of the ...   \n",
       "1      The best way to prevent illness from COVID-19 ...   \n",
       "2      It was first found in Wuhan City, Hubei Provin...   \n",
       "3      There is currently no evidence to suggest that...   \n",
       "4      By having them practice the same things you ha...   \n",
       "...                                                  ...   \n",
       "24235  A novel, or new, coronavirus is one that has n...   \n",
       "24236  Coronavirus are a type of virus - there are ma...   \n",
       "24237  A novel, or new, coronavirus is one that has n...   \n",
       "24238  They can, but it is not guaranteed; it depends...   \n",
       "24239                                                NaN   \n",
       "\n",
       "                                               source  rating  \n",
       "0      JHU Public Health Canda Public Health Services     1.0  \n",
       "1                               JHU Public Health CDC     0.0  \n",
       "2                               JHU Public Health CDC     0.0  \n",
       "3      JHU Public Health Canda Public Health Services     0.0  \n",
       "4                                   JHU Public Health     0.0  \n",
       "...                                               ...     ...  \n",
       "24235                           JHU Public Health CDC     0.0  \n",
       "24236                           JHU Public Health WHO     0.0  \n",
       "24237                           JHU Public Health CDC     0.0  \n",
       "24238                    JHU Public Health ScienceMag     0.0  \n",
       "24239                                             NaN     1.0  \n",
       "\n",
       "[24240 rows x 5 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read data as pandas DataFrame\n",
    "\n",
    "df = pd.read_csv(\"../../../BERT-FAQ/data/CovidFAQ/aligned_question_question_answer.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query_string</th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>source</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>How quickly can the global economy recover fro...</td>\n",
       "      <td>Can I get COVID-19 from animals when travellin...</td>\n",
       "      <td>Although the current spread and growth of the ...</td>\n",
       "      <td>JHU Public Health Canda Public Health Services</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>How quickly can the global economy recover fro...</td>\n",
       "      <td>How can I protect myself and others?</td>\n",
       "      <td>The best way to prevent illness from COVID-19 ...</td>\n",
       "      <td>JHU Public Health CDC</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How quickly can the global economy recover fro...</td>\n",
       "      <td>Where did COVID-19 come from?</td>\n",
       "      <td>It was first found in Wuhan City, Hubei Provin...</td>\n",
       "      <td>JHU Public Health CDC</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>How quickly can the global economy recover fro...</td>\n",
       "      <td>Can my pet or other animals get sick from COVI...</td>\n",
       "      <td>There is currently no evidence to suggest that...</td>\n",
       "      <td>JHU Public Health Canda Public Health Services</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>How quickly can the global economy recover fro...</td>\n",
       "      <td>How can I protect my child from COVID-19?</td>\n",
       "      <td>By having them practice the same things you ha...</td>\n",
       "      <td>JHU Public Health</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24235</th>\n",
       "      <td>austrian coronavirus</td>\n",
       "      <td>What is novel (new) coronavirus?</td>\n",
       "      <td>A novel, or new, coronavirus is one that has n...</td>\n",
       "      <td>JHU Public Health CDC</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24236</th>\n",
       "      <td>bbc coronavirus espanol</td>\n",
       "      <td>What is a coronavirus?</td>\n",
       "      <td>Coronavirus are a type of virus - there are ma...</td>\n",
       "      <td>JHU Public Health WHO</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24237</th>\n",
       "      <td>bbc coronavirus espanol</td>\n",
       "      <td>What is novel (new) coronavirus?</td>\n",
       "      <td>A novel, or new, coronavirus is one that has n...</td>\n",
       "      <td>JHU Public Health CDC</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24238</th>\n",
       "      <td>bbc coronavirus espanol</td>\n",
       "      <td>Do viruses get weaker over time? Will coronavi...</td>\n",
       "      <td>They can, but it is not guaranteed; it depends...</td>\n",
       "      <td>JHU Public Health ScienceMag</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24239</th>\n",
       "      <td>bbc coronavirus espanol</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24240 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            query_string  \\\n",
       "0      How quickly can the global economy recover fro...   \n",
       "1      How quickly can the global economy recover fro...   \n",
       "2      How quickly can the global economy recover fro...   \n",
       "3      How quickly can the global economy recover fro...   \n",
       "4      How quickly can the global economy recover fro...   \n",
       "...                                                  ...   \n",
       "24235                               austrian coronavirus   \n",
       "24236                            bbc coronavirus espanol   \n",
       "24237                            bbc coronavirus espanol   \n",
       "24238                            bbc coronavirus espanol   \n",
       "24239                            bbc coronavirus espanol   \n",
       "\n",
       "                                                question  \\\n",
       "0      Can I get COVID-19 from animals when travellin...   \n",
       "1                   How can I protect myself and others?   \n",
       "2                          Where did COVID-19 come from?   \n",
       "3      Can my pet or other animals get sick from COVI...   \n",
       "4              How can I protect my child from COVID-19?   \n",
       "...                                                  ...   \n",
       "24235                   What is novel (new) coronavirus?   \n",
       "24236                             What is a coronavirus?   \n",
       "24237                   What is novel (new) coronavirus?   \n",
       "24238  Do viruses get weaker over time? Will coronavi...   \n",
       "24239                                                NaN   \n",
       "\n",
       "                                                  answer  \\\n",
       "0      Although the current spread and growth of the ...   \n",
       "1      The best way to prevent illness from COVID-19 ...   \n",
       "2      It was first found in Wuhan City, Hubei Provin...   \n",
       "3      There is currently no evidence to suggest that...   \n",
       "4      By having them practice the same things you ha...   \n",
       "...                                                  ...   \n",
       "24235  A novel, or new, coronavirus is one that has n...   \n",
       "24236  Coronavirus are a type of virus - there are ma...   \n",
       "24237  A novel, or new, coronavirus is one that has n...   \n",
       "24238  They can, but it is not guaranteed; it depends...   \n",
       "24239                                                NaN   \n",
       "\n",
       "                                               source  rating  \n",
       "0      JHU Public Health Canda Public Health Services     1.0  \n",
       "1                               JHU Public Health CDC     0.0  \n",
       "2                               JHU Public Health CDC     0.0  \n",
       "3      JHU Public Health Canda Public Health Services     0.0  \n",
       "4                                   JHU Public Health     0.0  \n",
       "...                                               ...     ...  \n",
       "24235                           JHU Public Health CDC     0.0  \n",
       "24236                           JHU Public Health WHO     0.0  \n",
       "24237                           JHU Public Health CDC     0.0  \n",
       "24238                    JHU Public Health ScienceMag     0.0  \n",
       "24239                                             NaN     1.0  \n",
       "\n",
       "[24240 rows x 5 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# rename colnames question1: query_string, qustion2: question\n",
    "df = df.rename(columns={'question1': 'query_string', 'question2': 'question'})\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(24240, 5)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>100.000000</th>\n",
       "      <td>1525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99.833333</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99.714286</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99.500000</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99.333333</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.333333</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.000000</th>\n",
       "      <td>358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.500000</th>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.142857</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.000000</th>\n",
       "      <td>18156</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>231 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            rating\n",
       "100.000000    1525\n",
       "99.833333        1\n",
       "99.714286        1\n",
       "99.500000        1\n",
       "99.333333        1\n",
       "...            ...\n",
       "1.333333         2\n",
       "1.000000       358\n",
       "0.500000        24\n",
       "0.142857         1\n",
       "0.000000     18156\n",
       "\n",
       "[231 rows x 1 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check all rating scores\n",
    "df.rating.value_counts().sort_index(ascending=False).to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query_string</th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>source</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>What does Corona and Covid mean?</td>\n",
       "      <td>What is COVID-19?</td>\n",
       "      <td>COVID-19 is a new coronavirus that we have not...</td>\n",
       "      <td>JHU Public Health</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>when will the social distancing end? and what ...</td>\n",
       "      <td>What is a coronavirus?</td>\n",
       "      <td>Coronavirus are a type of virus - there are ma...</td>\n",
       "      <td>JHU Public Health WHO</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>How do I go grocery shopping?</td>\n",
       "      <td>Any advice about how to minimize risk during g...</td>\n",
       "      <td>Currently, there is no evidence of food or foo...</td>\n",
       "      <td>JHU Public Health FDA</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>what is covid 19</td>\n",
       "      <td>What is COVID-19?</td>\n",
       "      <td>COVID-19 is a new coronavirus that we have not...</td>\n",
       "      <td>JHU Public Health</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>What is COVID-19's definition?</td>\n",
       "      <td>What is COVID-19?</td>\n",
       "      <td>COVID-19 is a new coronavirus that we have not...</td>\n",
       "      <td>JHU Public Health</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24178</th>\n",
       "      <td>will coronavirus stop in summer</td>\n",
       "      <td>Will warm weather / summer / heat stop outbrea...</td>\n",
       "      <td>We do not know. Some viruses, like the common ...</td>\n",
       "      <td>JHU Public Health</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24183</th>\n",
       "      <td>will coronavirus stop in the summer</td>\n",
       "      <td>Will warm weather / summer / heat stop outbrea...</td>\n",
       "      <td>We do not know. Some viruses, like the common ...</td>\n",
       "      <td>JHU Public Health</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24200</th>\n",
       "      <td>will coronavirus survive in the summer</td>\n",
       "      <td>Will warm weather / summer / heat stop outbrea...</td>\n",
       "      <td>We do not know. Some viruses, like the common ...</td>\n",
       "      <td>JHU Public Health</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24202</th>\n",
       "      <td>will coronavirus survive on surfaces</td>\n",
       "      <td>How long does the virus live on surfaces, like...</td>\n",
       "      <td>A recent study shows that the virus can live i...</td>\n",
       "      <td>JHU Public Health JHU</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24212</th>\n",
       "      <td>world coronavirus die in summer</td>\n",
       "      <td>Will warm weather / summer / heat stop outbrea...</td>\n",
       "      <td>We do not know. Some viruses, like the common ...</td>\n",
       "      <td>JHU Public Health</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1525 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            query_string  \\\n",
       "5                       What does Corona and Covid mean?   \n",
       "10     when will the social distancing end? and what ...   \n",
       "21                        How do I go grocery shopping?    \n",
       "28                                      what is covid 19   \n",
       "33                        What is COVID-19's definition?   \n",
       "...                                                  ...   \n",
       "24178                    will coronavirus stop in summer   \n",
       "24183                will coronavirus stop in the summer   \n",
       "24200             will coronavirus survive in the summer   \n",
       "24202               will coronavirus survive on surfaces   \n",
       "24212                    world coronavirus die in summer   \n",
       "\n",
       "                                                question  \\\n",
       "5                                     What is COVID-19?    \n",
       "10                                What is a coronavirus?   \n",
       "21     Any advice about how to minimize risk during g...   \n",
       "28                                    What is COVID-19?    \n",
       "33                                    What is COVID-19?    \n",
       "...                                                  ...   \n",
       "24178  Will warm weather / summer / heat stop outbrea...   \n",
       "24183  Will warm weather / summer / heat stop outbrea...   \n",
       "24200  Will warm weather / summer / heat stop outbrea...   \n",
       "24202  How long does the virus live on surfaces, like...   \n",
       "24212  Will warm weather / summer / heat stop outbrea...   \n",
       "\n",
       "                                                  answer  \\\n",
       "5      COVID-19 is a new coronavirus that we have not...   \n",
       "10     Coronavirus are a type of virus - there are ma...   \n",
       "21     Currently, there is no evidence of food or foo...   \n",
       "28     COVID-19 is a new coronavirus that we have not...   \n",
       "33     COVID-19 is a new coronavirus that we have not...   \n",
       "...                                                  ...   \n",
       "24178  We do not know. Some viruses, like the common ...   \n",
       "24183  We do not know. Some viruses, like the common ...   \n",
       "24200  We do not know. Some viruses, like the common ...   \n",
       "24202  A recent study shows that the virus can live i...   \n",
       "24212  We do not know. Some viruses, like the common ...   \n",
       "\n",
       "                      source  rating  \n",
       "5          JHU Public Health   100.0  \n",
       "10     JHU Public Health WHO   100.0  \n",
       "21     JHU Public Health FDA   100.0  \n",
       "28         JHU Public Health   100.0  \n",
       "33         JHU Public Health   100.0  \n",
       "...                      ...     ...  \n",
       "24178      JHU Public Health   100.0  \n",
       "24183      JHU Public Health   100.0  \n",
       "24200      JHU Public Health   100.0  \n",
       "24202  JHU Public Health JHU   100.0  \n",
       "24212      JHU Public Health   100.0  \n",
       "\n",
       "[1525 rows x 5 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# keep only related question, query_string, answer for FAQ (rating = 100)\n",
    "\n",
    "df_by_rating = df[df['rating'] == 100]\n",
    "df_by_rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-7-f98f43fae200>:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_by_rating.dropna(inplace=True)\n"
     ]
    }
   ],
   "source": [
    "# drop null values\n",
    "df_by_rating.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1403, 5)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_by_rating.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100.0    1403\n",
       "Name: rating, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_by_rating.rating.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create instance of CovidFAQ_Parser and generate query_answer_pairs\n",
    "covidfaq_parser = CovidFAQ_Parser()\n",
    "covidfaq_parser.extract_data(df_by_rating)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "68"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unique_questions = df_by_rating.question.unique()\n",
    "len(unique_questions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1078"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unique_queries = df_by_rating.query_string.unique()\n",
    "len(unique_queries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# get faq pairs\n",
    "faq_pairs = covidfaq_parser.faq_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'What is COVID-19? ',\n",
       "  'answer': 'COVID-19 is a new coronavirus that we have not seen previously; it is not the same as the flu or common cold. Coronaviruses are types of viruses that cause illnesses. These include the common cold, flu, to more severe diseases like Middle East Respiratory Syndrom (MERs-CoV) and Severe Acute Respiratory Syndrome (SARS-CoV).   [[Would you like to know more about COVID-19 symptoms?]]',\n",
       "  'id': '1'},\n",
       " {'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'What is a coronavirus?',\n",
       "  'answer': 'Coronavirus are a type of virus - there are many kinds, and some of them can cause disease. Examples of coronaviruses include the common cold, flu, to more severe diseases like Middle East Respiratory Syndrome (MERS-CoV) and Severe Acute Respiratory Syndrome (SARS-CoV).   COVID19 is a novel coronavirus that has been recently identified.   [[Would you like more information on COVID19?]] [[Would you like to know the symptoms of COVID19?]]    ',\n",
       "  'id': '2'},\n",
       " {'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'Any advice about how to minimize risk during grocery shopping trips or getting takeout.',\n",
       "  'answer': 'Currently, there is no evidence of food or food packaging being associated with transmission of COVID-19. Like other viruses, it is possible that the virus that causes COVID-19 can survive on surfaces or objects. For that reason, it is critical to wash your hands often, including utensils. Currently, there is no evidence of food or food packaging being associated with transmission of COVID-19. However, the virus that causes COVID-19 is spreading from person-to-person in some communities in the U.S. The CDC recommends that if you are sick, stay home until you are better and no longer pose a risk of infecting others. Anyone handling, preparing and serving food should always follow safe food handling procedures, such as washing hands and surfaces often.',\n",
       "  'id': '3'},\n",
       " {'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'Why do I have to stay at home?',\n",
       "  'answer': 'Socially distancing is a practice that aims to prevent those who are sick from coming into contact with those who are healthy. This is to reduce the opportunity for disease transmission. As COVID-19 spreads person-to-person in close contact, we want to reduce the chance of infection amongst populations that are vulnerable. This includes older adults (over the age of 65) and those with suppressed immune systems. It is also to help reduce the burden on healthcar esystems and workers, as there are not enough workers and beds to care for patients if everyone were to have COVID-19 at the same time.',\n",
       "  'id': '4'},\n",
       " {'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'What is social distancing and how does it help to minimize COVID-19?',\n",
       "  'answer': 'Social distancing is one of the most effective ways to reduce the spread of illness during an outbreak, but it is sometimes hard to understand what this means given the vague language. What it means is to stay at home as much as possible, and limit your outside exposure. This does mean meaking changes to your everyday routine to minimize contacts with others, including:   avoiding non-essential gatherings avoiding common greetings, such as handshakes avoiding crowded places such as concerts, arenas, conferences and festivals limiting contact with people at higher risk like older adults and those in poor health keeping a distance of at least 2 arms-length (approximately 6 feet) from others  For example - if you need to go grocery shopping, limit to once a week. If you go on walks, make sure you maintain a 6 foot distance between yourself and others.',\n",
       "  'id': '5'},\n",
       " {'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'Where did COVID-19 come from?',\n",
       "  'answer': 'It was first found in Wuhan City, Hubei Province, China. The first cases are linked to a live animal market but now COVID-19 is able to spread person-to-person.',\n",
       "  'id': '6'},\n",
       " {'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'What are the symptoms of COVID-19?',\n",
       "  'answer': \"Reported illnesses have ranged from mild symptoms to severe illness and death from confirmed COVID-19 cases. Confirmed means that these people were tested positive for COVID-19. These are the symptoms that can appear 2 to 14 days after exposure: fever, cough, shortness of breath, pneumonia in both lungs. Those who are infected with COVID-19 may also have little to no symptoms.   However, if you're experiencing difficulty breathing or shortness of breath, persistent pain or pressure in the chest, new confusion or inability to arouse, and/or bluish lips or face - please seek medical attention immediately. This is not an all inclusive list - if you are experiencing symptoms that are severe or concerning, talk to your medical provider.\",\n",
       "  'id': '7'},\n",
       " {'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'Should I be wearing a face mask to prevent me getting COVID-19?',\n",
       "  'answer': 'Masks are only helpful in preventing people who have coronavirus (COVID19) from spreading it to others. If the person is healthy, there is likely no need for them to wear a face mask - as there is little evidence showing that these masks used by healthy people will prevent transmission to public.   If the person is coughing, sneezing, and exhibiting other symptoms of COVID-19, wearing a mask can be recommended. It is also recommended for those who are caring for someone who has or is suspected to have COVID-19. However, masks are only really effective if it is used in combination with frequent hand-washing with soap and water, or with an alcohol-based sanitizer.',\n",
       "  'id': '8'},\n",
       " {'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'Should I wear a face mask?',\n",
       "  'answer': 'You should only wear a face mask if a healthcare professional recommends it. Generally, those who have COVID-19 and are showing symptoms should wear face-masks to prevent others from the risk of getting infected  Please do not stock significantly on face masks as they are also crucial for health workers and those who are taking care of people infected with COVID-19.',\n",
       "  'id': '9'},\n",
       " {'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'Should the general population wear masks to protect themselves from COVID-19?',\n",
       "  'answer': 'If you are a healthy individual, the use of a mask is not recommended for preventing the spread of COVID-19.  Wearing a mask when you are not ill may give a false sense of security. There is a potential risk of infection with improper mask use and disposal. They also need to be changed frequently.  However, your health care provider may recommend you wear a mask if you are experiencing symptoms of COVID-19 while you are seeking or waiting for care. In this instance, masks are an appropriate part of infection prevention and control measures. The mask acts as a barrier and helps stop the tiny droplets from spreading you when you cough or sneeze.',\n",
       "  'id': '10'}]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "faq_pairs[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "68"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "covidfaq_parser.num_faq_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get user_query_pairs\n",
    "user_query_pairs = covidfaq_parser.user_query_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 1,\n",
       "  'query_type': 'user_query',\n",
       "  'question': 'What does Corona and Covid mean?',\n",
       "  'answer': 'COVID-19 is a new coronavirus that we have not seen previously; it is not the same as the flu or common cold. Coronaviruses are types of viruses that cause illnesses. These include the common cold, flu, to more severe diseases like Middle East Respiratory Syndrom (MERs-CoV) and Severe Acute Respiratory Syndrome (SARS-CoV).   [[Would you like to know more about COVID-19 symptoms?]]',\n",
       "  'id': '69'},\n",
       " {'label': 1,\n",
       "  'query_type': 'user_query',\n",
       "  'question': 'when will the social distancing end? and what is the economic consequent of the pandemic?',\n",
       "  'answer': 'Coronavirus are a type of virus - there are many kinds, and some of them can cause disease. Examples of coronaviruses include the common cold, flu, to more severe diseases like Middle East Respiratory Syndrome (MERS-CoV) and Severe Acute Respiratory Syndrome (SARS-CoV).   COVID19 is a novel coronavirus that has been recently identified.   [[Would you like more information on COVID19?]] [[Would you like to know the symptoms of COVID19?]]    ',\n",
       "  'id': '70'},\n",
       " {'label': 1,\n",
       "  'query_type': 'user_query',\n",
       "  'question': 'How do I go grocery shopping? ',\n",
       "  'answer': 'Currently, there is no evidence of food or food packaging being associated with transmission of COVID-19. Like other viruses, it is possible that the virus that causes COVID-19 can survive on surfaces or objects. For that reason, it is critical to wash your hands often, including utensils. Currently, there is no evidence of food or food packaging being associated with transmission of COVID-19. However, the virus that causes COVID-19 is spreading from person-to-person in some communities in the U.S. The CDC recommends that if you are sick, stay home until you are better and no longer pose a risk of infecting others. Anyone handling, preparing and serving food should always follow safe food handling procedures, such as washing hands and surfaces often.',\n",
       "  'id': '71'},\n",
       " {'label': 1,\n",
       "  'query_type': 'user_query',\n",
       "  'question': 'what is covid 19',\n",
       "  'answer': 'COVID-19 is a new coronavirus that we have not seen previously; it is not the same as the flu or common cold. Coronaviruses are types of viruses that cause illnesses. These include the common cold, flu, to more severe diseases like Middle East Respiratory Syndrom (MERs-CoV) and Severe Acute Respiratory Syndrome (SARS-CoV).   [[Would you like to know more about COVID-19 symptoms?]]',\n",
       "  'id': '72'},\n",
       " {'label': 1,\n",
       "  'query_type': 'user_query',\n",
       "  'question': \"What is COVID-19's definition?\",\n",
       "  'answer': 'COVID-19 is a new coronavirus that we have not seen previously; it is not the same as the flu or common cold. Coronaviruses are types of viruses that cause illnesses. These include the common cold, flu, to more severe diseases like Middle East Respiratory Syndrom (MERs-CoV) and Severe Acute Respiratory Syndrome (SARS-CoV).   [[Would you like to know more about COVID-19 symptoms?]]',\n",
       "  'id': '73'},\n",
       " {'label': 1,\n",
       "  'query_type': 'user_query',\n",
       "  'question': 'Should I stay home ?',\n",
       "  'answer': 'Socially distancing is a practice that aims to prevent those who are sick from coming into contact with those who are healthy. This is to reduce the opportunity for disease transmission. As COVID-19 spreads person-to-person in close contact, we want to reduce the chance of infection amongst populations that are vulnerable. This includes older adults (over the age of 65) and those with suppressed immune systems. It is also to help reduce the burden on healthcar esystems and workers, as there are not enough workers and beds to care for patients if everyone were to have COVID-19 at the same time.',\n",
       "  'id': '74'},\n",
       " {'label': 1,\n",
       "  'query_type': 'user_query',\n",
       "  'question': 'Should I practice social distancing? ',\n",
       "  'answer': 'Social distancing is one of the most effective ways to reduce the spread of illness during an outbreak, but it is sometimes hard to understand what this means given the vague language. What it means is to stay at home as much as possible, and limit your outside exposure. This does mean meaking changes to your everyday routine to minimize contacts with others, including:   avoiding non-essential gatherings avoiding common greetings, such as handshakes avoiding crowded places such as concerts, arenas, conferences and festivals limiting contact with people at higher risk like older adults and those in poor health keeping a distance of at least 2 arms-length (approximately 6 feet) from others  For example - if you need to go grocery shopping, limit to once a week. If you go on walks, make sure you maintain a 6 foot distance between yourself and others.',\n",
       "  'id': '75'},\n",
       " {'label': 1,\n",
       "  'query_type': 'user_query',\n",
       "  'question': 'How did it started',\n",
       "  'answer': 'It was first found in Wuhan City, Hubei Province, China. The first cases are linked to a live animal market but now COVID-19 is able to spread person-to-person.',\n",
       "  'id': '76'},\n",
       " {'label': 1,\n",
       "  'query_type': 'user_query',\n",
       "  'question': 'what are the 3 symptoms of covid-19',\n",
       "  'answer': \"Reported illnesses have ranged from mild symptoms to severe illness and death from confirmed COVID-19 cases. Confirmed means that these people were tested positive for COVID-19. These are the symptoms that can appear 2 to 14 days after exposure: fever, cough, shortness of breath, pneumonia in both lungs. Those who are infected with COVID-19 may also have little to no symptoms.   However, if you're experiencing difficulty breathing or shortness of breath, persistent pain or pressure in the chest, new confusion or inability to arouse, and/or bluish lips or face - please seek medical attention immediately. This is not an all inclusive list - if you are experiencing symptoms that are severe or concerning, talk to your medical provider.\",\n",
       "  'id': '77'},\n",
       " {'label': 1,\n",
       "  'query_type': 'user_query',\n",
       "  'question': 'Should I wear a mask?',\n",
       "  'answer': 'Masks are only helpful in preventing people who have coronavirus (COVID19) from spreading it to others. If the person is healthy, there is likely no need for them to wear a face mask - as there is little evidence showing that these masks used by healthy people will prevent transmission to public.   If the person is coughing, sneezing, and exhibiting other symptoms of COVID-19, wearing a mask can be recommended. It is also recommended for those who are caring for someone who has or is suspected to have COVID-19. However, masks are only really effective if it is used in combination with frequent hand-washing with soap and water, or with an alcohol-based sanitizer.',\n",
       "  'id': '78'}]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_query_pairs[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1387"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "covidfaq_parser.num_user_query_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get query_answer_pairs\n",
    "query_answer_pairs = covidfaq_parser.query_answer_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'What is COVID-19? ',\n",
       "  'answer': 'COVID-19 is a new coronavirus that we have not seen previously; it is not the same as the flu or common cold. Coronaviruses are types of viruses that cause illnesses. These include the common cold, flu, to more severe diseases like Middle East Respiratory Syndrom (MERs-CoV) and Severe Acute Respiratory Syndrome (SARS-CoV).   [[Would you like to know more about COVID-19 symptoms?]]',\n",
       "  'id': '1'},\n",
       " {'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'What is a coronavirus?',\n",
       "  'answer': 'Coronavirus are a type of virus - there are many kinds, and some of them can cause disease. Examples of coronaviruses include the common cold, flu, to more severe diseases like Middle East Respiratory Syndrome (MERS-CoV) and Severe Acute Respiratory Syndrome (SARS-CoV).   COVID19 is a novel coronavirus that has been recently identified.   [[Would you like more information on COVID19?]] [[Would you like to know the symptoms of COVID19?]]    ',\n",
       "  'id': '2'},\n",
       " {'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'Any advice about how to minimize risk during grocery shopping trips or getting takeout.',\n",
       "  'answer': 'Currently, there is no evidence of food or food packaging being associated with transmission of COVID-19. Like other viruses, it is possible that the virus that causes COVID-19 can survive on surfaces or objects. For that reason, it is critical to wash your hands often, including utensils. Currently, there is no evidence of food or food packaging being associated with transmission of COVID-19. However, the virus that causes COVID-19 is spreading from person-to-person in some communities in the U.S. The CDC recommends that if you are sick, stay home until you are better and no longer pose a risk of infecting others. Anyone handling, preparing and serving food should always follow safe food handling procedures, such as washing hands and surfaces often.',\n",
       "  'id': '3'},\n",
       " {'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'Why do I have to stay at home?',\n",
       "  'answer': 'Socially distancing is a practice that aims to prevent those who are sick from coming into contact with those who are healthy. This is to reduce the opportunity for disease transmission. As COVID-19 spreads person-to-person in close contact, we want to reduce the chance of infection amongst populations that are vulnerable. This includes older adults (over the age of 65) and those with suppressed immune systems. It is also to help reduce the burden on healthcar esystems and workers, as there are not enough workers and beds to care for patients if everyone were to have COVID-19 at the same time.',\n",
       "  'id': '4'},\n",
       " {'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'What is social distancing and how does it help to minimize COVID-19?',\n",
       "  'answer': 'Social distancing is one of the most effective ways to reduce the spread of illness during an outbreak, but it is sometimes hard to understand what this means given the vague language. What it means is to stay at home as much as possible, and limit your outside exposure. This does mean meaking changes to your everyday routine to minimize contacts with others, including:   avoiding non-essential gatherings avoiding common greetings, such as handshakes avoiding crowded places such as concerts, arenas, conferences and festivals limiting contact with people at higher risk like older adults and those in poor health keeping a distance of at least 2 arms-length (approximately 6 feet) from others  For example - if you need to go grocery shopping, limit to once a week. If you go on walks, make sure you maintain a 6 foot distance between yourself and others.',\n",
       "  'id': '5'},\n",
       " {'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'Where did COVID-19 come from?',\n",
       "  'answer': 'It was first found in Wuhan City, Hubei Province, China. The first cases are linked to a live animal market but now COVID-19 is able to spread person-to-person.',\n",
       "  'id': '6'},\n",
       " {'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'What are the symptoms of COVID-19?',\n",
       "  'answer': \"Reported illnesses have ranged from mild symptoms to severe illness and death from confirmed COVID-19 cases. Confirmed means that these people were tested positive for COVID-19. These are the symptoms that can appear 2 to 14 days after exposure: fever, cough, shortness of breath, pneumonia in both lungs. Those who are infected with COVID-19 may also have little to no symptoms.   However, if you're experiencing difficulty breathing or shortness of breath, persistent pain or pressure in the chest, new confusion or inability to arouse, and/or bluish lips or face - please seek medical attention immediately. This is not an all inclusive list - if you are experiencing symptoms that are severe or concerning, talk to your medical provider.\",\n",
       "  'id': '7'},\n",
       " {'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'Should I be wearing a face mask to prevent me getting COVID-19?',\n",
       "  'answer': 'Masks are only helpful in preventing people who have coronavirus (COVID19) from spreading it to others. If the person is healthy, there is likely no need for them to wear a face mask - as there is little evidence showing that these masks used by healthy people will prevent transmission to public.   If the person is coughing, sneezing, and exhibiting other symptoms of COVID-19, wearing a mask can be recommended. It is also recommended for those who are caring for someone who has or is suspected to have COVID-19. However, masks are only really effective if it is used in combination with frequent hand-washing with soap and water, or with an alcohol-based sanitizer.',\n",
       "  'id': '8'},\n",
       " {'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'Should I wear a face mask?',\n",
       "  'answer': 'You should only wear a face mask if a healthcare professional recommends it. Generally, those who have COVID-19 and are showing symptoms should wear face-masks to prevent others from the risk of getting infected  Please do not stock significantly on face masks as they are also crucial for health workers and those who are taking care of people infected with COVID-19.',\n",
       "  'id': '9'},\n",
       " {'label': 1,\n",
       "  'query_type': 'faq',\n",
       "  'question': 'Should the general population wear masks to protect themselves from COVID-19?',\n",
       "  'answer': 'If you are a healthy individual, the use of a mask is not recommended for preventing the spread of COVID-19.  Wearing a mask when you are not ill may give a false sense of security. There is a potential risk of infection with improper mask use and disposal. They also need to be changed frequently.  However, your health care provider may recommend you wear a mask if you are experiencing symptoms of COVID-19 while you are seeking or waiting for care. In this instance, masks are an appropriate part of infection prevention and control measures. The mask acts as a barrier and helps stop the tiny droplets from spreading you when you cough or sneeze.',\n",
       "  'id': '10'}]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query_answer_pairs[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1455"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "covidfaq_parser.num_query_answer_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dump data to json file\n",
    "\n",
    "dump_to_json(covidfaq_parser.query_answer_pairs, '../../../BERT-FAQ/data/CovidFAQ/query_answer_pairs.json', sort_keys=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}