Prepare_LDA_data.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python385jvsc74a57bd03ffa5b9ea856ed8ffaf5105c7cda503ceb4538c7650d33f63eca5c51fb008d20",
   "display_name": "Python 3.8.5 64-bit ('base': conda)"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "###################### before 0413\n",
    "# import pickle\n",
    "\n",
    "# with open(\"../../data/token/Step2/2018_clean_token.pkl\",\"rb\") as f:\n",
    "#     t_0 = pickle.load(f)\n",
    "# with open(\"../../data/token/Step2/2019_clean_token.pkl\",\"rb\") as f:\n",
    "#     t_1 = pickle.load(f)\n",
    "# with open(\"../../data/token/Step2/2020_clean_token.pkl\",\"rb\") as f:\n",
    "#     t_2 = pickle.load(f)\n",
    "# paper_list  =  t_0 + t_1 +t_2\n",
    "# with open(\".../data/token/clean_data.pkl\",\"wb\") as f:\n",
    "#     pickle.dump(paper_list,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# doc =[]\n",
    "# LDA_corpus = []\n",
    "# for papers in paper_list:\n",
    "#     doc =[]\n",
    "#     for docs in papers:\n",
    "#         for word in docs:\n",
    "#             doc.append(word)\n",
    "#             # break\n",
    "#         # break\n",
    "#     LDA_corpus.append(doc)\n",
    "#     # break\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(\"../../data/LDA_data/Step2_clean_data.pkl\",\"wb\") as f:\n",
    "#     pickle.dump(LDA_corpus,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open(\"../../data/tk_new/tk_token_1.pkl\",'rb') as f:\n",
    "    paper_list =pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "LDA_corpus =[]\n",
    "for docs in paper_list:\n",
    "    \n",
    "    if docs['doc_type']==3:\n",
    "        # print(docs['cl_token'])\n",
    "        # break\n",
    "        doc = []\n",
    "        for tokens in docs['cl_token']:\n",
    "            for word in tokens:\n",
    "                doc.append(word)\n",
    "                # break\n",
    "            #   break\n",
    "        LDA_corpus.append(doc)\n",
    "    # break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"../../data/LDA_data/tk_token_1.pkl\",\"wb\") as f:\n",
    "    pickle.dump(LDA_corpus,f)"
   ]
  }
 ]
}