arxiv_2_selection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cee84b07-af34-42ed-9286-d1d1856db132",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "arxiv_output_dir = './arxiv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1789805a-6bf5-4b11-a168-14426c87f485",
   "metadata": {},
   "outputs": [],
   "source": [
    "processing_dates = [\n",
    "    '0001',\n",
    "    '0002',\n",
    "    '0003',\n",
    "    '0004',\n",
    "    '0005',\n",
    "    '0006',\n",
    "    '0007',\n",
    "    '0008',\n",
    "    '0009',\n",
    "    '0010',\n",
    "    '0011',\n",
    "    '0012',\n",
    "    '0501',\n",
    "    '1001',\n",
    "    '1501',\n",
    "    '2001',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83264aa5-9fc0-433b-93b9-35e5a3215937",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "\n",
    "def save_random_selection(path, data_list, force=False):\n",
    "    \"\"\"\n",
    "    Randomly select 150 entries from the list and save them into a file as one item per line.\n",
    "\n",
    "    Parameters:\n",
    "    - data_list (list): The list from which to select entries.\n",
    "    - path (str): The path to the file where the selected entries should be saved.\n",
    "\n",
    "    Returns:\n",
    "    - list: The selected entries.\n",
    "    \"\"\"\n",
    "    # If the file already exists, read its content and return it\n",
    "    if os.path.exists(path) and not force:\n",
    "        with open(path, \"r\") as file:\n",
    "            existing_content = [line.strip() for line in file.readlines()]\n",
    "        print('Existing file loaded.')\n",
    "        return existing_content\n",
    "\n",
    "    # Randomly select 150 entries from the list\n",
    "    selected_entries = random.sample([i for i in data_list if i.endswith('.html')], 150)\n",
    "    selected_entries.sort()\n",
    "\n",
    "    # Save the selected entries to the file\n",
    "    with open(path, \"w\") as file:\n",
    "        for entry in selected_entries:\n",
    "            file.write(str(entry) + '\\n')\n",
    "\n",
    "    print('Random selection saved.')\n",
    "\n",
    "    return selected_entries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c6a7960-6380-4074-bd18-ecc18c42b4c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "def transfer_selected_files(selection_list, datedir):\n",
    "    os.makedirs(os.path.join(arxiv_output_dir, datedir, 'html_selected'), exist_ok=True)\n",
    "    os.makedirs(os.path.join(arxiv_output_dir, datedir, 'pdf_selected'), exist_ok=True)\n",
    "\n",
    "    pbar = tqdm(selection_list, desc=\"Moving selected files\")\n",
    "    for html_filename in pbar:\n",
    "        html_filename_stem = os.path.splitext(html_filename)[0]\n",
    "        html_path = os.path.join(arxiv_output_dir, datedir, 'html', html_filename)\n",
    "        pdf_path = os.path.join(arxiv_output_dir, datedir, 'pdf', html_filename_stem + '.pdf')\n",
    "        target_html_path = os.path.join(arxiv_output_dir, datedir, 'html_selected', html_filename)\n",
    "        target_pdf_path = os.path.join(arxiv_output_dir, datedir, 'pdf_selected', html_filename_stem + '.pdf')\n",
    "        if not os.path.exists(pdf_path):\n",
    "            print(f'PDF not exists. ({pdf_path})')\n",
    "            continue\n",
    "        if not os.path.exists(target_html_path):\n",
    "            shutil.copy(html_path, target_html_path)\n",
    "        if not os.path.exists(target_pdf_path):\n",
    "            shutil.copy(pdf_path, target_pdf_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d83fcdd9-9bb6-4f13-8f7e-33f3c9f6a7b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "for date_folder_name in processing_dates:\n",
    "    curr_html = os.listdir(os.path.join(arxiv_output_dir, date_folder_name, 'html'))\n",
    "    curr_pdf = os.listdir(os.path.join(arxiv_output_dir, date_folder_name, 'pdf'))\n",
    "    print(date_folder_name, len(curr_html), len(curr_pdf))\n",
    "    \n",
    "    curr_html_selection = save_random_selection(os.path.join(arxiv_output_dir, date_folder_name, 'html_selection.txt'), curr_html)\n",
    "    transfer_selected_files(curr_html_selection, date_folder_name)\n",
    "    \n",
    "    print(date_folder_name, 'done!')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}