DEMO_ToxicSpanDetection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "73212e71-10bb-4425-b8f0-380e9de1dfa4",
   "metadata": {
    "tags": []
   },
   "source": [
    "# DEMO ToxicSpanDetection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "30d053c0-f26b-4f31-8920-3cf3eb41668d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from TextProcessingModule.toxicity_filter import SpanDetector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4869f88a-6507-4d84-b663-bc9a1544c543",
   "metadata": {},
   "outputs": [],
   "source": [
    "spandetector = SpanDetector()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "207f0d09-2dbd-498a-aaa5-56276f2e9c78",
   "metadata": {},
   "outputs": [],
   "source": [
    "toxic_span = \"fucking\"\n",
    "example = f\"This is a {toxic_span} example. What the fuck?!\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f226d0a8-9ace-48a7-80e1-af88d4f44f92",
   "metadata": {},
   "source": [
    "### Return Toxic Spans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e2876a17-697d-42bc-8e23-2b4cf6718934",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73b5f9f2489f4d58b0b8d6d32526e0f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f9c01b3e47c64a8988556ae7f28f19f3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Prediction:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[(10, 16), (36, 39)]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spandetector.detect(example)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfd70c01-60e8-48de-ad5c-e88a92ef961e",
   "metadata": {},
   "source": [
    "### Return List of Toxic Characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e45a0f2a-7dc3-41fd-b27e-edca52d135a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fda0a16b3a1a41e5a84f6c99192725da",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ebdbbd53912b4adfaf6851ebdb4954e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Prediction:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[10, 11, 12, 13, 14, 15, 16, 36, 37, 38, 39]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spandetector.detect(example, return_spans=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e0ad0ce-440e-4ead-8da0-9331fc48fd03",
   "metadata": {},
   "source": [
    "### Return Toxic Tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "fd8f110b-c291-4bb7-99d2-5d0fd056f49a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c47a4c975264056a68f9f30563f2434",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da6eadbf64c740e9bb6620e878114c03",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Prediction:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "['fucking', 'fuck']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spandetector.return_toxic_token(example)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d48161e-8a44-4850-9396-648c58723b0f",
   "metadata": {},
   "source": [
    "### Replace spans by a specific character"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "122f943f-62d8-445b-9bba-ef8ab7ca4234",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "362e403e49824632a6b1d9e573b1f7b4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "90c27df8a3074e00bdc722860dd9b465",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Prediction:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'This is a ******* example. What the ****?!'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spandetector.replace_spans(example)  # default: char='*'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3fe5ae6-d525-401b-b3f3-9d3d01ccacb2",
   "metadata": {},
   "source": [
    "### Tagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "047a96e4-c56b-4f48-b756-5b71f3ac01f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef12ab49da3d4553beb6f1a2854b73e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "698460af719041fc887455b94252a113",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Prediction:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'This is a <TOX>fucking</TOX> example. What the <TOX>fuck</TOX>?!'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spandetector.tag_spans(example)  # default: tag='TOX'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1393fe03-12e8-4613-8701-dfe1e447ecbe",
   "metadata": {},
   "source": [
    "### HTML Highlighting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b12b0ef0-1e3b-4b1f-8889-4c240e1bbd57",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f5290bdc2bc431dbca4562506b0f7b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c127d90d7cb744968e5fe9f5808b14e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Prediction:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'This is a <span style=\"color:#217FF8\">fucking</span> example. What the <span style=\"color:#217FF8\">fuck</span>?!'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spandetector.highlight_spans(example)  # default: color='lightblue'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2a794112-9c55-48a7-b9e6-81767678adf4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "45837ecac5804a9bbac45e10dc9ca9d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7fa952b90fa34693893bcff9a3e6f7fc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running Prediction:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "This is a <span style=\"color:#217FF8\">fucking</span> example. What the <span style=\"color:#217FF8\">fuck</span>?!"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import display, HTML\n",
    "display(HTML(spandetector.highlight_spans(example)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "941cf144-2797-4938-acff-c28d7df3c562",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}