TweetSentiment.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": " TweetSentiment.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "mount_file_id": "15XPV3-4KZvL9w7HZ4aDWtY-T__JxHTdt",
      "authorship_tag": "ABX9TyO63ifykD2MmQt8xu9qc/st",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/birajsilwal/TweetSentiment/blob/master/TweetSentiment.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lxdxVZdGln9U"
      },
      "source": [
        "import re"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b2sqo5nG5brz"
      },
      "source": [
        "# Filter tweets class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ntir5YuYoR5u"
      },
      "source": [
        "\n",
        "'''\n",
        "this class filters (removes tweet tags, punctuations,\n",
        "irrelevant words such as a, an, the) from the tweets\n",
        "and returns array of clean filtered tweets\n",
        "'''\n",
        "class FilterTweets:\n",
        "\n",
        "    def filterTweets(self, file_path):\n",
        "      final_tweets = []\n",
        "      tweets = []\n",
        "      tweets_raw = open(file_path)\n",
        "      for tweet in tweets_raw:\n",
        "          tweet = tweet.lower()\n",
        "          tweet = re.sub(\":|;|\\(|\\)|!|\\'s|\\\"|\\.|n\\'t|,|\\?|i\\'m|i\\'ve\", \"\", tweet)\n",
        "          tweet = re.sub(\" an | the | is | of | a | i | was | and |\"\n",
        "                          \" on | in | off | all | it | me | you | to |\"\n",
        "                          \" into | we | your | that | they | can | could |\"\n",
        "                          \" should | do | does | for | my | at | so |\"\n",
        "                          \" if | has | have | had | from | such | are |\"\n",
        "                          \" not | this | now | but | go | day |\"\n",
        "                          \"-|_| up | down | these | today | lol |\"\n",
        "                          \" lmao | af | get | got | here | there | who |\"\n",
        "                          \" what | am | no | why | with | us | our | bro |\"\n",
        "                          \" too | then | ur | zero | ah | see | saw \", \" \", tweet)\n",
        "          tweets.append(tweet)\n",
        "\n",
        "\n",
        "      # cleaning up twice because of space issue\n",
        "      tweets1 = []\n",
        "      for tweet in tweets:\n",
        "          tweet = tweet.lower()\n",
        "          tweet = re.sub(\" an | the | is | of | a | i | was | and |\"\n",
        "                          \" on | in | off | all | it | me | you | to |\"\n",
        "                          \" into | we | your | that | they | can | could |\"\n",
        "                          \" should | do | does | for | my | at | so |\"\n",
        "                          \" if | has | have | had | from | such | are |\"\n",
        "                          \" not | this | now | but | go | day |\"\n",
        "                          \"-|_| up | down | these | today | lol |\"\n",
        "                          \" lmao | af | get | got | here | there | who |\"\n",
        "                          \" what | am | no | why | with | us | our | bro |\"\n",
        "                          \" too | then | ur | zero | ah | see | saw \", \" \", tweet)\n",
        "          tweets1.append(tweet)\n",
        "\n",
        "\n",
        "      '''\n",
        "      if there contains a substring starting with http or @,\n",
        "      remove that substring from the tweet\n",
        "      '''\n",
        "      for str in tweets1:\n",
        "          index = 0\n",
        "          string_arr = str.split()\n",
        "          for string in string_arr:\n",
        "              if 'http' in string or '@' in string:\n",
        "                  string_arr[index] = string.replace(string, '')\n",
        "              index += 1\n",
        "\n",
        "          tweet = ' '\n",
        "          tweet = tweet.join(string_arr)\n",
        "          final_tweets.append(tweet)\n",
        "          # print(tweet)\n",
        "      return final_tweets"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XpNCTjW88VRG"
      },
      "source": [
        "# Filtered datasets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uB3h2aVGomL7",
        "outputId": "c8dc5721-e716-4316-f73a-41b7f940fcd4"
      },
      "source": [
        "filter_tweets = FilterTweets()\n",
        "\n",
        "tweets_training_filtered = filter_tweets.filterTweets('/content/drive/MyDrive/Colab Notebooks/Tweets/noCR_train.txt')\n",
        "print(len(tweets_training_filtered))\n",
        "\n",
        "tweets_testing_filtered = filter_tweets.filterTweets('/content/drive/MyDrive/Colab Notebooks/Tweets/noCR_test.txt')\n",
        "print(len(tweets_testing_filtered))"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "10271\n",
            "1145\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IZGIgdr-OLOD"
      },
      "source": [
        ""
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q11axTfSJw5A"
      },
      "source": [
        "# Tweet class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kWdGnFjIHBhX"
      },
      "source": [
        "class Tweet:\n",
        "  def __init__(self, tweet, zero_or_one):\n",
        "      self.tweet = tweet\n",
        "      self.zero_or_one = zero_or_one\n",
        "      self.sentiment = self.get_sentiment()\n",
        "\n",
        "  \n",
        "  def get_sentiment(self):\n",
        "    if self.zero_or_one == '0':\n",
        "      return \"NEGATIVE\"\n",
        "    elif self.zero_or_one == '1':\n",
        "      return \"POSITIVE\" \n"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hhrtZpLRIrLV"
      },
      "source": [
        "# creating tweet objects for each tweet\n",
        "tweets_training = []\n",
        "tweets_testing = []\n",
        "\n",
        "for tweet in tweets_training_filtered:\n",
        "  string_arr = tweet.split()\n",
        "  zero_or_one = string_arr.pop(0)\n",
        "\n",
        "  tweet1 = ' '\n",
        "  tweet1 = tweet1.join(string_arr)\n",
        "  tweets_training.append(Tweet(tweet1, zero_or_one))\n",
        "\n",
        "\n",
        "for tweet in tweets_testing_filtered:\n",
        "  string_arr = tweet.split()\n",
        "  zero_or_one = string_arr.pop(0)\n",
        "  \n",
        "  tweet1 = ' '\n",
        "  tweet1 = tweet1.join(string_arr)\n",
        "  tweets_testing.append(Tweet(tweet1, zero_or_one))"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mCK6E_bEYy25"
      },
      "source": [
        "# x and y values for both training and testing dataset\n",
        "train_x = [x.tweet for x in tweets_training]\n",
        "train_y = [x.sentiment for x in tweets_training]\n",
        "\n",
        "test_x = [x.tweet for x in tweets_testing]\n",
        "test_y = [x.sentiment for x in tweets_testing]"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9EKo4VGg0a2e"
      },
      "source": [
        ""
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qVWi9rMtZfQP"
      },
      "source": [
        "def update_dict():\n",
        "  dict_tweet = {}\n",
        "  val = 1\n",
        "\n",
        "  for tweet in train_x:\n",
        "      tweet_arr = tweet.split()\n",
        "\n",
        "      for x in range(len(tweet_arr)):\n",
        "          key = tweet_arr[x]\n",
        "          '''\n",
        "          see if the key already exists in the dictionary\n",
        "          if yes, increase the value by 1 otherwise\n",
        "          new key and val.\n",
        "          '''\n",
        "          if key in dict_tweet:\n",
        "              update_val = dict_tweet[key] + 1\n",
        "              update_dict = {key: update_val}\n",
        "              dict_tweet.update(update_dict)\n",
        "          else:\n",
        "              dict_tweet[key] = val"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m0bIxtDmX3UZ"
      },
      "source": [
        "# Vectorization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kHSCDS_SX86y"
      },
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "\n",
        "vectorizer = CountVectorizer()\n",
        "train_x_vectors = vectorizer.fit_transform(train_x)\n",
        "test_x_vectors = vectorizer.transform(test_x) "
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FsHPg3OZ2crP"
      },
      "source": [
        "# Naive Bayes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aVZOX6w_9pAC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8779b4bb-60fa-4454-8692-c6a52f724e54"
      },
      "source": [
        "from sklearn.naive_bayes import GaussianNB\n",
        "\n",
        "clf = GaussianNB()\n",
        "clf.fit(train_x_vectors.toarray(), train_y)\n",
        "\n",
        "clf.predict(test_x_vectors[99].toarray())"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['NEGATIVE'], dtype='<U8')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CtDWup_q194Y",
        "outputId": "9ecb6b89-5e7d-453f-fd37-5aab7d87612e"
      },
      "source": [
        "print(clf.score(test_x_vectors.toarray(), test_y))"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.6541484716157205\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UprxQAe_2Ry5",
        "outputId": "28c8d486-f7b2-4c68-d329-c88ae14c76dc"
      },
      "source": [
        "test_set = [\"Thanks for your contribution and kind comment\", 'horrible waste of time']\n",
        "new_test = vectorizer.transform(test_set)\n",
        "\n",
        "clf.predict(new_test.toarray()) "
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['POSITIVE', 'NEGATIVE'], dtype='<U8')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    }
  ]
}