main_team_qstart_amazonml.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### MoonDream Vision Language Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fRB2-hBxRCOz",
        "outputId": "7e16c319-5ecc-4fa4-8e3e-404856206aca"
      },
      "outputs": [],
      "source": [
        "!git clone https://github.com/vikhyat/moondream"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Installing dependencies for moondream\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V4IV7yaTRhAh",
        "outputId": "7f22391d-185e-4f3a-b3f6-efa27bd42502"
      },
      "outputs": [],
      "source": [
        "!pip install -r ./moondream/requirements.txt"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Main working script"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eVwkSDQAQ4Zt",
        "outputId": "1245c163-04db-4f32-8b07-8ec79c8fdfb9"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import subprocess\n",
        "import os\n",
        "import requests\n",
        "import re\n",
        "\n",
        "csv_input_path = r\"/content/test.csv\"  # Path to the test.csv file\n",
        "image_folder = r\"/content/image\"  # Path to the folder containing the images\n",
        "csv_output_path = r\"/content/output.csv\"  # Output path\n",
        "exec_path = r\"/content/moondream/sample.py\"  # Path of the sample.py from the moondream repo\n",
        "pyrun_name = \"python\"  # Python executable name\n",
        "\n",
        "data = pd.read_csv(csv_input_path)\n",
        "output_data = []\n",
        "\n",
        "def download_and_save_image(image_url, image_name):\n",
        "    try:\n",
        "        response = requests.get(image_url)\n",
        "        response.raise_for_status()  # Raise an exception for bad status codes\n",
        "\n",
        "        image_name = image_folder + \"/\" + str(image_name) + \".jpg\"\n",
        "        with open(str(image_name), 'wb') as file:\n",
        "            file.write(response.content)\n",
        "\n",
        "        print(f\"Image downloaded successfully as {image_name}\")\n",
        "        return True\n",
        "    except requests.exceptions.RequestException as e:\n",
        "        print(f\"Error downloading image: {e}\")\n",
        "        return False\n",
        "\n",
        "def clean_output(unclean_value, value_type):\n",
        "    \"\"\"Cleans the unit from a messy input string and returns a standardized output.\"\"\"\n",
        "    unclean_value = str(unclean_value).lower()\n",
        "\n",
        "    unit_mapping = {\n",
        "        \"width\": [\"centimetre\", \"foot\", \"millimetre\", \"metre\", \"inch\", \"yard\"],\n",
        "        \"depth\": [\"centimetre\", \"foot\", \"millimetre\", \"metre\", \"inch\", \"yard\"],\n",
        "        \"height\": [\"centimetre\", \"foot\", \"millimetre\", \"metre\", \"inch\", \"yard\"],\n",
        "        \"item_weight\": [\"milligram\", \"kilogram\", \"microgram\", \"gram\", \"ounce\", \"ton\", \"pound\"],\n",
        "        \"maximum_weight_recommendation\": [\"milligram\", \"kilogram\", \"microgram\", \"gram\", \"ounce\", \"ton\", \"pound\"],\n",
        "        \"voltage\": [\"millivolt\", \"kilovolt\", \"volt\"],\n",
        "        \"wattage\": [\"kilowatt\", \"watt\"],\n",
        "        \"item_volume\": [\"cubic foot\", \"microlitre\", \"cup\", \"fluid ounce\",\n",
        "                        \"centilitre\", \"imperial gallon\", \"pint\", \"decilitre\",\n",
        "                        \"litre\", \"millilitre\", \"quart\", \"cubic inch\", \"gallon\"]\n",
        "    }\n",
        "\n",
        "    unit_alternatives = {\n",
        "        \"centimetre\": [\"cm\", \"cms\", \"centimeters\", \"centimeter\"],\n",
        "        \"foot\": [\"ft\", \"feet\"],\n",
        "        \"millimetre\": [\"mm\", \"mms\", \"millimeters\", \"millimeter\"],\n",
        "        \"metre\": [\"m\", \"ms\", \"meters\", \"meter\"],\n",
        "        \"inch\": [\"in\", \"inches\", '\"'],\n",
        "        \"yard\": [\"yd\", \"yds\", \"yards\"],\n",
        "        \"milligram\": [\"mg\", \"mgs\", \"milligrams\"],\n",
        "        \"kilogram\": [\"kg\", \"kgs\", \"kilograms\"],\n",
        "        \"microgram\": [\"mcg\", \"mcgs\", \"micrograms\"],\n",
        "        \"gram\": [\"g\", \"gs\", \"grams\"],\n",
        "        \"ounce\": [\"oz\", \"ozs\", \"ounces\"],\n",
        "        \"ton\": [\"tons\", \"tonnes\"],\n",
        "        \"pound\": [\"lb\", \"lbs\", \"pounds\"],\n",
        "        \"millivolt\": [\"mv\", \"mvs\", \"millivolts\"],\n",
        "        \"kilovolt\": [\"kv\", \"kvs\", \"kilovolts\"],\n",
        "        \"volt\": [\"v\", \"vs\", \"volts\"],\n",
        "        \"kilowatt\": [\"kw\", \"kws\", \"kilowatts\"],\n",
        "        \"watt\": [\"w\", \"ws\", \"watts\"],\n",
        "        \"cubic foot\": [\"ft3\", \"cu ft\", \"cubic feet\"],\n",
        "        \"microlitre\": [\"µl\", \"ul\", \"microlitres\"],\n",
        "        \"cup\": [\"cups\"],\n",
        "        \"fluid ounce\": [\"fl oz\", \"fl. oz.\", \"fluid ounces\"],\n",
        "        \"centilitre\": [\"cl\", \"cls\", \"centilitres\"],\n",
        "        \"imperial gallon\": [\"imp gal\", \"imperial gallons\"],\n",
        "        \"pint\": [\"pt\", \"pts\", \"pints\"],\n",
        "        \"decilitre\": [\"dl\", \"dls\", \"decilitres\"],\n",
        "        \"litre\": [\"l\", \"ls\", \"liters\", \"liter\"],\n",
        "        \"millilitre\": [\"ml\", \"mls\", \"millilitres\"],\n",
        "        \"quart\": [\"qt\", \"qts\", \"quarts\"],\n",
        "        \"cubic inch\": [\"in3\", \"cu in\", \"cubic inches\"],\n",
        "        \"gallon\": [\"gal\", \"gals\", \"gallons\"]\n",
        "    }\n",
        "\n",
        "    if re.match(r\"^\\d+\\.?\\d*$\", unclean_value):\n",
        "        return f\"{unclean_value} {unit_mapping[value_type][0]}\"\n",
        "\n",
        "    for unit in unit_mapping[value_type]:\n",
        "        for alt_unit in [unit] + unit_alternatives.get(unit, []):\n",
        "            pattern1 = r\"(\\d+\\.?\\d*)\\s*({}s?)\\b\".format(alt_unit)\n",
        "            pattern2 = r\"(\\d+\\.?\\d*)({}s?)\\b\".format(alt_unit[:3])\n",
        "            pattern3 = r\"(\\d+\\.?\\d*)({}s?)\".format(alt_unit)\n",
        "\n",
        "            match = re.search(pattern1, unclean_value, re.IGNORECASE) or \\\n",
        "                    re.search(pattern2, unclean_value, re.IGNORECASE) or \\\n",
        "                    re.search(pattern3, unclean_value, re.IGNORECASE)\n",
        "\n",
        "            if match:\n",
        "                value = match.group(1)\n",
        "                cleaned_unit = unit\n",
        "                return f\"{value} {cleaned_unit}\"\n",
        "\n",
        "    print(f\"Could not clean value: {unclean_value}\")\n",
        "    return \"\"\n",
        "\n",
        "# Start processing from index 0\n",
        "for index, row in data.iterrows():\n",
        "    if index < 0:\n",
        "        continue\n",
        "\n",
        "    actual_index = row['index']\n",
        "    image_index = row['group_id']\n",
        "    entity_name = row['entity_name']\n",
        "    image_path = os.path.join(image_folder, f\"{image_index}.jpg\")\n",
        "    print(f\"Processing index: {actual_index}\")\n",
        "\n",
        "    image_url = row['image_link']\n",
        "    image_name = image_index\n",
        "    download_and_save_image(image_url, image_name)\n",
        "\n",
        "    if not os.path.exists(image_path):\n",
        "        print(f\"Image {image_index} not found, skipping.\")\n",
        "        output_data.append([image_index, \"\"])\n",
        "        continue\n",
        "\n",
        "    prompt = f\"Extract {entity_name} from image. Do not give any other text/ information. If there are multiple values, give only the most important value along with the unit, separated by space. Answer: \"\n",
        "    prompt += '{'\n",
        "\n",
        "    try:\n",
        "        result = subprocess.run(\n",
        "            [pyrun_name, exec_path, '--image', image_path, '--prompt', prompt],\n",
        "            capture_output=True, text=True, timeout=120\n",
        "        )\n",
        "\n",
        "        model_output = result.stdout.strip()\n",
        "        model_output = model_output.split('{')[-1].strip()\n",
        "        model_output = clean_output(model_output, entity_name)\n",
        "        print(f\"Model Output: {model_output}\")\n",
        "\n",
        "        if not model_output:\n",
        "            model_output = \"\"\n",
        "\n",
        "        output_data.append([actual_index, model_output])\n",
        "\n",
        "        # Save output data incrementally\n",
        "        output_df = pd.DataFrame(output_data, columns=['index', 'prediction'])\n",
        "        output_df.to_csv(csv_output_path, index=False)\n",
        "        print(f\"Incremental save to {csv_output_path}\")\n",
        "\n",
        "    except Exception as e:\n",
        "        print(f\"Error processing image {image_index}: {e}\")\n",
        "        output_data.append([actual_index, \"\"])\n",
        "\n",
        "print(\"Process complete. Final output saved to\", csv_output_path)\n"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}