Untitled.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Using The Super Resolution Convolutional Neural Network for Image Restoration\n",
    "# \n",
    "# The goal of super-resolution (SR) is to recover a high resolution image from a low resolution input, or as they might say on any modern crime show, enhance!\n",
    "# \n",
    "# To accomplish this goal, we will be deploying the super-resolution convolution neural network (SRCNN) using Keras. This network was published in the paper, \"Image Super-Resolution Using Deep Convolutional Networks\" by Chao Dong, et al. in 2014. You can read the full paper at https://arxiv.org/abs/1501.00092.\n",
    "# \n",
    "# As the title suggests, the SRCNN is a deep convolutional neural network that learns end-to-end mapping of low resolution to high resolution images. As a result, we can use it to improve the image quality of low resolution images. To evaluate the performance of this network, we will be using three image quality metrics: peak signal to noise ratio (PSNR), mean squared error (MSE), and the structural similarity (SSIM) index.\n",
    "# \n",
    "# In this particular project, we will be using OpenCV to pre and post process our images.We will frequently be converting our images back and forth between the RGB, BGR, and YCrCb color spaces. This is necessary because the SRCNN network was trained on the luminance (Y) channel in the YCrCb color space."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import keras\n",
    "import numpy\n",
    "import matplotlib.pyplot as plt\n",
    "import cv2\n",
    "import skimage\n",
    "from skimage.measure import compare_ssim as ssim\n",
    "import numpy as np\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Conv2D,Dense,MaxPooling2D\n",
    "from keras.optimizers import Adam,SGD\n",
    "import math\n",
    "import os\n",
    "from IPython import get_ipython\n",
    "get_ipython().run_line_magic('matplotlib', 'inline')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define a function for peak signal-to-noise ratio (PSNR)\n",
    "def psnr(target, ref):\n",
    "         \n",
    "    # assume RGB image\n",
    "    target_data = target.astype(float)\n",
    "    ref_data = ref.astype(float)\n",
    "\n",
    "    diff = ref_data - target_data\n",
    "    diff = diff.flatten('C')\n",
    "\n",
    "    rmse = math.sqrt(np.mean(diff ** 2.))\n",
    "\n",
    "    return 20 * math.log10(255. / rmse)\n",
    "\n",
    "# define function for mean squared error (MSE)\n",
    "def mse(target, ref):\n",
    "    # the MSE between the two images is the sum of the squared difference between the two images\n",
    "    err = np.sum((target.astype('float') - ref.astype('float')) ** 2)\n",
    "    err /= float(target.shape[0] * target.shape[1])\n",
    "    \n",
    "    return err\n",
    "\n",
    "# define function that combines all three image quality metrics\n",
    "def compare_images(target, ref):\n",
    "    scores = []\n",
    "    scores.append(psnr(target, ref))\n",
    "    scores.append(mse(target, ref))\n",
    "    scores.append(ssim(target, ref, multichannel =True))\n",
    "    \n",
    "    return scores\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def prepare_images(path, factor):\n",
    "    \n",
    "    # loop through the files in the directory\n",
    "    for file in os.listdir(path):\n",
    "        \n",
    "        # open the file\n",
    "        img = cv2.imread(path + '/' + file)\n",
    "        \n",
    "        # find old and new image dimensions\n",
    "        h, w, _ = img.shape\n",
    "        new_height = h // factor\n",
    "        new_width = w // factor\n",
    "        \n",
    "        # resize the image - down\n",
    "        img = cv2.resize(img, (new_width, new_height), interpolation = cv2.INTER_LINEAR)\n",
    "        \n",
    "        # resize the image - up\n",
    "        img = cv2.resize(img, (w, h), interpolation = cv2.INTER_LINEAR)\n",
    "        \n",
    "        # save the image\n",
    "        print('Saving {}'.format(file))\n",
    "        cv2.imwrite('images/{}'.format(file), img)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving baboon.bmp\n",
      "Saving baby_GT.bmp\n",
      "Saving barbara.bmp\n",
      "Saving bird_GT.bmp\n",
      "Saving butterfly_GT.bmp\n",
      "Saving comic.bmp\n",
      "Saving face.bmp\n",
      "Saving flowers.bmp\n",
      "Saving foreman.bmp\n",
      "Saving head_GT.bmp\n",
      "Saving IMG_0948.JPG\n",
      "Saving Indian-Army-4-696x392.jpg\n",
      "Saving lenna.bmp\n",
      "Saving monarch.bmp\n",
      "Saving pepper.bmp\n",
      "Saving ppt3.bmp\n",
      "Saving woman_GT.bmp\n",
      "Saving zebra.bmp\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Dell\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:28: UserWarning: DEPRECATED: skimage.measure.compare_ssim has been moved to skimage.metrics.structural_similarity. It will be removed from skimage.measure in version 0.18.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "baboon.bmp\n",
      "PSNR: 22.157084083442548\n",
      "MSE: 1187.1161333333334\n",
      "SSIM: 0.629277587900277\n",
      "\n",
      "baby_GT.bmp\n",
      "PSNR: 34.37180640966199\n",
      "MSE: 71.28874588012695\n",
      "SSIM: 0.9356987872724932\n",
      "\n",
      "barbara.bmp\n",
      "PSNR: 25.906629837568126\n",
      "MSE: 500.65508535879627\n",
      "SSIM: 0.8098632646406401\n",
      "\n",
      "bird_GT.bmp\n",
      "PSNR: 32.896644728720005\n",
      "MSE: 100.12375819830247\n",
      "SSIM: 0.9533644866026473\n",
      "\n",
      "butterfly_GT.bmp\n",
      "PSNR: 24.782076560337416\n",
      "MSE: 648.6254119873047\n",
      "SSIM: 0.8791344763843051\n",
      "\n",
      "comic.bmp\n",
      "PSNR: 23.799861502225532\n",
      "MSE: 813.2338836565096\n",
      "SSIM: 0.8347335416398209\n",
      "\n",
      "face.bmp\n",
      "PSNR: 30.99220650287191\n",
      "MSE: 155.23189718546524\n",
      "SSIM: 0.8008439492289884\n",
      "\n",
      "flowers.bmp\n",
      "PSNR: 27.454504805386147\n",
      "MSE: 350.55093922651935\n",
      "SSIM: 0.8697286286974628\n",
      "\n",
      "foreman.bmp\n",
      "PSNR: 30.14456532664372\n",
      "MSE: 188.6883483270202\n",
      "SSIM: 0.933268417388899\n",
      "\n",
      "head_GT.bmp\n",
      "PSNR: 31.020502848237534\n",
      "MSE: 154.2237755102041\n",
      "SSIM: 0.8011121330733371\n",
      "\n",
      "IMG_0948.JPG\n",
      "PSNR: 43.70295884240723\n",
      "MSE: 8.315832467246159\n",
      "SSIM: 0.9943531164106147\n",
      "\n",
      "Indian-Army-4-696x392.jpg\n",
      "PSNR: 28.668338927989744\n",
      "MSE: 265.0743607787943\n",
      "SSIM: 0.9108414290340613\n",
      "\n",
      "lenna.bmp\n",
      "PSNR: 31.47349297867539\n",
      "MSE: 138.94800567626953\n",
      "SSIM: 0.8460989200521499\n",
      "\n",
      "monarch.bmp\n",
      "PSNR: 30.196242365288896\n",
      "MSE: 186.45643615722656\n",
      "SSIM: 0.9439574293434104\n",
      "\n",
      "pepper.bmp\n",
      "PSNR: 29.88947161686106\n",
      "MSE: 200.1033935546875\n",
      "SSIM: 0.8357937568464359\n",
      "\n",
      "ppt3.bmp\n",
      "PSNR: 24.84926168950471\n",
      "MSE: 638.6684263912582\n",
      "SSIM: 0.9284023942315316\n",
      "\n",
      "woman_GT.bmp\n",
      "PSNR: 29.326236280817465\n",
      "MSE: 227.812729498164\n",
      "SSIM: 0.9335397280466592\n",
      "\n",
      "zebra.bmp\n",
      "PSNR: 27.909840639329513\n",
      "MSE: 315.6585459528818\n",
      "SSIM: 0.8911656209329116\n",
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "prepare_images('source images/', 2)\n",
    "\n",
    "\n",
    "# In[59]:\n",
    "\n",
    "\n",
    "for file in os.listdir('images/'):\n",
    "    \n",
    "    # open target and reference images\n",
    "    target = cv2.imread('images/{}'.format(file))\n",
    "    ref = cv2.imread('source images/{}'.format(file))\n",
    "    \n",
    "    # calculate score\n",
    "    scores = compare_images(target, ref)\n",
    "\n",
    "    # print all three scores with new line characters (\\n) \n",
    "    print('{}\\nPSNR: {}\\nMSE: {}\\nSSIM: {}\\n'.format(file, scores[0], scores[1], scores[2]))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def SRCNN():\n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(filters=128,kernel_size=(9,9),kernel_initializer='glorot_uniform',activation='relu',\n",
    "                     padding='valid',use_bias=True,input_shape=(None,None,1)))\n",
    "    \n",
    "    model.add(Conv2D(filters=64,kernel_size=(3,3),kernel_initializer='glorot_uniform',activation='relu',\n",
    "                     padding='same',use_bias=True))\n",
    "    \n",
    "    model.add(Conv2D(filters=1,kernel_size=(5,5),kernel_initializer='glorot_uniform',activation='linear',\n",
    "                     padding='valid',use_bias=True))\n",
    "    adam = Adam(lr=0.0003)\n",
    "    \n",
    "    model.compile(optimizer=adam,loss='mean_squared_error',metrics=['mean_squared_error'])\n",
    "    \n",
    "    return model   \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, None, None, 128)   10496     \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, None, None, 64)    73792     \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, None, None, 1)     1601      \n",
      "=================================================================\n",
      "Total params: 85,889\n",
      "Trainable params: 85,889\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "srcnn = SRCNN()\n",
    "srcnn.summary()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def modcrop(img, scale):\n",
    "    tmpsz = img.shape\n",
    "    sz = tmpsz[0:2]\n",
    "    sz = sz - np.mod(sz, scale)\n",
    "    img = img[0:sz[0], 1:sz[1]]\n",
    "    return img\n",
    "\n",
    "\n",
    "def shave(image, border):\n",
    "    img = image[border: -border, border: -border]\n",
    "    return img\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\Dell\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Dell\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:28: UserWarning: DEPRECATED: skimage.measure.compare_ssim has been moved to skimage.metrics.structural_similarity. It will be removed from skimage.measure in version 0.18.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Degraded Image: \n",
      "PSNR: 24.752462626436284\n",
      "MSE: 653.0634118967453\n",
      "SSIM: 0.8788410517538944\n",
      "\n",
      "Reconstructed Image: \n",
      "PSNR: 30.382141759863842\n",
      "MSE: 178.64359079005544\n",
      "SSIM: 0.9519721030434253\n",
      "\n",
      "Saving baboon.bmp\n",
      "Saving baby_GT.bmp\n",
      "Saving barbara.bmp\n",
      "Saving bird_GT.bmp\n",
      "Saving butterfly_GT.bmp\n",
      "Saving comic.bmp\n",
      "Saving face.bmp\n",
      "Saving flowers.bmp\n",
      "Saving foreman.bmp\n",
      "Saving head_GT.bmp\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "def predict(image_path):\n",
    "    \n",
    "    # load the srcnn model with weights\n",
    "    srcnn = SRCNN()\n",
    "    srcnn.load_weights('weights.h5')\n",
    "    \n",
    "    # load the degraded and reference images\n",
    "    path, file = os.path.split(image_path)\n",
    "    degraded = cv2.imread(image_path)\n",
    "    ref = cv2.imread('source images/{}'.format(file))\n",
    "    \n",
    "    # preprocess the image with modcrop\n",
    "    ref = modcrop(ref, 3)\n",
    "    degraded = modcrop(degraded, 3)\n",
    "    \n",
    "    # convert the image to YCrCb - (srcnn trained on Y channel)\n",
    "    temp = cv2.cvtColor(degraded, cv2.COLOR_BGR2YCrCb)\n",
    "    \n",
    "    # create image slice and normalize  \n",
    "    Y = numpy.zeros((1, temp.shape[0], temp.shape[1], 1), dtype=float)\n",
    "    Y[0, :, :, 0] = temp[:, :, 0].astype(float) / 255\n",
    "    \n",
    "    # perform super-resolution with srcnn\n",
    "    pre = srcnn.predict(Y, batch_size=1)\n",
    "    \n",
    "    # post-process output\n",
    "    pre *= 255\n",
    "    pre[pre[:] > 255] = 255\n",
    "    pre[pre[:] < 0] = 0\n",
    "    pre = pre.astype(np.uint8)\n",
    "    \n",
    "    # copy Y channel back to image and convert to BGR\n",
    "    temp = shave(temp, 6)\n",
    "    temp[:, :, 0] = pre[0, :, :, 0]\n",
    "    output = cv2.cvtColor(temp, cv2.COLOR_YCrCb2BGR)\n",
    "    \n",
    "    # remove border from reference and degraged image\n",
    "    ref = shave(ref.astype(np.uint8), 6)\n",
    "    degraded = shave(degraded.astype(np.uint8), 6)\n",
    "    \n",
    "    # image quality calculations\n",
    "    scores = []\n",
    "    scores.append(compare_images(degraded, ref))\n",
    "    scores.append(compare_images(output, ref))\n",
    "    \n",
    "    # return images and scores\n",
    "    return ref, degraded, output, scores\n",
    "\n",
    "\n",
    "# In[64]:\n",
    "\n",
    "\n",
    "ref, degraded, output, scores = predict('images/butterfly_GT.bmp')\n",
    "\n",
    "# print all scores for all images\n",
    "print('Degraded Image: \\nPSNR: {}\\nMSE: {}\\nSSIM: {}\\n'.format(scores[0][0], scores[0][1], scores[0][2]))\n",
    "print('Reconstructed Image: \\nPSNR: {}\\nMSE: {}\\nSSIM: {}\\n'.format(scores[1][0], scores[1][1], scores[1][2]))\n",
    "\n",
    "\n",
    "# display images as subplots\n",
    "fig, axs = plt.subplots(1, 3, figsize=(20, 8))\n",
    "axs[0].imshow(cv2.cvtColor(ref, cv2.COLOR_BGR2RGB))\n",
    "axs[0].set_title('Original')\n",
    "axs[1].imshow(cv2.cvtColor(degraded, cv2.COLOR_BGR2RGB))\n",
    "axs[1].set_title('Degraded')\n",
    "axs[2].imshow(cv2.cvtColor(output, cv2.COLOR_BGR2RGB))\n",
    "axs[2].set_title('SRCNN')\n",
    "\n",
    "# remove the x and y ticks\n",
    "for ax in axs:\n",
    "    ax.set_xticks([])\n",
    "    ax.set_yticks([])\n",
    "\n",
    "\n",
    "# In[65]:\n",
    "\n",
    "\n",
    "for file in os.listdir('images'):\n",
    "    \n",
    "    # perform super-resolution\n",
    "    ref, degraded, output, scores = predict('images/{}'.format(file))\n",
    "    \n",
    "    # display images as subplots\n",
    "    fig, axs = plt.subplots(1, 3, figsize=(20, 8))\n",
    "    axs[0].imshow(cv2.cvtColor(ref, cv2.COLOR_BGR2RGB))\n",
    "    axs[0].set_title('Original')\n",
    "    axs[1].imshow(cv2.cvtColor(degraded, cv2.COLOR_BGR2RGB))\n",
    "    axs[1].set_title('Degraded')\n",
    "    axs[1].set(xlabel = 'PSNR: {}\\nMSE: {} \\nSSIM: {}'.format(scores[0][0], scores[0][1], scores[0][2]))\n",
    "    axs[2].imshow(cv2.cvtColor(output, cv2.COLOR_BGR2RGB))\n",
    "    axs[2].set_title('SRCNN')\n",
    "    axs[2].set(xlabel = 'PSNR: {} \\nMSE: {} \\nSSIM: {}'.format(scores[1][0], scores[1][1], scores[1][2]))\n",
    "\n",
    "    # remove the x and y ticks\n",
    "    for ax in axs:\n",
    "        ax.set_xticks([])\n",
    "        ax.set_yticks([])\n",
    "      \n",
    "    print('Saving {}'.format(file))\n",
    "    fig.savefig('Outputs/{}.png'.format(os.path.splitext(file)[0])) \n",
    "    plt.close()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}