src/methods/david_alexis/functions.py

+
+import numpy as np
+
+
+# interpolation kernels
+w1 = np.array([[0,0,-1,0,0], [0,0,2,0,0], [-1,2,4,2,-1], [0,0,2,0,0], [0,0,-1,0,0]])/8 # G at R location
+w2 = w1 #G at B location
+w3 = np.array([[0,0,0.5,0,0], [0,-1,0,-1,0], [-1,4,5,4,-1], [0,-1,0,-1,0], [0,0,0.5,0,0]])/8 # R at G location (Brow, Rcol)
+w4 = np.array([[0,0,-1,0,0], [0,-1,4,-1,0], [0.5,0,5,0,0.5], [0,-1,4,-1,0], [0,0,-1,0,0]])/8 # R at G location (Rrow, Bcol)
+w5 = np.array([[0,0,-3/2,0,0], [0,2,0,2,0], [-3/2,0,6,0,-1], [0,2,0,2,0], [0,0,-3/2,0,0]])/8 # R at B location (Brow, Bcol)
+w6 = w3 #R at G location (Brow, Rcol)
+w7 = w4 #B at G location (Rrow, Bcol)
+w8 = w5 #B at R location (Rrow, Rcol)
+w_s = [w1, w2, w3, w4, w5, w6, w7, w8]
+
+def conv(A,B):
+    return np.sum(A*B)
+
+def bayer_gradient_interpolation(y, op, w=w_s):
+    """
+    Second interpolation method: convolution with a 2D kernels (5x5) using multi-channel interpolations
+    y: input image
+    w: interpolation kernels
+    """
+    y_padded = np.pad(y, ((2,2),(2,2)), 'constant', constant_values=0)
+    size_padded = y_padded.shape
+
+    #separate channels
+    y_0 = y*op.mask[:, :, 0]
+    y_1 = y*op.mask[:, :, 1]
+    y_2 = y*op.mask[:, :, 2]
+
+    for i in range(2,size_padded[0]-2):
+        for j in range(2,size_padded[1]-2):
+
+            # determine the location of the pixel
+            if op.mask[i-2,j-2,1] == 1: # at G
+                if op.mask[i-2,j-3,0] == 1: #R row
+                    y_0[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w3)
+                    y_2[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w7)
+                else: #B row
+                    y_0[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w4)
+                    y_2[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w6)
+
+            elif op.mask[i-2,j-2,0] == 1: # at R
+                y_1[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w1)
+                y_2[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w8)
+
+            else: # at B
+                y_1[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w2)
+                y_0[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w5)
+
+    y_res = np.zeros((y.shape[0], y.shape[1], 3))
+    y_res[:, :, 0] = y_0
+    y_res[:, :, 1] = y_1
+    y_res[:, :, 2] = y_2
+
+    return y_res
+
+
+
+import numpy as np
+def conv(A,B):
+    return np.sum(A*B)
+def quad_gradient_interpolation(y, op):
+    """
+    Second interpolation method: convolution with a 2D kernels (5x5) using multi-channel interpolations
+    y: input image
+    w: interpolation kernels
+    """
+    y_padded = np.pad(y, ((2,2),(2,2)), 'constant', constant_values=0)
+    size_padded = y_padded.shape
+
+    mask_padded = np.pad(op.mask, ((2,2),(2,2),(0,0)), 'constant', constant_values=-1)
+
+    #separate channels
+    y_0 = y*op.mask[:, :, 0]
+    y_1 = y*op.mask[:, :, 1]
+    y_2 = y*op.mask[:, :, 2]
+
+    #interpolation kernels
+    # green quad kernels
+    wg11 = np.zeros((5,5))
+    wg11[0,2] = -3
+    wg11[1,2] = 13
+    wg11[2,4] = 2
+    wg11 += wg11.T
+    wg11 = wg11/24
+
+    # rotate the kernel
+    wg12 = np.rot90(wg11,3)
+    wg22 = np.rot90(wg11,2)
+    wg21 = np.rot90(wg11,1)
+
+    #red/blue quad kernels
+        # center part 
+    w_c11 = np.zeros((5,5))
+    w_c11[0:2,0:2] = np.array([[-1,-1],[-1,9]])/6
+    w_c21 = np.rot90(w_c11,1)
+    w_c22 = np.rot90(w_c11,2)
+    w_c12 = np.rot90(w_c11,3)
+        # left part
+    w_l11 = np.zeros((5,5))
+    w_l11[0:2, 2] = np.array([-3,13])
+    w_l11[4,2] = 2
+    w_l11 = w_l11/12
+    w_l22 = np.rot90(w_l11,2)
+        # top part
+    w_t11 = np.rot90(w_l11,1)
+    w_t22 = np.rot90(w_t11,2)
+
+
+    for i in range(2,size_padded[0]-2):
+        for j in range(2,size_padded[1]-2):
+
+            # determine the location of the pixel
+            if mask_padded[i,j,1] == 1: # at G
+                if mask_padded[i-1,j,0] + mask_padded[i+1,j,0] == 1: # at R column
+                    if mask_padded[i,j-1,2] == 1:
+                        y_2[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w_t11)
+                    elif mask_padded[i,j+1,2] == 1:
+                        y_2[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w_t22)
+                    if mask_padded[i-1,j,0] == 1:
+                        y_0[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w_l11)
+                    elif mask_padded[i+1,j,0] == 1:
+                        y_0[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w_l22)
+
+                else:
+                    if mask_padded[i,j-1,0] == 1:
+                        y_0[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w_t11)
+                    elif mask_padded[i,j+1,0] == 1:
+                        y_0[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w_t22)
+                    if mask_padded[i-1,j,2] == 1:
+                        y_2[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w_l11)
+                    elif mask_padded[i+1,j,2] == 1:
+                        y_2[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w_l22)
+
+
+            elif mask_padded[i,j,1] == 0: # at R or B
+                if mask_padded[i-1,j,1] ==1:
+                    if mask_padded[i,j-1,1] ==1:
+                        y_1[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],wg11)
+                    else:
+                        y_1[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],wg12)
+                else:
+                    if mask_padded[i,j-1,1] ==1:
+                        y_1[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],wg21)
+                    else:
+                        y_1[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],wg22)
+                
+                if mask_padded[i,j,0] == 1: #at R
+                    if mask_padded[i-1,j-1,2] ==1: #center (1,1)
+                        y_2[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w_c11)
+                    elif mask_padded[i-1,j+1,2] ==1: #center (1,2)
+                        y_2[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w_c12)
+                    elif mask_padded[i+1,j-1,2] ==1: #center (2,1)
+                        y_2[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w_c21)
+                    elif mask_padded[i+1,j+1,2] ==1: #center (2,2)
+                        y_2[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w_c22)
+                    
+                elif mask_padded[i,j,2] == 1: #at B
+                    if mask_padded[i-1,j-1,0] ==1: #center (1,1)
+                        y_0[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w_c11)   
+                    elif mask_padded[i-1,j+1,0] ==1: #center (1,2)
+                        y_0[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w_c12)
+                    elif mask_padded[i+1,j-1,0] ==1: #center (2,1)
+                        y_0[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w_c21)
+                    elif mask_padded[i+1,j+1,0] ==1: #center (2,2)
+                        y_0[i-2,j-2] = conv(y_padded[i-2:i+3, j-2:j+3],w_c22)
+
+    y_res = np.zeros((y.shape[0], y.shape[1], 3))
+    y_res[:, :, 0] = y_0
+    y_res[:, :, 1] = y_1
+    y_res[:, :, 2] = y_2
+
+    return y_res
+
+from scipy import signal
+def interpolate(y, op, mode = 'bayer'):
+    """
+    First interpolation method: convolution with a 2D kernel
+    y: input image
+    op: object of class CFA
+    """
+    #separate channels
+    y_0 = y*op.mask[:, :, 0]
+    y_1 = y*op.mask[:, :, 1]
+    y_2 = y*op.mask[:, :, 2]
+
+    #convolution 2D
+    
+    if mode == 'bayer':
+        a = np.array([1,2,1])
+    elif mode == 'quad_bayer':
+        a = np.array([1,2,3,2,1])
+    else:
+        raise Exception('Mode not supported')
+    
+    # create the 2D kernel
+    w = a[:, None] * a[None, :]
+    w = w / np.sum(w)
+
+    #interpolate green
+    y_1_interpolated = signal.convolve2d(y_1, w, mode='same', boundary='fill', fillvalue=0)
+
+    #interpolate red
+    y_0_interpolated = signal.convolve2d(y_0, w, mode='same', boundary='fill', fillvalue=0)
+
+    # interpolate blue
+    y_2_interpolated = signal.convolve2d(y_2, w, mode='same', boundary='fill', fillvalue=0)
+
+    y_res = np.zeros((y.shape[0], y.shape[1], 3))   
+    y_res[:,:,0] = y_0_interpolated*4
+    y_res[:,:,1] = y_1_interpolated*2
+    y_res[:,:,2] = y_2_interpolated*4
+
+    return y_res

src/methods/david_alexis/reconstruct.py

+"""The main file for the reconstruction.
+"""
+
+import numpy as np
+
+from src.forward_model import CFA
+from src.methods.david_alexis.functions import bayer_gradient_interpolation, quad_gradient_interpolation
+
+
+def run_reconstruction(y: np.ndarray, cfa: str) -> np.ndarray:
+    """Performs demosaicking on y.
+
+    Args:
+        y (np.ndarray): Mosaicked image to be reconstructed.
+        cfa (str): Name of the CFA. Can be bayer or quad_bayer.
+
+    Returns:
+        np.ndarray: Demosaicked image.
+    """
+    input_shape = (y.shape[0], y.shape[1], 3)
+    op = CFA(cfa, input_shape)
+
+    if cfa == "bayer": 
+        res = bayer_gradient_interpolation(y, op)
+    else:
+        res = quad_gradient_interpolation(y, op)
+
+    return res
+
+
+
+# Author: Alexis DAVID

src/methods/domer/forward_model.py

+"""A file containing the forward operator.
+This file should NOT be modified.
+"""
+
+
+import numpy as np
+
+from src.checks import check_cfa, check_rgb
+
+
+class CFA():
+    def __init__(self, cfa: str, input_shape: tuple) -> None:
+        """Constructor of the forward operator's class.
+
+        Args:
+            cfa (str): Name of the pattern. Either bayer or quad_bayer.
+            input_shape (tuple): Shape of the input images of the operator.
+        """
+        check_cfa(cfa)
+    
+        self.cfa = cfa
+        self.input_shape = input_shape
+        self.output_shape = input_shape[:-1]
+
+        if self.cfa == 'bayer':
+            self.mask = get_bayer_mask(input_shape)
+
+        elif self.cfa == 'quad_bayer':
+            self.mask = get_quad_bayer_mask(input_shape)
+
+
+    def direct(self, x: np.ndarray) -> np.ndarray:
+        """Applies the CFA operation to the image x.
+
+        Args:
+            x (np.ndarray): Input image.
+
+        Returns:
+            np.ndarray: Output image.
+        """
+        check_rgb(x)
+    
+        return np.sum(x * self.mask, axis=2)
+
+
+    def adjoint(self, y: np.ndarray) -> np.ndarray:
+        """Applies the adjoint of the CFA operation.
+
+        Args:
+            y (np.ndarray): Input image.
+
+        Returns:
+            np.ndarray: Output image.
+        """
+        return self.mask * y[..., np.newaxis]
+
+
+def get_bayer_mask(input_shape: tuple) -> np.ndarray:
+    """Return the mask of the Bayer CFA.
+
+    Args:
+        input_shape (tuple): Shape of the mask.
+
+    Returns:
+        np.ndarray: Mask.
+    """
+    res = np.kron(np.ones((input_shape[0], input_shape[1], 1)), [0, 1, 0])
+
+    res[::2, 1::2] = [1, 0, 0]
+    res[1::2, ::2] = [0, 0, 1]
+
+    return res
+
+
+def get_quad_bayer_mask(input_shape: tuple) -> np.ndarray:
+    """Return the mask of the quad_bayer CFA.
+
+    Args:
+        input_shape (tuple): Shape of the mask.
+
+    Returns:
+        np.ndarray: Mask.
+    """
+    res = np.kron(np.ones((input_shape[0], input_shape[1], 1)), [0, 1, 0])
+
+    res[::4, 2::4] = [1, 0, 0]
+    res[::4, 3::4] = [1, 0, 0]
+    res[1::4, 2::4] = [1, 0, 0]
+    res[1::4, 3::4] = [1, 0, 0]
+
+    res[2::4, ::4] = [0, 0, 1]
+    res[2::4, 1::4] = [0, 0, 1]
+    res[3::4, ::4] = [0, 0, 1]
+    res[3::4, 1::4] = [0, 0, 1]
+
+    return res
+
+
+####
+####
+####
+
+####      ####                ####        #############
+####      ######              ####      ##################
+####      ########            ####      ####################
+####      ##########          ####      ####        ########
+####      ############        ####      ####            ####
+####      ####  ########      ####      ####            ####
+####      ####    ########    ####      ####            ####
+####      ####      ########  ####      ####            ####
+####      ####  ##    ######  ####      ####          ######
+####      ####  ####      ##  ####      ####    ############
+####      ####  ######        ####      ####    ##########
+####      ####  ##########    ####      ####    ########
+####      ####      ########  ####      ####
+####      ####        ############      ####
+####      ####          ##########      ####
+####      ####            ########      ####
+####      ####              ######      ####
+
+# 2023
+# Authors: Mauro Dalla Mura and Matthieu Muller

src/methods/domer/image-colorization/LICENSE

+MIT License
+
+Copyright (c) 2020 priyavrat-misra
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

src/methods/domer/image-colorization/README.md

+# Image Colorization
+
+![cover](https://github.com/priyavrat-misra/image-colorization/blob/master/images/colorized/bnw_col.png?raw=true "an eye-candy")
+
+## Contents
+- [Overview](#overview)
+- [Approach](#approach)
+- [Steps](#steps)
+- [Results](#results)
+- [TL;DR](#tldr)
+- [Setup](#setup)
+- [Usage](#usage)
+- [Todo](#todo)
+
+<br>
+
+## Overview
+> This project is a Deep Convolutional Neural Network approach to solve the task of image colorization.
+> The goal is to produce a colored image given a grayscale image.<br>
+> At it's heart, it uses Convolutional Auto-Encoders to solve this task.
+> First few layers of [ResNet-18](https://arxiv.org/abs/1512.03385) model are used as the Encoder,
+> and the Decoder consists of a series of Deconvolution layers (i.e., upsample layers followed by convolutions) and residual connections.<br>
+> The model is trained on a subset of [MIT Places365](http://places2.csail.mit.edu/index.html) dataset, consisting of `41000` images of landscapes and scenes.
+
+## Approach
+> The images in the dataset are in RGB Colorspace.
+> Before loading the images, the images are converted to [LAB colorspace](https://en.wikipedia.org/wiki/CIELAB_color_space).
+> This colorspace contains exactly the same information as RGB.<br>
+> It has 3 channels, `Lightness, A and B`.
+> The lightness channel can be used as the grayscale equivalent of a colored image,
+> the rest 2 channels (A and B) contain the color information.<br>
+>
+> In a nutshell, the training process follows these steps:
+>> 1. The lightness channel is separated from the other 2 channels and used as the model's input.
+>> 2. The model predicts the A and B channels (or 'AB' for short).
+>> 3. The loss is calculated by comparing the predicted AB and the corresponding original AB of the input image.
+>
+> More about the training process can be found [here](https://github.com/priyavrat-misra/image-colorization/blob/master/train.ipynb "train.ipynb").
+
+## Steps
+> 1. [Defining a model architecture:](https://github.com/priyavrat-misra/image-colorization/blob/master/network.py "network.py")
+>    - The model follows an Auto-Encoder kind of architecture i.e., it has an `encoder` and a `decoder` part.
+>    - The encoder is used to _extract features_ of an image whereas,
+>    - the decoder is used to upsample the features. In other words, it increases the _spacial resolution_.
+>    - In here, the layers of the encoder are taken from ResNet-18 model, and the first conv layer is modified to take a single channel as input (i.e., grayscale or lightness) rather than 3 channels.
+>    - The decoder uses nearest neighbor upsampling (for increasing the spacial resolution),
+>     followed by convolutional layers (for dealing with the depth).
+>    - A more detailed visualization of the model architecture can be seen [here](https://github.com/priyavrat-misra/image-colorization/blob/master/images/architecture.png?raw=true 'after all "A picture is worth a thousand words" :)').
+> 2. [Defining a custom dataloader:](https://github.com/priyavrat-misra/image-colorization/blob/master/utils.py "utils.GrayscaleImageFolder")
+>    - when loading the images, it converts them to LAB, and returns L and AB separately.
+>    - it does few data processing tasks as well like applying tranforms and normalization.
+> 3. [Training the model:](https://github.com/priyavrat-misra/image-colorization/blob/master/train.ipynb "train.ipynb")
+>    - The model is trained for 64 epochs with [Adam Optimization](https://arxiv.org/abs/1412.6980).
+>    - For calculating the loss between the predicted AB and the original AB, Mean Squared Error is used.
+> 4. [Inference:](https://github.com/priyavrat-misra/image-colorization/blob/master/inference.ipynb "inference.ipynb")
+>    - Inference is done with unseen images and the results look promising, or should I say "natural"? :)
+
+## Results
+> ![results](https://github.com/priyavrat-misra/image-colorization/blob/master/images/results.png?raw=true)
+> _<sup>More colorized examples can be found in [here](https://github.com/priyavrat-misra/image-colorization/blob/master/images/colorized/).<sup>_
+
+## TL;DR
+> Given an image, the model can colorize it.
+
+## Setup
+- Clone and change directory:
+```bash
+git clone "https://github.com/priyavrat-misra/image-colorization.git"
+cd image-colorization/
+```
+- Dependencies:
+```bash
+pip install -r requirements.txt
+```
+
+## Usage
+```bash
+python colorize.py --img-path <path/to/image.jpg> --out-path <path/to/output.jpg> --res 360
+# or the short-way:
+python colorize.py -i <path/to/image.jpg> -o <path/to/output.jpg> -r 360
+```
+
+_Note:_
+> - As the model is trained with 224x224 images, it gives best results when `--res` is set to lower resolutions (<=480) and okay-ish when set around ~720.
+> - Setting `--res` higher than that of input image won't increase the output's quality.
+
+<br>
+
+## Todo
+- [x] define & train a model architecture
+- [x] add argparse support
+- [x] define a more residual architecture
+- [x] use pretrained resnet-18 params for the layers used in the encoder & train the model
+- [x] check how the colorization effect varies with image resolution
+- [x] separate the model from the checkpoint file to a different file
+- [x] complete README.md
+- [ ] deploy with flask
+- [ ] _after that, host it maybe?_
+
+<br>
+
+For any queries, feel free to reach me out on [LinkedIn](https://linkedin.com/in/priyavrat-misra/).
\ No newline at end of file

src/methods/domer/image-colorization/colorize.py

+import argparse
+import torch
+from torchvision.utils import save_image
+from network import ColorizeNet
+from utils import load_gray, to_rgb
+import os
+
+parser = argparse.ArgumentParser(description='colorizes an image')
+parser.add_argument('-i', '--img_path', type=str, metavar='', required=True,
+                    help='path and/or name of grayscale image to colorize')
+parser.add_argument('-r', '--res', type=int, metavar='',
+                    help='resizes the input to given resolution {default:360}')
+parser.add_argument('-o', '--out_path', type=str, metavar='', required=True,
+                    help='name to which the colorized image to be saved')
+
+device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
+
+current_directory = os.getcwd()
+c_path='src/methods/domer/image-colorization/models/model.pth'
+c_path = os.path.join(current_directory, c_path)
+model = ColorizeNet()
+model.load_state_dict(
+    
+    torch.load(c_path, map_location='cpu')
+)
+
+
+def main():
+    args = parser.parse_args()
+    img_l = load_gray(args.img_path, shape=args.res)
+
+    model.eval()
+    with torch.no_grad():
+        img_ab = model(img_l)
+
+    img_rgb = to_rgb(img_l, img_ab)
+    save_image(torch.from_numpy(img_rgb.transpose(2, 0, 1)),
+               args.out_path)
+    print(f'>>> colorized image saved to "{args.out_path}"')
+
+
+if __name__ == '__main__':
+    main()