src/methods/domer/image-colorization/network.py

+import torch.nn as nn
+import torchvision.models as models
+
+
+class BasicBlock(nn.Module):
+    def __init__(self, in_channels, out_channels,
+                 activation=None, upsample=None):
+        super().__init__()
+
+        self.conv1 = nn.Conv2d(
+            in_channels=in_channels, out_channels=out_channels,
+            kernel_size=5, stride=1, padding=2, bias=False
+        )
+        self.bn1 = nn.BatchNorm2d(num_features=out_channels)
+        self.relu = nn.ReLU(inplace=True)
+        self.conv2 = nn.Conv2d(
+            in_channels=out_channels, out_channels=out_channels,
+            kernel_size=3, stride=1, padding=1, bias=False
+        )
+        self.bn2 = nn.BatchNorm2d(num_features=out_channels)
+
+        if activation is not None:
+            self.activation = activation
+        else:
+            self.res_conv = nn.Conv2d(
+                in_channels=in_channels, out_channels=out_channels,
+                kernel_size=1, stride=1, bias=False
+            )
+            self.res_bn = nn.BatchNorm2d(num_features=out_channels)
+
+        self.upsample = upsample
+
+    def forward(self, t):
+
+        res = t
+        t = self.conv1(t)
+        t = self.bn1(t)
+        t = self.relu(t)
+        t = self.conv2(t)
+        t = self.bn2(t)
+
+        if self.upsample is not None:
+            res = self.res_conv(res)
+            res = self.res_bn(res)
+            t += res
+            t = self.relu(t)
+            t = self.upsample(t)
+        else:
+            t += res
+            t = self.activation(t)
+
+        return t
+
+
+class ColorizeNet(nn.Module):
+    def __init__(self):
+        super().__init__()
+
+        # make pretrained=True before starting the training process
+        resnet18 = models.resnet18(pretrained=False)
+        # change first conv layer to accept single channel (grayscale)
+        resnet18.conv1.weight = nn.Parameter(
+            resnet18.conv1.weight.mean(dim=1).unsqueeze(dim=1))
+
+        # use first 3 layers of ResNet-18 as encoder
+        self.encoder = nn.Sequential(
+            *list(resnet18.children())[:6]
+        )
+        self.decoder = nn.Sequential(
+            self._make_layer(BasicBlock, 128, 64, nn.ReLU(inplace=True)),
+            self._make_layer(BasicBlock, 64, 32, nn.ReLU(inplace=True)),
+            self._make_layer(BasicBlock, 32, 2, nn.Sigmoid())
+        )
+
+    def _make_layer(self, block, in_channels, out_channels, activation):
+        upsample = nn.Upsample(scale_factor=2, mode='nearest')
+        layers = []
+        layers.append(block(in_channels, out_channels, upsample=upsample))
+        layers.append(block(out_channels, out_channels, activation=activation))
+
+        return nn.Sequential(*layers)
+
+    def forward(self, t):
+        t = self.encoder(t)
+        t = self.decoder(t)
+
+        return t

src/methods/domer/image-colorization/requirements.txt

+matplotlib==3.3.2
+numpy==1.19.4
+Pillow==8.0.1
+scikit-image==0.17.2
+torch==1.7.0+cpu
+torchvision==0.8.1+cpu
+tqdm==4.51.0
\ No newline at end of file

src/methods/domer/image-colorization/train.ipynb

+%% Cell type:code id: tags:
+
+``` 
+# from google.colab import drive
+# drive.mount('/content/drive')
+```
+
+%% Output
+
+    Mounted at /content/drive
+
+%% Cell type:code id: tags:
+
+``` 
+# Download and unzip (2.2GB)
+# !wget http://data.csail.mit.edu/places/places205/testSetPlaces205_resize.tar.gz
+# !mkdir images/
+# !tar -xzf drive/MyDrive/testSetPlaces205_resize.tar.gz -C 'images/'
+```
+
+%% Cell type:code id: tags:
+
+``` 
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import torchvision.transforms as transforms
+
+from tqdm import tqdm
+from network import ColorizeNet
+from utils import GrayscaleImageFolder
+```
+
+%% Cell type:code id: tags:
+
+``` 
+device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
+```
+
+%% Cell type:code id: tags:
+
+``` 
+transform = transforms.Compose([
+    transforms.RandomResizedCrop(224),
+    transforms.RandomHorizontalFlip()
+])
+
+train_set = GrayscaleImageFolder('images/', transform=transform)
+train_loader = torch.utils.data.DataLoader(train_set, batch_size=128, shuffle=True, num_workers=4)
+```
+
+%% Cell type:code id: tags:
+
+``` 
+criterion = nn.MSELoss()
+
+model = ColorizeNet().to(device)
+optimizer = optim.Adam(model.parameters(), lr=0.001)
+# scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[8, 24], gamma=1/3)
+scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=1/3, patience=5, verbose=True)
+```
+
+%% Cell type:code id: tags:
+
+``` 
+try:
+    checkpoint = torch.load('drive/MyDrive/checkpoint_0.001.pth', map_location=device)
+    start_epoch = checkpoint['next_epoch']
+    model.load_state_dict(checkpoint['model'])
+    optimizer.load_state_dict(checkpoint['optimizer'])
+    scheduler.load_state_dict(checkpoint['scheduler'])
+
+except:
+    start_epoch = 0
+```
+
+%% Cell type:code id: tags:
+
+``` 
+num_epochs = 64
+for epoch in range(start_epoch, num_epochs):
+    train_loss = 0
+    loop = tqdm(train_loader)
+    for batch in loop:
+        in_gray, in_ab = batch[0].to(device), batch[1].to(device)
+        out_ab = model(in_gray)
+        loss = criterion(out_ab, in_ab)
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+
+        train_loss += loss.item()*in_gray.size(0)
+        loop.set_description(f'Epoch [{epoch+1:2d}/{num_epochs}]')
+        loop.set_postfix(loss=train_loss)
+
+    scheduler.step(train_loss)
+    checkpoint = {
+        'next_epoch': epoch + 1,
+        'model': model.state_dict(),
+        'optimizer': optimizer.state_dict(),
+        'scheduler': scheduler.state_dict()
+    }
+    torch.save(checkpoint, f'drive/MyDrive/checkpoint_{scheduler._last_lr[0]}.pth')
+```
+
+%% Output
+
+    Epoch [39/64]: 100%|██████████| 321/321 [12:00<00:00,  2.24s/it, loss=97.4]
+    Epoch [40/64]: 100%|██████████| 321/321 [11:30<00:00,  2.15s/it, loss=98]
+    Epoch [41/64]: 100%|██████████| 321/321 [11:18<00:00,  2.12s/it, loss=97.4]
+    Epoch [42/64]: 100%|██████████| 321/321 [11:28<00:00,  2.15s/it, loss=103]
+    Epoch [43/64]: 100%|██████████| 321/321 [11:23<00:00,  2.13s/it, loss=98.4]
+    Epoch [44/64]: 100%|██████████| 321/321 [11:15<00:00,  2.10s/it, loss=97.3]
+    Epoch [45/64]: 100%|██████████| 321/321 [11:29<00:00,  2.15s/it, loss=97.6]
+    Epoch [46/64]: 100%|██████████| 321/321 [11:25<00:00,  2.14s/it, loss=96.5]
+    Epoch [47/64]: 100%|██████████| 321/321 [11:40<00:00,  2.18s/it, loss=96.7]
+    Epoch [48/64]: 100%|██████████| 321/321 [11:44<00:00,  2.19s/it, loss=96.3]
+    Epoch [49/64]: 100%|██████████| 321/321 [11:13<00:00,  2.10s/it, loss=96.5]
+    Epoch [50/64]: 100%|██████████| 321/321 [11:09<00:00,  2.09s/it, loss=96.4]
+    Epoch [51/64]: 100%|██████████| 321/321 [11:09<00:00,  2.09s/it, loss=95.7]
+    Epoch [52/64]: 100%|██████████| 321/321 [11:13<00:00,  2.10s/it, loss=95.9]
+    Epoch [53/64]: 100%|██████████| 321/321 [11:08<00:00,  2.08s/it, loss=95.8]
+    Epoch [54/64]: 100%|██████████| 321/321 [11:31<00:00,  2.15s/it, loss=95.4]
+    Epoch [55/64]: 100%|██████████| 321/321 [11:54<00:00,  2.23s/it, loss=95.5]
+    Epoch [56/64]: 100%|██████████| 321/321 [12:11<00:00,  2.28s/it, loss=95.5]
+    Epoch [57/64]: 100%|██████████| 321/321 [12:00<00:00,  2.24s/it, loss=94.9]
+    Epoch [58/64]: 100%|██████████| 321/321 [11:54<00:00,  2.23s/it, loss=94.9]
+    Epoch [59/64]: 100%|██████████| 321/321 [11:56<00:00,  2.23s/it, loss=94.9]
+    Epoch [60/64]: 100%|██████████| 321/321 [12:06<00:00,  2.26s/it, loss=94.3]
+    Epoch [61/64]: 100%|██████████| 321/321 [12:03<00:00,  2.26s/it, loss=95]
+    Epoch [62/64]: 100%|██████████| 321/321 [12:01<00:00,  2.25s/it, loss=95]
+    Epoch [63/64]: 100%|██████████| 321/321 [12:05<00:00,  2.26s/it, loss=96.7]
+    Epoch [64/64]: 100%|██████████| 321/321 [12:10<00:00,  2.27s/it, loss=95]
+
+%% Cell type:code id: tags:
+
+``` 
+torch.save(model.state_dict(), f'./models/model.pth')  # save the model separately from the checkpoint file
+```

src/methods/domer/image-colorization/utils.py

+import torch
+from torchvision import transforms, datasets
+import numpy as np
+from PIL import Image
+from skimage.color import rgb2lab, rgb2gray, lab2rgb
+
+
+def count_params(model):
+    '''
+    returns the number of trainable parameters in some model
+    '''
+    return sum(p.numel() for p in model.parameters() if p.requires_grad)
+
+
+class GrayscaleImageFolder(datasets.ImageFolder):
+    '''
+    Custom dataloader for various operations on images before loading them.
+    '''
+
+    def __getitem__(self, index):
+        path, target = self.imgs[index]
+        img = self.loader(path)
+        if self.transform is not None:
+            img_orig = self.transform(img)  # apply transforms
+            img_orig = np.asarray(img_orig)  # convert to numpy array
+            img_lab = rgb2lab(img_orig)  # convert RGB image to LAB
+            img_ab = img_lab[:, :, 1:3]  # separate AB channels from LAB
+            img_ab = (img_ab + 128) / 255  # normalize the pixel values
+            # transpose image from HxWxC to CxHxW and turn it into a tensor
+            img_ab = torch.from_numpy(img_ab.transpose((2, 0, 1))).float()
+            img_orig = rgb2gray(img_orig)  # convert RGB to grayscale
+            # add a channel axis to grascale image and turn it into a tensor
+            img_orig = torch.from_numpy(img_orig).unsqueeze(0).float()
+
+        if self.target_transform is not None:
+            target = self.target_transform(target)
+
+        return img_orig, img_ab, target
+
+
+def load_gray(path, max_size=360, shape=None):
+    '''
+    load an image as grayscale, change the shape as per input,
+    perform transformations and convert it to model compatable shape.
+    '''
+    img_gray = Image.open(path).convert('L')
+
+    if max(img_gray.size) > max_size:
+        size = max_size
+    else:
+        size = max(img_gray.size)
+
+    if shape is not None:
+        size = shape
+
+    img_transform = transforms.Compose([
+        transforms.Resize(size),
+        transforms.ToTensor()
+    ])
+
+    img_gray = img_transform(img_gray).unsqueeze(0)
+    return img_gray
+
+
+def to_rgb(img_l, img_ab):
+    '''
+    concatinates Lightness (grayscale) and AB channels,
+    and converts the resulting LAB image to RGB
+    '''
+    if img_l.shape == img_ab.shape:
+        img_lab = torch.cat((img_l, img_ab), 1).numpy().squeeze()
+    else:
+        img_lab = torch.cat(
+            (img_l, img_ab[:, :, :img_l.size(2), :img_l.size(3)]),
+            dim=1
+        ).numpy().squeeze()
+
+    img_lab = img_lab.transpose(1, 2, 0)  # transpose image to HxWxC
+    img_lab[:, :, 0] = img_lab[:, :, 0] * 100  # range pixel values from 0-100
+    img_lab[:, :, 1:] = img_lab[:, :, 1:] * 255 - 128  # un-normalize
+    img_rgb = lab2rgb(img_lab.astype(np.float64))  # convert LAB image to RGB
+
+    return img_rgb

src/methods/domer/reconstruct.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Tue Jan 30 19:35:12 2024
+
+@author: stani1
+"""
+
+"""The main file for the baseline reconstruction.
+This file should NOT be modified.
+"""
+
+
+import numpy as np
+from PIL import Image
+import subprocess
+import os
+
+
+from src.forward_model import CFA
+from src.methods.baseline.demo_reconstruction import naive_interpolation
+
+
+def run_reconstruction(y: np.ndarray, cfa: str) -> np.ndarray:
+    """Performs demosaicking on y.
+
+    Args:
+        y (np.ndarray): Mosaicked image to be reconstructed.
+        cfa (str): Name of the CFA. Can be bayer or quad_bayer.
+
+    Returns:
+        np.ndarray: Demosaicked image.
+    """
+    current_directory = os.getcwd() #we take the current directory to be sure we will have the correct one
+    input_path='src/methods/domer/image-colorization/input/0.jpg'
+    input_path = os.path.join(current_directory, input_path)#location to store the image
+    output_path='src/methods/domer/image-colorization/output/0.jpg'
+    output_path = os.path.join(current_directory, output_path)
+    # venv_path='src/methods/domer/image-colorization/image-colorization/bin/activate'
+    # venv_path = os.path.join(current_directory, venv_path) #activation of the venv, but no need for a lot of versions of libraries
+    y2=Image.fromarray(y)
+    y2=y2.convert('L')
+    y2.save(input_path)
+    resolution=np.shape(y)[0]
+    # activation_venv = ['source ', venv_path]
+    # subprocess.run(activation_venv, shell=True)
+    n=y2.size[0]
+    # for i in range (n) :
+    #     for j in range (n) :
+    #         y2.putpixel((i,j),y.getpixel((i,j))*255)
+    command = [
+        'python', 'src/methods/domer/image-colorization/colorize.py',
+        '-i', input_path,
+        '-o', output_path,
+        '-r', str(resolution)
+    ]
+    subprocess.run(command) #calling of the NN with the location of the input and output and the resolution
+    # deactivation_venv = f"deactivate"
+    # subprocess.run(deactivation_venv, shell=True)
+    res=Image.open(output_path) #We take the result of the NN
+    res=np.asarray(res)#reseting the result in a numpy array
+
+    return res
+
+

src/methods/dzik_erwan/functions.py

+import numpy as np 
+from scipy.ndimage import convolve 
+
+from src.forward_model import CFA
+
+def malvar2004(y,op): 
+    "Code inspiered by KelSolaar's GitHub repo:"
+    "https://github.com/colour-science/colour-demosaicing/tree/master"
+
+    z = op.adjoint(y)
+    mask = op.mask
+
+    R = z[:,:,0]
+    G = z[:,:,1]
+    B = z[:,:,2]
+
+    R_m, G_m, B_m = mask[:,:,0], mask[:,:,1], mask[:,:,2]
+
+    
+    G = np.where(np.logical_or(R_m == 1, B_m == 1), convolve(y, GR_GB), G)
+
+
+    #Logical array that verify the conditon of the different filters 
+    # Red rows.
+    R_r = np.transpose(np.any(R_m == 1, axis=1)[None]) * np.ones(R.shape)
+    # Red columns.
+    R_c = np.any(R_m == 1, axis=0)[None] * np.ones(R.shape)
+    # Blue rows.
+    B_r = np.transpose(np.any(B_m == 1, axis=1)[None]) * np.ones(B.shape)
+    # Blue columns
+    B_c = np.any(B_m == 1, axis=0)[None] * np.ones(B.shape)
+
+    #Results of the convolutions
+    RBg_RBBR = convolve(y, Rg_RB_Bg_BR)
+    RBg_BRRB = convolve(y, Rg_BR_Bg_RB)
+    RBgr_BBRR = convolve(y, Rb_BB_Br_RR)
+
+    
+    R = np.where(np.logical_and(R_r == 1, B_c == 1), RBg_RBBR, R)
+    R = np.where(np.logical_and(B_r == 1, R_c == 1), RBg_BRRB, R)
+
+    B = np.where(np.logical_and(B_r == 1, R_c == 1), RBg_RBBR, B)
+    B = np.where(np.logical_and(R_r == 1, B_c == 1), RBg_BRRB, B)
+
+    R = np.where(np.logical_and(B_r == 1, B_c == 1), RBgr_BBRR, R)
+    B = np.where(np.logical_and(R_r == 1, R_c == 1), RBgr_BBRR, B)
+
+    
+    res = np.clip(np.stack((R,G,B), axis = 2), 0, 1)
+
+
+    return res
+
+def quad_bayer_to_bayer(cfa_quad_bayer):
+    "Quad to Bayer by swapping method"
+
+    cfa_bayer = np.flip(np.copy(cfa_quad_bayer), 1)
+    h,w = cfa_bayer.shape
+    for col in range(1, w, 4):
+        temp = np.copy(cfa_bayer[:, col])
+        cfa_bayer[:, col] = cfa_bayer[:, col + 1]
+        cfa_bayer[:, col + 1] = temp
+    del temp
+
+
+    for row in range(1, h, 4):
+        temp = np.copy(cfa_bayer[row, :])
+        cfa_bayer[row, :] = cfa_bayer[row + 1, :]
+        cfa_bayer[row + 1, :] = temp
+    del temp
+
+    for row in range(1, h, 4):
+        for col in range(2, w, 4):
+            g1 = cfa_bayer[row, col]
+            cfa_bayer[row, col] = cfa_bayer[row + 1, col -1]
+            cfa_bayer[row + 1, col -1] = g1
+
+    return np.flip(cfa_bayer,1)
+
+# G at R locations or G at B locations
+GR_GB = (1/8) * np.asarray([
+    [0., 0., -1., 0., 0.],
+    [0., 0., 2., 0., 0.],
+    [-1., 2., 4., 2., -1.],
+    [0., 0., 2., 0., 0.],
+    [0., 0., -1., 0., 0.],
+])
+
+#R at green in R row, B column or B at green in B row, R colunm
+Rg_RB_Bg_BR = (1/8) * np.asarray([
+    [0., 0., .5, 0., 0.],
+    [0., -1., 0., -1., 0.],
+    [-1., 4., 5., 4., -1.],
+    [0., -1., 0., -1., 0.],
+    [0., 0., .5, 0., 0.],
+])
+
+# R at green in B row, R column or B at green in R row, B column 
+Rg_BR_Bg_RB = (1/8) * np.asarray([
+    [0., 0., -1, 0., 0.],
+    [0., -1., 4., -1., 0.],
+    [.5, 0., 5., 0., .5],
+    [0., -1., 4., -1., 0.],
+    [0., 0., -1, 0., 0.],
+])
+
+#R at blue in B row, B column or B at red in R row, R column
+Rb_BB_Br_RR = (1/8) * np.asarray([
+    [0., 0., -1.5, 0., 0.],
+    [0., 2., 0., 2., 0.],
+    [-1.5, 0., 6., 0., -1.5],
+    [0., 2., 0., 2., 0.],
+    [0., 0., -1.5, 0., 0.],
+])
+