03.1-Batch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Celeb Faces Dataset (CelebA)\n",
    "=================================================\n",
    "---\n",
    "Introduction au Deep Learning  (IDLE) - S. Arias, E. Maldonado, JL. Parouty - CNRS/SARI/DEVLOG - 2020  \n",
    "\n",
    "We'll do the same thing again but with a more interesting dataset:  CelebFaces  \n",
    "About this dataset : http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html\n",
    "\n",
    "## Episode 1 : Preparation of data - Batch mode\n",
    "\n",
    " - Save enhanced datasets in h5 file format\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1 - Import and init\n",
    "### 1.2 - Import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from skimage import io, transform\n",
    "\n",
    "import os,time,sys,json,glob\n",
    "import csv\n",
    "import math, random\n",
    "\n",
    "from importlib import reload\n",
    "\n",
    "sys.path.append('..')\n",
    "import fidle.pwk as ooo\n",
    "\n",
    "ooo.init()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2 - Directories and files :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "place, dataset_dir = ooo.good_place( { 'GRICAD' : f'{os.getenv(\"SCRATCH_DIR\",\"\")}/PROJECTS/pr-fidle/datasets/celeba',\n",
    "                                       'IDRIS'  : f'{os.getenv(\"WORK\",\"\")}/datasets/celeba'    } )\n",
    "\n",
    "dataset_csv  = f'{dataset_dir}/list_attr_celeba.csv'\n",
    "dataset_img  = f'{dataset_dir}/img_align_celeba'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2 - Read filenames catalog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_desc = pd.read_csv(dataset_csv, header=0)\n",
    "dataset_desc = dataset_desc.reindex(np.random.permutation(dataset_desc.index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3 - Save as clusters of n images"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.2 - Cooking function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_and_save( dataset_img, dataset_desc, \n",
    "                   cluster_size=1000, cluster_dir='./dataset_cluster', cluster_name='images',\n",
    "                   image_size=(128,128)):\n",
    "    \n",
    "    def save_cluster(imgs,desc,cols,id):\n",
    "        file_img  = f'{cluster_dir}/{cluster_name}-{id:03d}.npy'\n",
    "        file_desc = f'{cluster_dir}/{cluster_name}-{id:03d}.csv'\n",
    "        np.save(file_img,  np.array(imgs))\n",
    "        df=pd.DataFrame(data=desc,columns=cols)\n",
    "        df.to_csv(file_desc, index=False)\n",
    "        return [],[],id+1\n",
    "    \n",
    "    start_time = time.time()\n",
    "    cols = list(dataset_desc.columns)\n",
    "\n",
    "    # ---- Check if cluster files exist\n",
    "    #\n",
    "    if os.path.isfile(f'{cluster_dir}/images-000.npy'):\n",
    "        print('\\n*** Oops. There are already clusters in the target folder!\\n')\n",
    "        return 0,0\n",
    "    \n",
    "    # ---- Create cluster_dir\n",
    "    #\n",
    "    os.makedirs(cluster_dir, mode=0o750, exist_ok=True)\n",
    "    \n",
    "    # ---- Read and save clusters\n",
    "    #\n",
    "    imgs, desc, cluster_id = [],[],0\n",
    "    #\n",
    "    for i,row in dataset_desc.iterrows():\n",
    "        #\n",
    "        filename = f'{dataset_img}/{row.image_id}'\n",
    "        #\n",
    "        # ---- Read image, resize (and normalize)\n",
    "        #\n",
    "        img = io.imread(filename)\n",
    "        img = transform.resize(img, image_size)\n",
    "        #\n",
    "        # ---- Add image and description\n",
    "        #\n",
    "        imgs.append( img )\n",
    "        desc.append( row.values )\n",
    "        #\n",
    "        # ---- Progress bar\n",
    "        #\n",
    "        ooo.update_progress(f'Cluster {cluster_id:03d} :',len(imgs),cluster_size)\n",
    "        #\n",
    "        # ---- Save cluster if full\n",
    "        #\n",
    "        if len(imgs)==cluster_size:\n",
    "            imgs,desc,cluster_id=save_cluster(imgs,desc,cols, cluster_id)\n",
    "\n",
    "    # ---- Save uncomplete cluster\n",
    "    if len(imgs)>0 : imgs,desc,cluster_id=save_cluster(imgs,desc,cols,cluster_id)\n",
    "\n",
    "    duration=time.time()-start_time\n",
    "    return cluster_id,duration\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.3 - Cluster building"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ---- Cluster size\n",
    "\n",
    "cluster_size_train = 10000\n",
    "cluster_size_test  = 10000\n",
    "image_size         = (192,160)\n",
    "\n",
    "# ---- Clusters location\n",
    "\n",
    "train_dir  = f'{dataset_dir}/clusters-M.train'\n",
    "test_dir   = f'{dataset_dir}/clusters-M.test'\n",
    "\n",
    "# ---- x_train, x_test\n",
    "#\n",
    "n1,d1 = read_and_save(dataset_img, dataset_desc[:200000],\n",
    "                      cluster_size = cluster_size_train, \n",
    "                      cluster_dir  = train_dir,\n",
    "                      image_size   = image_size )\n",
    "\n",
    "n2,d2 = read_and_save(dataset_img, dataset_desc[200000:],\n",
    "                      cluster_size = cluster_size_test, \n",
    "                      cluster_dir  = test_dir,\n",
    "                      image_size   = image_size )\n",
    "        \n",
    "print(f'\\n\\nDuration : {d1+d2:.2f} s or {ooo.hdelay(d1+d2)}')\n",
    "print(f'Train clusters : {train_dir}')\n",
    "print(f'Test  clusters : {test_dir}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----\n",
    "That's all folks !"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}