{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Celeb Faces Dataset (CelebA)\n",
"=================================================\n",
"---\n",
"Introduction au Deep Learning (IDLE) - S. Arias, E. Maldonado, JL. Parouty - CNRS/SARI/DEVLOG - 2020 \n",
"\n",
"We'll do the same thing again but with a more interesting dataset: CelebFaces \n",
"About this dataset : http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html\n",
"\n",
"## Episode 1 : Preparation of data - Batch mode\n",
"\n",
" - Save enhanced datasets in h5 file format\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 1 - Import and init\n",
"### 1.2 - Import"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<style>\n",
"\n",
"div.warn { \n",
" background-color: #fcf2f2;\n",
" border-color: #dFb5b4;\n",
" border-left: 5px solid #dfb5b4;\n",
" padding: 0.5em;\n",
" font-weight: bold;\n",
" font-size: 1.1em;;\n",
" }\n",
"\n",
"\n",
"\n",
"div.nota { \n",
" background-color: #DAFFDE;\n",
" border-left: 5px solid #92CC99;\n",
" padding: 0.5em;\n",
" }\n",
"\n",
"\n",
"\n",
"</style>\n",
"\n"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"FIDLE 2020 - Practical Work Module\n",
"Version : 0.2.8\n",
"Run time : Thursday 13 February 2020, 23:50:25\n",
"TensorFlow version : 2.0.0\n",
"Keras version : 2.2.4-tf\n"
]
}
],
"source": [
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"import pandas as pd\n",
"from skimage import io, transform\n",
"\n",
"import os,time,sys,json,glob\n",
"import csv\n",
"import math, random\n",
"\n",
"from importlib import reload\n",
"\n",
"sys.path.append('..')\n",
"import fidle.pwk as ooo\n",
"\n",
"ooo.init()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 1.2 - Directories and files :"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Well, we should be at IDRIS !\n",
"We are going to use: /gpfswork/rech/mlh/uja62cb/datasets/celeba\n"
]
}
],
"source": [
"place, dataset_dir = ooo.good_place( { 'GRICAD' : f'{os.getenv(\"SCRATCH_DIR\",\"\")}/PROJECTS/pr-fidle/datasets/celeba',\n",
" 'IDRIS' : f'{os.getenv(\"WORK\",\"\")}/datasets/celeba' } )\n",
"\n",
"dataset_csv = f'{dataset_dir}/list_attr_celeba.csv'\n",
"dataset_img = f'{dataset_dir}/img_align_celeba'"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 2 - Read filenames catalog"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"dataset_desc = pd.read_csv(dataset_csv, header=0)\n",
"dataset_desc = dataset_desc.reindex(np.random.permutation(dataset_desc.index))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 3 - Save as clusters of n images"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 4.2 - Cooking function"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"def read_and_save( dataset_img, dataset_desc, \n",
" cluster_size=1000, cluster_dir='./dataset_cluster', cluster_name='images',\n",
" image_size=(128,128)):\n",
" \n",
" def save_cluster(imgs,desc,cols,id):\n",
" file_img = f'{cluster_dir}/{cluster_name}-{id:03d}.npy'\n",
" file_desc = f'{cluster_dir}/{cluster_name}-{id:03d}.csv'\n",
" np.save(file_img, np.array(imgs))\n",
" df=pd.DataFrame(data=desc,columns=cols)\n",
" df.to_csv(file_desc, index=False)\n",
" return [],[],id+1\n",
" \n",
" start_time = time.time()\n",
" cols = list(dataset_desc.columns)\n",
"\n",
" # ---- Check if cluster files exist\n",
" #\n",
" if os.path.isfile(f'{cluster_dir}/images-000.npy'):\n",
" print('\\n*** Oops. There are already clusters in the target folder!\\n')\n",
" return 0,0\n",
" \n",
" # ---- Create cluster_dir\n",
" #\n",
" os.makedirs(cluster_dir, mode=0o750, exist_ok=True)\n",
" \n",
" # ---- Read and save clusters\n",
" #\n",
" imgs, desc, cluster_id = [],[],0\n",
" #\n",
" for i,row in dataset_desc.iterrows():\n",
" #\n",
" filename = f'{dataset_img}/{row.image_id}'\n",
" #\n",
" # ---- Read image, resize (and normalize)\n",
" #\n",
" img = io.imread(filename)\n",
" img = transform.resize(img, image_size)\n",
" #\n",
" # ---- Add image and description\n",
" #\n",
" imgs.append( img )\n",
" desc.append( row.values )\n",
" #\n",
" # ---- Progress bar\n",
" #\n",
" ooo.update_progress(f'Cluster {cluster_id:03d} :',len(imgs),cluster_size)\n",
" #\n",
" # ---- Save cluster if full\n",
" #\n",
" if len(imgs)==cluster_size:\n",
" imgs,desc,cluster_id=save_cluster(imgs,desc,cols, cluster_id)\n",
"\n",
" # ---- Save uncomplete cluster\n",
" if len(imgs)>0 : imgs,desc,cluster_id=save_cluster(imgs,desc,cols,cluster_id)\n",
"\n",
" duration=time.time()-start_time\n",
" return cluster_id,duration\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 4.3 - Cluster building"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"*** Oops. There are already clusters in the target folder!\n",
"\n",
"\n",
"*** Oops. There are already clusters in the target folder!\n",
"\n",
"\n",
"\n",
"Duration : 0.00 s or 0:00:00\n",
"Train clusters : /gpfswork/rech/mlh/uja62cb/datasets/celeba/clusters-M.train\n",
"Test clusters : /gpfswork/rech/mlh/uja62cb/datasets/celeba/clusters-M.test\n"
]
}
],
"source": [
"# ---- Cluster size\n",
"\n",
"cluster_size_train = 10000\n",
"cluster_size_test = 10000\n",
"image_size = (192,160)\n",
"\n",
"# ---- Clusters location\n",
"\n",
"train_dir = f'{dataset_dir}/clusters-M.train'\n",
"test_dir = f'{dataset_dir}/clusters-M.test'\n",
"\n",
"# ---- x_train, x_test\n",
"#\n",
"n1,d1 = read_and_save(dataset_img, dataset_desc[:200000],\n",
" cluster_size = cluster_size_train, \n",
" cluster_dir = train_dir,\n",
" image_size = image_size )\n",
"\n",
"n2,d2 = read_and_save(dataset_img, dataset_desc[200000:],\n",
" cluster_size = cluster_size_test, \n",
" cluster_dir = test_dir,\n",
" image_size = image_size )\n",
" \n",
"print(f'\\n\\nDuration : {d1+d2:.2f} s or {ooo.hdelay(d1+d2)}')\n",
"print(f'Train clusters : {train_dir}')\n",
"print(f'Test clusters : {test_dir}')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"----\n",
"That's all folks !"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}