$#&@! de VAE...

VAE/01-VAE-with-MNIST.ipynb

@@ -174,7 +174,7 @@
     "image_periodicity = 1      # for each epoch\n",
     "chkpt_periodicity = 2      # for each epoch\n",
     "initial_epoch     = 0\n",
-    "dataset_size      = 1      # 1 mean 100%"
+    "k_size            = 1      # 1 mean using 100% of the dataset"
    ]
   },
   {
@@ -403,7 +403,7 @@
     "          image_periodicity = image_periodicity,\n",
     "          chkpt_periodicity = chkpt_periodicity,\n",
     "          initial_epoch     = initial_epoch,\n",
-    "          dataset_size      = dataset_size\n",
+    "          k_size            = k_size\n",
     "         )"
    ]
   },

VAE/modules/data_generator.py

@@ -3,7 +3,7 @@
 #     _____ _     _ _
 #    |  ___(_) __| | | ___
 #    | |_  | |/ _` | |/ _ \
-#    |  _| | | (_| | |  __/       Data_generator
+#    |  _| | | (_| | |  __/       DataGenerator
 #    |_|   |_|\__,_|_|\___|       for clustered CelebA sataset
 # ------------------------------------------------------------------
 # Formation Introduction au Deep Learning  (FIDLE)
@@ -19,11 +19,11 @@ import os,glob
 
 from tensorflow.keras.utils import Sequence
 
-class Data_generator(Sequence):
+class DataGenerator(Sequence):
 
-    version = 0.1
+    version = 0.4
     
-    def __init__(self, clusters_dir='./data', batch_size=32, debug=False):
+    def __init__(self, clusters_dir='./data', batch_size=32, debug=False, k_size=1):
         '''
         Instanciation of the data generator
         args:
@@ -31,6 +31,7 @@ class Data_generator(Sequence):
             batch_size  : Batch size (32)
             debug       : debug mode (False)
         '''
+        if debug : self.about()
         #
         # ---- Get the list of clusters
         #      
@@ -44,8 +45,13 @@ class Data_generator(Sequence):
         for c in clusters_name:
             df = pd.read_csv(c+'.csv', header=0)
             dataset_size+=len(df.index)
+        #
+        # ---- If we only want to use a part of the dataset...
+        #
+        dataset_size = int(dataset_size * k_size)
+        #
         if debug: 
-            print(f'Clusters nb  : {len(clusters_name)} files')
+            print(f'\nClusters nb  : {len(clusters_name)} files')
             print(f'Dataset size : {dataset_size}')
             print(f'Batch size   : {batch_size}')
 
@@ -99,6 +105,11 @@ class Data_generator(Sequence):
         return batch, batch
     
     
+    def on_epoch_end(self):
+        self.cluster_i = clusters_size
+        self.read_next_cluster()
+    
+    
     def read_next_cluster(self):
         #
         # ---- Get the next cluster name
@@ -123,10 +134,9 @@ class Data_generator(Sequence):
         self.cluster_i = i
         #
         if self.debug: print(f'\n[Load {self.cluster_i:02d},s={len(self.data):3d}] ',end='')
-
-    
+          
         
     @classmethod
     def about(cls):
-        print('\nFIDLE 2020 - Data_generator')
+        print('\nFIDLE 2020 - DataGenerator')
         print('Version              :', cls.version)
\ No newline at end of file

VAE/modules/vae.py

@@ -23,10 +23,9 @@ from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard
 from tensorflow.keras.optimizers import Adam
 from tensorflow.keras.utils import plot_model
 
-import tensorflow.keras.datasets.imdb as imdb
+from modules.callbacks      import ImagesCallback
+from modules.data_generator import DataGenerator
 
-import modules.callbacks
-from modules.callbacks import ImagesCallback
 
 import os, json, time, datetime
 
@@ -163,20 +162,33 @@ class VariationalAutoencoder():
         print(f'Optimizer is Adam with learning_rate={learning_rate:}')
     
     
+    
     def train(self, 
-              x_train,x_test,
+              x_train=None,
+              x_test=None,
+              data_generator=None,
               batch_size=32, 
               epochs=200, 
               image_periodicity=1,
               chkpt_periodicity=2,
               initial_epoch=0,
-              dataset_size=1
+              k_size=1
              ):
 
-        # ---- Dataset size
-        n_train = int(x_train.shape[0] * dataset_size)
-        n_test  = int(x_test.shape[0]  * dataset_size)
-
+        # ---- Data given or via generator
+        mode_data = (data_generator is None)
+        
+        # ---- Size of the dataset we are going to use
+        #      k_size ==1 : mean 100%
+        #      ** Cannot be use with data generator **
+        #
+        if mode_data:
+            n_train = int(x_train.shape[0] * k_size)
+            n_test  = int(x_test.shape[0]  * k_size)
+        else:
+            n_train = len(data_generator)
+            n_test  = int(x_test.shape[0])
+            
         # ---- Need by callbacks
         self.n_train    = n_train
         self.n_test     = n_test
@@ -201,20 +213,38 @@ class VariationalAutoencoder():
 
         # ---- Let's go...
         start_time   = time.time()
-        self.history = self.model.fit(x_train[:n_train], x_train[:n_train],
-                                      batch_size = batch_size,
-                                      shuffle = True,
-                                      epochs = epochs,
-                                      initial_epoch = initial_epoch,
-                                      callbacks = callbacks_list,
-                                      validation_data = (x_test[:n_test], x_test[:n_test])
-                                      )
+        
+        if mode_data:
+            #
+            # ---- With pure data (x_train) -----------------------------------------
+            #                             
+            self.history = self.model.fit(x_train[:n_train], x_train[:n_train],
+                                          batch_size = batch_size,
+                                          shuffle = True,
+                                          epochs = epochs,
+                                          initial_epoch = initial_epoch,
+                                          callbacks = callbacks_list,
+                                          validation_data = (x_test[:n_test], x_test[:n_test])
+                                          )
+            #
+        else:
+            # ---- With Data Generator ----------------------------------------------
+            #
+            self.history = self.model.fit(data_generator,
+                                          shuffle = True,
+                                          epochs = epochs,
+                                          initial_epoch = initial_epoch,
+                                          callbacks = callbacks_list
+#                                           validation_data = (x_test, x_test)
+                                         )
+        
         end_time  = time.time()
         dt  = end_time-start_time
         dth = str(datetime.timedelta(seconds=int(dt)))
         self.duration = dt
         print(f'\nTrain duration : {dt:.2f} sec. - {dth:}')
-        
+
+
     def plot_model(self):
         d=self.run_directory+'/figs'
         plot_model(self.model,   to_file=f'{d}/model.png',   show_shapes = True, show_layer_names = True, expand_nested=True)