Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
# ------------------------------------------------------------------
# _____ _ _ _
# | ___(_) __| | | ___
# | |_ | |/ _` | |/ _ \
# | _| | | (_| | | __/
# |_| |_|\__,_|_|\___| Dataset reader
# ------------------------------------------------------------------
# Formation Introduction au Deep Learning (FIDLE) - CNRS/MIAI/UGA
# ------------------------------------------------------------------
# JL Parouty 2023
import h5py
import os
import fidle
def read_dataset(enhanced_dir, dataset_name, scale=1):
'''
Reads h5 dataset
Args:
filename : datasets filename
dataset_name : dataset name, without .h5
Returns:
x_train,y_train, x_test,y_test data, x_meta,y_meta
'''
# ---- Read dataset
#
chrono=fidle.Chrono()
chrono.start()
filename = f'{enhanced_dir}/{dataset_name}.h5'
with h5py.File(filename,'r') as f:
x_train = f['x_train'][:]
y_train = f['y_train'][:]
x_test = f['x_test'][:]
y_test = f['y_test'][:]
x_meta = f['x_meta'][:]
y_meta = f['y_meta'][:]
# ---- Rescale
#
print('Original shape :', x_train.shape, y_train.shape)
x_train,y_train, x_test,y_test = fidle.utils.rescale_dataset(x_train,y_train,x_test,y_test, scale=scale)
print('Rescaled shape :', x_train.shape, y_train.shape)
# ---- Shuffle
#
x_train,y_train=fidle.utils.shuffle_np_dataset(x_train,y_train)
# ---- done
#
duration = chrono.get_delay()
size = fidle.utils.hsize(os.path.getsize(filename))
print(f'\nDataset "{dataset_name}" is loaded and shuffled. ({size} in {duration})')
return x_train,y_train, x_test,y_test, x_meta,y_meta
print('Module my_loader loaded.')