datadump.sh 4.97 KB
Newer Older
1
2
3
#!/bin/bash

# Auteur: Jonathan Schaeffer <jonathan.schaeffer@univ-grenoble-alpes.fr>
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
4
5
6
#
# This script dumps precious RESIF data to an IRODS server
#
7
8
9
10
11
12
13

# Print a message using logger
# Arguments:
# $1 : message
# $2 : severity (optionnal, default is INFO)
log(){
    [[ -z $2 ]] && severity="info" || severity=$2
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
14
    logger -t resifdd -p local3.$severity "$1"
15
16
}

Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
17
18
19
20
21
22
####################
#
# Preliminary tests
#
####################

23
24
25
26
27
# Test for working directory
if [[ ! -w $RESIFDD_WORKDIR ]] ; then
    log "RESIFDD_WORKDIR $RESIFDD_WORKDIR not writable. Check permissions"
    exit 1
fi
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
28
# test the data directory where to dump everything from
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
if [[ -z $RESIFDD_DATADIR ]]; then
    log "Variable RESIFDD_DATADIR must be set to the RESIF mountpoint" "error"
    exit 1
fi
if [[ ! -d $RESIFDD_DATADIR ]]; then
    log "Variable RESIFDD_DATADIR set to $RESIFDD_DATADIR must be a directory" "error"
    exit 1
fi

# Get the snapshot name for this month
MONTH=$(date +%Y-%m)
SNAPSHOT_DIR="${RESIFDD_DATADIR}/validated_seismic_data/.snapshot/monthly.${MONTH}-01_0000"

if [[ ! -d $SNAPSHOT_DIR ]]; then
    log "Snapshot directory $SNAPSHOT_DIR does not exist" "error"
    exit 1
fi

# Checking IRODS environment
Jonathan Schaeffer's avatar
debug    
Jonathan Schaeffer committed
48
if [[ ! -f ~/.irods/.irodsA ]] ; then
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
49
50
51
52
53
54
    log "The irods scrambled password file is not present. Please run iinit and provide the password"
    exit 1
fi

####################
#
55
# Start dumping validated data
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
56
57
58
#
####################

Jonathan Schaeffer's avatar
Debug    
Jonathan Schaeffer committed
59
cd $SNAPSHOT_DIR
60

Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
61
62
log "Starting dump of validated data"

Jonathan Schaeffer's avatar
Debug    
Jonathan Schaeffer committed
63
for dir in $(find . -maxdepth 3 -type d); do
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
64
    # First clean workspace
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
65
    rm -f $RESIFDD_WORKDIR/*.tar
66
    IFS='/' read -r -a YNS <<< $dir
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
67
68
69
70
    [ ! ${#YNS[@]} -eq 4 ] && continue
    YEAR=${YNS[1]}
    NETWORK=${YNS[2]}
    STATION=${YNS[3]}
Jonathan Schaeffer's avatar
debug    
Jonathan Schaeffer committed
71
    KEY=${YEAR}_${NETWORK}_${STATION}
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
72
    log "[$KEY] Creating tar on $RESIFDD_WORKDIR/$KEY.tar"
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
73
74
    tar cf $RESIFDD_WORKDIR/$KEY.tar $dir
    if [[ $? -ne 0 ]]; then
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
75
76
77
78
79
80
        # Something went wrong creating archive. Exit
        log "[$KEY] tar reported an error. Abort" error
        exit 1
    fi
    SIZE=$(( $(stat -c %s $RESIFDD_WORKDIR/$KEY.tar) / 1024 / 1024 / 1024 ))
    # Check if file exists on irods server
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
81
82
    irods_latest=$(ils -L $KEY/latest.tar)
    log "[$KEY] irods latest file is : $irods_latest" "debug"
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
83
84
    if [[ $? -eq 0 ]]; then
        log "[$KEY] latest.tar already exists on iRODS server. Let's compare hashes"
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
85
86
        local_sha=$(sha256sum $RESIFDD_WORKDIR/$KEY.tar | awk '{print $1}' | xxd -r -p | base64)
        irods_sha=$(ichksum $KEY/latest.tar | awk -F':' '/sha2:/ {print $2; exit;}')
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
87
        # Compare to local hash
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
88

Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
89
        # If the hashes differs, then move distant file and push this one
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
90
        if [[ "$local_sha" = "$irods_sha" ]]; then
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
91
            log "[$KEY] The archive on irods is the same as our version. Skipping."
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
92
93
            continue
        fi
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
94
        log "[$KEY] local checksum: $local_sha" "debug"
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
95
        log "[$KEY] idos checksum:  $irods_sha" "debug"
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
96

Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
97
        log "[$KEY] Local checksum differs from last dump. Let's dump it"
Jonathan Schaeffer's avatar
Ajout    
Jonathan Schaeffer committed
98
        imv $KEY/previous.tar $KEY/previous_to_delete.tar
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
99
100
101
102
        imv $KEY/latest.tar $KEY/previous.tar  # Rename latest distant dump tp "previous"
        if [[ $? -ne 0 ]]; then
            # Something wrong moving remote file arround
            log "[$KEY] There where something wrong moving $KEY/previous.tar, cancel iput" "error"
Jonathan Schaeffer's avatar
Ajout    
Jonathan Schaeffer committed
103
            irm -f $KEY/previous_to_delete.tar
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
104
105
106
            # zabbixsnd $KEY
            continue
        fi
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
107
108
    fi
    # Send file to IRODS
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
109
    log "[$KEY] Sending file to iRODS ($SIZE GB)"
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
110
    imkdir -p $KEY
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
111
112
    # Décommenter le bloc et les redirections pour traiter stdout et stderr différemment
    # {
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
113
114
    iput \
        --lfrestart $RESIFDD_WORKDIR/$KEY.lfrestart \
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
115
        --retries 5 -T -K -f \
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
116
        $RESIFDD_WORKDIR/$KEY.tar $KEY/latest.tar # 2>&1 1>&3 3>&- | logger -t 'resifdd' -p local3.error; } 3>&1 1>&2 | logger -t 'resifdd' -p local3.info
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
117
118
    RC=$?
    if [[ $RC -ne 0 ]]; then
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
119
        log "[$KEY] Error sending file to irods. The command was: iput --lfrestart $RESIFDD_WORKDIR/$KEY.lfrestart --retries 5 -T -K -f $RESIFDD_WORKDIR/$KEY.tar $KEY/latest.tar" 'error'
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
120
121
        log "[$KEY] Rolling back from $KEY/previous.tar to $KEY/latest.tar"
        # Roll back
Jonathan Schaeffer's avatar
Ajout    
Jonathan Schaeffer committed
122
        # TODO : section à tester
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
123
124
        irm -f  $KEY/latest.tar
        imv  $KEY/previous.tar $KEY/latest.tar
Jonathan Schaeffer's avatar
Ajout    
Jonathan Schaeffer committed
125
126
127
        # Recover previous from the safety copy
        imv $KEY/previous_to_delete.tar $KEY/previous.tar
        # Alert to zabbix
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
128
    fi
Jonathan Schaeffer's avatar
Ajout    
Jonathan Schaeffer committed
129
130
    # Cleaning up the safety previous file
    irm -f $KEY/previous_to_delete.tar
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
131
    log "[$KEY] Sent successfully"
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
132
    log "Cleaning generated tar $RESIFDD_WORKDIR/$YEAR_$NETWORK_$STATION.tar"
133
done
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
134
135
136
137
138
139
140
141

log "Dump of validated data done"
log "Starting dump of metadata"
tar cf $RESIFDD_DATADIR/validated_seismic_metadata/.snapshot/monthly.${MONTH}-01_0000 $RESIFDD_WORKDIR/validated_metadata.tar
irm validated_metadata/previous.tar
imv validated_metadata/latest.tar validated_metadata/previous.tar
iput $RESIFDD_WORKDIR/validated_metadata.tar validated_metadata/latest.tar
log "Dump of metadata done"