resifdatadump 6.95 KB
Newer Older
1
2
#!/bin/bash

Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
3
4
exec 1> >(logger -s -t $(basename $0)) 2>&1

5
# Auteur: Jonathan Schaeffer <jonathan.schaeffer@univ-grenoble-alpes.fr>
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
6
7
8
#
# This script dumps precious RESIF data to an IRODS server
#
9

Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
10
11
12
13
14
15
16
####################
#
# Zabbix sender
#
###################

zabbix(){
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
17
    zabbix_sender -k resifdatadump.failed -s $(hostname -a) -o $1 -z monitoring.osug.fr
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
18
19
}

Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
20
21
22
23
24
25
26
27
# Push data to irods as a staging file
# Argument is the distant directory to push to.
# If something goes wrong, roll back
# If everything goes fine, then validate staging data
irods_push(){
    KEY=$1
    SIZE=$(stat -c %s $RESIFDD_WORKDIR/$KEY.tar)
    SIZEMB=$(($SIZE/1024/1024))
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
28
    echo "[$KEY] Sending data to iRODS ($SIZEMB MB)"
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
    imkdir -p $KEY
    if [[ $? -ne 0 ]]; then
        echo "[$KEY] Error 002 creating remote directory. Manual action has to be taken."
        zabbix "${KEY}:Error 002"
        exit 1
    fi

    COMMAND="iput --retries 5 -T -K -f -X $RESIFDD_WORKDIR/${KEY}.restart $RESIFDD_WORKDIR/${KEY}.tar $KEY/staging.tar"
    eval $COMMAND
    if [[ $? -ne 0 ]]; then
        echo "[$KEY] Error 011 sending file to irods. The command was: $COMMAND"
        # Roll back
        irods_rollback $KEY
        # Alert to zabbix
        zabbix "${KEY}:Error 011"
    else
        irods_commit $KEY
    fi
}
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
48
49
50
51
52
53
54
55

# In case of any problem, this function rolls every distant file operation bask
# 1. tries to revover latest.tar from previous.tar
# 2. tries to recover previous.tar from previous_to_delete.tar
# Argument is the distant directory to work with
irods_rollback(){
    KEY=$1
    echo "[$KEY] Rollback : try to recover latest.tar from previous"
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
56
    ils ${KEY}/previous.tar 2>/dev/null && (
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
57
58
59
60
61
62
63
64
        imv ${KEY}/previous.tar ${KEY}/latest.tar
        if [[ $? -ne 0 ]]; then
            echo "[$KEY] Error $? recovering latest.tar Exit 1"
            return 1
        fi
    )
    echo "[$KEY] Rollback : OK"
    echo "[$KEY] Rollback : try to recover previous.tar from previous_to_delete"
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
65
    ils ${KEY}/previous_to_delete.tar 2>/dev/null && (
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
66
67
68
69
70
71
72
73
74
75
        imv ${KEY}/previous_to_delete.tar ${KEY}/previous.tar
        if [[ $? -ne 0 ]]; then
            echo "[$KEY] Error $? recovering previous.tar Exit 1"
            return 1
        fi
    )
    echo "[$KEY] Rollback : OK"
    return 0
}

Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
# When the transfer to irods is successfull, we can move file arounds remotely :
# 1. $KEY/previous.tar move to trash
# 2. $KEY/latest.tar move to $KEY/previous.tar
# 3. $KEY/staging.tar move to $KEY/latest.tar
irods_commit(){
    KEY=$1
    echo "[$KEY] If previous.tar exists, move it around but keep it safe"
    ils $KEY/previous.tar 2>/dev/null && (
        imv $KEY/previous.tar $KEY/previous_to_delete.tar
        if [[ $? -ne 0 ]]; then
            echo "[$KEY] Error 003 moving previous.tar around. Corrective action has to be taken manualy"
            zabbix "${KEY}:Error 003"
            return 1
        fi
    )

    echo "[$KEY] If latest.tar exists, move it to previous.tar"
    ils $KEY/latest.tar 2>/dev/null && (
        imv $KEY/latest.tar $KEY/previous.tar
        if [[ $? -ne 0 ]]; then
            echo "[$KEY] Error 004 moving latest.tar to previous.tar. Corrective action has to be taken manualy"
            zabbix "${KEY}:Error 004"
            return 1
        fi
    )
    echp "[$KEY] Validate staging.tar by moving it to staging.tar"
    imv $KEY/staging.tar $KEY/latest.tar
    if [[ $? -ne 0 ]]; then
        echo "[$KEY] Error 005 moving statging.tar to latest.tar. Corrective action has to be taken manualy"
        zabbix "${KEY}:Error 005"
        return 1
    fi
    ils $KEY/previous_to_delete.tar 2>/dev/null && irm -f $KEY/previous_to_delete.tar
}

Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
111
112
113
114
115
116
####################
#
# Preliminary tests
#
####################

Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
117
118
119
120
121
122
# Checking IRODS environment
if [[ ! -f ~/.irods/.irodsA ]] ; then
    echo "The irods scrambled password file is not present. Please run iinit and provide the password. Exit 1"
    exit 1
fi

123
124
# Test for working directory
if [[ ! -w $RESIFDD_WORKDIR ]] ; then
125
    echo "RESIFDD_WORKDIR \"$RESIFDD_WORKDIR\" not writable. Check permissions. Exit 1"
126
127
    exit 1
fi
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
128
# test the data directory where to dump everything from
129
if [[ -z $RESIFDD_DATADIR ]]; then
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
130
    echo "Variable RESIFDD_DATADIR must be set to the RESIF mountpoint. Exit 1"
131
132
133
    exit 1
fi
if [[ ! -d $RESIFDD_DATADIR ]]; then
134
    echo "Variable RESIFDD_DATADIR set to \"$RESIFDD_DATADIR\" must be a directory. Exit 1"
135
136
137
138
139
    exit 1
fi

# Get the snapshot name for this month
MONTH=$(date +%Y-%m)
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
140
SNAPSHOT_DIR=$RESIFDD_DATADIR/validated_seismic_seismic_metadata/.snapshot/monthly.${MONTH}-01_0000
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
141
142
143
144
145
if [[ ! -d $SNAPSHOT_DIR ]]; then
    echo "Error 000 Snapshot directory $SNAPSHOT_DIR does not exist"
    exit 1
fi

Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
146
KEY="validated-seismic-metadata"
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
147

Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
148
149
echo "[$KEY] Starting dump from ${SNAPSHOT_DIR}"
tar cf  $RESIFDD_WORKDIR/$KEY.tar --exclude portalproducts -C $SNAPSHOT_DIR $SNAPSHOT_DIR
150
if [[ $? -ne 0 ]]; then
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
151
152
    echo "[$KEY] Error 001 while creating tar archive."
    zabbix "${KEY}:Error 001"
153
154
155
    exit 1
fi

Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
156
157
158
159
160
irods_push $KEY
echo "[$KEY] Cleaning workdir $RESIFDD_WORKDIR"
rm -f $RESIFDD_WORKDIR/$KEY.tar $RESIFDD_WORKDIR/${KEY}.restart
echo "[$KEY] Dump terminated :"
ils -l $KEY
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
161

Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
162
163
164
165
166
####################
#
# Start dumping validated data
#
####################
167

Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
168
SNAPSHOT_DIR="${RESIFDD_DATADIR}/validated_seismic_data/.snapshot/monthly.${MONTH}-01_0000"
169
if [[ ! -d $SNAPSHOT_DIR ]]; then
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
170
    echo "Error 006 Snapshot directory $SNAPSHOT_DIR does not exist"
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
171
172
173
    exit 1
fi

Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
174
echo "Starting dump of validated data"
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
175
176
cd $SNAPSHOT_DIR
for dir in $(find . -maxdepth 3 -type d | sort); do
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
177
    # First clean workspace
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
178
    rm -rf $RESIFDD_WORKDIR/*
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
179
    # Parse path to get year, station and network
180
    IFS='/' read -r -a YNS <<< $dir
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
181
182
183
184
    [ ! ${#YNS[@]} -eq 4 ] && continue
    YEAR=${YNS[1]}
    NETWORK=${YNS[2]}
    STATION=${YNS[3]}
Jonathan Schaeffer's avatar
debug    
Jonathan Schaeffer committed
185
    KEY=${YEAR}_${NETWORK}_${STATION}
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
186
    echo "[$KEY] Creating tar on $RESIFDD_WORKDIR/$KEY.tar"
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
187
188
    tar cf $RESIFDD_WORKDIR/$KEY.tar $dir
    if [[ $? -ne 0 ]]; then
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
189
        # Something went wrong creating archive. Exit
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
190
        echo "[$KEY] Error 007 creating tar"
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
191
        # Send key to zabbix
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
192
        zabbix "$KEY:Error 007"
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
193
        continue
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
194
195
    fi
    # Check if file exists on irods server
Jonathan Schaeffer's avatar
Bugfix    
Jonathan Schaeffer committed
196
    ils -L $KEY/latest.tar > /dev/null 2>&1
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
197
    if [[ $? -eq 0 ]]; then
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
198
        echo "[$KEY] latest.tar already exists on iRODS server. Let's compare hashes"
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
199
200
        local_sha=$(sha256sum $RESIFDD_WORKDIR/$KEY.tar | awk '{print $1}' | xxd -r -p | base64)
        irods_sha=$(ichksum $KEY/latest.tar | awk -F':' '/sha2:/ {print $2; exit;}')
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
201
        echo "[$KEY] local checksum: $local_sha"
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
202
        echo "[$KEY] irods checksum: $irods_sha"
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
203
        # If the hashes differs, then move distant file and push this one
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
204
        if [[ "$local_sha" = "$irods_sha" ]]; then
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
205
            echo "[$KEY] The archive on irods is the same as our version. Skipping."
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
206
207
208
            continue
        fi
    fi
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
209
    # Send latest archive file to IRODS
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
210
211
212
    irods_push $KEY
    echo "[$KEY] Cleaning workdir $RESIFDD_WORKDIR"
    rm -f $RESIFDD_WORKDIR/$KEY.tar $RESIFDD_WORKDIR/${KEY}.restart
213
done
Jonathan Schaeffer's avatar
Jonathan Schaeffer committed
214
echo "Dump of validated data done"