...
 
Commits (2)
......@@ -5,7 +5,7 @@
# This line tells to redirect all outputs to logger and stdout
exec 1> >(logger -s -t $(basename $0)) 2>&1
set -u
# set -u
set -a
####################
#
......@@ -44,15 +44,17 @@ format_report(){
}
# Push data to irods as a staging file
# Argument is the distant directory to push to.
# Arguments :
# 1. the distant directory to push to
# 2. the hash of the file to push
# If something goes wrong, roll back
# If everything goes fine, then validate staging data
irods_push(){
KEY=$1
SHA=$2
SIZE=$(stat -c %s $RESIFDD_WORKDIR/$KEY.tar)
SIZEMB=$(($SIZE/1024/1024))
echo "[$KEY] Sending data to iRODS ($SIZEMB MB)"
echo "[$KEY] Sending data to iRODS ($SIZEMB MB, sha2: $SHA)"
for n in $(seq 1 5); do [ $n -gt 1 ] && sleep 10 ; imkdir -p $KEY && s=0 && break || s=$?; done
if [[ $s -ne 0 ]]; then
echo "[$KEY] Error 002 creating remote directory."
......@@ -61,11 +63,9 @@ irods_push(){
return 1
fi
COMMAND="iput --retries 5 -T -K -f -X $RESIFDD_WORKDIR/${KEY}.restart $RESIFDD_WORKDIR/${KEY}.tar $KEY/staging.tar"
COMMAND="iput --retries 5 -T -f -X $RESIFDD_WORKDIR/${KEY}.restart $RESIFDD_WORKDIR/${KEY}.tar $KEY/staging.tar"
start=$(date +%s)
eval $COMMAND
IPUTRC=$?
if [[ $IPUTRC -ne 0 ]]; then
if $COMMAND; then
echo "[$KEY] Error 011 sending file to irods. The command was: $COMMAND"
format_report $KEY $SIZEMB $(date +%Y-%m-%dT%H:%M:%S --date=@$start) - - 'Error 011. Transfer cancelled' >> $LOCAL_REPORT
# Roll back
......@@ -76,10 +76,19 @@ irods_push(){
else
duration=$(($(date +%s)-$start))
throughput=$(($SIZEMB / $duration ))
echo "[$KEY] staging.tar data sent, let's commit everything on irods server"
format_report $KEY $SIZEMB $(date +%Y-%m-%dT%H:%M:%S --date=@$start) $duration $throughput 'OK' >> $LOCAL_REPORT
irods_commit $KEY
zabbix_ok "$KEY|${SIZEMB}MB|${duration}s|${throughput}MB/s"
# Compute remote checksum
echo "[$KEY] staging.tar data sent, let's compute remote checksum"
irods_sha=$(ichksum $KEY/staging.tar | awk -F':' '/sha2:/ {print $2; exit;}')
echo "[$KEY] $irods_sha"
if [[ "$SHA" = "$irods_sha" ]]; then
echo "[$KEY] checksums match, commit remote data"
format_report $KEY $SIZEMB $(date +%Y-%m-%dT%H:%M:%S --date=@$start) $duration $throughput 'OK' >> $LOCAL_REPORT
irods_commit $KEY
zabbix_ok "$KEY|${SIZEMB}MB|${duration}s|${throughput}MB/s"
else
echo "[$KEY] Error 012, checksum mismatch"
zabbix_err "${KEY} Error 012"
fi
rm $RESIFDD_WORKDIR/${KEY}.*
fi
# Send report to irods. Do some locking here
......@@ -185,14 +194,14 @@ pack_and_send() {
echo "[$KEY] Error 007 creating tar"
rm -f $RESIFDD_WORKDIR/$KEY.tar
# Send key to zabbix_err
zabbix_err "$KEY:Error 007"
zabbix_err "$KEY Error 007"
return 1
fi
local_sha=$(sha256sum $RESIFDD_WORKDIR/$KEY.tar | awk '{print $1}' | xxd -r -p | base64)
# Check if file exists on irods server
ils -L $KEY/latest.tar > /dev/null 2>&1
if [[ $? -eq 0 ]]; then
echo "[$KEY] latest.tar already exists on iRODS server. Let's compare hashes"
local_sha=$(sha256sum $RESIFDD_WORKDIR/$KEY.tar | awk '{print $1}' | xxd -r -p | base64)
irods_sha=$(ichksum $KEY/latest.tar | awk -F':' '/sha2:/ {print $2; exit;}')
echo "[$KEY] local checksum: $local_sha"
echo "[$KEY] irods checksum: $irods_sha"
......@@ -206,7 +215,7 @@ pack_and_send() {
fi
fi
# Send latest archive file to IRODS
irods_push $KEY
irods_push $KEY $local_sha
}
export -f pack_and_send # Necessary for call with GNU parallel
......@@ -296,7 +305,8 @@ if [[ "x$DUMP_METADATA" = "xyes" ]]; then
zabbix_err "${KEY} Error 001"
exit 1
fi
irods_push $KEY
local_sha=$(sha256sum $RESIFDD_WORKDIR/$KEY.tar | awk '{print $1}' | xxd -r -p | base64)
irods_push $KEY $local_sha
echo "[$KEY] Dump terminated :"
ils -l $KEY
fi
......