Commit 8c271745 authored by Jonathan Schaeffer's avatar Jonathan Schaeffer

Push archive based on content's age

parent 11c3a591
......@@ -80,9 +80,11 @@ irods_push(){
throughput=$(($SIZEMB / $duration ))
# Check integrity.
# We do it separately from transfer because for big files it can be very long, and the network can cut
local_sha=$(sha256sum $RESIFDD_WORKDIR/$KEY.tar | awk '{print $1}' | xxd -r -p | base64)
echo "[$KEY] local checksum: $local_sha"
irods_sha=$(ichksum $KEY/staging.tar | awk -F':' '/sha2:/ {print $2; exit;}')
if [[ "$LOCAL_SHA" != "$irods_sha" ]]; then
echo "[$KEY] Error 012 distant file is corrupted (localsha $LOCAL_SHA irodssha $irods_sha). Rollback"
if [[ "$local_sha" != "$irods_sha" ]]; then
echo "[$KEY] Error 012 distant file is corrupted (localsha $local_sha irodssha $irods_sha). Rollback"
irods_rollback $KEY
zabbix_err "${KEY}:Error 012"
fi
......@@ -133,7 +135,7 @@ irods_rollback(){
irods_commit(){
KEY=$1
echo "[$KEY] If previous.tar exists, move it around but keep it safe"
ils $KEY/previous.tar 2>/dev/null && (
ils $KEY/previous.tar 2&1>/dev/null && (
imv $KEY/previous.tar $KEY/previous_to_delete.tar
if [[ $? -ne 0 ]]; then
echo "[$KEY] Error 003 moving previous.tar around. Corrective action has to be taken manualy"
......@@ -143,7 +145,7 @@ irods_commit(){
)
echo "[$KEY] If latest.tar exists, move it to previous.tar"
ils $KEY/latest.tar 2>/dev/null && (
ils $KEY/latest.tar 2&1>/dev/null && (
imv $KEY/latest.tar $KEY/previous.tar
if [[ $? -ne 0 ]]; then
echo "[$KEY] Error 004 moving latest.tar to previous.tar. Corrective action has to be taken manualy"
......@@ -184,6 +186,26 @@ pack_and_send() {
return 0
fi
fi
# Check if file exists on irods server and get creation date
# days=$(( $(date +%j --date $(date +%Y-%m-01)) - $(date +%j --date $(ils -l 2001_YB_F13 | awk '/latest.tar/ {print substr($5, 0,11)}')) ))
ilsr=$(ils -L $KEY/latest.tar)
if [[ $? -eq 0 ]]; then
echo "[$KEY] latest.tar already exists on iRODS server. Check if something changed localy since last push."
# Diff in days between current month (snapshot) and latest irods push
days=$(( $(date +%j --date $(ils -l 2001_YB_F13 | awk '/latest.tar/ {print substr($5, 0,11)}')) - $(date +%j --date $(date +%Y-%m-01)) ))
if [[ $days -ge 0 ]]; then
# We have an uptodate dump (newer than snapshot)
echo "[$KEY] latest.tar is up to date"
return 0
fi
# find files modified in the last $days. If none, we can skip because latests irods version is up to date.
items_modified=$(find $dir -ctime $days | wc -l)
if [[ $items_modified -eq 0 ]]; then
echo "[$KEY] $items_modified new file to update since $days days. Skipping."
format_report $KEY "-" $(date +%Y-%m-%dT%H:%M:%S) "-" "-" "Skipped" >> $LOCAL_REPORT
return 0
fi
fi
echo "[$KEY] Creating tar on $RESIFDD_WORKDIR/$KEY.tar"
echo "[$KEY] tar cf $RESIFDD_WORKDIR/$KEY.tar -C ${dir%$YEAR/$NETWORK/$STATION} $dir"
tar cf $RESIFDD_WORKDIR/$KEY.tar -C ${dir%$YEAR/$NETWORK/$STATION} $dir
......@@ -194,27 +216,10 @@ pack_and_send() {
zabbix_err "$KEY:Error 007"
return 1
fi
# Check if file exists on irods server
ils -L $KEY/latest.tar > /dev/null 2>&1
if [[ $? -eq 0 ]]; then
echo "[$KEY] latest.tar already exists on iRODS server. Let's compare hashes"
local_sha=$(sha256sum $RESIFDD_WORKDIR/$KEY.tar | awk '{print $1}' | xxd -r -p | base64)
irods_sha=$(ichksum $KEY/latest.tar | awk -F':' '/sha2:/ {print $2; exit;}')
echo "[$KEY] local checksum: $local_sha"
echo "[$KEY] irods checksum: $irods_sha"
# If the hashes differs, then move distant file and push this one
if [[ "$local_sha" = "$irods_sha" ]]; then
echo "[$KEY] The archive on irods is the same as our version. Skipping."
format_report $KEY "-" $(date +%Y-%m-%dT%H:%M:%S) "-" "-" "Skipped" >> $LOCAL_REPORT
return 0
fi
fi
# Send latest archive file to IRODS
irods_push $KEY $local_sha
irods_push $KEY
rm $RESIFDD_WORKDIR/$KEY*
}
export -f pack_and_send # Necessary for call with GNU parallel
####################
#
......@@ -290,8 +295,7 @@ else
zabbix_err "${KEY}:Error 001"
exit 1
fi
local_sha=$(sha256sum $RESIFDD_WORKDIR/$KEY.tar | awk '{print $1}' | xxd -r -p | base64)
irods_push $KEY $local_sha
irods_push $KEY
echo "[$KEY] Dump terminated :"
ils -l $KEY
fi
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment