Skip to content
GitLab
Commit 1590a76e authored by Jonathan Schaeffer's avatar Jonathan Schaeffer
Browse files

Clean rollback

parent c168bae1
Hide whitespace changes
Inline Side-by-side
src/resifdatadump
View file @ 1590a76e
......@@ -7,6 +7,45 @@ exec 1> >(logger -s -t $(basename $0)) 2>&1
# This script dumps precious RESIF data to an IRODS server
#
####################
#
# Zabbix sender
#
###################
zabbix(){
}
# In case of any problem, this function rolls every distant file operation bask
# 1. tries to revover latest.tar from previous.tar
# 2. tries to recover previous.tar from previous_to_delete.tar
# Argument is the distant directory to work with
irods_rollback(){
KEY=$1
echo "[$KEY] Rollback : try to recover latest.tar from previous"
ils ${KEY}/previous.tar && (
imv ${KEY}/previous.tar ${KEY}/latest.tar
if [[ $? -ne 0 ]]; then
echo "[$KEY] Error $? recovering latest.tar Exit 1"
return 1
fi
)
echo "[$KEY] Rollback : OK"
echo "[$KEY] Rollback : try to recover previous.tar from previous_to_delete"
ils ${KEY}/previous_to_delete.tar && (
imv ${KEY}/previous_to_delete.tar ${KEY}/previous.tar
if [[ $? -ne 0 ]]; then
echo "[$KEY] Error $? recovering previous.tar Exit 1"
return 1
fi
)
echo "[$KEY] Rollback : OK"
return 0
}
####################
#
# Preliminary tests
......@@ -32,7 +71,7 @@ fi
MONTH=$(date +%Y-%m)
SNAPSHOT_DIR=$RESIFDD_DATADIR/validated_seismic_seismic_metadata/.snapshot/monthly.${MONTH}-01_0000
echo "Starting dump of metadata from ${SNAPSHOT_DIR}"
tar cf $RESIFDD_WORKDIR/validated_seismic_metadata.tar -C $SNAPSHOT_DIR $SNAPSHOT_DIR
tar cf $RESIFDD_WORKDIR/validated_seismic_metadata.tar --exclude portalproducts -C $SNAPSHOT_DIR $SNAPSHOT_DIR
if [[ $? -ne 0 ]]; then
echo "Error creating tar archive. Exit 1"
exit 1
......@@ -45,33 +84,29 @@ if [[ $? -ne 0 ]]; then
exit 1
fi
# If previous.tar exist, move it around but keep it safe
ils validated_seismic_metadata/previous.tar && imv validated_seismic_metadata/previous.tar validated_seismic_metadata/previous_to_delete.tar
if [[ $? -ne 0 ]]; then
echo "Error $? moving previous.tar around Exit 1"
exit 1
fi
# If latest.tar exist, move it to previous.tar
ils validated_seismic_metadata/latest.tar && imv validated_seismic_metadata/latest.tar validated_seismic_metadata/previous.tar
if [[ $? -ne 0 ]]; then
echo "Error $? moving latest.tar to previous.tar Exit 1"
exit 1
fi
iput $RESIFDD_WORKDIR/validated_seismic_metadata.tar validated_seismic_metadata/latest.tar
if [[ $? -ne 0 ]]; then
echo "Error $? Something went wrong pushing validated metadata, Roll back"
imv validated_seismic_metadata/previous.tar validated_seismic_metadata/latest.tar
echo "If previous.tar exist, move it around but keep it safe"
ils validated_seismic_metadata/previous.tar && (
imv validated_seismic_metadata/previous.tar validated_seismic_metadata/previous_to_delete.tar
if [[ $? -ne 0 ]]; then
echo "Error $? recovering latest.tar Exit 1"
echo "Error $? moving previous.tar around Exit 1"
exit 1
fi
imv validated_seismic_metadata/previous_to_delete.tar validated_seismic_metadata/previous.tar
)
echo "If latest.tar exist, move it to previous.tar"
ils validated_seismic_metadata/latest.tar && (
imv validated_seismic_metadata/latest.tar validated_seismic_metadata/previous.tar
if [[ $? -ne 0 ]]; then
echo "Error $? recovering previous.tar Exit 1"
echo "Error $? moving latest.tar to previous.tar Exit 1"
exit 1
fi
)
echp "Push metadata archive"
iput -N1 $RESIFDD_WORKDIR/validated_seismic_metadata.tar validated_seismic_metadata/latest.tar
if [[ $? -ne 0 ]]; then
echo "Error $? Something went wrong pushing validated metadata, Roll back"
irods_rollback validated_seismic_metadata
fi
# Cleaning
irm validated_seismic_metadata/previous_to_delete.tar
......@@ -98,13 +133,12 @@ fi
#
####################
cd $SNAPSHOT_DIR
echo "Starting dump of validated data"
for dir in $(find . -maxdepth 3 -type d); do
cd $SNAPSHOT_DIR
for dir in $(find . -maxdepth 3 -type d | sort); do
# First clean workspace
rm -f $RESIFDD_WORKDIR/*.tar
# Parse path to get year, station and network
IFS='/' read -r -a YNS <<< $dir
[ ! ${#YNS[@]} -eq 4 ] && continue
YEAR=${YNS[1]}
......@@ -115,40 +149,53 @@ for dir in $(find . -maxdepth 3 -type d); do
tar cf $RESIFDD_WORKDIR/$KEY.tar $dir
if [[ $? -ne 0 ]]; then
# Something went wrong creating archive. Exit
echo "[$KEY] tar reported an error. Exit 1"
exit 1
echo "[$KEY] Error creating tar"
# Send key to zabbix
zabbix $KEY
continue
fi
SIZE=$(( $(stat -c %s $RESIFDD_WORKDIR/$KEY.tar) / 1024 / 1024 / 1024 ))
# Check if file exists on irods server
irods_latest=$(ils -L $KEY/latest.tar)
echo "[$KEY] irods latest file is : $irods_latest"
# Latest archive exist, lets move it to previous.
if [[ $? -eq 0 ]]; then
echo "[$KEY] latest.tar already exists on iRODS server. Let's compare hashes"
local_sha=$(sha256sum $RESIFDD_WORKDIR/$KEY.tar | awk '{print $1}' | xxd -r -p | base64)
irods_sha=$(ichksum $KEY/latest.tar | awk -F':' '/sha2:/ {print $2; exit;}')
# Compare to local hash
# If the hashes differs, then move distant file and push this one
if [[ "$local_sha" = "$irods_sha" ]]; then
echo "[$KEY] The archive on irods is the same as our version. Skipping."
continue
fi
echo "[$KEY] local checksum: $local_sha" "debug"
echo "[$KEY] idos checksum: $irods_sha" "debug"
echo "[$KEY] local checksum: $local_sha"
echo "[$KEY] idos checksum: $irods_sha"
echo "[$KEY] Local checksum differs from last dump. Let's dump it"
imv $KEY/previous.tar $KEY/previous_to_delete.tar
# Put previous.tar aside in case we need to revover it.
ils $KEY/previous.tar && (
imv $KEY/previous.tar $KEY/previous_to_delete.tar
if [[ $? -ne 0 ]]; then
echo "[$KEY] Error putting previous.tar aside. $KEY archive will not be sent"
zabbix $KEY
continue
fi
)
imv $KEY/latest.tar $KEY/previous.tar # Rename latest distant dump tp "previous"
if [[ $? -ne 0 ]]; then
# Something wrong moving remote file arround
echo "[$KEY] Error, something wrong moving $KEY/previous.tar, cancel iput."
irm -f $KEY/previous_to_delete.tar
# zabbixsnd $KEY
ils $KEY/previous_to_delete.tar && (
imv $KEY/previous_to_delete.tar $KEY/previous.tar
if [[ $? -ne 0 ]]; then
echo "[KEY] Error restoring previous.tar"
fi
)
zabbix $KEY
continue
fi
fi
# Send file to IRODS
echo "[$KEY] Sending file to iRODS ($SIZE GB)"
# Send latest archive file to IRODS
echo "[$KEY] Sending latest.tar to iRODS ($SIZE GB)"
imkdir -p $KEY
# Décommenter le bloc et les redirections pour traiter stdout et stderr différemment
iput \
......@@ -156,14 +203,10 @@ for dir in $(find . -maxdepth 3 -type d); do
$RESIFDD_WORKDIR/$KEY.tar $KEY/latest.tar
if [[ $? -ne 0 ]]; then
echo "[$KEY] Error sending file to irods. The command was: iput --lfrestart $RESIFDD_WORKDIR/$KEY.lfrestart --retries 5 -T -K -f $RESIFDD_WORKDIR/$KEY.tar $KEY/latest.tar"
echo "[$KEY] Rolling back from $KEY/previous.tar to $KEY/latest.tar"
# Roll back
# TODO : section à tester
irm -f $KEY/latest.tar
imv $KEY/previous.tar $KEY/latest.tar
# Recover previous from the safety copy
imv $KEY/previous_to_delete.tar $KEY/previous.tar
irods_rollback $KEY
# Alert to zabbix
zabbix $KEY
fi
# Cleaning up the safety previous file
irm -f $KEY/previous_to_delete.tar
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment