Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
OSUG
RESIF
datadump
Commits
1590a76e
Commit
1590a76e
authored
Jul 09, 2019
by
Jonathan Schaeffer
Browse files
Clean rollback
parent
c168bae1
Changes
1
Hide whitespace changes
Inline
Side-by-side
src/resifdatadump
View file @
1590a76e
...
...
@@ -7,6 +7,45 @@ exec 1> >(logger -s -t $(basename $0)) 2>&1
# This script dumps precious RESIF data to an IRODS server
#
####################
#
# Zabbix sender
#
###################
zabbix
(){
}
# In case of any problem, this function rolls every distant file operation bask
# 1. tries to revover latest.tar from previous.tar
# 2. tries to recover previous.tar from previous_to_delete.tar
# Argument is the distant directory to work with
irods_rollback
(){
KEY
=
$1
echo
"[
$KEY
] Rollback : try to recover latest.tar from previous"
ils
${
KEY
}
/previous.tar
&&
(
imv
${
KEY
}
/previous.tar
${
KEY
}
/latest.tar
if
[[
$?
-ne
0
]]
;
then
echo
"[
$KEY
] Error
$?
recovering latest.tar Exit 1"
return
1
fi
)
echo
"[
$KEY
] Rollback : OK"
echo
"[
$KEY
] Rollback : try to recover previous.tar from previous_to_delete"
ils
${
KEY
}
/previous_to_delete.tar
&&
(
imv
${
KEY
}
/previous_to_delete.tar
${
KEY
}
/previous.tar
if
[[
$?
-ne
0
]]
;
then
echo
"[
$KEY
] Error
$?
recovering previous.tar Exit 1"
return
1
fi
)
echo
"[
$KEY
] Rollback : OK"
return
0
}
####################
#
# Preliminary tests
...
...
@@ -32,7 +71,7 @@ fi
MONTH
=
$(
date
+%Y-%m
)
SNAPSHOT_DIR
=
$RESIFDD_DATADIR
/validated_seismic_seismic_metadata/.snapshot/monthly.
${
MONTH
}
-01_0000
echo
"Starting dump of metadata from
${
SNAPSHOT_DIR
}
"
tar
cf
$RESIFDD_WORKDIR
/validated_seismic_metadata.tar
-C
$SNAPSHOT_DIR
$SNAPSHOT_DIR
tar
cf
$RESIFDD_WORKDIR
/validated_seismic_metadata.tar
--exclude
portalproducts
-C
$SNAPSHOT_DIR
$SNAPSHOT_DIR
if
[[
$?
-ne
0
]]
;
then
echo
"Error creating tar archive. Exit 1"
exit
1
...
...
@@ -45,33 +84,29 @@ if [[ $? -ne 0 ]]; then
exit
1
fi
# If previous.tar exist, move it around but keep it safe
ils validated_seismic_metadata/previous.tar
&&
imv validated_seismic_metadata/previous.tar validated_seismic_metadata/previous_to_delete.tar
if
[[
$?
-ne
0
]]
;
then
echo
"Error
$?
moving previous.tar around Exit 1"
exit
1
fi
# If latest.tar exist, move it to previous.tar
ils validated_seismic_metadata/latest.tar
&&
imv validated_seismic_metadata/latest.tar validated_seismic_metadata/previous.tar
if
[[
$?
-ne
0
]]
;
then
echo
"Error
$?
moving latest.tar to previous.tar Exit 1"
exit
1
fi
iput
$RESIFDD_WORKDIR
/validated_seismic_metadata.tar validated_seismic_metadata/latest.tar
if
[[
$?
-ne
0
]]
;
then
echo
"Error
$?
Something went wrong pushing validated metadata, Roll back"
imv validated_seismic_metadata/previous.tar validated_seismic_metadata/latest.tar
echo
"If previous.tar exist, move it around but keep it safe"
ils validated_seismic_metadata/previous.tar
&&
(
imv validated_seismic_metadata/previous.tar validated_seismic_metadata/previous_to_delete.tar
if
[[
$?
-ne
0
]]
;
then
echo
"Error
$?
recovering latest.tar
Exit 1"
echo
"Error
$?
moving previous.tar around
Exit 1"
exit
1
fi
imv validated_seismic_metadata/previous_to_delete.tar validated_seismic_metadata/previous.tar
)
echo
"If latest.tar exist, move it to previous.tar"
ils validated_seismic_metadata/latest.tar
&&
(
imv validated_seismic_metadata/latest.tar validated_seismic_metadata/previous.tar
if
[[
$?
-ne
0
]]
;
then
echo
"Error
$?
recovering
previous.tar Exit 1"
echo
"Error
$?
moving latest.tar to
previous.tar Exit 1"
exit
1
fi
)
echp
"Push metadata archive"
iput
-N1
$RESIFDD_WORKDIR
/validated_seismic_metadata.tar validated_seismic_metadata/latest.tar
if
[[
$?
-ne
0
]]
;
then
echo
"Error
$?
Something went wrong pushing validated metadata, Roll back"
irods_rollback validated_seismic_metadata
fi
# Cleaning
irm validated_seismic_metadata/previous_to_delete.tar
...
...
@@ -98,13 +133,12 @@ fi
#
####################
cd
$SNAPSHOT_DIR
echo
"Starting dump of validated data"
for
dir
in
$(
find
.
-maxdepth
3
-type
d
)
;
do
cd
$SNAPSHOT_DIR
for
dir
in
$(
find
.
-maxdepth
3
-type
d
|
sort
)
;
do
# First clean workspace
rm
-f
$RESIFDD_WORKDIR
/
*
.tar
# Parse path to get year, station and network
IFS
=
'/'
read
-r
-a
YNS
<<<
$dir
[
!
${#
YNS
[@]
}
-eq
4
]
&&
continue
YEAR
=
${
YNS
[1]
}
...
...
@@ -115,40 +149,53 @@ for dir in $(find . -maxdepth 3 -type d); do
tar
cf
$RESIFDD_WORKDIR
/
$KEY
.tar
$dir
if
[[
$?
-ne
0
]]
;
then
# Something went wrong creating archive. Exit
echo
"[
$KEY
] tar reported an error. Exit 1"
exit
1
echo
"[
$KEY
] Error creating tar"
# Send key to zabbix
zabbix
$KEY
continue
fi
SIZE
=
$((
$(
stat
-c
%s
$RESIFDD_WORKDIR
/
$KEY
.tar
)
/
1024
/
1024
/
1024
))
# Check if file exists on irods server
irods_latest
=
$(
ils
-L
$KEY
/latest.tar
)
echo
"[
$KEY
] irods latest file is :
$irods_latest
"
# Latest archive exist, lets move it to previous.
if
[[
$?
-eq
0
]]
;
then
echo
"[
$KEY
] latest.tar already exists on iRODS server. Let's compare hashes"
local_sha
=
$(
sha256sum
$RESIFDD_WORKDIR
/
$KEY
.tar |
awk
'{print $1}'
| xxd
-r
-p
|
base64
)
irods_sha
=
$(
ichksum
$KEY
/latest.tar |
awk
-F
':'
'/sha2:/ {print $2; exit;}'
)
# Compare to local hash
# If the hashes differs, then move distant file and push this one
if
[[
"
$local_sha
"
=
"
$irods_sha
"
]]
;
then
echo
"[
$KEY
] The archive on irods is the same as our version. Skipping."
continue
fi
echo
"[
$KEY
] local checksum:
$local_sha
"
"debug"
echo
"[
$KEY
] idos checksum:
$irods_sha
"
"debug"
echo
"[
$KEY
] local checksum:
$local_sha
"
echo
"[
$KEY
] idos checksum:
$irods_sha
"
echo
"[
$KEY
] Local checksum differs from last dump. Let's dump it"
imv
$KEY
/previous.tar
$KEY
/previous_to_delete.tar
# Put previous.tar aside in case we need to revover it.
ils
$KEY
/previous.tar
&&
(
imv
$KEY
/previous.tar
$KEY
/previous_to_delete.tar
if
[[
$?
-ne
0
]]
;
then
echo
"[
$KEY
] Error putting previous.tar aside.
$KEY
archive will not be sent"
zabbix
$KEY
continue
fi
)
imv
$KEY
/latest.tar
$KEY
/previous.tar
# Rename latest distant dump tp "previous"
if
[[
$?
-ne
0
]]
;
then
# Something wrong moving remote file arround
echo
"[
$KEY
] Error, something wrong moving
$KEY
/previous.tar, cancel iput."
irm
-f
$KEY
/previous_to_delete.tar
# zabbixsnd $KEY
ils
$KEY
/previous_to_delete.tar
&&
(
imv
$KEY
/previous_to_delete.tar
$KEY
/previous.tar
if
[[
$?
-ne
0
]]
;
then
echo
"[KEY] Error restoring previous.tar"
fi
)
zabbix
$KEY
continue
fi
fi
# Send file to IRODS
echo
"[
$KEY
] Sending
file
to iRODS (
$SIZE
GB)"
# Send
latest archive
file to IRODS
echo
"[
$KEY
] Sending
latest.tar
to iRODS (
$SIZE
GB)"
imkdir
-p
$KEY
# Décommenter le bloc et les redirections pour traiter stdout et stderr différemment
iput
\
...
...
@@ -156,14 +203,10 @@ for dir in $(find . -maxdepth 3 -type d); do
$RESIFDD_WORKDIR
/
$KEY
.tar
$KEY
/latest.tar
if
[[
$?
-ne
0
]]
;
then
echo
"[
$KEY
] Error sending file to irods. The command was: iput --lfrestart
$RESIFDD_WORKDIR
/
$KEY
.lfrestart --retries 5 -T -K -f
$RESIFDD_WORKDIR
/
$KEY
.tar
$KEY
/latest.tar"
echo
"[
$KEY
] Rolling back from
$KEY
/previous.tar to
$KEY
/latest.tar"
# Roll back
# TODO : section à tester
irm
-f
$KEY
/latest.tar
imv
$KEY
/previous.tar
$KEY
/latest.tar
# Recover previous from the safety copy
imv
$KEY
/previous_to_delete.tar
$KEY
/previous.tar
irods_rollback
$KEY
# Alert to zabbix
zabbix
$KEY
fi
# Cleaning up the safety previous file
irm
-f
$KEY
/previous_to_delete.tar
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment