Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
OSUG
RESIF
datadump
Commits
fae5167a
Commit
fae5167a
authored
Jul 09, 2019
by
Jonathan Schaeffer
Browse files
Nettoyage des messages
parent
78647ff8
Changes
1
Hide whitespace changes
Inline
Side-by-side
src/resifdatadump
View file @
fae5167a
...
...
@@ -26,7 +26,7 @@ zabbix(){
irods_rollback
(){
KEY
=
$1
echo
"[
$KEY
] Rollback : try to recover latest.tar from previous"
ils
${
KEY
}
/previous.tar
&&
(
ils
${
KEY
}
/previous.tar
2>/dev/null
&&
(
imv
${
KEY
}
/previous.tar
${
KEY
}
/latest.tar
if
[[
$?
-ne
0
]]
;
then
echo
"[
$KEY
] Error
$?
recovering latest.tar Exit 1"
...
...
@@ -35,7 +35,7 @@ irods_rollback(){
)
echo
"[
$KEY
] Rollback : OK"
echo
"[
$KEY
] Rollback : try to recover previous.tar from previous_to_delete"
ils
${
KEY
}
/previous_to_delete.tar
&&
(
ils
${
KEY
}
/previous_to_delete.tar
2>/dev/null
&&
(
imv
${
KEY
}
/previous_to_delete.tar
${
KEY
}
/previous.tar
if
[[
$?
-ne
0
]]
;
then
echo
"[
$KEY
] Error
$?
recovering previous.tar Exit 1"
...
...
@@ -88,6 +88,9 @@ if [[ $? -ne 0 ]]; then
exit
1
fi
SIZE
=
$(
stat
-c
%s
$RESIFDD_WORKDIR
/
$KEY
.tar
)
SIZEGB
=
$((
$(
stat
-c
%s
$RESIFDD_WORKDIR
/
$KEY
.tar
)
/
1024
/
1024
/
1024
))
# Creating remote directory if not exist
imkdir
-p
validated_seismic_metadata
if
[[
$?
-ne
0
]]
;
then
...
...
@@ -96,7 +99,7 @@ if [[ $? -ne 0 ]]; then
fi
echo
"[METADATA] If previous.tar exists, move it around but keep it safe"
ils validated_seismic_metadata/previous.tar
&&
(
ils validated_seismic_metadata/previous.tar
2>/dev/null
&&
(
imv validated_seismic_metadata/previous.tar validated_seismic_metadata/previous_to_delete.tar
if
[[
$?
-ne
0
]]
;
then
echo
"[METADATA] Error 003 moving previous.tar around Exit 1"
...
...
@@ -105,7 +108,7 @@ ils validated_seismic_metadata/previous.tar && (
)
echo
"[METADATA] If latest.tar exist, move it to previous.tar"
ils validated_seismic_metadata/latest.tar
&&
(
ils validated_seismic_metadata/latest.tar
2>/dev/null
&&
(
imv validated_seismic_metadata/latest.tar validated_seismic_metadata/previous.tar
if
[[
$?
-ne
0
]]
;
then
echo
"[METADATA] Error 004 moving latest.tar to previous.tar Exit 1"
...
...
@@ -113,7 +116,7 @@ ils validated_seismic_metadata/latest.tar && (
fi
)
echo
"Push metadata archive"
echo
"Push metadata archive
(
$SIZEGB
GB)
"
iput
-N1
$RESIFDD_WORKDIR
/validated_seismic_metadata.tar validated_seismic_metadata/latest.tar
if
[[
$?
-ne
0
]]
;
then
echo
"[METADATA] Error 005 Something went wrong pushing validated metadata, Roll back"
...
...
@@ -122,7 +125,7 @@ fi
# Cleaning
irm validated_seismic_metadata/previous_to_delete.tar
rm
$RESIFDD_WORKDIR
/validated_seismic_metadata.tar
echo
"[METADATA] Dump terminate :"
echo
"[METADATA] Dump terminate
d
:"
ils
-l
validated_seismic_metadata
...
...
@@ -160,12 +163,12 @@ for dir in $(find . -maxdepth 3 -type d | sort); do
zabbix
$KEY
continue
fi
SIZE
=
$((
$(
stat
-c
%s
$RESIFDD_WORKDIR
/
$KEY
.tar
)
/
1024
/
1024
/
1024
))
SIZE
=
$(
stat
-c
%s
$RESIFDD_WORKDIR
/
$KEY
.tar
)
SIZEGB
=
$((
$(
stat
-c
%s
$RESIFDD_WORKDIR
/
$KEY
.tar
)
/
1024
/
1024
/
1024
))
# Check if file exists on irods server
irods_latest
=
$(
ils
-L
$KEY
/latest.tar
)
echo
"[
$KEY
] irods latest file is :
$irods_latest
"
# Latest archive exist, lets move it to previous.
ils
-L
$KEY
/latest.tar 2>&1/dev/null
if
[[
$?
-eq
0
]]
;
then
# Latest archive exist, lets move it to previous.
echo
"[
$KEY
] latest.tar already exists on iRODS server. Let's compare hashes"
local_sha
=
$(
sha256sum
$RESIFDD_WORKDIR
/
$KEY
.tar |
awk
'{print $1}'
| xxd
-r
-p
|
base64
)
irods_sha
=
$(
ichksum
$KEY
/latest.tar |
awk
-F
':'
'/sha2:/ {print $2; exit;}'
)
...
...
@@ -178,7 +181,7 @@ for dir in $(find . -maxdepth 3 -type d | sort); do
echo
"[
$KEY
] idos checksum:
$irods_sha
"
echo
"[
$KEY
] Local checksum differs from last dump. Let's dump it"
# Put previous.tar aside in case we need to revover it.
ils
$KEY
/previous.tar
&&
(
ils
$KEY
/previous.tar
2>/dev/null
&&
(
irm
-f
$KEY
/previous_to_delete.tar
imv
$KEY
/previous.tar
$KEY
/previous_to_delete.tar
if
[[
$?
-ne
0
]]
;
then
...
...
@@ -191,7 +194,7 @@ for dir in $(find . -maxdepth 3 -type d | sort); do
if
[[
$?
-ne
0
]]
;
then
# Something wrong moving remote file arround
echo
"[
$KEY
] Error 009 something wrong moving
$KEY
/previous.tar, cancel iput."
ils
$KEY
/previous_to_delete.tar
&&
(
ils
$KEY
/previous_to_delete.tar
2>/dev/null
&&
(
imv
$KEY
/previous_to_delete.tar
$KEY
/previous.tar
if
[[
$?
-ne
0
]]
;
then
echo
"[KEY] Error 010 restoring previous.tar"
...
...
@@ -202,11 +205,11 @@ for dir in $(find . -maxdepth 3 -type d | sort); do
fi
fi
# Send latest archive file to IRODS
echo
"[
$KEY
] Sending latest.tar to iRODS (
$SIZE
GB)"
echo
"[
$KEY
] Sending latest.tar to iRODS (
$SIZE
GB
GB)"
imkdir
-p
$KEY
# Décommenter le bloc et les redirections pour traiter stdout et stderr différemment
iput
\
--retries
5
-T
-K
-f
\
--retries
5
-T
-K
-f
-X
$RESIFDD_WORKDIR
/irods_restart
-N1
\
$RESIFDD_WORKDIR
/
$KEY
.tar
$KEY
/latest.tar
if
[[
$?
-ne
0
]]
;
then
echo
"[
$KEY
] Error 011 sending file to irods. The command was: iput --lfrestart
$RESIFDD_WORKDIR
/
$KEY
.lfrestart --retries 5 -T -K -f
$RESIFDD_WORKDIR
/
$KEY
.tar
$KEY
/latest.tar"
...
...
@@ -216,10 +219,10 @@ for dir in $(find . -maxdepth 3 -type d | sort); do
zabbix
$KEY
fi
# Cleaning up the safety previous file
irm
-f
$KEY
/previous_to_delete.tar
echo
"[
$KEY
] Sent successfully"
irm
-f
$KEY
/previous_to_delete.tar
2>/dev/null
echo
"[
$KEY
] Sent
$SIZE
bytes
successfully"
# TODO Checksums check
echo
"[
$KEY
] Cleaning local generated tar"
rm
-f
$RESIFDD_WORKDIR
/
$KEY
.tar
rm
-f
$RESIFDD_WORKDIR
/
$KEY
.tar
$RESIFDD_WORKDIR
/irods_restart
done
echo
"Dump of validated data done"
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment