Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
OSUG
RESIF
datadump
Commits
8c271745
Commit
8c271745
authored
Dec 16, 2019
by
Jonathan Schaeffer
Browse files
Push archive based on content's age
parent
11c3a591
Changes
1
Hide whitespace changes
Inline
Side-by-side
src/resifdatadump
View file @
8c271745
...
...
@@ -80,9 +80,11 @@ irods_push(){
throughput
=
$((
$SIZEMB
/
$duration
))
# Check integrity.
# We do it separately from transfer because for big files it can be very long, and the network can cut
local_sha
=
$(
sha256sum
$RESIFDD_WORKDIR
/
$KEY
.tar |
awk
'{print $1}'
| xxd
-r
-p
|
base64
)
echo
"[
$KEY
] local checksum:
$local_sha
"
irods_sha
=
$(
ichksum
$KEY
/staging.tar |
awk
-F
':'
'/sha2:/ {print $2; exit;}'
)
if
[[
"
$
LOCAL_SHA
"
!=
"
$irods_sha
"
]]
;
then
echo
"[
$KEY
] Error 012 distant file is corrupted (localsha
$
LOCAL_SHA
irodssha
$irods_sha
). Rollback"
if
[[
"
$
local_sha
"
!=
"
$irods_sha
"
]]
;
then
echo
"[
$KEY
] Error 012 distant file is corrupted (localsha
$
local_sha
irodssha
$irods_sha
). Rollback"
irods_rollback
$KEY
zabbix_err
"
${
KEY
}
:Error 012"
fi
...
...
@@ -133,7 +135,7 @@ irods_rollback(){
irods_commit
(){
KEY
=
$1
echo
"[
$KEY
] If previous.tar exists, move it around but keep it safe"
ils
$KEY
/previous.tar 2>/dev/null
&&
(
ils
$KEY
/previous.tar 2
&1
>/dev/null
&&
(
imv
$KEY
/previous.tar
$KEY
/previous_to_delete.tar
if
[[
$?
-ne
0
]]
;
then
echo
"[
$KEY
] Error 003 moving previous.tar around. Corrective action has to be taken manualy"
...
...
@@ -143,7 +145,7 @@ irods_commit(){
)
echo
"[
$KEY
] If latest.tar exists, move it to previous.tar"
ils
$KEY
/latest.tar 2>/dev/null
&&
(
ils
$KEY
/latest.tar 2
&1
>/dev/null
&&
(
imv
$KEY
/latest.tar
$KEY
/previous.tar
if
[[
$?
-ne
0
]]
;
then
echo
"[
$KEY
] Error 004 moving latest.tar to previous.tar. Corrective action has to be taken manualy"
...
...
@@ -184,6 +186,26 @@ pack_and_send() {
return
0
fi
fi
# Check if file exists on irods server and get creation date
# days=$(( $(date +%j --date $(date +%Y-%m-01)) - $(date +%j --date $(ils -l 2001_YB_F13 | awk '/latest.tar/ {print substr($5, 0,11)}')) ))
ilsr
=
$(
ils
-L
$KEY
/latest.tar
)
if
[[
$?
-eq
0
]]
;
then
echo
"[
$KEY
] latest.tar already exists on iRODS server. Check if something changed localy since last push."
# Diff in days between current month (snapshot) and latest irods push
days
=
$((
$(
date
+%j
--date
$(
ils
-l
2001_YB_F13 |
awk
'/latest.tar/ {print substr($5, 0,11)}'
))
-
$(
date
+%j
--date
$(
date
+%Y-%m-01
))
))
if
[[
$days
-ge
0
]]
;
then
# We have an uptodate dump (newer than snapshot)
echo
"[
$KEY
] latest.tar is up to date"
return
0
fi
# find files modified in the last $days. If none, we can skip because latests irods version is up to date.
items_modified
=
$(
find
$dir
-ctime
$days
|
wc
-l
)
if
[[
$items_modified
-eq
0
]]
;
then
echo
"[
$KEY
]
$items_modified
new file to update since
$days
days. Skipping."
format_report
$KEY
"-"
$(
date
+%Y-%m-%dT%H:%M:%S
)
"-"
"-"
"Skipped"
>>
$LOCAL_REPORT
return
0
fi
fi
echo
"[
$KEY
] Creating tar on
$RESIFDD_WORKDIR
/
$KEY
.tar"
echo
"[
$KEY
] tar cf
$RESIFDD_WORKDIR
/
$KEY
.tar -C
${
dir
%
$YEAR
/
$NETWORK
/
$STATION
}
$dir
"
tar
cf
$RESIFDD_WORKDIR
/
$KEY
.tar
-C
${
dir
%
$YEAR
/
$NETWORK
/
$STATION
}
$dir
...
...
@@ -194,27 +216,10 @@ pack_and_send() {
zabbix_err
"
$KEY
:Error 007"
return
1
fi
# Check if file exists on irods server
ils
-L
$KEY
/latest.tar
>
/dev/null 2>&1
if
[[
$?
-eq
0
]]
;
then
echo
"[
$KEY
] latest.tar already exists on iRODS server. Let's compare hashes"
local_sha
=
$(
sha256sum
$RESIFDD_WORKDIR
/
$KEY
.tar |
awk
'{print $1}'
| xxd
-r
-p
|
base64
)
irods_sha
=
$(
ichksum
$KEY
/latest.tar |
awk
-F
':'
'/sha2:/ {print $2; exit;}'
)
echo
"[
$KEY
] local checksum:
$local_sha
"
echo
"[
$KEY
] irods checksum:
$irods_sha
"
# If the hashes differs, then move distant file and push this one
if
[[
"
$local_sha
"
=
"
$irods_sha
"
]]
;
then
echo
"[
$KEY
] The archive on irods is the same as our version. Skipping."
format_report
$KEY
"-"
$(
date
+%Y-%m-%dT%H:%M:%S
)
"-"
"-"
"Skipped"
>>
$LOCAL_REPORT
return
0
fi
fi
# Send latest archive file to IRODS
irods_push
$KEY
$local_sha
irods_push
$KEY
rm
$RESIFDD_WORKDIR
/
$KEY
*
}
export
-f
pack_and_send
# Necessary for call with GNU parallel
####################
#
...
...
@@ -290,8 +295,7 @@ else
zabbix_err
"
${
KEY
}
:Error 001"
exit
1
fi
local_sha
=
$(
sha256sum
$RESIFDD_WORKDIR
/
$KEY
.tar |
awk
'{print $1}'
| xxd
-r
-p
|
base64
)
irods_push
$KEY
$local_sha
irods_push
$KEY
echo
"[
$KEY
] Dump terminated :"
ils
-l
$KEY
fi
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment