Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
OSUG
RESIF
datadump
Commits
1788d4de
Commit
1788d4de
authored
Jul 19, 2019
by
Jonathan Schaeffer
Browse files
Ajout de l'option STARTAT
parent
e597242c
Changes
1
Hide whitespace changes
Inline
Side-by-side
src/resifdatadump
View file @
1788d4de
...
...
@@ -163,6 +163,9 @@ pack_and_send() {
NETWORK
=
${
YNS
[-2]
}
STATION
=
${
YNS
[-1]
}
KEY
=
${
YEAR
}
_
${
NETWORK
}
_
${
STATION
}
if
[[
$YEAR
-lt
$RESIFDD_START_AT
]]
;
then
return
0
fi
echo
"[
$KEY
] Starting job
$2
"
# Test if in recovery mode, we should send or not
if
[[
-r
$RECOVERY_FILE
]]
;
then
...
...
@@ -173,8 +176,8 @@ pack_and_send() {
fi
fi
echo
"[
$KEY
] Creating tar on
$RESIFDD_WORKDIR
/
$KEY
.tar"
echo
"[
$KEY
] tar cf
$RESIFDD_WORKDIR
/
$KEY
.tar -C
${
dir
%
$YEAR
/
$NETWORK
/
$STATION
}
$YEAR
/
$NETWORK
/
$STATION
"
tar
cf
$RESIFDD_WORKDIR
/
$KEY
.tar
-C
${
dir
%
$YEAR
/
$NETWORK
/
$STATION
}
$YEAR
/
$NETWORK
/
$STATION
echo
"[
$KEY
] tar cf
$RESIFDD_WORKDIR
/
$KEY
.tar -C
${
dir
%
$YEAR
/
$NETWORK
/
$STATION
}
./
$YEAR
/
$NETWORK
/
$STATION
"
tar
cf
$RESIFDD_WORKDIR
/
$KEY
.tar
-C
${
dir
%
$YEAR
/
$NETWORK
/
$STATION
}
./
$YEAR
/
$NETWORK
/
$STATION
if
[[
$?
-ne
0
]]
;
then
# Something went wrong creating archive. Exit
echo
"[
$KEY
] Error 007 creating tar"
...
...
@@ -195,6 +198,7 @@ pack_and_send() {
echo
"[
$KEY
] The archive on irods is the same as our version. Skipping."
SIZE
=
$(
stat
-c
%s
$RESIFDD_WORKDIR
/
$KEY
.tar
)
format_report
$KEY
$((
$SIZE
/
1024
/
1024
))
$(
date
+%Y-%m-%dT%H:%M:%S
)
"-"
"-"
"Skipped"
>>
$LOCAL_REPORT
rm
$RESIFDD_WORKDIR
/
$KEY
*
return
0
fi
fi
...
...
@@ -232,6 +236,7 @@ if [[ ! -d $RESIFDD_DATADIR ]]; then
exit
1
fi
####################
#
# Option ContinueFrom
...
...
@@ -243,14 +248,15 @@ if [[ -r ${RESIFDD_CONTINUE_FROM_FILE} ]]; then
cp
$RESIFDD_CONTINUE_FROM_FILE
$RESIFDD_WORKDIR
/recovery.
$$
RECOVERY_FILE
=
$RESIFDD_WORKDIR
/recovery.
$$
echo
"Now using
$RESIFDD_WORKDIR
/recovery.
$$
as recovery file"
LOCAL_REPORT
=
$RESIFDD_CONTINUE_FROM_FILE
else
echo
"No recovery file present. Dumping everything now"
LOCAL_REPORT
=
$RESIFDD_WORKDIR
/report_
$(
date
+%Y%m%d-%H%M
)
.csv
format_report
"Year_Network_Station"
"Size(MB)"
"Dumpdate"
"Duration(s)"
"Throughput(MB/s)"
"Comment"
>
$LOCAL_REPORT
fi
# Header for the report :
IRODS_REPORT
=
reports/
$(
date
+%Y%m%d-%H%M
)
.csv
LOCAL_REPORT
=
$RESIFDD_WORKDIR
/report_
$(
date
+%Y%m%d-%H%M
)
.csv
format_report
"Year_Network_Station"
"Size(MB)"
"Dumpdate"
"Duration(s)"
"Throughput(MB/s)"
"Comment"
>
$LOCAL_REPORT
imkdir
-p
reports
iput
-f
$LOCAL_REPORT
$IRODS_REPORT
rm
$RESIFDD_WORKDIR
/report.lock
...
...
@@ -298,5 +304,5 @@ if [[ ! -d $SNAPSHOT_DIR ]]; then
fi
echo
"Starting dump of validated data with 4 jobs"
find
$SNAPSHOT_DIR
-maxdepth
3
-mindepth
3
-type
d |
sort
| parallel
--jobs
4
--max-args
1 pack_and_send
{}
{
%
}
find
$SNAPSHOT_DIR
-maxdepth
3
-mindepth
3
-type
d |
sort
| parallel
--jobs
6
--max-args
1 pack_and_send
{}
{
%
}
echo
"Dump of validated data done"
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment