Commit 15eddfdc authored by Jonathan Schaeffer's avatar Jonathan Schaeffer

Derniers ajustements sur version fonctionnelle

Cette version a tourné pendant le mois d'aout et a permis de pousser
toutes nos données
parent 1788d4de
......@@ -5,7 +5,7 @@
# This line tells to redirect all outputs to logger and stdout
exec 1> >(logger -s -t $(basename $0)) 2>&1
set -u
set -a
####################
#
......@@ -55,7 +55,7 @@ irods_push(){
for n in $(seq 1 5); do [ $n -gt 1 ] && sleep 10 ; imkdir -p $KEY && s=0 && break || s=$?; done
if [[ $s -ne 0 ]]; then
echo "[$KEY] Error 002 creating remote directory. Manual action has to be taken."
echo "[$KEY] Error 002 creating remote directory."
format_report $KEY $SIZEMB $(date +%Y-%m-%dT%H:%M:%S) - - 'Error 010. imkdir failed' >> $LOCAL_REPORT
zabbix_err "${KEY}:Error 002"
return 1
......@@ -70,6 +70,7 @@ irods_push(){
format_report $KEY $SIZEMB $(date +%Y-%m-%dT%H:%M:%S --date=@$start) - - 'Error 011. Transfer cancelled' >> $LOCAL_REPORT
# Roll back
irods_rollback $KEY
rm $RESIFDD_WORKDIR/${KEY}.tar
# Alert to zabbix
zabbix_err "${KEY}:Error 011"
else
......@@ -78,8 +79,9 @@ irods_push(){
echo "[$KEY] staging.tar data sent, let's commit everything on irods server"
format_report $KEY $SIZEMB $(date +%Y-%m-%dT%H:%M:%S --date=@$start) $duration $throughput 'OK' >> $LOCAL_REPORT
irods_commit $KEY
zabbix_ok "$KEY|${SIZEMB}MB|${duration}s|${throughput}MB/s"
rm $RESIFDD_WORKDIR/${KEY}.*
fi
zabbix_ok "$KEY|${SIZEMB}MB|${duration}s|${throughput}MB/s"
# Send report to irods. Do some locking here
(
flock -e 200
......@@ -181,6 +183,7 @@ pack_and_send() {
if [[ $? -ne 0 ]]; then
# Something went wrong creating archive. Exit
echo "[$KEY] Error 007 creating tar"
rm -f $RESIFDD_WORKDIR/$KEY.tar
# Send key to zabbix_err
zabbix_err "$KEY:Error 007"
return 1
......@@ -204,8 +207,6 @@ pack_and_send() {
fi
# Send latest archive file to IRODS
irods_push $KEY
rm $RESIFDD_WORKDIR/$KEY*
}
export -f pack_and_send # Necessary for call with GNU parallel
......@@ -260,6 +261,7 @@ IRODS_REPORT=reports/$(date +%Y%m%d-%H%M).csv
imkdir -p reports
iput -f $LOCAL_REPORT $IRODS_REPORT
rm $RESIFDD_WORKDIR/report.lock
rm $RESIFDD_WORKDIR/*.tar
##################
#
......@@ -273,7 +275,7 @@ if [[ -r $RECOVERY_FILE ]] && egrep -q -e ".*($KEY ).*( OK | Skipped ).*" $RECOV
else
# Get the snapshot name for this month
MONTH=$(date +%Y-%m)
SNAPSHOT_DIR=$(ls -d $RESIFDD_DATADIR/validated_seismic_metadata/.snapshot/weekly.${MONTH}*|tail -1)
SNAPSHOT_DIR=$(ls -d $RESIFDD_DATADIR/validated_seismic_metadata/.snapshot/monthly.${MONTH}*|tail -1)
if [[ ! -d $SNAPSHOT_DIR ]]; then
echo "Error 000 Snapshot directory $SNAPSHOT_DIR does not exist"
exit 1
......@@ -304,5 +306,5 @@ if [[ ! -d $SNAPSHOT_DIR ]]; then
fi
echo "Starting dump of validated data with 4 jobs"
find $SNAPSHOT_DIR -maxdepth 3 -mindepth 3 -type d | sort | parallel --jobs 6 --max-args 1 pack_and_send {} {%}
find $SNAPSHOT_DIR -maxdepth 3 -mindepth 3 -type d | sort | parallel --jobs 4 --max-args 1 pack_and_send {} {%}
echo "Dump of validated data done"
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment