Vous avez reçu un message "Your GitLab account has been locked ..." ? Pas d'inquiétude : lisez cet article https://docs.gricad-pages.univ-grenoble-alpes.fr/help/unlock/

Commit 3093d8c1 authored by Jonathan Schaeffer's avatar Jonathan Schaeffer
Browse files

Creating a report

parent 403ae20d
......@@ -5,7 +5,6 @@
# This line tells to redirect all outputs to logger and stdout
exec 1> >(logger -s -t $(basename $0)) 2>&1
####################
#
# Zabbix sender
......@@ -16,6 +15,28 @@ zabbix(){
zabbix_sender -k resifdatadump.failed -s $(hostname -a) -o $1 -z monitoring.osug.fr
}
# Format a report line. Parameters is the list of data to append to the report :
# "2019_FR_PLOP" "123456" '2015-07-15T06:51:02' duration throughput "feaaaaaa" "OK"
format_report(){
IFS='_' read -r -a YNS <<< $1
if [[ ${#YNS[@]} -eq 3 ]]; then
Y=${YNS[0]}
N=${YNS[1]}
S=${YNS[2]}
else
Y=${YNS[0]}
N="-"
S="-"
fi
SIZE=$2
DATE=$3
DURATION=$4
THROUGHPUT=$5
SHA=$6
COMMENT=${@:7}
printf " %6s | %9s | %9s | %13s | %16s | %6s | %6s | %18s | %s" $Y $N $S $SIZE $DATE $DURATION $THROUGHPUT $SHA "${COMMENT}"
}
# Push data to irods as a staging file
# Argument is the distant directory to push to.
# If something goes wrong, roll back
......@@ -33,18 +54,24 @@ irods_push(){
fi
COMMAND="iput --retries 5 -T -K -f -X $RESIFDD_WORKDIR/${KEY}.restart $RESIFDD_WORKDIR/${KEY}.tar $KEY/staging.tar"
start=$(date +%s)
eval $COMMAND
duration=$(($(date +%s)-$start))
throughput=$(($SIZEMB / $duration ))
if [[ $? -ne 0 ]]; then
echo "[$KEY] Error 011 sending file to irods. The command was: $COMMAND"
format_report $KEY $SIZEMB $(date +%Y-%m-%dT%H%M%S --date=@$start) $duration $throughput 'Error 011. Transfer Cancelled' >> $LOCAL_REPORT
# Roll back
irods_rollback $KEY
# Alert to zabbix
zabbix "${KEY}:Error 011"
else
echo "[$KEY] staging.tar data sent, let's commit everything on irods server"
format_report $KEY $SIZEMB $(date +%Y-%m-%dT%H%M%S --date=@$start) $duration $throughput 'OK' >> $LOCAL_REPORT
irods_commit $KEY
fi
# Send report to irods
iput -f $LOCAL_REPORT $IRODS_REPORT
}
# In case of any problem, this function rolls every distant file operation bask
......@@ -136,6 +163,7 @@ if [[ ! -d $RESIFDD_DATADIR ]]; then
exit 1
fi
# Get the snapshot name for this month
MONTH=$(date +%Y-%m)
SNAPSHOT_DIR=$(ls -d $RESIFDD_DATADIR/validated_seismic_seismic_metadata/.snapshot/monthly.${MONTH}*|tail -1)
......@@ -144,6 +172,14 @@ if [[ ! -d $SNAPSHOT_DIR ]]; then
exit 1
fi
# Header for the report :
IRODS_REPORT=reports/$(date +%Y-%m-%d).csv
LOCAL_REPORT=$RESIFDD_WORKDIR/report.csv
format_report "Year_Network_Station" "Size(MB)" "Dumpdate" "Duration(s)" "Throughput(MB/s)" "Comment" > $LOCAL_REPORT
imkdir -p reports
iput $LOCAL_REPORT $IRODS_REPORT
KEY="validated-seismic-metadata"
echo "[$KEY] Starting dump from ${SNAPSHOT_DIR}"
......@@ -155,8 +191,6 @@ if [[ $? -ne 0 ]]; then
fi
irods_push $KEY
echo "[$KEY] Cleaning workdir $RESIFDD_WORKDIR"
rm -f $RESIFDD_WORKDIR/$KEY.tar $RESIFDD_WORKDIR/${KEY}.restart
echo "[$KEY] Dump terminated :"
ils -l $KEY
......@@ -176,7 +210,7 @@ echo "Starting dump of validated data"
cd $SNAPSHOT_DIR
for dir in $(find . -maxdepth 3 -type d | sort); do
# First clean workspace
rm -rf $RESIFDD_WORKDIR/*
rm -rf $RESIFDD_WORKDIR/*.tar $RESIFDD_WORKDIR/*.restart
# Parse path to get year, station and network
IFS='/' read -r -a YNS <<< $dir
[ ! ${#YNS[@]} -eq 4 ] && continue
......@@ -209,7 +243,5 @@ for dir in $(find . -maxdepth 3 -type d | sort); do
fi
# Send latest archive file to IRODS
irods_push $KEY
echo "[$KEY] Cleaning workdir $RESIFDD_WORKDIR"
rm -f $RESIFDD_WORKDIR/$KEY.tar $RESIFDD_WORKDIR/${KEY}.restart
done
echo "Dump of validated data done"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment