Commit 3093d8c1 authored by Jonathan Schaeffer's avatar Jonathan Schaeffer
Browse files

Creating a report

parent 403ae20d
...@@ -5,7 +5,6 @@ ...@@ -5,7 +5,6 @@
# This line tells to redirect all outputs to logger and stdout # This line tells to redirect all outputs to logger and stdout
exec 1> >(logger -s -t $(basename $0)) 2>&1 exec 1> >(logger -s -t $(basename $0)) 2>&1
#################### ####################
# #
# Zabbix sender # Zabbix sender
...@@ -16,6 +15,28 @@ zabbix(){ ...@@ -16,6 +15,28 @@ zabbix(){
zabbix_sender -k resifdatadump.failed -s $(hostname -a) -o $1 -z monitoring.osug.fr zabbix_sender -k resifdatadump.failed -s $(hostname -a) -o $1 -z monitoring.osug.fr
} }
# Format a report line. Parameters is the list of data to append to the report :
# "2019_FR_PLOP" "123456" '2015-07-15T06:51:02' duration throughput "feaaaaaa" "OK"
format_report(){
IFS='_' read -r -a YNS <<< $1
if [[ ${#YNS[@]} -eq 3 ]]; then
Y=${YNS[0]}
N=${YNS[1]}
S=${YNS[2]}
else
Y=${YNS[0]}
N="-"
S="-"
fi
SIZE=$2
DATE=$3
DURATION=$4
THROUGHPUT=$5
SHA=$6
COMMENT=${@:7}
printf " %6s | %9s | %9s | %13s | %16s | %6s | %6s | %18s | %s" $Y $N $S $SIZE $DATE $DURATION $THROUGHPUT $SHA "${COMMENT}"
}
# Push data to irods as a staging file # Push data to irods as a staging file
# Argument is the distant directory to push to. # Argument is the distant directory to push to.
# If something goes wrong, roll back # If something goes wrong, roll back
...@@ -33,18 +54,24 @@ irods_push(){ ...@@ -33,18 +54,24 @@ irods_push(){
fi fi
COMMAND="iput --retries 5 -T -K -f -X $RESIFDD_WORKDIR/${KEY}.restart $RESIFDD_WORKDIR/${KEY}.tar $KEY/staging.tar" COMMAND="iput --retries 5 -T -K -f -X $RESIFDD_WORKDIR/${KEY}.restart $RESIFDD_WORKDIR/${KEY}.tar $KEY/staging.tar"
start=$(date +%s)
eval $COMMAND eval $COMMAND
duration=$(($(date +%s)-$start))
throughput=$(($SIZEMB / $duration ))
if [[ $? -ne 0 ]]; then if [[ $? -ne 0 ]]; then
echo "[$KEY] Error 011 sending file to irods. The command was: $COMMAND" echo "[$KEY] Error 011 sending file to irods. The command was: $COMMAND"
format_report $KEY $SIZEMB $(date +%Y-%m-%dT%H%M%S --date=@$start) $duration $throughput 'Error 011. Transfer Cancelled' >> $LOCAL_REPORT
# Roll back # Roll back
irods_rollback $KEY irods_rollback $KEY
# Alert to zabbix # Alert to zabbix
zabbix "${KEY}:Error 011" zabbix "${KEY}:Error 011"
else else
echo "[$KEY] staging.tar data sent, let's commit everything on irods server" echo "[$KEY] staging.tar data sent, let's commit everything on irods server"
format_report $KEY $SIZEMB $(date +%Y-%m-%dT%H%M%S --date=@$start) $duration $throughput 'OK' >> $LOCAL_REPORT
irods_commit $KEY irods_commit $KEY
fi fi
# Send report to irods
iput -f $LOCAL_REPORT $IRODS_REPORT
} }
# In case of any problem, this function rolls every distant file operation bask # In case of any problem, this function rolls every distant file operation bask
...@@ -136,6 +163,7 @@ if [[ ! -d $RESIFDD_DATADIR ]]; then ...@@ -136,6 +163,7 @@ if [[ ! -d $RESIFDD_DATADIR ]]; then
exit 1 exit 1
fi fi
# Get the snapshot name for this month # Get the snapshot name for this month
MONTH=$(date +%Y-%m) MONTH=$(date +%Y-%m)
SNAPSHOT_DIR=$(ls -d $RESIFDD_DATADIR/validated_seismic_seismic_metadata/.snapshot/monthly.${MONTH}*|tail -1) SNAPSHOT_DIR=$(ls -d $RESIFDD_DATADIR/validated_seismic_seismic_metadata/.snapshot/monthly.${MONTH}*|tail -1)
...@@ -144,6 +172,14 @@ if [[ ! -d $SNAPSHOT_DIR ]]; then ...@@ -144,6 +172,14 @@ if [[ ! -d $SNAPSHOT_DIR ]]; then
exit 1 exit 1
fi fi
# Header for the report :
IRODS_REPORT=reports/$(date +%Y-%m-%d).csv
LOCAL_REPORT=$RESIFDD_WORKDIR/report.csv
format_report "Year_Network_Station" "Size(MB)" "Dumpdate" "Duration(s)" "Throughput(MB/s)" "Comment" > $LOCAL_REPORT
imkdir -p reports
iput $LOCAL_REPORT $IRODS_REPORT
KEY="validated-seismic-metadata" KEY="validated-seismic-metadata"
echo "[$KEY] Starting dump from ${SNAPSHOT_DIR}" echo "[$KEY] Starting dump from ${SNAPSHOT_DIR}"
...@@ -155,8 +191,6 @@ if [[ $? -ne 0 ]]; then ...@@ -155,8 +191,6 @@ if [[ $? -ne 0 ]]; then
fi fi
irods_push $KEY irods_push $KEY
echo "[$KEY] Cleaning workdir $RESIFDD_WORKDIR"
rm -f $RESIFDD_WORKDIR/$KEY.tar $RESIFDD_WORKDIR/${KEY}.restart
echo "[$KEY] Dump terminated :" echo "[$KEY] Dump terminated :"
ils -l $KEY ils -l $KEY
...@@ -176,7 +210,7 @@ echo "Starting dump of validated data" ...@@ -176,7 +210,7 @@ echo "Starting dump of validated data"
cd $SNAPSHOT_DIR cd $SNAPSHOT_DIR
for dir in $(find . -maxdepth 3 -type d | sort); do for dir in $(find . -maxdepth 3 -type d | sort); do
# First clean workspace # First clean workspace
rm -rf $RESIFDD_WORKDIR/* rm -rf $RESIFDD_WORKDIR/*.tar $RESIFDD_WORKDIR/*.restart
# Parse path to get year, station and network # Parse path to get year, station and network
IFS='/' read -r -a YNS <<< $dir IFS='/' read -r -a YNS <<< $dir
[ ! ${#YNS[@]} -eq 4 ] && continue [ ! ${#YNS[@]} -eq 4 ] && continue
...@@ -209,7 +243,5 @@ for dir in $(find . -maxdepth 3 -type d | sort); do ...@@ -209,7 +243,5 @@ for dir in $(find . -maxdepth 3 -type d | sort); do
fi fi
# Send latest archive file to IRODS # Send latest archive file to IRODS
irods_push $KEY irods_push $KEY
echo "[$KEY] Cleaning workdir $RESIFDD_WORKDIR"
rm -f $RESIFDD_WORKDIR/$KEY.tar $RESIFDD_WORKDIR/${KEY}.restart
done done
echo "Dump of validated data done" echo "Dump of validated data done"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment