Skip to content

GitLab

Vous avez reçu un message "Your GitLab account has been locked ..." ? Pas d'inquiétude : lisez cet article https://docs.gricad-pages.univ-grenoble-alpes.fr/help/unlock/

Commit 9739b65a authored by Jonathan Schaeffer's avatar Jonathan Schaeffer
Browse files

Logger mieux 

https://urbanautomaton.com/blog/2014/09/09/redirecting-bash-script-output-to-syslog/
parent 13caf446
Hide whitespace changes
Inline Side-by-side
Showing with 37 additions and 43 deletions
src/datadump.sh
View file @ 9739b65a
#!/bin/bash
exec 1> >(logger -s -t $(basename $0)) 2>&1
# Auteur: Jonathan Schaeffer <jonathan.schaeffer@univ-grenoble-alpes.fr>
#
# This script dumps precious RESIF data to an IRODS server
#
# Print a message using logger
# Arguments:
# $1 : message
# $2 : severity (optionnal, default is INFO)
log(){
[[ -z $2 ]] && severity="info" || severity=$2
logger -t resifdd -p local3.$severity "$1"
}
####################
#
# Preliminary tests
......@@ -22,46 +15,47 @@ log(){
# Test for working directory
if [[ ! -w $RESIFDD_WORKDIR ]] ; then
log "RESIFDD_WORKDIR $RESIFDD_WORKDIR not writable. Check permissions"
echo "RESIFDD_WORKDIR $RESIFDD_WORKDIR not writable. Check permissions. Exit 1"
exit 1
fi
# test the data directory where to dump everything from
if [[ -z $RESIFDD_DATADIR ]]; then
log "Variable RESIFDD_DATADIR must be set to the RESIF mountpoint" "error"
echo "Variable RESIFDD_DATADIR must be set to the RESIF mountpoint. Exit 1"
exit 1
fi
if [[ ! -d $RESIFDD_DATADIR ]]; then
log "Variable RESIFDD_DATADIR set to $RESIFDD_DATADIR must be a directory" "error"
echo "Variable RESIFDD_DATADIR set to $RESIFDD_DATADIR must be a directory. Exit 1"
exit 1
fi
# Get the snapshot name for this month
MONTH=$(date +%Y-%m)
log "Starting dump of metadata from ${RESIFDD_DATADIR}/validated_seismic_metadata/.snapshot/monthly.${MONTH}-01_0000"
tar cf $RESIFDD_WORKDIR/validated_metadata.tar -C $RESIFDD_DATADIR/validated_seismic_metadata/.snapshot/monthly.${MONTH}-01_0000 $RESIFDD_DATADIR/validated_seismic_metadata/.snapshot/monthly.${MONTH}-01_0000
imv validated_metadata/previous.tar validated_metadata/previous_to_delete.tar
imv validated_metadata/latest.tar validated_metadata/previous.tar
iput $RESIFDD_WORKDIR/validated_metadata.tar validated_metadata/latest.tar
SNAPSHOT_DIR=$RESIFDD_DATADIR/validated_seismic_seismic_metadata/.snapshot/monthly.${MONTH}-01_0000
echo "Starting dump of metadata from ${SNAPSHOT_DIR}"
tar cf $RESIFDD_WORKDIR/validated_seismic_metadata.tar -C $SNAPSHOT_DIR $SNAPSHOT_DIR
imkdir -p validated_seismic_metadata
imv validated_seismic_metadata/previous.tar validated_seismic_metadata/previous_to_delete.tar
imv validated_seismic_metadata/latest.tar validated_seismic_metadata/previous.tar
iput $RESIFDD_WORKDIR/validated_seismic_metadata.tar validated_seismic_metadata/latest.tar
if [[ $? -ne 0 ]]; then
# Something went wrong, roll back
imv validated_metadata/previous.tar validated_metadata/latest.tar
imv validated_metadata/previous_to_delete.tar validated_metadata/previous.tar
imv validated_seismic_metadata/previous.tar validated_seismic_metadata/latest.tar
imv validated_seismic_metadata/previous_to_delete.tar validated_seismic_metadata/previous.tar
fi
log "Dump of metadata done"
echo "Dump of metadata done"
SNAPSHOT_DIR="${RESIFDD_DATADIR}/validated_seismic_data/.snapshot/monthly.${MONTH}-01_0000"
if [[ ! -d $SNAPSHOT_DIR ]]; then
log "Snapshot directory $SNAPSHOT_DIR does not exist" "error"
echo "Error: Snapshot directory $SNAPSHOT_DIR does not exist"
exit 1
fi
# Checking IRODS environment
if [[ ! -f ~/.irods/.irodsA ]] ; then
log "The irods scrambled password file is not present. Please run iinit and provide the password"
echo "The irods scrambled password file is not present. Please run iinit and provide the password. Exit 1"
exit 1
fi
......@@ -73,7 +67,7 @@ fi
cd $SNAPSHOT_DIR
log "Starting dump of validated data"
echo "Starting dump of validated data"
for dir in $(find . -maxdepth 3 -type d); do
# First clean workspace
......@@ -84,54 +78,52 @@ for dir in $(find . -maxdepth 3 -type d); do
NETWORK=${YNS[2]}
STATION=${YNS[3]}
KEY=${YEAR}_${NETWORK}_${STATION}
log "[$KEY] Creating tar on $RESIFDD_WORKDIR/$KEY.tar"
echo "[$KEY] Creating tar on $RESIFDD_WORKDIR/$KEY.tar"
tar cf $RESIFDD_WORKDIR/$KEY.tar $dir
if [[ $? -ne 0 ]]; then
# Something went wrong creating archive. Exit
log "[$KEY] tar reported an error. Abort" error
echo "[$KEY] tar reported an error. Exit 1"
exit 1
fi
SIZE=$(( $(stat -c %s $RESIFDD_WORKDIR/$KEY.tar) / 1024 / 1024 / 1024 ))
# Check if file exists on irods server
irods_latest=$(ils -L $KEY/latest.tar)
log "[$KEY] irods latest file is : $irods_latest" "debug"
echo "[$KEY] irods latest file is : $irods_latest"
if [[ $? -eq 0 ]]; then
log "[$KEY] latest.tar already exists on iRODS server. Let's compare hashes"
echo "[$KEY] latest.tar already exists on iRODS server. Let's compare hashes"
local_sha=$(sha256sum $RESIFDD_WORKDIR/$KEY.tar | awk '{print $1}' | xxd -r -p | base64)
irods_sha=$(ichksum $KEY/latest.tar | awk -F':' '/sha2:/ {print $2; exit;}')
# Compare to local hash
# If the hashes differs, then move distant file and push this one
if [[ "$local_sha" = "$irods_sha" ]]; then
log "[$KEY] The archive on irods is the same as our version. Skipping."
echo "[$KEY] The archive on irods is the same as our version. Skipping."
continue
fi
log "[$KEY] local checksum: $local_sha" "debug"
log "[$KEY] idos checksum: $irods_sha" "debug"
echo "[$KEY] local checksum: $local_sha" "debug"
echo "[$KEY] idos checksum: $irods_sha" "debug"
log "[$KEY] Local checksum differs from last dump. Let's dump it"
echo "[$KEY] Local checksum differs from last dump. Let's dump it"
imv $KEY/previous.tar $KEY/previous_to_delete.tar
imv $KEY/latest.tar $KEY/previous.tar # Rename latest distant dump tp "previous"
if [[ $? -ne 0 ]]; then
# Something wrong moving remote file arround
log "[$KEY] There where something wrong moving $KEY/previous.tar, cancel iput" "error"
echo "[$KEY] Error, something wrong moving $KEY/previous.tar, cancel iput."
irm -f $KEY/previous_to_delete.tar
# zabbixsnd $KEY
continue
fi
fi
# Send file to IRODS
log "[$KEY] Sending file to iRODS ($SIZE GB)"
echo "[$KEY] Sending file to iRODS ($SIZE GB)"
imkdir -p $KEY
# Décommenter le bloc et les redirections pour traiter stdout et stderr différemment
{
iput \
--retries 5 -T -K -f \
$RESIFDD_WORKDIR/$KEY.tar $KEY/latest.tar 2>&1 1>&3 3>&- | logger -t 'resifdd' -p local3.error; } 3>&1 1>&2 | logger -t 'resifdd' -p local3.info
RC=$?
if [[ $RC -ne 0 ]]; then
log "[$KEY] Error sending file to irods. The command was: iput --lfrestart $RESIFDD_WORKDIR/$KEY.lfrestart --retries 5 -T -K -f $RESIFDD_WORKDIR/$KEY.tar $KEY/latest.tar" 'error'
log "[$KEY] Rolling back from $KEY/previous.tar to $KEY/latest.tar"
$RESIFDD_WORKDIR/$KEY.tar $KEY/latest.tar
if [[ $? -ne 0 ]]; then
echo "[$KEY] Error sending file to irods. The command was: iput --lfrestart $RESIFDD_WORKDIR/$KEY.lfrestart --retries 5 -T -K -f $RESIFDD_WORKDIR/$KEY.tar $KEY/latest.tar"
echo "[$KEY] Rolling back from $KEY/previous.tar to $KEY/latest.tar"
# Roll back
# TODO : section à tester
irm -f $KEY/latest.tar
......@@ -142,7 +134,9 @@ for dir in $(find . -maxdepth 3 -type d); do
fi
# Cleaning up the safety previous file
irm -f $KEY/previous_to_delete.tar
log "[$KEY] Sent successfully"
log "Cleaning generated tar $RESIFDD_WORKDIR/$YEAR_$NETWORK_$STATION.tar"
echo "[$KEY] Sent successfully"
# TODO Checksums check
echo "[$KEY] Cleaning local generated tar"
rm -f $RESIFDD_WORKDIR/$KEY.tar
done
log "Dump of validated data done"
echo "Dump of validated data done"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment