resifdatadump

#!/bin/bash

exec 1> >(logger -s -t $(basename $0)) 2>&1

# Auteur: Jonathan Schaeffer <jonathan.schaeffer@univ-grenoble-alpes.fr>
#
# This script dumps precious RESIF data to an IRODS server
#

####################
#
# Zabbix sender
#
###################

zabbix(){
    zabbix_sender -k resifdatadump.failed -s $(hostname -a) -o $1 -z monitoring.osug.fr
}

# Push data to irods as a staging file
# Argument is the distant directory to push to.
# If something goes wrong, roll back
# If everything goes fine, then validate staging data
irods_push(){
    KEY=$1
    SIZE=$(stat -c %s $RESIFDD_WORKDIR/$KEY.tar)
    SIZEMB=$(($SIZE/1024/1024))
    echo "[$KEY] Sending data to iRODS ($SIZEMB MB)"
    imkdir -p $KEY
    if [[ $? -ne 0 ]]; then
        echo "[$KEY] Error 002 creating remote directory. Manual action has to be taken."
        zabbix "${KEY}:Error 002"
        exit 1
    fi

    COMMAND="iput --retries 5 -T -K -f -X $RESIFDD_WORKDIR/${KEY}.restart $RESIFDD_WORKDIR/${KEY}.tar $KEY/staging.tar"
    eval $COMMAND
    if [[ $? -ne 0 ]]; then
        echo "[$KEY] Error 011 sending file to irods. The command was: $COMMAND"
        # Roll back
        irods_rollback $KEY
        # Alert to zabbix
        zabbix "${KEY}:Error 011"
    else
        irods_commit $KEY
    fi
}

# In case of any problem, this function rolls every distant file operation bask
# 1. tries to revover latest.tar from previous.tar
# 2. tries to recover previous.tar from previous_to_delete.tar
# Argument is the distant directory to work with
irods_rollback(){
    KEY=$1
    echo "[$KEY] Rollback : try to recover latest.tar from previous"
    ils ${KEY}/previous.tar 2>/dev/null && (
        imv ${KEY}/previous.tar ${KEY}/latest.tar
        if [[ $? -ne 0 ]]; then
            echo "[$KEY] Error $? recovering latest.tar Exit 1"
            return 1
        fi
    )
    echo "[$KEY] Rollback : OK"
    echo "[$KEY] Rollback : try to recover previous.tar from previous_to_delete"
    ils ${KEY}/previous_to_delete.tar 2>/dev/null && (
        imv ${KEY}/previous_to_delete.tar ${KEY}/previous.tar
        if [[ $? -ne 0 ]]; then
            echo "[$KEY] Error $? recovering previous.tar Exit 1"
            return 1
        fi
    )
    echo "[$KEY] Rollback : OK"
    return 0
}

# When the transfer to irods is successfull, we can move file arounds remotely :
# 1. $KEY/previous.tar move to trash
# 2. $KEY/latest.tar move to $KEY/previous.tar
# 3. $KEY/staging.tar move to $KEY/latest.tar
irods_commit(){
    KEY=$1
    echo "[$KEY] If previous.tar exists, move it around but keep it safe"
    ils $KEY/previous.tar 2>/dev/null && (
        imv $KEY/previous.tar $KEY/previous_to_delete.tar
        if [[ $? -ne 0 ]]; then
            echo "[$KEY] Error 003 moving previous.tar around. Corrective action has to be taken manualy"
            zabbix "${KEY}:Error 003"
            return 1
        fi
    )

    echo "[$KEY] If latest.tar exists, move it to previous.tar"
    ils $KEY/latest.tar 2>/dev/null && (
        imv $KEY/latest.tar $KEY/previous.tar
        if [[ $? -ne 0 ]]; then
            echo "[$KEY] Error 004 moving latest.tar to previous.tar. Corrective action has to be taken manualy"
            zabbix "${KEY}:Error 004"
            return 1
        fi
    )
    echp "[$KEY] Validate staging.tar by moving it to staging.tar"
    imv $KEY/staging.tar $KEY/latest.tar
    if [[ $? -ne 0 ]]; then
        echo "[$KEY] Error 005 moving statging.tar to latest.tar. Corrective action has to be taken manualy"
        zabbix "${KEY}:Error 005"
        return 1
    fi
    ils $KEY/previous_to_delete.tar 2>/dev/null && irm -f $KEY/previous_to_delete.tar
}

####################
#
# Preliminary tests
#
####################

# Checking IRODS environment
if [[ ! -f ~/.irods/.irodsA ]] ; then
    echo "The irods scrambled password file is not present. Please run iinit and provide the password. Exit 1"
    exit 1
fi

# Test for working directory
if [[ ! -w $RESIFDD_WORKDIR ]] ; then
    echo "RESIFDD_WORKDIR \"$RESIFDD_WORKDIR\" not writable. Check permissions. Exit 1"
    exit 1
fi
# test the data directory where to dump everything from
if [[ -z $RESIFDD_DATADIR ]]; then
    echo "Variable RESIFDD_DATADIR must be set to the RESIF mountpoint. Exit 1"
    exit 1
fi
if [[ ! -d $RESIFDD_DATADIR ]]; then
    echo "Variable RESIFDD_DATADIR set to \"$RESIFDD_DATADIR\" must be a directory. Exit 1"
    exit 1
fi

# Get the snapshot name for this month
MONTH=$(date +%Y-%m)
SNAPSHOT_DIR=$RESIFDD_DATADIR/validated_seismic_seismic_metadata/.snapshot/monthly.${MONTH}-01_0000
if [[ ! -d $SNAPSHOT_DIR ]]; then
    echo "Error 000 Snapshot directory $SNAPSHOT_DIR does not exist"
    exit 1
fi

KEY="validated-seismic-metadata"

echo "[$KEY] Starting dump from ${SNAPSHOT_DIR}"
tar cf  $RESIFDD_WORKDIR/$KEY.tar --exclude portalproducts -C $SNAPSHOT_DIR $SNAPSHOT_DIR
if [[ $? -ne 0 ]]; then
    echo "[$KEY] Error 001 while creating tar archive."
    zabbix "${KEY}:Error 001"
    exit 1
fi

irods_push $KEY
echo "[$KEY] Cleaning workdir $RESIFDD_WORKDIR"
rm -f $RESIFDD_WORKDIR/$KEY.tar $RESIFDD_WORKDIR/${KEY}.restart
echo "[$KEY] Dump terminated :"
ils -l $KEY

####################
#
# Start dumping validated data
#
####################

SNAPSHOT_DIR="${RESIFDD_DATADIR}/validated_seismic_data/.snapshot/monthly.${MONTH}-01_0000"
if [[ ! -d $SNAPSHOT_DIR ]]; then
    echo "Error 006 Snapshot directory $SNAPSHOT_DIR does not exist"
    exit 1
fi

echo "Starting dump of validated data"
cd $SNAPSHOT_DIR
for dir in $(find . -maxdepth 3 -type d | sort); do
    # First clean workspace
    rm -rf $RESIFDD_WORKDIR/*
    # Parse path to get year, station and network
    IFS='/' read -r -a YNS <<< $dir
    [ ! ${#YNS[@]} -eq 4 ] && continue
    YEAR=${YNS[1]}
    NETWORK=${YNS[2]}
    STATION=${YNS[3]}
    KEY=${YEAR}_${NETWORK}_${STATION}
    echo "[$KEY] Creating tar on $RESIFDD_WORKDIR/$KEY.tar"
    tar cf $RESIFDD_WORKDIR/$KEY.tar $dir
    if [[ $? -ne 0 ]]; then
        # Something went wrong creating archive. Exit
        echo "[$KEY] Error 007 creating tar"
        # Send key to zabbix
        zabbix "$KEY:Error 007"
        continue
    fi
    # Check if file exists on irods server
    ils -L $KEY/latest.tar > /dev/null 2>&1
    if [[ $? -eq 0 ]]; then
        echo "[$KEY] latest.tar already exists on iRODS server. Let's compare hashes"
        local_sha=$(sha256sum $RESIFDD_WORKDIR/$KEY.tar | awk '{print $1}' | xxd -r -p | base64)
        irods_sha=$(ichksum $KEY/latest.tar | awk -F':' '/sha2:/ {print $2; exit;}')
        echo "[$KEY] local checksum: $local_sha"
        echo "[$KEY] irods checksum: $irods_sha"
        # If the hashes differs, then move distant file and push this one
        if [[ "$local_sha" = "$irods_sha" ]]; then
            echo "[$KEY] The archive on irods is the same as our version. Skipping."
            continue
        fi
    fi
    # Send latest archive file to IRODS
    irods_push $KEY
    echo "[$KEY] Cleaning workdir $RESIFDD_WORKDIR"
    rm -f $RESIFDD_WORKDIR/$KEY.tar $RESIFDD_WORKDIR/${KEY}.restart
done
echo "Dump of validated data done"