Vous avez reçu un message "Your GitLab account has been locked ..." ? Pas d'inquiétude : lisez cet article https://docs.gricad-pages.univ-grenoble-alpes.fr/help/unlock/

Commit 7db8906f authored by Jonathan Schaeffer's avatar Jonathan Schaeffer
Browse files

Getting better

parent 1026398d
......@@ -59,42 +59,53 @@ fi
cd $SNAPSHOT_DIR
log "Starting dump of validated data"
for dir in $(find . -maxdepth 3 -type d); do
IFS='/' read -r -a YNS <<< $dir
[ ! ${#YNS[@]} -eq 3 ] && continue
YEAR=${YNS[0]}
NETWORK=${YNS[1]}
STATION=${YNS[2]}
[ ! ${#YNS[@]} -eq 4 ] && continue
YEAR=${YNS[1]}
NETWORK=${YNS[2]}
STATION=${YNS[3]}
KEY=${YEAR}_${NETWORK}_${STATION}
log "Going to tar data from $KEY"
log "[$KEY] Creating tar on $RESIFDD_WORKDIR/$KEY.tar"
tar cf $RESIFDD_WORKDIR/$KEY.tar $dir
# Check if file exists on irods server
ils -l $KEY/latest.tar
if [[ $? -ne 0 ]]; then
# Something went wrong creating archive. Exit
log "[$KEY] tar reported an error. Abort" error
exit 1
fi
SIZE=$(( $(stat -c %s $RESIFDD_WORKDIR/$KEY.tar) / 1024 / 1024 / 1024 ))
# Check if file exists on irods server
ils -l $KEY/latest.tar > /dev/null 2>&1
if [[ $? -eq 0 ]]; then
log "[$KEY] latest.tar already exists on iRODS server. Let's compare hashes"
# Check hash on irods server
# Compare to local hash
# If the hashes differs, then move distant file and push this one
if [[ "$local_hash" = "$irods_hash" ]]; then
log "[$KEY] The archive on irods is already there. Skipping."
log "[$KEY] The archive on irods is the same as our version. Skipping."
continue
fi
log "[$KEY] Local checksum differs from last dump. Let's dump it" info
log "[$KEY] Local checksum differs from last dump. Let's dump it"
irm $KEY/previous.tar > /dev/null 2>&1 # Silently remove previous.tar, ignore errors
imv $KEY/latest.tar $KEY/previous.tar # Rename latest distant dump tp "previous"
imv $KEY/latest.tar $KEY/previous.tar > /dev/null 2>&1 # Rename latest distant dump tp "previous"
fi
# Send file to IRODS
log "[$KEY] Sending file to iRODS"
log "[$KEY] Sending file to iRODS ($SIZE GB)"
imkdir -p $KEY
iput \
--lfrestart $RESIFDD_WORKDIR/$KEY.lfrestart \
-X $RESIFDD_WORKDIR/$KEY.restart \
--retries 5 \
-T \ # reconnects after 10 minutes \
-b \ # bulk upload to reduce overhead \
-K \ # verify checksums \
--retries 5 -T -b -K \
$RESIFDD_WORKDIR/$KEY.tar $KEY/latest.tar
log "[$KEY] iput returned $?"
RC=$?
if [[ $RC -ne 0 ]]; then
log "[$KEY] error sending file to irods. The command was : iput --lfrestart $RESIFDD_WORKDIR/$KEY.lfrestart -X $RESIFDD_WORKDIR/$KEY.restart --retries 5 -T -b -K $RESIFDD_WORKDIR/$KEY.tar $KEY/latest.tar" error
exit 1
fi
log "Cleaning generated tar $RESIFDD_WORKDIR/$YEAR_$NETWORK_$STATION.tar"
rm $RESIFDD_WORKDIR/$KEY.tar
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment