Vous avez reçu un message "Your GitLab account has been locked ..." ? Pas d'inquiétude : lisez cet article https://docs.gricad-pages.univ-grenoble-alpes.fr/help/unlock/

Commit 254de23c authored by Jonathan Schaeffer's avatar Jonathan Schaeffer
Browse files

Correction des chemins à l'intérieur des archives

parent 8718c6d0
...@@ -80,11 +80,9 @@ irods_push(){ ...@@ -80,11 +80,9 @@ irods_push(){
throughput=$(($SIZEMB / $duration )) throughput=$(($SIZEMB / $duration ))
# Check integrity. # Check integrity.
# We do it separately from transfer because for big files it can be very long, and the network can cut # We do it separately from transfer because for big files it can be very long, and the network can cut
local_sha=$(sha256sum $RESIFDD_WORKDIR/$KEY.tar | awk '{print $1}' | xxd -r -p | base64)
echo "[$KEY] local checksum: $local_sha"
irods_sha=$(ichksum $KEY/staging.tar | awk -F':' '/sha2:/ {print $2; exit;}') irods_sha=$(ichksum $KEY/staging.tar | awk -F':' '/sha2:/ {print $2; exit;}')
if [[ "$local_sha" != "$irods_sha" ]]; then if [[ "$LOCAL_SHA" != "$irods_sha" ]]; then
echo "[$KEY] Error 012 distant file is corrupted (localsha $local_sha irodssha $irods_sha). Rollback" echo "[$KEY] Error 012 distant file is corrupted (localsha $LOCAL_SHA irodssha $irods_sha). Rollback"
irods_rollback $KEY irods_rollback $KEY
zabbix_err "${KEY}:Error 012" zabbix_err "${KEY}:Error 012"
fi fi
...@@ -135,7 +133,7 @@ irods_rollback(){ ...@@ -135,7 +133,7 @@ irods_rollback(){
irods_commit(){ irods_commit(){
KEY=$1 KEY=$1
echo "[$KEY] If previous.tar exists, move it around but keep it safe" echo "[$KEY] If previous.tar exists, move it around but keep it safe"
ils $KEY/previous.tar >/dev/null 2>&1 && ( ils $KEY/previous.tar 2>/dev/null && (
imv $KEY/previous.tar $KEY/previous_to_delete.tar imv $KEY/previous.tar $KEY/previous_to_delete.tar
if [[ $? -ne 0 ]]; then if [[ $? -ne 0 ]]; then
echo "[$KEY] Error 003 moving previous.tar around. Corrective action has to be taken manualy" echo "[$KEY] Error 003 moving previous.tar around. Corrective action has to be taken manualy"
...@@ -145,7 +143,7 @@ irods_commit(){ ...@@ -145,7 +143,7 @@ irods_commit(){
) )
echo "[$KEY] If latest.tar exists, move it to previous.tar" echo "[$KEY] If latest.tar exists, move it to previous.tar"
ils $KEY/latest.tar >/dev/null 2>&1 && ( ils $KEY/latest.tar 2>/dev/null && (
imv $KEY/latest.tar $KEY/previous.tar imv $KEY/latest.tar $KEY/previous.tar
if [[ $? -ne 0 ]]; then if [[ $? -ne 0 ]]; then
echo "[$KEY] Error 004 moving latest.tar to previous.tar. Corrective action has to be taken manualy" echo "[$KEY] Error 004 moving latest.tar to previous.tar. Corrective action has to be taken manualy"
...@@ -160,39 +158,7 @@ irods_commit(){ ...@@ -160,39 +158,7 @@ irods_commit(){
zabbix_err "${KEY}:Error 005" zabbix_err "${KEY}:Error 005"
return 1 return 1
fi fi
ils $KEY/previous_to_delete.tar >/dev/null 2>&1 && irm -f $KEY/previous_to_delete.tar ils $KEY/previous_to_delete.tar 2>/dev/null && irm -f $KEY/previous_to_delete.tar
}
##################
#
# Dump Metadata
#
##################
pack_and_send_metadata(){
KEY="validated-seismic-metadata"
if [[ -r $RECOVERY_FILE ]] && egrep -q -e ".*($KEY ).*( OK | Skipped ).*" $RECOVERY_FILE ; then
format_report $KEY "-" $(date +%Y-%m-%dT%H:%M:%S) "-" "-" "Skipped" >> $LOCAL_REPORT
else
# Get the snapshot name for this month
MONTH=$(date +%Y-%m)
SNAPSHOT_DIR=$(ls -d $RESIFDD_DATADIR/validated_seismic_metadata/.snapshot/weekly.${MONTH}*|tail -1)
if [[ ! -d $SNAPSHOT_DIR ]]; then
echo "Error 000 Snapshot directory $SNAPSHOT_DIR does not exist"
exit 1
fi
echo "[$KEY] Starting dump from ${SNAPSHOT_DIR}"
tar cf $RESIFDD_WORKDIR/$KEY.tar --exclude portalproducts -C $SNAPSHOT_DIR $SNAPSHOT_DIR
if [[ $? -ne 0 ]]; then
echo "[$KEY] Error 001 while creating tar archive."
zabbix_err "${KEY}:Error 001"
exit 1
fi
irods_push $KEY
echo "[$KEY] Dump terminated :"
ils -l $KEY
fi
} }
# Choose if the data should be processed or skipped # Choose if the data should be processed or skipped
...@@ -218,30 +184,10 @@ pack_and_send() { ...@@ -218,30 +184,10 @@ pack_and_send() {
return 0 return 0
fi fi
fi fi
# Check if file exists on irods server and get creation date
# days=$(( $(date +%j --date $(date +%Y-%m-01)) - $(date +%j --date $(ils -l 2001_YB_F13 | awk '/latest.tar/ {print substr($5, 0,11)}')) ))
ilsr=$(ils -L $KEY/latest.tar)
if [[ $? -eq 0 ]]; then
echo "[$KEY] latest.tar already exists on iRODS server. Check if something changed localy since last push."
# Diff in days between current month (snapshot) and latest irods push
days=$(( $(date +%j --date $(ils -l 2001_YB_F13 | awk '/latest.tar/ {print substr($5, 0,11)}')) - $(date +%j --date $(date +%Y-%m-01)) ))
if [[ $days -ge 0 ]]; then
# We have an uptodate dump (newer than snapshot)
echo "[$KEY] latest.tar is up to date"
format_report $KEY "-" $(date +%Y-%m-%dT%H:%M:%S) "-" "-" "Skipped" >> $LOCAL_REPORT
return 0
fi
# find files modified in the last $days. If none, we can skip because latests irods version is up to date.
items_modified=$(find $dir -ctime $days | wc -l)
if [[ $items_modified -eq 0 ]]; then
echo "[$KEY] $items_modified new file to update since $days days. Skipping."
format_report $KEY "-" $(date +%Y-%m-%dT%H:%M:%S) "-" "-" "Skipped" >> $LOCAL_REPORT
return 0
fi
fi
echo "[$KEY] Creating tar on $RESIFDD_WORKDIR/$KEY.tar" echo "[$KEY] Creating tar on $RESIFDD_WORKDIR/$KEY.tar"
echo "[$KEY] tar cf $RESIFDD_WORKDIR/$KEY.tar -C ${dir%$YEAR/$NETWORK/$STATION} $dir" tar_cmd="tar cf $RESIFDD_WORKDIR/$KEY.tar -C ${dir%$YEAR/$NETWORK/$STATION} ${YEAR}/${NETWORK}/${STATION}"
tar cf $RESIFDD_WORKDIR/$KEY.tar -C ${dir%$YEAR/$NETWORK/$STATION} $dir echo "[$KEY] $tar_cmd"
eval $tar_cmd
if [[ $? -ne 0 ]]; then if [[ $? -ne 0 ]]; then
# Something went wrong creating archive. Exit # Something went wrong creating archive. Exit
echo "[$KEY] Error 007 creating tar" echo "[$KEY] Error 007 creating tar"
...@@ -249,10 +195,27 @@ pack_and_send() { ...@@ -249,10 +195,27 @@ pack_and_send() {
zabbix_err "$KEY:Error 007" zabbix_err "$KEY:Error 007"
return 1 return 1
fi fi
# Check if file exists on irods server
ils -L $KEY/latest.tar > /dev/null 2>&1
if [[ $? -eq 0 ]]; then
echo "[$KEY] latest.tar already exists on iRODS server. Let's compare hashes"
local_sha=$(sha256sum $RESIFDD_WORKDIR/$KEY.tar | awk '{print $1}' | xxd -r -p | base64)
irods_sha=$(ichksum $KEY/latest.tar | awk -F':' '/sha2:/ {print $2; exit;}')
echo "[$KEY] local checksum: $local_sha"
echo "[$KEY] irods checksum: $irods_sha"
# If the hashes differs, then move distant file and push this one
if [[ "$local_sha" = "$irods_sha" ]]; then
echo "[$KEY] The archive on irods is the same as our version. Skipping."
format_report $KEY "-" $(date +%Y-%m-%dT%H:%M:%S) "-" "-" "Skipped" >> $LOCAL_REPORT
return 0
fi
fi
# Send latest archive file to IRODS # Send latest archive file to IRODS
irods_push $KEY irods_push $KEY $local_sha
rm $RESIFDD_WORKDIR/$KEY* rm $RESIFDD_WORKDIR/$KEY*
} }
export -f pack_and_send # Necessary for call with GNU parallel export -f pack_and_send # Necessary for call with GNU parallel
#################### ####################
# #
...@@ -293,25 +256,7 @@ if [[ -r ${RESIFDD_CONTINUE_FROM_FILE} ]]; then ...@@ -293,25 +256,7 @@ if [[ -r ${RESIFDD_CONTINUE_FROM_FILE} ]]; then
RECOVERY_FILE=$RESIFDD_WORKDIR/recovery.$$ RECOVERY_FILE=$RESIFDD_WORKDIR/recovery.$$
echo "Now using $RESIFDD_WORKDIR/recovery.$$ as recovery file" echo "Now using $RESIFDD_WORKDIR/recovery.$$ as recovery file"
else else
echo "No recovery file present. Dumping everything now" echo "No recovery file set"
fi
####################
#
# Option KEYFILE to send just a list of keys
#
####################
if [ $# -gt 0 ]; then
for var in "$@"; do
echo "Dumping $var"
MONTH=$(date +%Y-%m)
if [ "x$var" = "xvalidated-seismic-metadata" ]; then
pack_and_send_metadata
else
SNAPSHOT_DIR=$(ls -d ${RESIFDD_DATADIR}/validated_seismic_data/.snapshot/monthly.${MONTH}*|tail -1)
pack_and_send ${SNAPSHOT_DIR}/${var} 1
fi
done
exit 0
fi fi
# Header for the report : # Header for the report :
...@@ -321,8 +266,36 @@ format_report "Year_Network_Station" "Size(MB)" "Dumpdate" "Duration(s)" "Throug ...@@ -321,8 +266,36 @@ format_report "Year_Network_Station" "Size(MB)" "Dumpdate" "Duration(s)" "Throug
imkdir -p reports imkdir -p reports
iput -f $LOCAL_REPORT $IRODS_REPORT iput -f $LOCAL_REPORT $IRODS_REPORT
# First save metadata ##################
pack_and_send_metadata #
# Dump Metadata
#
##################
KEY="validated-seismic-metadata"
if [[ -r $RECOVERY_FILE ]] && egrep -q -e ".*($KEY ).*( OK | Skipped ).*" $RECOVERY_FILE ; then
format_report $KEY "-" $(date +%Y-%m-%dT%H:%M:%S) "-" "-" "Skipped" >> $LOCAL_REPORT
else
# Get the snapshot name for this month
MONTH=$(date +%Y-%m)
SNAPSHOT_DIR=$(ls -d $RESIFDD_DATADIR/validated_seismic_metadata/.snapshot/weekly.${MONTH}*|tail -1)
if [[ ! -d $SNAPSHOT_DIR ]]; then
echo "Error 000 Snapshot directory $SNAPSHOT_DIR does not exist"
exit 1
fi
echo "[$KEY] Starting dump from ${SNAPSHOT_DIR}"
tar cf $RESIFDD_WORKDIR/$KEY.tar --exclude portalproducts -C $SNAPSHOT_DIR $SNAPSHOT_DIR
if [[ $? -ne 0 ]]; then
echo "[$KEY] Error 001 while creating tar archive."
zabbix_err "${KEY}:Error 001"
exit 1
fi
local_sha=$(sha256sum $RESIFDD_WORKDIR/$KEY.tar | awk '{print $1}' | xxd -r -p | base64)
irods_push $KEY $local_sha
echo "[$KEY] Dump terminated :"
ils -l $KEY
fi
#################### ####################
# #
...@@ -330,13 +303,28 @@ pack_and_send_metadata ...@@ -330,13 +303,28 @@ pack_and_send_metadata
# #
#################### ####################
MONTH=$(date +%Y-%m)
SNAPSHOT_DIR=$(ls -d ${RESIFDD_DATADIR}/validated_seismic_data/.snapshot/monthly.${MONTH}*|tail -1) SNAPSHOT_DIR=$(ls -d ${RESIFDD_DATADIR}/validated_seismic_data/.snapshot/monthly.${MONTH}*|tail -1)
if [[ ! -d $SNAPSHOT_DIR ]]; then if [[ ! -d $SNAPSHOT_DIR ]]; then
echo "Error 006 Snapshot directory $SNAPSHOT_DIR does not exist" echo "Error 006 Snapshot directory $SNAPSHOT_DIR does not exist"
exit 1 exit 1
fi fi
echo "Starting dump of validated data with 3 jobs" echo "Starting dump of validated data with 4 jobs"
find $SNAPSHOT_DIR -maxdepth 3 -mindepth 3 -type d | sort | parallel --jobs 3 --max-args 1 pack_and_send {} {%} find $SNAPSHOT_DIR -maxdepth 3 -mindepth 3 -type d | sort | parallel --jobs 4 --max-args 1 pack_and_send {} {%}
echo "Dump of validated data done" echo "Dump of validated data done"
####################
#
# Start dumping cold data
#
####################
SNAPSHOT_DIR=$(ls -d ${RESIFDD_DATADIR}/cold_validated_seismic_data/.snapshot/weekly.$(date +%Y-%m-%d --date 'last sunday')_*|tail -1)
if [[ ! -d $SNAPSHOT_DIR ]]; then
echo "Error 006 Snapshot directory $SNAPSHOT_DIR does not exist"
exit 1
fi
echo "Starting dump of cold data with 4 jobs"
find $SNAPSHOT_DIR -maxdepth 3 -mindepth 3 -type d | sort | parallel --jobs 4 --max-args 1 pack_and_send {} {%}
echo "Dump of cold validated data done"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment