...
 
Commits (2)
......@@ -27,30 +27,14 @@ RESIFDD_WORKDIR=/osug-dc/resif RESIFDD_DATATIR=/scratch/resifdumper resifdatadum
### Options particulières de l'outil
Démarrer le transfert de toutes les données à partir de 2009
Lancer la sauvegarde d'une liste de stations :
``` shell
RESIFDD_WORKDIR=/osug-dc/resif RESIFDD_DATATIR=/scratch/resifdumper RESIFDD_START_AT=2009 src/resifdatadump
RESIFDD_DATADIR=/osug-dc/resif RESIFDD_WORKDIR=/scratch/resif_datadump src/resifdatadump 2011/RA/NCAD 2012/MT/THE
```
Transférer les données listées dans le fichier `RESIFDD_KEYFILE` :
Sauvegarder les métadonnées
``` shell
RESIFDD_DATADIR=/osug-dc/resif RESIFDD_WORKDIR=/scratch/resif_datadump src/resifdatadump validated_seismic_metadata
``` shell
RESIFDD_WORKDIR=/osug-dc/resif RESIFDD_DATATIR=/scratch/resifdumper RESIFDD_KEYFILE=/scratch/resifdumper/keys.txt src/resifdatadump
```
Le ficher doit contenir une clé par ligne, comme rapportée dans les logs :
``` textfile
2018_RA_CGBP
2018_FR_RUSF
2018_RA_NCAD
2018_RA_PYTO
2016_FR_RUSF
2017_RA_PYTO
2017_RA_CGBP
2017_FR_RUSF
2016_MT_CLP2
2016_RA_NCAD
```
On peut générer un fichier de clés avec le script python `src/scan_dupms.py` qui
This diff is collapsed.
......@@ -163,6 +163,38 @@ irods_commit(){
ils $KEY/previous_to_delete.tar >/dev/null 2>&1 && irm -f $KEY/previous_to_delete.tar
}
##################
#
# Dump Metadata
#
##################
pack_and_send_metadata(){
KEY="validated-seismic-metadata"
if [[ -r $RECOVERY_FILE ]] && egrep -q -e ".*($KEY ).*( OK | Skipped ).*" $RECOVERY_FILE ; then
format_report $KEY "-" $(date +%Y-%m-%dT%H:%M:%S) "-" "-" "Skipped" >> $LOCAL_REPORT
else
# Get the snapshot name for this month
MONTH=$(date +%Y-%m)
SNAPSHOT_DIR=$(ls -d $RESIFDD_DATADIR/validated_seismic_metadata/.snapshot/weekly.${MONTH}*|tail -1)
if [[ ! -d $SNAPSHOT_DIR ]]; then
echo "Error 000 Snapshot directory $SNAPSHOT_DIR does not exist"
exit 1
fi
echo "[$KEY] Starting dump from ${SNAPSHOT_DIR}"
tar cf $RESIFDD_WORKDIR/$KEY.tar --exclude portalproducts -C $SNAPSHOT_DIR $SNAPSHOT_DIR
if [[ $? -ne 0 ]]; then
echo "[$KEY] Error 001 while creating tar archive."
zabbix_err "${KEY}:Error 001"
exit 1
fi
irods_push $KEY
echo "[$KEY] Dump terminated :"
ils -l $KEY
fi
}
# Choose if the data should be processed or skipped
# Pack the data and send it to iRODS
# Argument is the path (abs or relative) to the data
......@@ -263,6 +295,24 @@ if [[ -r ${RESIFDD_CONTINUE_FROM_FILE} ]]; then
else
echo "No recovery file present. Dumping everything now"
fi
####################
#
# Option KEYFILE to send just a list of keys
#
####################
if [ $# -gt 0 ]; then
for var in "$@"; do
echo "Dumping $var"
MONTH=$(date +%Y-%m)
if [ "x$var" = "xvalidated-seismic-metadata" ]; then
pack_and_send_metadata
else
SNAPSHOT_DIR=$(ls -d ${RESIFDD_DATADIR}/validated_seismic_data/.snapshot/monthly.${MONTH}*|tail -1)
pack_and_send ${SNAPSHOT_DIR}/${var} 1
fi
done
exit 0
fi
# Header for the report :
IRODS_REPORT=reports/$(date +%Y%m%d-%H%M).csv
......@@ -271,35 +321,8 @@ format_report "Year_Network_Station" "Size(MB)" "Dumpdate" "Duration(s)" "Throug
imkdir -p reports
iput -f $LOCAL_REPORT $IRODS_REPORT
##################
#
# Dump Metadata
#
##################
KEY="validated-seismic-metadata"
if [[ -r $RECOVERY_FILE ]] && egrep -q -e ".*($KEY ).*( OK | Skipped ).*" $RECOVERY_FILE ; then
format_report $KEY "-" $(date +%Y-%m-%dT%H:%M:%S) "-" "-" "Skipped" >> $LOCAL_REPORT
else
# Get the snapshot name for this month
MONTH=$(date +%Y-%m)
SNAPSHOT_DIR=$(ls -d $RESIFDD_DATADIR/validated_seismic_metadata/.snapshot/weekly.${MONTH}*|tail -1)
if [[ ! -d $SNAPSHOT_DIR ]]; then
echo "Error 000 Snapshot directory $SNAPSHOT_DIR does not exist"
exit 1
fi
echo "[$KEY] Starting dump from ${SNAPSHOT_DIR}"
tar cf $RESIFDD_WORKDIR/$KEY.tar --exclude portalproducts -C $SNAPSHOT_DIR $SNAPSHOT_DIR
if [[ $? -ne 0 ]]; then
echo "[$KEY] Error 001 while creating tar archive."
zabbix_err "${KEY}:Error 001"
exit 1
fi
irods_push $KEY
echo "[$KEY] Dump terminated :"
ils -l $KEY
fi
# First save metadata
pack_and_send_metadata
####################
#
......@@ -307,6 +330,7 @@ fi
#
####################
MONTH=$(date +%Y-%m)
SNAPSHOT_DIR=$(ls -d ${RESIFDD_DATADIR}/validated_seismic_data/.snapshot/monthly.${MONTH}*|tail -1)
if [[ ! -d $SNAPSHOT_DIR ]]; then
echo "Error 006 Snapshot directory $SNAPSHOT_DIR does not exist"
......