Commit d341ba6b authored by Jonathan Schaeffer's avatar Jonathan Schaeffer
Browse files

plus d'options

parent 9b58eb16
......@@ -27,30 +27,14 @@ RESIFDD_WORKDIR=/osug-dc/resif RESIFDD_DATATIR=/scratch/resifdumper resifdatadum
### Options particulières de l'outil
Démarrer le transfert de toutes les données à partir de 2009
Lancer la sauvegarde d'une liste de stations :
``` shell
RESIFDD_WORKDIR=/osug-dc/resif RESIFDD_DATATIR=/scratch/resifdumper RESIFDD_START_AT=2009 src/resifdatadump
RESIFDD_DATADIR=/osug-dc/resif RESIFDD_WORKDIR=/scratch/resif_datadump src/resifdatadump 2011/RA/NCAD 2012/MT/THE
```
Transférer les données listées dans le fichier `RESIFDD_KEYFILE` :
Sauvegarder les métadonnées
``` shell
RESIFDD_DATADIR=/osug-dc/resif RESIFDD_WORKDIR=/scratch/resif_datadump src/resifdatadump validated_seismic_metadata
``` shell
RESIFDD_WORKDIR=/osug-dc/resif RESIFDD_DATATIR=/scratch/resifdumper RESIFDD_KEYFILE=/scratch/resifdumper/keys.txt src/resifdatadump
```
Le ficher doit contenir une clé par ligne, comme rapportée dans les logs :
``` textfile
2018_RA_CGBP
2018_FR_RUSF
2018_RA_NCAD
2018_RA_PYTO
2016_FR_RUSF
2017_RA_PYTO
2017_RA_CGBP
2017_FR_RUSF
2016_MT_CLP2
2016_RA_NCAD
```
On peut générer un fichier de clés avec le script python `src/scan_dupms.py` qui
This diff is collapsed.
This diff is collapsed.
irm reports/2019-07-16.csv
irm reports/2019-07-17.csv
irm reports/20190718-0507.csv
irm reports/20190718-0509.csv
irm reports/20190719-0630.csv
irm reports/20190719-0641.csv
irm reports/20190719-0642.csv
irm reports/20190719-0643.csv
irm reports/20190719-0711.csv
irm reports/20190719-0712.csv
irm reports/20190719-1216.csv
irm reports/20190719-1235.csv
irm reports/20190719-1444.csv
irm reports/20190721-1245.csv
irm reports/20190721-1944.csv
irm reports/20190722-0422.csv
irm reports/20190722-0743.csv
irm reports/20190722-0821.csv
irm reports/20190722-0904.csv
irm reports/20190722-0930.csv
irm reports/20190803-2302.csv
irm reports/20190804-0002.csv
irm reports/20190804-0102.csv
irm reports/20190804-0202.csv
irm reports/20190804-0302.csv
irm reports/20190804-0402.csv
irm reports/20190804-0502.csv
irm reports/20190804-0602.csv
irm reports/20190804-0802.csv
irm reports/20190804-0902.csv
irm reports/20190804-1002.csv
irm reports/20190804-1102.csv
irm reports/20190804-1202.csv
irm reports/20190804-1302.csv
irm reports/20190804-1403.csv
irm reports/20190804-1503.csv
irm reports/20190804-1603.csv
irm reports/20190804-1703.csv
irm reports/20190804-1803.csv
irm reports/20190804-1903.csv
irm reports/20190804-2003.csv
irm reports/20190804-2103.csv
irm reports/20190804-2203.csv
irm reports/20190804-2303.csv
irm reports/20190805-0003.csv
irm reports/20190805-0103.csv
irm reports/20190805-0203.csv
irm reports/20190805-0303.csv
irm reports/20190805-0403.csv
irm reports/20190805-0503.csv
irm reports/20190805-0603.csv
irm reports/20190805-0703.csv
irm reports/20190805-0803.csv
irm reports/20190805-0903.csv
irm reports/20190805-1003.csv
irm reports/20190805-1103.csv
irm reports/20190805-1203.csv
irm reports/20190805-1303.csv
irm reports/20190805-1403.csv
irm reports/20190805-1503.csv
irm reports/20190805-1603.csv
irm reports/20190805-1703.csv
irm reports/20190805-1803.csv
irm reports/20190805-1903.csv
irm reports/20190805-2003.csv
irm reports/20190805-2104.csv
irm reports/20190805-2204.csv
irm reports/20190805-2304.csv
irm reports/20190806-0004.csv
irm reports/20190806-0104.csv
irm reports/20190806-0204.csv
irm reports/20190806-0304.csv
irm reports/20190806-0404.csv
irm reports/20190806-0504.csv
irm reports/20190806-0604.csv
irm reports/20190806-0704.csv
irm reports/20190806-0804.csv
irm reports/20190806-0904.csv
irm reports/20190806-1004.csv
irm reports/20190806-1104.csv
irm reports/20190806-1204.csv
irm reports/20190806-1304.csv
irm reports/20190806-1404.csv
irm reports/20190806-1504.csv
irm reports/20190806-1604.csv
irm reports/20190806-1704.csv
irm reports/20190806-1804.csv
irm reports/20190806-1904.csv
irm reports/20190806-2004.csv
irm reports/20190806-2104.csv
irm reports/20190806-2204.csv
irm reports/20190806-2304.csv
irm reports/20190807-0004.csv
irm reports/20190807-0104.csv
irm reports/20190807-0204.csv
irm reports/20190807-0304.csv
irm reports/20190807-0404.csv
irm reports/20190807-0504.csv
irm reports/20190807-0604.csv
irm reports/20190807-0704.csv
irm reports/20190807-0804.csv
irm reports/20190807-0905.csv
irm reports/20190807-1005.csv
irm reports/20190807-1105.csv
irm reports/20190807-1205.csv
irm reports/20190807-1305.csv
irm reports/20190807-1405.csv
irm reports/20190807-1505.csv
irm reports/20190807-1605.csv
irm reports/20190807-1705.csv
irm reports/20190807-1805.csv
irm reports/20190807-1905.csv
irm reports/20190807-2005.csv
irm reports/20190807-2105.csv
irm reports/20190807-2205.csv
irm reports/20190807-2307.csv
irm reports/20190808-0007.csv
irm reports/20190808-0107.csv
irm reports/20190808-0207.csv
irm reports/20190808-0307.csv
irm reports/20190808-0407.csv
irm reports/20190808-0507.csv
irm reports/20190808-0607.csv
irm reports/20190808-0707.csv
irm reports/20190808-0807.csv
irm reports/20190808-0907.csv
irm reports/20190808-1007.csv
irm reports/20190808-1107.csv
irm reports/20190808-1207.csv
irm reports/20190808-1307.csv
irm reports/20190808-1407.csv
irm reports/20190808-1507.csv
irm reports/20190808-1607.csv
irm reports/20190808-1707.csv
irm reports/20190808-1807.csv
irm reports/20190808-1907.csv
irm reports/20190808-2007.csv
irm reports/20190808-2107.csv
irm reports/20190808-2207.csv
irm reports/20190808-2307.csv
irm reports/20190809-0007.csv
irm reports/20190809-0107.csv
irm reports/20190809-0208.csv
irm reports/20190809-0308.csv
irm reports/20190809-0408.csv
irm reports/20190809-0508.csv
irm reports/20190809-0608.csv
irm reports/20190809-0708.csv
irm reports/20190809-0808.csv
irm reports/20190809-0908.csv
irm reports/20190809-1008.csv
irm reports/20190809-1108.csv
irm reports/20190809-1208.csv
irm reports/20190809-1308.csv
irm reports/20190809-1408.csv
irm reports/20190809-1508.csv
irm reports/20190809-1608.csv
irm reports/20190809-1708.csv
irm reports/20190809-1808.csv
irm reports/20190809-1908.csv
irm reports/20190809-2008.csv
irm reports/20190809-2108.csv
irm reports/20190809-2208.csv
irm reports/20190809-2308.csv
irm reports/20190810-0008.csv
irm reports/20190810-0108.csv
irm reports/20190810-0208.csv
irm reports/20190810-0308.csv
irm reports/20190810-0408.csv
irm reports/20190810-0508.csv
irm reports/20190810-0608.csv
irm reports/20190810-0708.csv
irm reports/20190810-0808.csv
irm reports/20190810-0908.csv
irm reports/20190810-1008.csv
irm reports/20190810-1108.csv
irm reports/20190810-1209.csv
irm reports/20190810-1309.csv
irm reports/20190810-1409.csv
irm reports/20190810-1509.csv
irm reports/20190810-1609.csv
irm reports/20190810-1709.csv
irm reports/20190810-1809.csv
irm reports/20190810-1909.csv
irm reports/20190810-2009.csv
irm reports/20190810-2109.csv
irm reports/20190810-2209.csv
irm reports/20190810-2309.csv
irm reports/20190811-0009.csv
irm reports/20190811-0109.csv
irm reports/20190811-0209.csv
irm reports/20190811-0309.csv
irm reports/20190811-0409.csv
irm reports/20190811-0509.csv
irm reports/20190811-0609.csv
irm reports/20190811-0709.csv
irm reports/20190811-0809.csv
irm reports/20190811-0909.csv
irm reports/20190811-1009.csv
irm reports/20190811-1109.csv
irm reports/20190811-1209.csv
irm reports/20190811-1309.csv
irm reports/20190811-1409.csv
irm reports/20190811-1509.csv
irm reports/20190811-1609.csv
irm reports/20190811-1709.csv
irm reports/20190811-1809.csv
irm reports/20190811-1909.csv
irm reports/20190811-2009.csv
irm reports/20190811-2109.csv
irm reports/20190811-2209.csv
irm reports/20190811-2309.csv
irm reports/20190812-0009.csv
irm reports/20190812-0109.csv
irm reports/20190812-0209.csv
irm reports/20190812-0309.csv
irm reports/20190812-0409.csv
irm reports/20190812-0509.csv
irm reports/20190812-0609.csv
irm reports/20190812-0709.csv
irm reports/20190812-0809.csv
irm reports/20190812-0909.csv
irm reports/20190812-1010.csv
irm reports/20190812-1110.csv
irm reports/20190812-1210.csv
irm reports/20190812-1310.csv
irm reports/20190812-1410.csv
irm reports/20190812-1510.csv
irm reports/20190812-1610.csv
irm reports/20190812-1710.csv
irm reports/20190812-1810.csv
irm reports/20190812-1910.csv
irm reports/20190812-2010.csv
irm reports/20190812-2110.csv
irm reports/20190812-2210.csv
irm reports/20190812-2310.csv
irm reports/20190813-0010.csv
irm reports/20190813-0110.csv
irm reports/20190813-0210.csv
irm reports/20190813-0310.csv
irm reports/20190813-0410.csv
irm reports/20190813-0510.csv
irm reports/20190813-0610.csv
irm reports/20190813-0710.csv
irm reports/20190813-0810.csv
irm reports/20190813-0910.csv
irm reports/20190813-1010.csv
irm reports/20190813-1110.csv
irm reports/20190813-1210.csv
irm reports/20190813-1310.csv
irm reports/20190813-1410.csv
irm reports/20190813-1510.csv
irm reports/20190813-1610.csv
irm reports/20190813-1710.csv
irm reports/20190813-1810.csv
irm reports/20190813-1910.csv
irm reports/20190813-2010.csv
irm reports/20190813-2110.csv
irm reports/20190813-2210.csv
irm reports/20190813-2310.csv
irm reports/20190814-0011.csv
irm reports/20190814-0111.csv
irm reports/20190814-0211.csv
irm reports/20190814-0311.csv
irm reports/20190814-0411.csv
irm reports/20190814-0511.csv
irm reports/20190814-0611.csv
irm reports/20190814-0711.csv
irm reports/20190814-0811.csv
irm reports/20190814-0911.csv
irm reports/20190814-1011.csv
irm reports/20190814-1111.csv
irm reports/20190814-1211.csv
irm reports/20190814-1311.csv
irm reports/20190814-1411.csv
irm reports/20190814-1511.csv
irm reports/20190814-1612.csv
irm reports/20190814-1712.csv
irm reports/20190814-1812.csv
irm reports/20190814-1912.csv
irm reports/20190814-2012.csv
irm reports/20190814-2112.csv
irm reports/20190814-2212.csv
irm reports/20190814-2312.csv
irm reports/20190815-0012.csv
irm reports/20190815-0112.csv
irm reports/20190815-0212.csv
irm reports/20190815-0312.csv
irm reports/20190815-0412.csv
irm reports/20190815-0513.csv
irm reports/20190815-0613.csv
irm reports/20190815-0713.csv
irm reports/20190815-0813.csv
irm reports/20190815-0913.csv
irm reports/20190815-1013.csv
irm reports/20190815-1113.csv
irm reports/20190815-1213.csv
irm reports/20190815-1313.csv
irm reports/20190815-1413.csv
irm reports/20190815-1513.csv
irm reports/20190815-1613.csv
irm reports/20190815-1714.csv
irm reports/20190815-1814.csv
irm reports/20190815-1914.csv
irm reports/20190815-2014.csv
irm reports/20190815-2114.csv
irm reports/20190815-2214.csv
irm reports/20190815-2314.csv
irm reports/20190816-0014.csv
irm reports/20190816-0114.csv
irm reports/20190816-0214.csv
irm reports/20190816-0314.csv
irm reports/20190816-0414.csv
irm reports/20190816-0514.csv
irm reports/20190816-0614.csv
irm reports/20190816-0714.csv
irm reports/20190816-0814.csv
irm reports/20190816-0914.csv
irm reports/20190816-1014.csv
irm reports/20190816-1115.csv
irm reports/20190816-1215.csv
irm reports/20190816-1315.csv
irm reports/20190816-1415.csv
irm reports/20190816-1515.csv
irm reports/20190816-1615.csv
irm reports/20190816-1715.csv
irm reports/20190816-1815.csv
irm reports/20190816-1915.csv
irm reports/20190816-2015.csv
irm reports/20190816-2115.csv
irm reports/20190816-2215.csv
irm reports/20190816-2315.csv
irm reports/20190817-0015.csv
irm reports/20190817-0115.csv
irm reports/20190817-0215.csv
irm reports/20190817-0315.csv
irm reports/20190817-0416.csv
irm reports/20190817-0516.csv
irm reports/20190817-0616.csv
irm reports/20190817-0716.csv
irm reports/20190817-0816.csv
irm reports/20190817-0916.csv
irm reports/20190817-1016.csv
irm reports/20190817-1116.csv
irm reports/20190817-1216.csv
irm reports/20190817-1316.csv
irm reports/20190817-1416.csv
irm reports/20190817-1516.csv
irm reports/20190817-1616.csv
irm reports/20190817-1716.csv
irm reports/20190817-1816.csv
irm reports/20190817-1916.csv
irm reports/20190817-2016.csv
irm reports/20190817-2116.csv
irm reports/20190817-2216.csv
irm reports/20190817-2316.csv
irm reports/20190818-0016.csv
irm reports/20190818-0116.csv
irm reports/20190818-0216.csv
irm reports/20190818-0316.csv
irm reports/20190818-0416.csv
irm reports/20190818-0516.csv
irm reports/20190818-0616.csv
irm reports/20190818-0716.csv
irm reports/20190818-0816.csv
irm reports/20190818-0916.csv
irm reports/20190818-1016.csv
irm reports/20190818-1116.csv
irm reports/20190818-1216.csv
irm reports/20190818-1316.csv
irm reports/20190818-1416.csv
irm reports/20190818-1516.csv
irm reports/20190818-1616.csv
irm reports/20190818-1716.csv
irm reports/20190818-1817.csv
irm reports/20190818-1917.csv
irm reports/20190818-2017.csv
irm reports/20190818-2117.csv
irm reports/20190818-2217.csv
irm reports/20190818-2317.csv
irm reports/20190819-0017.csv
irm reports/20190819-0117.csv
irm reports/20190819-0217.csv
irm reports/20190819-0317.csv
irm reports/20190819-0417.csv
irm reports/20190819-0517.csv
irm reports/20190819-0617.csv
irm reports/20190819-0717.csv
......@@ -163,6 +163,38 @@ irods_commit(){
ils $KEY/previous_to_delete.tar >/dev/null 2>&1 && irm -f $KEY/previous_to_delete.tar
}
##################
#
# Dump Metadata
#
##################
pack_and_send_metadata(){
KEY="validated-seismic-metadata"
if [[ -r $RECOVERY_FILE ]] && egrep -q -e ".*($KEY ).*( OK | Skipped ).*" $RECOVERY_FILE ; then
format_report $KEY "-" $(date +%Y-%m-%dT%H:%M:%S) "-" "-" "Skipped" >> $LOCAL_REPORT
else
# Get the snapshot name for this month
MONTH=$(date +%Y-%m)
SNAPSHOT_DIR=$(ls -d $RESIFDD_DATADIR/validated_seismic_metadata/.snapshot/weekly.${MONTH}*|tail -1)
if [[ ! -d $SNAPSHOT_DIR ]]; then
echo "Error 000 Snapshot directory $SNAPSHOT_DIR does not exist"
exit 1
fi
echo "[$KEY] Starting dump from ${SNAPSHOT_DIR}"
tar cf $RESIFDD_WORKDIR/$KEY.tar --exclude portalproducts -C $SNAPSHOT_DIR $SNAPSHOT_DIR
if [[ $? -ne 0 ]]; then
echo "[$KEY] Error 001 while creating tar archive."
zabbix_err "${KEY}:Error 001"
exit 1
fi
irods_push $KEY
echo "[$KEY] Dump terminated :"
ils -l $KEY
fi
}
# Choose if the data should be processed or skipped
# Pack the data and send it to iRODS
# Argument is the path (abs or relative) to the data
......@@ -263,6 +295,24 @@ if [[ -r ${RESIFDD_CONTINUE_FROM_FILE} ]]; then
else
echo "No recovery file present. Dumping everything now"
fi
####################
#
# Option KEYFILE to send just a list of keys
#
####################
if [ $# -gt 0 ]; then
for var in "$@"; do
echo "Dumping $var"
MONTH=$(date +%Y-%m)
if [ "x$var" = "xvalidated-seismic-metadata" ]; then
pack_and_send_metadata
else
SNAPSHOT_DIR=$(ls -d ${RESIFDD_DATADIR}/validated_seismic_data/.snapshot/monthly.${MONTH}*|tail -1)
pack_and_send ${SNAPSHOT_DIR}/${var} 1
fi
done
exit 0
fi
# Header for the report :
IRODS_REPORT=reports/$(date +%Y%m%d-%H%M).csv
......@@ -271,35 +321,8 @@ format_report "Year_Network_Station" "Size(MB)" "Dumpdate" "Duration(s)" "Throug
imkdir -p reports
iput -f $LOCAL_REPORT $IRODS_REPORT
##################
#
# Dump Metadata
#
##################
KEY="validated-seismic-metadata"
if [[ -r $RECOVERY_FILE ]] && egrep -q -e ".*($KEY ).*( OK | Skipped ).*" $RECOVERY_FILE ; then
format_report $KEY "-" $(date +%Y-%m-%dT%H:%M:%S) "-" "-" "Skipped" >> $LOCAL_REPORT
else
# Get the snapshot name for this month
MONTH=$(date +%Y-%m)
SNAPSHOT_DIR=$(ls -d $RESIFDD_DATADIR/validated_seismic_metadata/.snapshot/weekly.${MONTH}*|tail -1)
if [[ ! -d $SNAPSHOT_DIR ]]; then
echo "Error 000 Snapshot directory $SNAPSHOT_DIR does not exist"
exit 1
fi
echo "[$KEY] Starting dump from ${SNAPSHOT_DIR}"
tar cf $RESIFDD_WORKDIR/$KEY.tar --exclude portalproducts -C $SNAPSHOT_DIR $SNAPSHOT_DIR
if [[ $? -ne 0 ]]; then
echo "[$KEY] Error 001 while creating tar archive."
zabbix_err "${KEY}:Error 001"
exit 1
fi
irods_push $KEY
echo "[$KEY] Dump terminated :"
ils -l $KEY
fi
# First save metadata
pack_and_send_metadata
####################
#
......@@ -307,6 +330,7 @@ fi
#
####################
MONTH=$(date +%Y-%m)
SNAPSHOT_DIR=$(ls -d ${RESIFDD_DATADIR}/validated_seismic_data/.snapshot/monthly.${MONTH}*|tail -1)
if [[ ! -d $SNAPSHOT_DIR ]]; then
echo "Error 006 Snapshot directory $SNAPSHOT_DIR does not exist"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment