plus d'options

README.md

@@ -27,30 +27,14 @@ RESIFDD_WORKDIR=/osug-dc/resif RESIFDD_DATATIR=/scratch/resifdumper resifdatadum
 
 ### Options particulières de l'outil
 
-Démarrer le transfert de toutes les données à partir de 2009
+Lancer la sauvegarde d'une liste de stations :
 ``` shell
-RESIFDD_WORKDIR=/osug-dc/resif RESIFDD_DATATIR=/scratch/resifdumper RESIFDD_START_AT=2009 src/resifdatadump
+RESIFDD_DATADIR=/osug-dc/resif RESIFDD_WORKDIR=/scratch/resif_datadump src/resifdatadump 2011/RA/NCAD 2012/MT/THE
 ```
 
-Transférer les données listées dans le fichier `RESIFDD_KEYFILE` :
-
+Sauvegarder les métadonnées
+``` shell
+RESIFDD_DATADIR=/osug-dc/resif RESIFDD_WORKDIR=/scratch/resif_datadump src/resifdatadump validated_seismic_metadata
 ``` shell
-RESIFDD_WORKDIR=/osug-dc/resif RESIFDD_DATATIR=/scratch/resifdumper RESIFDD_KEYFILE=/scratch/resifdumper/keys.txt src/resifdatadump
-```
-
-Le ficher doit contenir une clé par ligne, comme rapportée dans les logs :
-
-``` textfile
-2018_RA_CGBP
-2018_FR_RUSF
-2018_RA_NCAD
-2018_RA_PYTO
-2016_FR_RUSF
-2017_RA_PYTO
-2017_RA_CGBP
-2017_FR_RUSF
-2016_MT_CLP2
-2016_RA_NCAD
-```
 
 On peut générer un fichier de clés avec le script python `src/scan_dupms.py` qui

src/remover-reports.sh

+irm reports/2019-07-16.csv
+irm reports/2019-07-17.csv
+irm reports/20190718-0507.csv
+irm reports/20190718-0509.csv
+irm reports/20190719-0630.csv
+irm reports/20190719-0641.csv
+irm reports/20190719-0642.csv
+irm reports/20190719-0643.csv
+irm reports/20190719-0711.csv
+irm reports/20190719-0712.csv
+irm reports/20190719-1216.csv
+irm reports/20190719-1235.csv
+irm reports/20190719-1444.csv
+irm reports/20190721-1245.csv
+irm reports/20190721-1944.csv
+irm reports/20190722-0422.csv
+irm reports/20190722-0743.csv
+irm reports/20190722-0821.csv
+irm reports/20190722-0904.csv
+irm reports/20190722-0930.csv
+irm reports/20190803-2302.csv
+irm reports/20190804-0002.csv
+irm reports/20190804-0102.csv
+irm reports/20190804-0202.csv
+irm reports/20190804-0302.csv
+irm reports/20190804-0402.csv
+irm reports/20190804-0502.csv
+irm reports/20190804-0602.csv
+irm reports/20190804-0802.csv
+irm reports/20190804-0902.csv
+irm reports/20190804-1002.csv
+irm reports/20190804-1102.csv
+irm reports/20190804-1202.csv
+irm reports/20190804-1302.csv
+irm reports/20190804-1403.csv
+irm reports/20190804-1503.csv
+irm reports/20190804-1603.csv
+irm reports/20190804-1703.csv
+irm reports/20190804-1803.csv
+irm reports/20190804-1903.csv
+irm reports/20190804-2003.csv
+irm reports/20190804-2103.csv
+irm reports/20190804-2203.csv
+irm reports/20190804-2303.csv
+irm reports/20190805-0003.csv
+irm reports/20190805-0103.csv
+irm reports/20190805-0203.csv
+irm reports/20190805-0303.csv
+irm reports/20190805-0403.csv
+irm reports/20190805-0503.csv
+irm reports/20190805-0603.csv
+irm reports/20190805-0703.csv
+irm reports/20190805-0803.csv
+irm reports/20190805-0903.csv
+irm reports/20190805-1003.csv
+irm reports/20190805-1103.csv
+irm reports/20190805-1203.csv
+irm reports/20190805-1303.csv
+irm reports/20190805-1403.csv
+irm reports/20190805-1503.csv
+irm reports/20190805-1603.csv
+irm reports/20190805-1703.csv
+irm reports/20190805-1803.csv
+irm reports/20190805-1903.csv
+irm reports/20190805-2003.csv
+irm reports/20190805-2104.csv
+irm reports/20190805-2204.csv
+irm reports/20190805-2304.csv
+irm reports/20190806-0004.csv
+irm reports/20190806-0104.csv
+irm reports/20190806-0204.csv
+irm reports/20190806-0304.csv
+irm reports/20190806-0404.csv
+irm reports/20190806-0504.csv
+irm reports/20190806-0604.csv
+irm reports/20190806-0704.csv
+irm reports/20190806-0804.csv
+irm reports/20190806-0904.csv
+irm reports/20190806-1004.csv
+irm reports/20190806-1104.csv
+irm reports/20190806-1204.csv
+irm reports/20190806-1304.csv
+irm reports/20190806-1404.csv
+irm reports/20190806-1504.csv
+irm reports/20190806-1604.csv
+irm reports/20190806-1704.csv
+irm reports/20190806-1804.csv
+irm reports/20190806-1904.csv
+irm reports/20190806-2004.csv
+irm reports/20190806-2104.csv
+irm reports/20190806-2204.csv
+irm reports/20190806-2304.csv
+irm reports/20190807-0004.csv
+irm reports/20190807-0104.csv
+irm reports/20190807-0204.csv
+irm reports/20190807-0304.csv
+irm reports/20190807-0404.csv
+irm reports/20190807-0504.csv
+irm reports/20190807-0604.csv
+irm reports/20190807-0704.csv
+irm reports/20190807-0804.csv
+irm reports/20190807-0905.csv
+irm reports/20190807-1005.csv
+irm reports/20190807-1105.csv
+irm reports/20190807-1205.csv
+irm reports/20190807-1305.csv
+irm reports/20190807-1405.csv
+irm reports/20190807-1505.csv
+irm reports/20190807-1605.csv
+irm reports/20190807-1705.csv
+irm reports/20190807-1805.csv
+irm reports/20190807-1905.csv
+irm reports/20190807-2005.csv
+irm reports/20190807-2105.csv
+irm reports/20190807-2205.csv
+irm reports/20190807-2307.csv
+irm reports/20190808-0007.csv
+irm reports/20190808-0107.csv
+irm reports/20190808-0207.csv
+irm reports/20190808-0307.csv
+irm reports/20190808-0407.csv
+irm reports/20190808-0507.csv
+irm reports/20190808-0607.csv
+irm reports/20190808-0707.csv
+irm reports/20190808-0807.csv
+irm reports/20190808-0907.csv
+irm reports/20190808-1007.csv
+irm reports/20190808-1107.csv
+irm reports/20190808-1207.csv
+irm reports/20190808-1307.csv
+irm reports/20190808-1407.csv
+irm reports/20190808-1507.csv
+irm reports/20190808-1607.csv
+irm reports/20190808-1707.csv
+irm reports/20190808-1807.csv
+irm reports/20190808-1907.csv
+irm reports/20190808-2007.csv
+irm reports/20190808-2107.csv
+irm reports/20190808-2207.csv
+irm reports/20190808-2307.csv
+irm reports/20190809-0007.csv
+irm reports/20190809-0107.csv
+irm reports/20190809-0208.csv
+irm reports/20190809-0308.csv
+irm reports/20190809-0408.csv
+irm reports/20190809-0508.csv
+irm reports/20190809-0608.csv
+irm reports/20190809-0708.csv
+irm reports/20190809-0808.csv
+irm reports/20190809-0908.csv
+irm reports/20190809-1008.csv
+irm reports/20190809-1108.csv
+irm reports/20190809-1208.csv
+irm reports/20190809-1308.csv
+irm reports/20190809-1408.csv
+irm reports/20190809-1508.csv
+irm reports/20190809-1608.csv
+irm reports/20190809-1708.csv
+irm reports/20190809-1808.csv
+irm reports/20190809-1908.csv
+irm reports/20190809-2008.csv
+irm reports/20190809-2108.csv
+irm reports/20190809-2208.csv
+irm reports/20190809-2308.csv
+irm reports/20190810-0008.csv
+irm reports/20190810-0108.csv
+irm reports/20190810-0208.csv
+irm reports/20190810-0308.csv
+irm reports/20190810-0408.csv
+irm reports/20190810-0508.csv
+irm reports/20190810-0608.csv
+irm reports/20190810-0708.csv
+irm reports/20190810-0808.csv
+irm reports/20190810-0908.csv
+irm reports/20190810-1008.csv
+irm reports/20190810-1108.csv
+irm reports/20190810-1209.csv
+irm reports/20190810-1309.csv
+irm reports/20190810-1409.csv
+irm reports/20190810-1509.csv
+irm reports/20190810-1609.csv
+irm reports/20190810-1709.csv
+irm reports/20190810-1809.csv
+irm reports/20190810-1909.csv
+irm reports/20190810-2009.csv
+irm reports/20190810-2109.csv
+irm reports/20190810-2209.csv
+irm reports/20190810-2309.csv
+irm reports/20190811-0009.csv
+irm reports/20190811-0109.csv
+irm reports/20190811-0209.csv
+irm reports/20190811-0309.csv
+irm reports/20190811-0409.csv
+irm reports/20190811-0509.csv
+irm reports/20190811-0609.csv
+irm reports/20190811-0709.csv
+irm reports/20190811-0809.csv
+irm reports/20190811-0909.csv
+irm reports/20190811-1009.csv
+irm reports/20190811-1109.csv
+irm reports/20190811-1209.csv
+irm reports/20190811-1309.csv
+irm reports/20190811-1409.csv
+irm reports/20190811-1509.csv
+irm reports/20190811-1609.csv
+irm reports/20190811-1709.csv
+irm reports/20190811-1809.csv
+irm reports/20190811-1909.csv
+irm reports/20190811-2009.csv
+irm reports/20190811-2109.csv
+irm reports/20190811-2209.csv
+irm reports/20190811-2309.csv
+irm reports/20190812-0009.csv
+irm reports/20190812-0109.csv
+irm reports/20190812-0209.csv
+irm reports/20190812-0309.csv
+irm reports/20190812-0409.csv
+irm reports/20190812-0509.csv
+irm reports/20190812-0609.csv
+irm reports/20190812-0709.csv
+irm reports/20190812-0809.csv
+irm reports/20190812-0909.csv
+irm reports/20190812-1010.csv
+irm reports/20190812-1110.csv
+irm reports/20190812-1210.csv
+irm reports/20190812-1310.csv
+irm reports/20190812-1410.csv
+irm reports/20190812-1510.csv
+irm reports/20190812-1610.csv
+irm reports/20190812-1710.csv
+irm reports/20190812-1810.csv
+irm reports/20190812-1910.csv
+irm reports/20190812-2010.csv
+irm reports/20190812-2110.csv
+irm reports/20190812-2210.csv
+irm reports/20190812-2310.csv
+irm reports/20190813-0010.csv
+irm reports/20190813-0110.csv
+irm reports/20190813-0210.csv
+irm reports/20190813-0310.csv
+irm reports/20190813-0410.csv
+irm reports/20190813-0510.csv
+irm reports/20190813-0610.csv
+irm reports/20190813-0710.csv
+irm reports/20190813-0810.csv
+irm reports/20190813-0910.csv
+irm reports/20190813-1010.csv
+irm reports/20190813-1110.csv
+irm reports/20190813-1210.csv
+irm reports/20190813-1310.csv
+irm reports/20190813-1410.csv
+irm reports/20190813-1510.csv
+irm reports/20190813-1610.csv
+irm reports/20190813-1710.csv
+irm reports/20190813-1810.csv
+irm reports/20190813-1910.csv
+irm reports/20190813-2010.csv
+irm reports/20190813-2110.csv
+irm reports/20190813-2210.csv
+irm reports/20190813-2310.csv
+irm reports/20190814-0011.csv
+irm reports/20190814-0111.csv
+irm reports/20190814-0211.csv
+irm reports/20190814-0311.csv
+irm reports/20190814-0411.csv
+irm reports/20190814-0511.csv
+irm reports/20190814-0611.csv
+irm reports/20190814-0711.csv
+irm reports/20190814-0811.csv
+irm reports/20190814-0911.csv
+irm reports/20190814-1011.csv
+irm reports/20190814-1111.csv
+irm reports/20190814-1211.csv
+irm reports/20190814-1311.csv
+irm reports/20190814-1411.csv
+irm reports/20190814-1511.csv
+irm reports/20190814-1612.csv
+irm reports/20190814-1712.csv
+irm reports/20190814-1812.csv
+irm reports/20190814-1912.csv
+irm reports/20190814-2012.csv
+irm reports/20190814-2112.csv
+irm reports/20190814-2212.csv
+irm reports/20190814-2312.csv
+irm reports/20190815-0012.csv
+irm reports/20190815-0112.csv
+irm reports/20190815-0212.csv
+irm reports/20190815-0312.csv
+irm reports/20190815-0412.csv
+irm reports/20190815-0513.csv
+irm reports/20190815-0613.csv
+irm reports/20190815-0713.csv
+irm reports/20190815-0813.csv
+irm reports/20190815-0913.csv
+irm reports/20190815-1013.csv
+irm reports/20190815-1113.csv
+irm reports/20190815-1213.csv
+irm reports/20190815-1313.csv
+irm reports/20190815-1413.csv
+irm reports/20190815-1513.csv
+irm reports/20190815-1613.csv
+irm reports/20190815-1714.csv
+irm reports/20190815-1814.csv
+irm reports/20190815-1914.csv
+irm reports/20190815-2014.csv
+irm reports/20190815-2114.csv
+irm reports/20190815-2214.csv
+irm reports/20190815-2314.csv
+irm reports/20190816-0014.csv
+irm reports/20190816-0114.csv
+irm reports/20190816-0214.csv
+irm reports/20190816-0314.csv
+irm reports/20190816-0414.csv
+irm reports/20190816-0514.csv
+irm reports/20190816-0614.csv
+irm reports/20190816-0714.csv
+irm reports/20190816-0814.csv
+irm reports/20190816-0914.csv
+irm reports/20190816-1014.csv
+irm reports/20190816-1115.csv
+irm reports/20190816-1215.csv
+irm reports/20190816-1315.csv
+irm reports/20190816-1415.csv
+irm reports/20190816-1515.csv
+irm reports/20190816-1615.csv
+irm reports/20190816-1715.csv
+irm reports/20190816-1815.csv
+irm reports/20190816-1915.csv
+irm reports/20190816-2015.csv
+irm reports/20190816-2115.csv
+irm reports/20190816-2215.csv
+irm reports/20190816-2315.csv
+irm reports/20190817-0015.csv
+irm reports/20190817-0115.csv
+irm reports/20190817-0215.csv
+irm reports/20190817-0315.csv
+irm reports/20190817-0416.csv
+irm reports/20190817-0516.csv
+irm reports/20190817-0616.csv
+irm reports/20190817-0716.csv
+irm reports/20190817-0816.csv
+irm reports/20190817-0916.csv
+irm reports/20190817-1016.csv
+irm reports/20190817-1116.csv
+irm reports/20190817-1216.csv
+irm reports/20190817-1316.csv
+irm reports/20190817-1416.csv
+irm reports/20190817-1516.csv
+irm reports/20190817-1616.csv
+irm reports/20190817-1716.csv
+irm reports/20190817-1816.csv
+irm reports/20190817-1916.csv
+irm reports/20190817-2016.csv
+irm reports/20190817-2116.csv
+irm reports/20190817-2216.csv
+irm reports/20190817-2316.csv
+irm reports/20190818-0016.csv
+irm reports/20190818-0116.csv
+irm reports/20190818-0216.csv
+irm reports/20190818-0316.csv
+irm reports/20190818-0416.csv
+irm reports/20190818-0516.csv
+irm reports/20190818-0616.csv
+irm reports/20190818-0716.csv
+irm reports/20190818-0816.csv
+irm reports/20190818-0916.csv
+irm reports/20190818-1016.csv
+irm reports/20190818-1116.csv
+irm reports/20190818-1216.csv
+irm reports/20190818-1316.csv
+irm reports/20190818-1416.csv
+irm reports/20190818-1516.csv
+irm reports/20190818-1616.csv
+irm reports/20190818-1716.csv
+irm reports/20190818-1817.csv
+irm reports/20190818-1917.csv
+irm reports/20190818-2017.csv
+irm reports/20190818-2117.csv
+irm reports/20190818-2217.csv
+irm reports/20190818-2317.csv
+irm reports/20190819-0017.csv
+irm reports/20190819-0117.csv
+irm reports/20190819-0217.csv
+irm reports/20190819-0317.csv
+irm reports/20190819-0417.csv
+irm reports/20190819-0517.csv
+irm reports/20190819-0617.csv
+irm reports/20190819-0717.csv

src/resifdatadump

@@ -163,6 +163,38 @@ irods_commit(){
     ils $KEY/previous_to_delete.tar >/dev/null 2>&1 && irm -f $KEY/previous_to_delete.tar
 }
 
+##################
+#
+# Dump Metadata
+#
+##################
+pack_and_send_metadata(){
+    KEY="validated-seismic-metadata"
+    
+    if [[ -r $RECOVERY_FILE ]] && egrep -q -e ".*($KEY ).*( OK | Skipped ).*" $RECOVERY_FILE ; then
+        format_report $KEY "-" $(date +%Y-%m-%dT%H:%M:%S) "-" "-" "Skipped" >> $LOCAL_REPORT
+    else
+        # Get the snapshot name for this month
+        MONTH=$(date +%Y-%m)
+        SNAPSHOT_DIR=$(ls -d $RESIFDD_DATADIR/validated_seismic_metadata/.snapshot/weekly.${MONTH}*|tail -1)
+        if [[ ! -d $SNAPSHOT_DIR ]]; then
+            echo "Error 000 Snapshot directory $SNAPSHOT_DIR does not exist"
+            exit 1
+        fi
+    
+        echo "[$KEY] Starting dump from ${SNAPSHOT_DIR}"
+        tar cf  $RESIFDD_WORKDIR/$KEY.tar --exclude portalproducts -C $SNAPSHOT_DIR $SNAPSHOT_DIR
+        if [[ $? -ne 0 ]]; then
+            echo "[$KEY] Error 001 while creating tar archive."
+            zabbix_err "${KEY}:Error 001"
+            exit 1
+        fi
+        irods_push $KEY
+        echo "[$KEY] Dump terminated :"
+        ils -l $KEY
+    fi
+}
+
 # Choose if the data should be processed or skipped
 # Pack the data and send it to iRODS
 # Argument is the path (abs or relative) to the data
@@ -263,6 +295,24 @@ if [[ -r ${RESIFDD_CONTINUE_FROM_FILE} ]]; then
 else
     echo "No recovery file present. Dumping everything now"
 fi
+####################
+#
+# Option KEYFILE to send just a list of keys
+#
+####################
+if [ $# -gt 0 ]; then
+    for var in "$@"; do
+        echo "Dumping $var"
+        MONTH=$(date +%Y-%m)
+	if [ "x$var" = "xvalidated-seismic-metadata" ]; then
+            pack_and_send_metadata
+	else
+            SNAPSHOT_DIR=$(ls -d ${RESIFDD_DATADIR}/validated_seismic_data/.snapshot/monthly.${MONTH}*|tail -1)
+	    pack_and_send ${SNAPSHOT_DIR}/${var} 1
+        fi
+    done 
+    exit 0
+fi
 
 # Header for the report :
 IRODS_REPORT=reports/$(date +%Y%m%d-%H%M).csv
@@ -271,35 +321,8 @@ format_report "Year_Network_Station" "Size(MB)" "Dumpdate" "Duration(s)" "Throug
 imkdir -p reports
 iput -f $LOCAL_REPORT $IRODS_REPORT
 
-##################
-#
-# Dump Metadata
-#
-##################
-KEY="validated-seismic-metadata"
-
-if [[ -r $RECOVERY_FILE ]] && egrep -q -e ".*($KEY ).*( OK | Skipped ).*" $RECOVERY_FILE ; then
-    format_report $KEY "-" $(date +%Y-%m-%dT%H:%M:%S) "-" "-" "Skipped" >> $LOCAL_REPORT
-else
-    # Get the snapshot name for this month
-    MONTH=$(date +%Y-%m)
-    SNAPSHOT_DIR=$(ls -d $RESIFDD_DATADIR/validated_seismic_metadata/.snapshot/weekly.${MONTH}*|tail -1)
-    if [[ ! -d $SNAPSHOT_DIR ]]; then
-        echo "Error 000 Snapshot directory $SNAPSHOT_DIR does not exist"
-        exit 1
-    fi
-
-    echo "[$KEY] Starting dump from ${SNAPSHOT_DIR}"
-    tar cf  $RESIFDD_WORKDIR/$KEY.tar --exclude portalproducts -C $SNAPSHOT_DIR $SNAPSHOT_DIR
-    if [[ $? -ne 0 ]]; then
-        echo "[$KEY] Error 001 while creating tar archive."
-        zabbix_err "${KEY}:Error 001"
-        exit 1
-    fi
-    irods_push $KEY
-    echo "[$KEY] Dump terminated :"
-    ils -l $KEY
-fi
+# First save metadata
+pack_and_send_metadata
 
 ####################
 #
@@ -307,6 +330,7 @@ fi
 #
 ####################
 
+MONTH=$(date +%Y-%m)
 SNAPSHOT_DIR=$(ls -d ${RESIFDD_DATADIR}/validated_seismic_data/.snapshot/monthly.${MONTH}*|tail -1)
 if [[ ! -d $SNAPSHOT_DIR ]]; then
     echo "Error 006 Snapshot directory $SNAPSHOT_DIR does not exist"

src/test_age.sh

+ils -L