Push archive based on content's age

src/resifdatadump

@@ -80,9 +80,11 @@ irods_push(){
         throughput=$(($SIZEMB / $duration ))
     # Check integrity.
     # We do it separately from transfer because for big files it can be very long, and the network can cut
+        local_sha=$(sha256sum $RESIFDD_WORKDIR/$KEY.tar | awk '{print $1}' | xxd -r -p | base64)
+        echo "[$KEY] local checksum: $local_sha"
         irods_sha=$(ichksum $KEY/staging.tar | awk -F':' '/sha2:/ {print $2; exit;}')
-        if [[ "$LOCAL_SHA" != "$irods_sha" ]]; then
-            echo "[$KEY] Error 012 distant file is corrupted (localsha $LOCAL_SHA irodssha $irods_sha). Rollback"
+        if [[ "$local_sha" != "$irods_sha" ]]; then
+            echo "[$KEY] Error 012 distant file is corrupted (localsha $local_sha irodssha $irods_sha). Rollback"
             irods_rollback $KEY
             zabbix_err "${KEY}:Error 012"
         fi
@@ -133,7 +135,7 @@ irods_rollback(){
 irods_commit(){
     KEY=$1
     echo "[$KEY] If previous.tar exists, move it around but keep it safe"
-    ils $KEY/previous.tar 2>/dev/null && (
+    ils $KEY/previous.tar 2&1>/dev/null && (
         imv $KEY/previous.tar $KEY/previous_to_delete.tar
         if [[ $? -ne 0 ]]; then
             echo "[$KEY] Error 003 moving previous.tar around. Corrective action has to be taken manualy"
@@ -143,7 +145,7 @@ irods_commit(){
     )
 
     echo "[$KEY] If latest.tar exists, move it to previous.tar"
-    ils $KEY/latest.tar 2>/dev/null && (
+    ils $KEY/latest.tar 2&1>/dev/null && (
         imv $KEY/latest.tar $KEY/previous.tar
         if [[ $? -ne 0 ]]; then
             echo "[$KEY] Error 004 moving latest.tar to previous.tar. Corrective action has to be taken manualy"
@@ -184,6 +186,26 @@ pack_and_send() {
             return 0
         fi
     fi
+    # Check if file exists on irods server and get creation date
+    # days=$(( $(date +%j --date $(date +%Y-%m-01)) - $(date +%j --date $(ils -l 2001_YB_F13 | awk '/latest.tar/ {print substr($5, 0,11)}')) ))
+    ilsr=$(ils -L $KEY/latest.tar)
+    if [[ $? -eq 0 ]]; then
+        echo "[$KEY] latest.tar already exists on iRODS server. Check if something changed localy since last push."
+        # Diff in days between current month (snapshot) and latest irods push
+        days=$(( $(date +%j --date $(ils -l 2001_YB_F13 | awk '/latest.tar/ {print substr($5, 0,11)}')) - $(date +%j --date $(date +%Y-%m-01)) ))
+        if [[ $days -ge 0 ]]; then
+            # We have an uptodate dump (newer than snapshot)
+            echo "[$KEY] latest.tar is up to date"
+            return 0
+        fi
+        # find files modified in the last $days. If none, we can skip because latests irods version is up to date.
+        items_modified=$(find $dir -ctime $days | wc -l)
+        if [[ $items_modified -eq 0 ]]; then
+            echo "[$KEY] $items_modified new file to update since $days days. Skipping."
+            format_report $KEY "-" $(date +%Y-%m-%dT%H:%M:%S) "-" "-" "Skipped" >> $LOCAL_REPORT
+            return 0
+        fi
+    fi
     echo "[$KEY] Creating tar on $RESIFDD_WORKDIR/$KEY.tar"
     echo "[$KEY] tar cf $RESIFDD_WORKDIR/$KEY.tar -C ${dir%$YEAR/$NETWORK/$STATION} $dir"
     tar cf $RESIFDD_WORKDIR/$KEY.tar -C ${dir%$YEAR/$NETWORK/$STATION} $dir
@@ -194,27 +216,10 @@ pack_and_send() {
         zabbix_err "$KEY:Error 007"
         return 1
     fi
-    # Check if file exists on irods server
-    ils -L $KEY/latest.tar > /dev/null 2>&1
-    if [[ $? -eq 0 ]]; then
-        echo "[$KEY] latest.tar already exists on iRODS server. Let's compare hashes"
-        local_sha=$(sha256sum $RESIFDD_WORKDIR/$KEY.tar | awk '{print $1}' | xxd -r -p | base64)
-        irods_sha=$(ichksum $KEY/latest.tar | awk -F':' '/sha2:/ {print $2; exit;}')
-        echo "[$KEY] local checksum: $local_sha"
-        echo "[$KEY] irods checksum: $irods_sha"
-        # If the hashes differs, then move distant file and push this one
-        if [[ "$local_sha" = "$irods_sha" ]]; then
-            echo "[$KEY] The archive on irods is the same as our version. Skipping."
-            format_report $KEY "-" $(date +%Y-%m-%dT%H:%M:%S) "-" "-" "Skipped" >> $LOCAL_REPORT
-            return 0
-        fi
-    fi
     # Send latest archive file to IRODS
-    irods_push $KEY $local_sha
+    irods_push $KEY
     rm $RESIFDD_WORKDIR/$KEY*
-
 }
-
 export -f pack_and_send   # Necessary for call with GNU parallel
 ####################
 #
@@ -290,8 +295,7 @@ else
         zabbix_err "${KEY}:Error 001"
         exit 1
     fi
-    local_sha=$(sha256sum $RESIFDD_WORKDIR/$KEY.tar | awk '{print $1}' | xxd -r -p | base64)
-    irods_push $KEY $local_sha
+    irods_push $KEY
     echo "[$KEY] Dump terminated :"
     ils -l $KEY
 fi