No data.yaml necessary

resifdatareporter/resifdatareporter.py

@@ -7,7 +7,7 @@ import subprocess
 from time import gmtime, strftime
 import yaml
 from influxdb import InfluxDBClient
-import datetime
+from datetime import datetime, date, timedelta
 import psycopg2
 import click
 from fdsnextender import FdsnExtender
@@ -42,10 +42,10 @@ def scan_volume(path):
     volume = os.path.realpath(path)+'/'
     logger.debug("Volume %s", volume)
     # TODO mettre le niveau de profondeur (2) en option
-    starttime = datetime.datetime.now()
+    starttime = datetime.now()
     lines = subprocess.check_output(
         ["du", "--exclude", ".snapshot", "-b", "-d4", volume]).decode("utf-8").splitlines()
-    logger.debug("Volume scanned in %s", datetime.datetime.now() - starttime)
+    logger.debug("Volume scanned in %s", datetime.now() - starttime)
     for l in lines:
         logger.debug(l)
         (size, path) = l.split('\t')
@@ -67,7 +67,7 @@ def scan_volumes(volumes):
     # En sortie, une liste de dictionnaires :
     # [ {stat}, {stat}, ]
     volume_stats = []
-    starttime = datetime.datetime.now()
+    starttime = datetime.now()
     for volume in volumes:
         logger.debug("Preparing scan of volume %s", volume['path'])
         if 'path' in volume:
@@ -82,13 +82,15 @@ def scan_volumes(volumes):
             raise ValueError("Volume has no path key : %s" % (volume))
     # on applati la liste de listes :
     logger.info("All volumes scanned in %s",
-                 (datetime.datetime.now() - starttime))
+                 (datetime.now() - starttime))
     return [x for vol in volume_stats for x in vol]
 
 
 @click.command()
-@click.option('--config-file',  'configfile', type=click.File(), help='Configuration file path', envvar='CONFIG_FILE', show_default=True, default=f"{os.path.dirname(os.path.realpath(__file__))}/config.yml")
-def cli(configfile):
+@click.option('--config-file',  'configfile', type=click.File(), help='Configuration file path', envvar='CONFIG_FILE', show_default=True,
+              default=f"{os.path.dirname(os.path.realpath(__file__))}/config.yml")
+@click.option('--force-scan', flag_value=True, default=False, help='Force scanning of the archive')
+def cli(configfile, force_scan):
     """
     Command line interface. Stands as main
     """
@@ -107,36 +109,26 @@ def cli(configfile):
     logger = logging.getLogger("resif_data_reporter")
     logger.info("Starting")
     statistics = []
-    # Refresh or use cache ?
-    # Try to open data.yaml
-    cache_file = cfg['cache_file']
-    try:
-        if not cache_file.startswith('/'):
-            cache_file = os.path.split(configfile.name)[0]+'/'+cache_file
-        with open(cache_file, 'r') as ymlfile:
-            cache = yaml.load(ymlfile, Loader=yaml.SafeLoader)
-        # Compare volumes in cfg and in cache
-        if cfg['volumes'] == cache['volumes']:
-            # Get previous run data
-            previous_run_date = datetime.datetime.strptime(
-                cache['date'], "%Y-%m-%d").date()
-            # Compute cache age
-            if datetime.date.today() - previous_run_date > datetime.timedelta(days=(cfg['cache_ttl'])):
-                logger.info("Cache is old, let's scan volumes")
-            else:
-                logger.info(
-                    "Last data report made at %s. Younger than %s. We don not scan",
-                    previous_run_date, cfg['cache_ttl'])
-                sys.exit(0)
-
-    except FileNotFoundError:
-        logger.debug("Cache file %s not found, let's scan volumes." %
-                     cfg['cache_file'])
+    today = date.today().strftime("%Y-%m-%d")
+
+    if not force_scan:
+        # Get last stat date
+        conn = psycopg2.connect(dbname=cfg['postgres']['database'], user=cfg['postgres']['user'],
+                                host=cfg['postgres']['host'], password=cfg['postgres']['password'], port=cfg['postgres']['port'])
+        cur = conn.cursor()
+        cur.execute('select distinct date from dataholdings order by date desc limit 1;')
+        last_stat_date = cur.fetchone()[0]
+        conn.close()
+        if date.today() - last_stat_date > timedelta(days=(cfg['cache_ttl'])):
+            logger.info("Cache is old, let's scan volumes")
+        else:
+            logger.info(
+                "Last data report made at %s. Younger than %s. Don't scan",
+                last_stat_date, cfg['cache_ttl'])
+            sys.exit(0)
 
     statistics = scan_volumes(cfg['volumes'])
 
-    logger.debug(statistics)
-    today = datetime.date.today().strftime("%Y-%m-%d")
     # add the network_type (is the network permanent or not) to the statistic
     # also insert the extended network code.
     extender = FdsnExtender()