Commit 656383ec authored by Jonathan Schaeffer's avatar Jonathan Schaeffer
Browse files

No data.yaml necessary

parent 2cbae28f
...@@ -7,7 +7,7 @@ import subprocess ...@@ -7,7 +7,7 @@ import subprocess
from time import gmtime, strftime from time import gmtime, strftime
import yaml import yaml
from influxdb import InfluxDBClient from influxdb import InfluxDBClient
import datetime from datetime import datetime, date, timedelta
import psycopg2 import psycopg2
import click import click
from fdsnextender import FdsnExtender from fdsnextender import FdsnExtender
...@@ -42,10 +42,10 @@ def scan_volume(path): ...@@ -42,10 +42,10 @@ def scan_volume(path):
volume = os.path.realpath(path)+'/' volume = os.path.realpath(path)+'/'
logger.debug("Volume %s", volume) logger.debug("Volume %s", volume)
# TODO mettre le niveau de profondeur (2) en option # TODO mettre le niveau de profondeur (2) en option
starttime = datetime.datetime.now() starttime = datetime.now()
lines = subprocess.check_output( lines = subprocess.check_output(
["du", "--exclude", ".snapshot", "-b", "-d4", volume]).decode("utf-8").splitlines() ["du", "--exclude", ".snapshot", "-b", "-d4", volume]).decode("utf-8").splitlines()
logger.debug("Volume scanned in %s", datetime.datetime.now() - starttime) logger.debug("Volume scanned in %s", datetime.now() - starttime)
for l in lines: for l in lines:
logger.debug(l) logger.debug(l)
(size, path) = l.split('\t') (size, path) = l.split('\t')
...@@ -67,7 +67,7 @@ def scan_volumes(volumes): ...@@ -67,7 +67,7 @@ def scan_volumes(volumes):
# En sortie, une liste de dictionnaires : # En sortie, une liste de dictionnaires :
# [ {stat}, {stat}, ] # [ {stat}, {stat}, ]
volume_stats = [] volume_stats = []
starttime = datetime.datetime.now() starttime = datetime.now()
for volume in volumes: for volume in volumes:
logger.debug("Preparing scan of volume %s", volume['path']) logger.debug("Preparing scan of volume %s", volume['path'])
if 'path' in volume: if 'path' in volume:
...@@ -82,13 +82,15 @@ def scan_volumes(volumes): ...@@ -82,13 +82,15 @@ def scan_volumes(volumes):
raise ValueError("Volume has no path key : %s" % (volume)) raise ValueError("Volume has no path key : %s" % (volume))
# on applati la liste de listes : # on applati la liste de listes :
logger.info("All volumes scanned in %s", logger.info("All volumes scanned in %s",
(datetime.datetime.now() - starttime)) (datetime.now() - starttime))
return [x for vol in volume_stats for x in vol] return [x for vol in volume_stats for x in vol]
@click.command() @click.command()
@click.option('--config-file', 'configfile', type=click.File(), help='Configuration file path', envvar='CONFIG_FILE', show_default=True, default=f"{os.path.dirname(os.path.realpath(__file__))}/config.yml") @click.option('--config-file', 'configfile', type=click.File(), help='Configuration file path', envvar='CONFIG_FILE', show_default=True,
def cli(configfile): default=f"{os.path.dirname(os.path.realpath(__file__))}/config.yml")
@click.option('--force-scan', flag_value=True, default=False, help='Force scanning of the archive')
def cli(configfile, force_scan):
""" """
Command line interface. Stands as main Command line interface. Stands as main
""" """
...@@ -107,36 +109,26 @@ def cli(configfile): ...@@ -107,36 +109,26 @@ def cli(configfile):
logger = logging.getLogger("resif_data_reporter") logger = logging.getLogger("resif_data_reporter")
logger.info("Starting") logger.info("Starting")
statistics = [] statistics = []
# Refresh or use cache ? today = date.today().strftime("%Y-%m-%d")
# Try to open data.yaml
cache_file = cfg['cache_file'] if not force_scan:
try: # Get last stat date
if not cache_file.startswith('/'): conn = psycopg2.connect(dbname=cfg['postgres']['database'], user=cfg['postgres']['user'],
cache_file = os.path.split(configfile.name)[0]+'/'+cache_file host=cfg['postgres']['host'], password=cfg['postgres']['password'], port=cfg['postgres']['port'])
with open(cache_file, 'r') as ymlfile: cur = conn.cursor()
cache = yaml.load(ymlfile, Loader=yaml.SafeLoader) cur.execute('select distinct date from dataholdings order by date desc limit 1;')
# Compare volumes in cfg and in cache last_stat_date = cur.fetchone()[0]
if cfg['volumes'] == cache['volumes']: conn.close()
# Get previous run data if date.today() - last_stat_date > timedelta(days=(cfg['cache_ttl'])):
previous_run_date = datetime.datetime.strptime( logger.info("Cache is old, let's scan volumes")
cache['date'], "%Y-%m-%d").date() else:
# Compute cache age logger.info(
if datetime.date.today() - previous_run_date > datetime.timedelta(days=(cfg['cache_ttl'])): "Last data report made at %s. Younger than %s. Don't scan",
logger.info("Cache is old, let's scan volumes") last_stat_date, cfg['cache_ttl'])
else: sys.exit(0)
logger.info(
"Last data report made at %s. Younger than %s. We don not scan",
previous_run_date, cfg['cache_ttl'])
sys.exit(0)
except FileNotFoundError:
logger.debug("Cache file %s not found, let's scan volumes." %
cfg['cache_file'])
statistics = scan_volumes(cfg['volumes']) statistics = scan_volumes(cfg['volumes'])
logger.debug(statistics)
today = datetime.date.today().strftime("%Y-%m-%d")
# add the network_type (is the network permanent or not) to the statistic # add the network_type (is the network permanent or not) to the statistic
# also insert the extended network code. # also insert the extended network code.
extender = FdsnExtender() extender = FdsnExtender()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment