Commit 656383ec authored by Jonathan Schaeffer's avatar Jonathan Schaeffer
Browse files

No data.yaml necessary

parent 2cbae28f
......@@ -7,7 +7,7 @@ import subprocess
from time import gmtime, strftime
import yaml
from influxdb import InfluxDBClient
import datetime
from datetime import datetime, date, timedelta
import psycopg2
import click
from fdsnextender import FdsnExtender
......@@ -42,10 +42,10 @@ def scan_volume(path):
volume = os.path.realpath(path)+'/'
logger.debug("Volume %s", volume)
# TODO mettre le niveau de profondeur (2) en option
starttime =
starttime =
lines = subprocess.check_output(
["du", "--exclude", ".snapshot", "-b", "-d4", volume]).decode("utf-8").splitlines()
logger.debug("Volume scanned in %s", - starttime)
logger.debug("Volume scanned in %s", - starttime)
for l in lines:
(size, path) = l.split('\t')
......@@ -67,7 +67,7 @@ def scan_volumes(volumes):
# En sortie, une liste de dictionnaires :
# [ {stat}, {stat}, ]
volume_stats = []
starttime =
starttime =
for volume in volumes:
logger.debug("Preparing scan of volume %s", volume['path'])
if 'path' in volume:
......@@ -82,13 +82,15 @@ def scan_volumes(volumes):
raise ValueError("Volume has no path key : %s" % (volume))
# on applati la liste de listes :"All volumes scanned in %s",
( - starttime))
( - starttime))
return [x for vol in volume_stats for x in vol]
@click.option('--config-file', 'configfile', type=click.File(), help='Configuration file path', envvar='CONFIG_FILE', show_default=True, default=f"{os.path.dirname(os.path.realpath(__file__))}/config.yml")
def cli(configfile):
@click.option('--config-file', 'configfile', type=click.File(), help='Configuration file path', envvar='CONFIG_FILE', show_default=True,
@click.option('--force-scan', flag_value=True, default=False, help='Force scanning of the archive')
def cli(configfile, force_scan):
Command line interface. Stands as main
......@@ -107,36 +109,26 @@ def cli(configfile):
logger = logging.getLogger("resif_data_reporter")"Starting")
statistics = []
# Refresh or use cache ?
# Try to open data.yaml
cache_file = cfg['cache_file']
if not cache_file.startswith('/'):
cache_file = os.path.split([0]+'/'+cache_file
with open(cache_file, 'r') as ymlfile:
cache = yaml.load(ymlfile, Loader=yaml.SafeLoader)
# Compare volumes in cfg and in cache
if cfg['volumes'] == cache['volumes']:
# Get previous run data
previous_run_date = datetime.datetime.strptime(
cache['date'], "%Y-%m-%d").date()
# Compute cache age
if - previous_run_date > datetime.timedelta(days=(cfg['cache_ttl'])):
today ="%Y-%m-%d")
if not force_scan:
# Get last stat date
conn = psycopg2.connect(dbname=cfg['postgres']['database'], user=cfg['postgres']['user'],
host=cfg['postgres']['host'], password=cfg['postgres']['password'], port=cfg['postgres']['port'])
cur = conn.cursor()
cur.execute('select distinct date from dataholdings order by date desc limit 1;')
last_stat_date = cur.fetchone()[0]
if - last_stat_date > timedelta(days=(cfg['cache_ttl'])):"Cache is old, let's scan volumes")
"Last data report made at %s. Younger than %s. We don not scan",
previous_run_date, cfg['cache_ttl'])
"Last data report made at %s. Younger than %s. Don't scan",
last_stat_date, cfg['cache_ttl'])
except FileNotFoundError:
logger.debug("Cache file %s not found, let's scan volumes." %
statistics = scan_volumes(cfg['volumes'])
today ="%Y-%m-%d")
# add the network_type (is the network permanent or not) to the statistic
# also insert the extended network code.
extender = FdsnExtender()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment