Commit e7c3cafd authored by Jonathan Schaeffer's avatar Jonathan Schaeffer
Browse files

With fdsnextender

parent 930008ee
certifi==2018.10.15
chardet==3.0.4
idna==2.7
influxdb==5.2.0
python-dateutil==2.7.5
pytz==2018.7
PyYAML==5.1
requests==2.20.1
six==1.11.0
urllib3==1.24.1
psycopg2-binary==2.7.6.1
......@@ -12,6 +12,7 @@ import datetime
import psycopg2
from psycopg2.extras import execute_values
import click
from fdsnextender import FdsnExtender
logger = logging.getLogger(__name__)
......@@ -29,12 +30,13 @@ def dict_dumper(dict):
except:
return dict.__dict__
def scan_volume(path):
"""
Scanne un volume indiqué par son chemin (path).
La fonction lance une commande "du -d3 path" et analyse chaque ligne renvoyée.
Elle renvoie une liste de dictionnaires :
[ {year: 2011, network: 'G', size: '100', files: '14'}, ...]
[ {year: 2011, network: 'G', size: '100', files: '14', station: 'STAT'}, ...]
"""
data = []
volume = os.path.realpath(path)+'/'
......@@ -90,9 +92,6 @@ def cli(configfile):
except Error as e:
print(f"Error reading file {configfile}")
print("Config:")
print(configfile.name)
print(cfg)
validate_config(cfg)
# At this point we ensure that configuration is sane.
......@@ -132,20 +131,23 @@ def cli(configfile):
logger.debug(statistics)
today = datetime.date.today().strftime("%Y-%m-%d")
# add the network_type (is the network permanent or not) to the statistic
# also insert the extended network code.
extender = FdsnExtender()
for stat in statistics:
is_permanent = False
if stat['network'] in cfg['metadata']['permanent_networks']:
is_permanent = True
stat['is_permanent'] = is_permanent
stat['is_permanent'] = True
else :
stat['is_permanent'] = False
stat['network'] = extender.extend(stat['network'], int(stat['year']))
# TODO: à revoir, la ligne suivante, c'est la honte
stat['date'] = today
stat['date'] = today
logger.debug(stat)
# Open dump file and write the stats.
if use_cache == False:
try:
with open(os.path.dirname(os.path.realpath(__file__))+"/data.yaml", 'w') as outfile:
with open(os.path.split(configfile.name)[0]+"/data.yaml", 'w') as outfile:
yaml.dump({'date': today,
'volumes': cfg['volumes'],
'statistics': statistics
......@@ -180,7 +182,7 @@ def cli(configfile):
"station": stat['station'],
"permanent": bool(stat['is_permanent']),
"type": stat['type'],
"date": today
"date": stat['date']
},
"time": record_time,
"fields": {
......
from io import open
from setuptools import setup, find_packages
with open('resifdatareporter/__init__.py', 'r') as f:
for line in f:
if line.startswith('__version__'):
version = line.strip().split('=')[1].strip(' \'"')
break
else:
version = '0.0.1'
with open('README.md', 'r', encoding='utf-8') as f:
readme = f.read()
setup(
name='resifdatareporter',
version=version,
version='0.7.6',
description='Scans the resif data repository and compute metrics. Sends the result in influxdb or postgres',
long_description=readme,
long_description_content_type="text/markdown",
......@@ -26,6 +18,7 @@ setup(
license='GPL-3.0',
packages=find_packages(),
install_requires=[
'fdsnextender==0.3.1',
'Click==7.0',
'python-dateutil==2.7.5',
'PyYAML==5.1',
......
......@@ -2,7 +2,7 @@ import pytest
from random import getrandbits, choices
import string
import subprocess
from resifdatareporter import resifdatareporter
import resifdatareporter
import yaml
from pprint import pprint
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment