Commit c70a6e8a authored by Jonathan Schaeffer's avatar Jonathan Schaeffer
Browse files

Remove influxdb support

parent c41dd4e1
......@@ -14,7 +14,7 @@ Dans le rapport final, on souhaite distinguer les réseaux permanents des expér
Les données collectées sont écrites en YAML dans un fichier cache.
Les données sont ensuite écrites dans une base postgres ou influxdb.
Les données sont ensuite écrites dans une base postgres.
## Configuration
......@@ -42,10 +42,6 @@ CREATE TABLE dataholdings (network varchar(2),
date date);
```
### Influxdb
Mêmes questions.
## Tester
......@@ -65,8 +61,6 @@ docker pull postgres
docker run --name pg -p 5432:5432 -e POSTGRES_PASSWORD=mysecretpassword -d postgres
```
### Lancer un docker pour influxdb
## Configuration
......@@ -84,15 +78,6 @@ Un fichier de Configuration pour les tests :
user: postgres
password: mysecretpassword
influxdb:
measurement: resifstats
server: localhost
port: 8086
ssl: True
verify_ssl: False
user: user
password: secret
database: sandbox
metadata: # Information about the gathered data. Used to tag the timeserie values
permanent_networks: # List all permanent networks here. Otherwise, they will be considered as temporary
- CL
......
......@@ -5,9 +5,11 @@
logger_file: "logger.conf" # relative or absolute path for the logger configuration file
cache_ttl: 15 # Cache validity time in day
volumes: # list of directories to scan
- path: /path/1
- name: path1
path: /path/1
type: validated
- path: /data/all/example
- name: path2
path: /data/all/example
type: bud
postgres:
host: postgres-server # Host serving postgres
......@@ -15,27 +17,3 @@ postgres:
database: statistics # Database having the dataholdings table
user: statistics
password: statistics_secret
influxdb: # influxdb database to send reports
measurement: validated_data # measurement name
server: osug-influxdb.u-ga.fr # server name
port: 8086 # port number
ssl: True # use SSL ?
verify_ssl: False # verify SSL certificate ?
user: resifmonitor # user to connect to
password: iiKa5GfvDzWDXyE797 # password for the user
database: sandbox # database name
tags: # list of tags to add to metrics, key values pairs
- host: localhost
metadata: # Information about the gathered data. Used to tag the timeserie values
permanent_networks: # List all permanent networks here. Otherwise, they will be considered as temporary
- CL
- GL
- MT
- MQ
- ND
- PF
- RD
- FR
- G
- RA
- WI
......@@ -9,7 +9,6 @@ import re
from time import gmtime, strftime
from datetime import datetime, date, timedelta
import yaml
from influxdb import InfluxDBClient
import psycopg2
import click
from fdsnextender import FdsnExtender
......@@ -157,61 +156,12 @@ def cli(configfile, force_scan, dryrun, verbose):
for stat in statistics:
cur.execute(
"""
INSERT INTO dataholdings (network, year, station, channel, quality, type, size, is_permanent, date)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
INSERT INTO dataholdings (network, year, station, channel, quality, type, size, is_permanent, volume, date)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (network,year,station,channel,type,date) DO UPDATE SET size = EXCLUDED.size;
""",
(stat['network'], stat['year'], stat['station'], stat['channel'], stat['quality'], stat['type'], stat['size'], stat['is_permanent'], stat['date']))
(stat['network'], stat['year'], stat['station'], stat['channel'], stat['quality'], stat['type'], stat['size'], stat['is_permanent'], stat['volume'], stat['date']))
conn.commit()
if 'influxdb' in cfg:
logger.info('Writing in influxdb')
influxdb_json_data = []
# Compose json data
record_time = strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())
for stat in statistics:
influxdb_json_data.append(
{"measurement": cfg['influxdb']['measurement'],
"tags": {
"year": int(stat['year']),
"network": stat['network'],
"station": stat['station'],
"channel": stat['channel'],
"quality": stat['quality'],
"permanent": bool(stat['is_permanent']),
"type": stat['type'],
"date": stat['date']
},
"time": record_time,
"fields": {
"size": int(stat['size'])
}
}
)
logger.info(influxdb_json_data)
# Now, send this data to influxdb
try:
logger.info("Sending data to influxdb")
logger.debug("host = %s", cfg['influxdb']['server'])
logger.debug("port = %s", str(cfg['influxdb']['port']))
logger.debug("database = %s", cfg['influxdb']['database'])
logger.debug("username = %s", cfg['influxdb']['user'])
client = InfluxDBClient(host=cfg['influxdb']['server'],
port=cfg['influxdb']['port'],
database=cfg['influxdb']['database'],
username=cfg['influxdb']['user'],
password=cfg['influxdb']['password'],
ssl=cfg['influxdb']['ssl'],
verify_ssl=cfg['influxdb']['verify_ssl']
)
client.write_points(influxdb_json_data)
except Exception as e:
logger.error("Unexpected error writing data to influxdb")
logger.error(e)
if __name__ == "__main__":
cli()
......@@ -6,7 +6,7 @@ with open('README.md', 'r', encoding='utf-8') as f:
setup(
name='resifdatareporter',
version='0.19.0',
version='0.20.0',
description='Scans the resif data repository and compute metrics. Sends the result in influxdb or postgres',
long_description=readme,
long_description_content_type="text/markdown",
......@@ -23,7 +23,6 @@ setup(
'python-dateutil==2.7.5',
'PyYAML==5.1',
'psycopg2-binary==2.8.5',
'influxdb==5.2.1',
],
keywords=[
'',
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment