Remove influxdb support

README.md

@@ -14,7 +14,7 @@ Dans le rapport final, on souhaite distinguer les réseaux permanents des expér
 
 Les données collectées sont écrites en YAML dans un fichier cache.
 
-Les données sont ensuite écrites dans une base postgres ou influxdb.
+Les données sont ensuite écrites dans une base postgres.
 
 ## Configuration
 
@@ -42,10 +42,6 @@ CREATE TABLE dataholdings (network varchar(2),
                           date date);
 ```
 
-### Influxdb
-
-Mêmes questions.
-
 
 ## Tester
 
@@ -65,8 +61,6 @@ docker pull postgres
 docker run --name pg -p 5432:5432 -e POSTGRES_PASSWORD=mysecretpassword -d postgres
 ```
 
-### Lancer un docker pour influxdb
-
 
 ## Configuration
 
@@ -84,15 +78,6 @@ Un fichier de Configuration pour les tests :
       user: postgres
       password: mysecretpassword
 
-    influxdb:
-      measurement: resifstats
-      server: localhost
-      port: 8086
-      ssl: True
-      verify_ssl: False
-      user: user
-      password: secret
-      database: sandbox
     metadata:    # Information about the gathered data. Used to tag the timeserie values
       permanent_networks: #  List all permanent networks here. Otherwise, they will be considered as temporary
         - CL

config.yml.example

@@ -5,9 +5,11 @@
 logger_file: "logger.conf"             # relative or absolute path for the logger configuration file
 cache_ttl: 15                          # Cache validity time in day
 volumes:                               # list of directories to scan
-  - path: /path/1
+  - name: path1
+    path: /path/1
     type: validated
-  - path: /data/all/example
+  - name: path2
+    path: /data/all/example
     type: bud
 postgres:
   host: postgres-server                # Host serving postgres
@@ -15,27 +17,3 @@ postgres:
   database: statistics                 # Database having the dataholdings table
   user: statistics
   password: statistics_secret
-influxdb:                              # influxdb database to send reports
-  measurement: validated_data          # measurement name
-  server: osug-influxdb.u-ga.fr        # server name
-  port: 8086                           # port number
-  ssl: True                            # use SSL ?
-  verify_ssl: False                    # verify SSL certificate ?
-  user: resifmonitor                   # user to connect to
-  password: iiKa5GfvDzWDXyE797         # password for the user
-  database: sandbox                    # database name
-  tags:                                # list of tags to add to metrics, key values pairs
-    - host: localhost
-metadata:                              # Information about the gathered data. Used to tag the timeserie values
-  permanent_networks:                  # List all permanent networks here. Otherwise, they will be considered as temporary
-    - CL
-    - GL
-    - MT
-    - MQ
-    - ND
-    - PF
-    - RD
-    - FR
-    - G
-    - RA
-    - WI

resifdatareporter/resifdatareporter.py

@@ -9,7 +9,6 @@ import re
 from time import gmtime, strftime
 from datetime import datetime, date, timedelta
 import yaml
-from influxdb import InfluxDBClient
 import psycopg2
 import click
 from fdsnextender import FdsnExtender
@@ -157,61 +156,12 @@ def cli(configfile, force_scan, dryrun, verbose):
         for stat in statistics:
             cur.execute(
                 """
-                INSERT INTO dataholdings (network, year, station, channel, quality, type, size, is_permanent, date)
-                VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
+                INSERT INTO dataholdings (network, year, station, channel, quality, type, size, is_permanent, volume, date)
+                VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
                 ON CONFLICT (network,year,station,channel,type,date) DO UPDATE SET size = EXCLUDED.size;
                 """,
-                (stat['network'], stat['year'], stat['station'], stat['channel'], stat['quality'], stat['type'], stat['size'], stat['is_permanent'], stat['date']))
+                (stat['network'], stat['year'], stat['station'], stat['channel'], stat['quality'], stat['type'], stat['size'], stat['is_permanent'], stat['volume'], stat['date']))
         conn.commit()
 
-    if 'influxdb' in cfg:
-        logger.info('Writing in influxdb')
-        influxdb_json_data = []
-        # Compose json data
-        record_time = strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())
-        for stat in statistics:
-            influxdb_json_data.append(
-                {"measurement": cfg['influxdb']['measurement'],
-                 "tags": {
-                     "year": int(stat['year']),
-                     "network": stat['network'],
-                     "station": stat['station'],
-                     "channel": stat['channel'],
-                     "quality": stat['quality'],
-                     "permanent": bool(stat['is_permanent']),
-                     "type": stat['type'],
-                     "date": stat['date']
-                 },
-                 "time": record_time,
-                 "fields": {
-                     "size": int(stat['size'])
-                 }
-                }
-            )
-        logger.info(influxdb_json_data)
-
-        # Now, send this data to influxdb
-        try:
-            logger.info("Sending data to influxdb")
-
-            logger.debug("host     = %s", cfg['influxdb']['server'])
-            logger.debug("port     = %s", str(cfg['influxdb']['port']))
-            logger.debug("database = %s", cfg['influxdb']['database'])
-            logger.debug("username = %s", cfg['influxdb']['user'])
-
-            client = InfluxDBClient(host=cfg['influxdb']['server'],
-                                    port=cfg['influxdb']['port'],
-                                    database=cfg['influxdb']['database'],
-                                    username=cfg['influxdb']['user'],
-                                    password=cfg['influxdb']['password'],
-                                    ssl=cfg['influxdb']['ssl'],
-                                    verify_ssl=cfg['influxdb']['verify_ssl']
-                                    )
-
-            client.write_points(influxdb_json_data)
-        except Exception as e:
-            logger.error("Unexpected error writing data to influxdb")
-            logger.error(e)
-
 if __name__ == "__main__":
     cli()

setup.py

@@ -6,7 +6,7 @@ with open('README.md', 'r', encoding='utf-8') as f:
 
 setup(
     name='resifdatareporter',
-    version='0.19.0',
+    version='0.20.0',
     description='Scans the resif data repository and compute metrics. Sends the result in influxdb or postgres',
     long_description=readme,
     long_description_content_type="text/markdown",
@@ -23,7 +23,6 @@ setup(
         'python-dateutil==2.7.5',
         'PyYAML==5.1',
         'psycopg2-binary==2.8.5',
-        'influxdb==5.2.1',
     ],
     keywords=[
         '',