Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
OSUG
RESIF
resif_data_reporter
Commits
c70a6e8a
Commit
c70a6e8a
authored
Aug 10, 2020
by
Jonathan Schaeffer
Browse files
Remove influxdb support
parent
c41dd4e1
Changes
4
Hide whitespace changes
Inline
Side-by-side
README.md
View file @
c70a6e8a
...
...
@@ -14,7 +14,7 @@ Dans le rapport final, on souhaite distinguer les réseaux permanents des expér
Les données collectées sont écrites en YAML dans un fichier cache.
Les données sont ensuite écrites dans une base postgres
ou influxdb
.
Les données sont ensuite écrites dans une base postgres.
## Configuration
...
...
@@ -42,10 +42,6 @@ CREATE TABLE dataholdings (network varchar(2),
date
date
);
```
### Influxdb
Mêmes questions.
## Tester
...
...
@@ -65,8 +61,6 @@ docker pull postgres
docker run
--name
pg
-p
5432:5432
-e
POSTGRES_PASSWORD
=
mysecretpassword
-d
postgres
```
### Lancer un docker pour influxdb
## Configuration
...
...
@@ -84,15 +78,6 @@ Un fichier de Configuration pour les tests :
user
:
postgres
password
:
mysecretpassword
influxdb
:
measurement
:
resifstats
server
:
localhost
port
:
8086
ssl
:
True
verify_ssl
:
False
user
:
user
password
:
secret
database
:
sandbox
metadata
:
# Information about the gathered data. Used to tag the timeserie values
permanent_networks
:
# List all permanent networks here. Otherwise, they will be considered as temporary
-
CL
...
...
config.yml.example
View file @
c70a6e8a
...
...
@@ -5,9 +5,11 @@
logger_file: "logger.conf" # relative or absolute path for the logger configuration file
cache_ttl: 15 # Cache validity time in day
volumes: # list of directories to scan
- path: /path/1
- name: path1
path: /path/1
type: validated
- path: /data/all/example
- name: path2
path: /data/all/example
type: bud
postgres:
host: postgres-server # Host serving postgres
...
...
@@ -15,27 +17,3 @@ postgres:
database: statistics # Database having the dataholdings table
user: statistics
password: statistics_secret
influxdb: # influxdb database to send reports
measurement: validated_data # measurement name
server: osug-influxdb.u-ga.fr # server name
port: 8086 # port number
ssl: True # use SSL ?
verify_ssl: False # verify SSL certificate ?
user: resifmonitor # user to connect to
password: iiKa5GfvDzWDXyE797 # password for the user
database: sandbox # database name
tags: # list of tags to add to metrics, key values pairs
- host: localhost
metadata: # Information about the gathered data. Used to tag the timeserie values
permanent_networks: # List all permanent networks here. Otherwise, they will be considered as temporary
- CL
- GL
- MT
- MQ
- ND
- PF
- RD
- FR
- G
- RA
- WI
resifdatareporter/resifdatareporter.py
View file @
c70a6e8a
...
...
@@ -9,7 +9,6 @@ import re
from
time
import
gmtime
,
strftime
from
datetime
import
datetime
,
date
,
timedelta
import
yaml
from
influxdb
import
InfluxDBClient
import
psycopg2
import
click
from
fdsnextender
import
FdsnExtender
...
...
@@ -157,61 +156,12 @@ def cli(configfile, force_scan, dryrun, verbose):
for
stat
in
statistics
:
cur
.
execute
(
"""
INSERT INTO dataholdings (network, year, station, channel, quality, type, size, is_permanent, date)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
INSERT INTO dataholdings (network, year, station, channel, quality, type, size, is_permanent,
volume,
date)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s,
%s,
%s)
ON CONFLICT (network,year,station,channel,type,date) DO UPDATE SET size = EXCLUDED.size;
"""
,
(
stat
[
'network'
],
stat
[
'year'
],
stat
[
'station'
],
stat
[
'channel'
],
stat
[
'quality'
],
stat
[
'type'
],
stat
[
'size'
],
stat
[
'is_permanent'
],
stat
[
'date'
]))
(
stat
[
'network'
],
stat
[
'year'
],
stat
[
'station'
],
stat
[
'channel'
],
stat
[
'quality'
],
stat
[
'type'
],
stat
[
'size'
],
stat
[
'is_permanent'
],
stat
[
'volume'
],
stat
[
'date'
]))
conn
.
commit
()
if
'influxdb'
in
cfg
:
logger
.
info
(
'Writing in influxdb'
)
influxdb_json_data
=
[]
# Compose json data
record_time
=
strftime
(
"%Y-%m-%dT%H:%M:%SZ"
,
gmtime
())
for
stat
in
statistics
:
influxdb_json_data
.
append
(
{
"measurement"
:
cfg
[
'influxdb'
][
'measurement'
],
"tags"
:
{
"year"
:
int
(
stat
[
'year'
]),
"network"
:
stat
[
'network'
],
"station"
:
stat
[
'station'
],
"channel"
:
stat
[
'channel'
],
"quality"
:
stat
[
'quality'
],
"permanent"
:
bool
(
stat
[
'is_permanent'
]),
"type"
:
stat
[
'type'
],
"date"
:
stat
[
'date'
]
},
"time"
:
record_time
,
"fields"
:
{
"size"
:
int
(
stat
[
'size'
])
}
}
)
logger
.
info
(
influxdb_json_data
)
# Now, send this data to influxdb
try
:
logger
.
info
(
"Sending data to influxdb"
)
logger
.
debug
(
"host = %s"
,
cfg
[
'influxdb'
][
'server'
])
logger
.
debug
(
"port = %s"
,
str
(
cfg
[
'influxdb'
][
'port'
]))
logger
.
debug
(
"database = %s"
,
cfg
[
'influxdb'
][
'database'
])
logger
.
debug
(
"username = %s"
,
cfg
[
'influxdb'
][
'user'
])
client
=
InfluxDBClient
(
host
=
cfg
[
'influxdb'
][
'server'
],
port
=
cfg
[
'influxdb'
][
'port'
],
database
=
cfg
[
'influxdb'
][
'database'
],
username
=
cfg
[
'influxdb'
][
'user'
],
password
=
cfg
[
'influxdb'
][
'password'
],
ssl
=
cfg
[
'influxdb'
][
'ssl'
],
verify_ssl
=
cfg
[
'influxdb'
][
'verify_ssl'
]
)
client
.
write_points
(
influxdb_json_data
)
except
Exception
as
e
:
logger
.
error
(
"Unexpected error writing data to influxdb"
)
logger
.
error
(
e
)
if
__name__
==
"__main__"
:
cli
()
setup.py
View file @
c70a6e8a
...
...
@@ -6,7 +6,7 @@ with open('README.md', 'r', encoding='utf-8') as f:
setup
(
name
=
'resifdatareporter'
,
version
=
'0.
19
.0'
,
version
=
'0.
20
.0'
,
description
=
'Scans the resif data repository and compute metrics. Sends the result in influxdb or postgres'
,
long_description
=
readme
,
long_description_content_type
=
"text/markdown"
,
...
...
@@ -23,7 +23,6 @@ setup(
'python-dateutil==2.7.5'
,
'PyYAML==5.1'
,
'psycopg2-binary==2.8.5'
,
'influxdb==5.2.1'
,
],
keywords
=
[
''
,
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment