Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
OSUG
RESIF
resif_data_reporter
Commits
656383ec
Commit
656383ec
authored
May 07, 2020
by
Jonathan Schaeffer
Browse files
No data.yaml necessary
parent
2cbae28f
Changes
1
Hide whitespace changes
Inline
Side-by-side
resifdatareporter/resifdatareporter.py
View file @
656383ec
...
...
@@ -7,7 +7,7 @@ import subprocess
from
time
import
gmtime
,
strftime
import
yaml
from
influxdb
import
InfluxDBClient
import
datetime
from
datetime
import
datetime
,
date
,
timedelta
import
psycopg2
import
click
from
fdsnextender
import
FdsnExtender
...
...
@@ -42,10 +42,10 @@ def scan_volume(path):
volume
=
os
.
path
.
realpath
(
path
)
+
'/'
logger
.
debug
(
"Volume %s"
,
volume
)
# TODO mettre le niveau de profondeur (2) en option
starttime
=
datetime
.
datetime
.
now
()
starttime
=
datetime
.
now
()
lines
=
subprocess
.
check_output
(
[
"du"
,
"--exclude"
,
".snapshot"
,
"-b"
,
"-d4"
,
volume
]).
decode
(
"utf-8"
).
splitlines
()
logger
.
debug
(
"Volume scanned in %s"
,
datetime
.
datetime
.
now
()
-
starttime
)
logger
.
debug
(
"Volume scanned in %s"
,
datetime
.
now
()
-
starttime
)
for
l
in
lines
:
logger
.
debug
(
l
)
(
size
,
path
)
=
l
.
split
(
'
\t
'
)
...
...
@@ -67,7 +67,7 @@ def scan_volumes(volumes):
# En sortie, une liste de dictionnaires :
# [ {stat}, {stat}, ]
volume_stats
=
[]
starttime
=
datetime
.
datetime
.
now
()
starttime
=
datetime
.
now
()
for
volume
in
volumes
:
logger
.
debug
(
"Preparing scan of volume %s"
,
volume
[
'path'
])
if
'path'
in
volume
:
...
...
@@ -82,13 +82,15 @@ def scan_volumes(volumes):
raise
ValueError
(
"Volume has no path key : %s"
%
(
volume
))
# on applati la liste de listes :
logger
.
info
(
"All volumes scanned in %s"
,
(
datetime
.
datetime
.
now
()
-
starttime
))
(
datetime
.
now
()
-
starttime
))
return
[
x
for
vol
in
volume_stats
for
x
in
vol
]
@
click
.
command
()
@
click
.
option
(
'--config-file'
,
'configfile'
,
type
=
click
.
File
(),
help
=
'Configuration file path'
,
envvar
=
'CONFIG_FILE'
,
show_default
=
True
,
default
=
f
"
{
os
.
path
.
dirname
(
os
.
path
.
realpath
(
__file__
))
}
/config.yml"
)
def
cli
(
configfile
):
@
click
.
option
(
'--config-file'
,
'configfile'
,
type
=
click
.
File
(),
help
=
'Configuration file path'
,
envvar
=
'CONFIG_FILE'
,
show_default
=
True
,
default
=
f
"
{
os
.
path
.
dirname
(
os
.
path
.
realpath
(
__file__
))
}
/config.yml"
)
@
click
.
option
(
'--force-scan'
,
flag_value
=
True
,
default
=
False
,
help
=
'Force scanning of the archive'
)
def
cli
(
configfile
,
force_scan
):
"""
Command line interface. Stands as main
"""
...
...
@@ -107,36 +109,26 @@ def cli(configfile):
logger
=
logging
.
getLogger
(
"resif_data_reporter"
)
logger
.
info
(
"Starting"
)
statistics
=
[]
# Refresh or use cache ?
# Try to open data.yaml
cache_file
=
cfg
[
'cache_file'
]
try
:
if
not
cache_file
.
startswith
(
'/'
):
cache_file
=
os
.
path
.
split
(
configfile
.
name
)[
0
]
+
'/'
+
cache_file
with
open
(
cache_file
,
'r'
)
as
ymlfile
:
cache
=
yaml
.
load
(
ymlfile
,
Loader
=
yaml
.
SafeLoader
)
# Compare volumes in cfg and in cache
if
cfg
[
'volumes'
]
==
cache
[
'volumes'
]:
# Get previous run data
previous_run_date
=
datetime
.
datetime
.
strptime
(
cache
[
'date'
],
"%Y-%m-%d"
).
date
()
# Compute cache age
if
datetime
.
date
.
today
()
-
previous_run_date
>
datetime
.
timedelta
(
days
=
(
cfg
[
'cache_ttl'
])):
logger
.
info
(
"Cache is old, let's scan volumes"
)
else
:
logger
.
info
(
"Last data report made at %s. Younger than %s. We don not scan"
,
previous_run_date
,
cfg
[
'cache_ttl'
])
sys
.
exit
(
0
)
except
FileNotFoundError
:
logger
.
debug
(
"Cache file %s not found, let's scan volumes."
%
cfg
[
'cache_file'
])
today
=
date
.
today
().
strftime
(
"%Y-%m-%d"
)
if
not
force_scan
:
# Get last stat date
conn
=
psycopg2
.
connect
(
dbname
=
cfg
[
'postgres'
][
'database'
],
user
=
cfg
[
'postgres'
][
'user'
],
host
=
cfg
[
'postgres'
][
'host'
],
password
=
cfg
[
'postgres'
][
'password'
],
port
=
cfg
[
'postgres'
][
'port'
])
cur
=
conn
.
cursor
()
cur
.
execute
(
'select distinct date from dataholdings order by date desc limit 1;'
)
last_stat_date
=
cur
.
fetchone
()[
0
]
conn
.
close
()
if
date
.
today
()
-
last_stat_date
>
timedelta
(
days
=
(
cfg
[
'cache_ttl'
])):
logger
.
info
(
"Cache is old, let's scan volumes"
)
else
:
logger
.
info
(
"Last data report made at %s. Younger than %s. Don't scan"
,
last_stat_date
,
cfg
[
'cache_ttl'
])
sys
.
exit
(
0
)
statistics
=
scan_volumes
(
cfg
[
'volumes'
])
logger
.
debug
(
statistics
)
today
=
datetime
.
date
.
today
().
strftime
(
"%Y-%m-%d"
)
# add the network_type (is the network permanent or not) to the statistic
# also insert the extended network code.
extender
=
FdsnExtender
()
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment