Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
OSUG
RESIF
ws-statistics
Commits
95245814
Commit
95245814
authored
Jul 03, 2020
by
Jerome Touvier
Browse files
Merge branch 'upadate' into 'master'
add timeseries See merge request
!2
parents
8a638fde
61617353
Changes
10
Hide whitespace changes
Inline
Side-by-side
apps/README.md
View file @
95245814
...
...
@@ -8,12 +8,12 @@ Ce service donne accès aux statistiques de RESIF-DC.
où :
request-options :: (request=<
storage|send|country
>)
request-options :: (request=<
country|send|storage|timeseries
>)
channel-options :: [net=<network> & sta=<station> & loc=<location> & cha=<channel>]
date-options :: [starttime=<date>] & [endtime=<date>]
storage-options :: [year=<year>] & [type=<type>]
send-options :: [country=<country_code>] & [media=<all|dataselect|seedlink>]
output-options ::
[timeseries=<true|false>] &
[format=<csv|request|sync|text>]
output-options :: [format=<csv|request|sync|text>]
(..) requis
[..] optionnel
...
...
apps/README_EN.md
View file @
95245814
...
...
@@ -8,12 +8,12 @@ This service provides access to the RESIF-DC statistics.
where :
request-options :: (request=<
storage|send|country
>)
request-options :: (request=<
country|send|storage|timeseries
>)
channel-options :: [net=<network> & sta=<station> & loc=<location> & cha=<channel>]
date-options :: [starttime=<date>] & [endtime=<date>]
storage-options :: [year=<year>] & [type=<type>]
send-options :: [country=<country_code>] & [media=<all|dataselect|seedlink>]
output-options ::
[timeseries=<true|false>] &
[format=<csv|request|sync|text>]
output-options :: [format=<csv|request|sync|text>]
(..) required
[..] optional
...
...
apps/constants.py
View file @
95245814
...
...
@@ -17,7 +17,6 @@ parameters = {
"year"
:
None
,
"type"
:
"all"
,
"request"
:
None
,
"timeseries"
:
"F"
,
"media"
:
"all"
,
"granularity"
:
None
,
"orderby"
:
"nslc"
,
...
...
@@ -35,6 +34,6 @@ ALIAS = [
(
"endtime"
,
"end"
),
]
BOOLEANS
=
[
"timeseries"
]
BOOLEANS
=
[]
FLOATS
=
[]
NOT_NONE
=
[
"request"
]
apps/globals.py
View file @
95245814
STRING_TRUE
=
(
"yes"
,
"true"
,
"t"
,
"y"
,
"1"
,
""
)
STRING_FALSE
=
(
"no"
,
"false"
,
"f"
,
"n"
,
"0"
)
REQUEST
=
(
"storage"
,
"send"
,
"country"
)
REQUEST
=
(
"storage"
,
"send"
,
"country"
,
"timeseries"
)
DATATYPE
=
(
"all"
,
"bud"
,
"validated"
)
GRANULARITY
=
(
"year"
,
"month"
)
ORDERBY
=
(
"nslc"
,
"time"
,
"country"
,
"protocol"
)
...
...
apps/model.py
View file @
95245814
...
...
@@ -40,6 +40,8 @@ def check_parameters(params):
# media parameter validation
if
params
[
"media"
]:
if
params
[
"media"
]
==
"all"
:
params
[
"media"
]
=
"dataselect,seedlink"
params
[
"media"
]
=
params
[
"media"
].
split
(
","
)
for
ind
,
media
in
enumerate
(
params
[
"media"
]):
if
not
is_valid_media
(
media
):
...
...
apps/output.py
View file @
95245814
...
...
@@ -5,7 +5,7 @@ import time
from
datetime
import
datetime
import
psycopg2
from
flask
import
make_response
from
flask
import
current_app
,
make_response
from
apps.globals
import
Error
from
apps.utils
import
error_request
...
...
@@ -22,13 +22,11 @@ def is_like_or_equal(params, key):
return
" OR "
.
join
(
subquery
)
def
get_table
(
media
):
if
media
==
"seedlink"
:
return
"ringserver_events"
return
"dataselectvol"
def
get_table
():
return
"sent_data_summary_weekly"
def
sql_common_string
(
params
,
begin
=
""
,
date_s
=
"date"
):
def
sql_common_string
(
params
,
begin
=
""
):
s
=
begin
# network, station, location, channel parameters
s
=
f
"""
{
s
}
WHERE (
{
is_like_or_equal
(
params
,
"network"
)
}
)"""
...
...
@@ -42,10 +40,10 @@ def sql_common_string(params, begin="", date_s="date"):
# starttime, endtime parameters
if
params
[
"start"
]:
start
=
datetime
.
strftime
(
params
[
"start"
],
"%Y-%m-%d"
)
s
=
f
"""
{
s
}
AND
{
date
_s
}
>= '
{
start
}
'"""
s
=
f
"""
{
s
}
AND date >= '
{
start
}
'"""
if
params
[
"end"
]:
end
=
datetime
.
strftime
(
params
[
"end"
],
"%Y-%m-%d"
)
s
=
f
"""
{
s
}
AND
{
date
_s
}
<= '
{
end
}
'"""
s
=
f
"""
{
s
}
AND date <= '
{
end
}
'"""
return
s
...
...
@@ -65,42 +63,45 @@ def sql_request(params):
s
=
f
"""
{
s
}
ORDER BY
{
columns
}
, date DESC"""
return
s
.
replace
(
"?"
,
"_"
).
replace
(
"*"
,
"%"
)
else
:
select
=
list
()
for
media
in
params
[
"media"
]:
date_s
=
"time"
if
media
==
"seedlink"
else
"date"
if
params
[
"request"
]
==
"country"
:
s
=
f
"SELECT country, count(
{
date_s
}
) FROM
{
get_table
(
media
)
}
"
if
params
[
"request"
]
==
"send"
:
s
=
f
"SELECT sum(bytes) FROM
{
get_table
(
media
)
}
"
if
params
[
"timeseries"
]:
s
=
f
"SELECT date, network, station, location, channel, bytes, country FROM
{
get_table
(
media
)
}
"
s
=
sql_common_string
(
params
,
s
,
date_s
)
if
params
[
"request"
]
==
"country"
:
s
=
f
"""
{
s
}
AND (
{
is_like_or_equal
(
params
,
"country"
)
}
) GROUP BY country"""
if
params
[
"request"
]
==
"send"
:
s
=
f
"""
{
s
}
AND (
{
is_like_or_equal
(
params
,
"country"
)
}
)"""
# if params["granularity"]:
# granularity = "52w" if params["granularity"] == "year" else "4w"
# s = f"""{s} , time({granularity})"""
select
.
append
(
s
)
select
=
" UNION "
.
join
(
select
)
return
select
.
replace
(
"?"
,
"_"
).
replace
(
"*"
,
"%"
)
if
params
[
"request"
]
==
"country"
:
s
=
f
"SELECT country, count(date) FROM
{
get_table
()
}
"
elif
params
[
"request"
]
==
"send"
:
s
=
f
"SELECT sum(bytes) FROM
{
get_table
()
}
"
elif
params
[
"request"
]
==
"timeseries"
:
s
=
f
"""SELECT date, country, sum(bytes), sum(hll_cardinality(clients))::INTEGER FROM
{
get_table
()
}
"""
s
=
sql_common_string
(
params
,
s
)
s
=
f
"""
{
s
}
AND (
{
is_like_or_equal
(
params
,
"country"
)
}
)"""
if
"seedlink"
in
params
[
"media"
]
and
"dataselect"
in
params
[
"media"
]:
s
=
f
"
{
s
}
AND (protocol = 'seedlink' OR protocol = 'dataselect')"
elif
"seedlink"
in
params
[
"media"
]:
s
=
f
"
{
s
}
AND protocol = 'seedlink'"
elif
"dataselect"
in
params
[
"media"
]:
s
=
f
"
{
s
}
AND protocol = 'dataselect'"
if
params
[
"request"
]
==
"country"
:
s
=
f
"""
{
s
}
GROUP BY country"""
elif
params
[
"request"
]
==
"timeseries"
:
if
"seedlink"
in
params
[
"media"
]
and
"dataselect"
in
params
[
"media"
]:
s
=
f
"""
{
s
}
GROUP BY date, country ORDER BY date"""
else
:
s
=
f
"""
{
s
}
GROUP BY date, country, protocol ORDER BY date"""
return
s
.
replace
(
"?"
,
"_"
).
replace
(
"*"
,
"%"
)
def
collect_data
(
params
):
"""
Connect to the PostgreSQL RESIF database
"""
"""
Get the result of the SQL query.
"""
tic
=
time
.
time
()
logging
.
debug
(
"Start collecting data..."
)
with
psycopg2
.
connect
(
os
.
getenv
(
"PG_DB
URI"
)
)
as
conn
:
with
psycopg2
.
connect
(
current_app
.
config
[
"DATABASE_
URI"
]
)
as
conn
:
logging
.
debug
(
conn
.
get_dsn_parameters
())
logging
.
debug
(
f
"Postgres version :
{
conn
.
server_version
}
"
)
with
conn
.
cursor
()
as
curs
:
SQL_SELECT
=
sql_request
(
params
)
curs
.
execute
(
SQL_SELECT
)
logging
.
debug
(
f
"
{
SQL_SELECT
}
"
)
select
=
sql_request
(
params
)
logging
.
debug
(
select
)
curs
.
execute
(
select
)
logging
.
debug
(
curs
.
statusmessage
)
return
curs
.
fetchall
()
logging
.
debug
(
f
"Get data in
{
tictac
(
tic
)
}
seconds."
)
...
...
@@ -140,18 +141,10 @@ def get_header(params):
header
=
[
"SEEDLINK (in bytes)"
]
elif
"dataselect"
in
params
[
"media"
]:
header
=
[
"DATASELECT (in bytes)"
]
elif
params
[
"timeseries"
]:
header
=
[
"date"
,
"network"
,
"station"
,
"location"
,
"channel"
,
"bytes"
,
"country"
,
]
else
:
header
=
[
"SEEDLINK and DATASELECT (in bytes)"
]
elif
params
[
"request"
]
==
"timeseries"
:
header
=
[
"time"
,
"country"
,
"bytes"
,
"clients"
]
return
header
...
...
@@ -219,7 +212,7 @@ def get_statistics(params):
nrows
=
len
(
data
)
logging
.
debug
(
f
"Number of collected rows:
{
nrows
}
"
)
if
params
[
"request"
]
!=
"storage"
and
not
params
[
"timeseries"
]
:
if
params
[
"request"
]
in
(
"send"
,
"country"
)
:
data
=
sum_results
(
params
,
data
)
return
get_response
(
params
,
data
)
...
...
config.py
View file @
95245814
...
...
@@ -9,7 +9,6 @@ class Config:
Each parameter can be overriden directly by an environment variable.
"""
PGHOST
=
PGUSER
=
PGPORT
=
PGDATABASE
=
""
RUNMODE
=
os
.
environ
.
get
(
"RUNMODE"
)
if
RUNMODE
==
"production"
:
PGHOST
=
"resif-pgprod.u-ga.fr"
...
...
@@ -27,10 +26,9 @@ class Config:
PGUSER
=
os
.
environ
.
get
(
"PGUSER"
)
or
PGUSER
PGPORT
=
os
.
environ
.
get
(
"PGPORT"
)
or
PGPORT
PGDATABASE
=
os
.
environ
.
get
(
"PGDATABASE"
)
or
PGDATABASE
os
.
environ
[
"PG_DBURI"
]
=
f
"postgresql://
{
PGUSER
}
@
{
PGHOST
}
:
{
PGPORT
}
/
{
PGDATABASE
}
"
except
NameError
as
ne
:
print
(
ne
)
except
NameError
:
print
(
"Missing environment variables. Either RUNMODE=(test|production) or PGHOST, PGUSER, PGPORT and PGDATABASE should be set."
)
raise
DATABASE_URI
=
f
"postgresql://
{
PGUSER
}
@
{
PGHOST
}
:
{
PGPORT
}
/
{
PGDATABASE
}
"
start.py
View file @
95245814
...
...
@@ -17,15 +17,7 @@ logging.basicConfig(format=fmt, level=loglevel)
app
.
config
.
from_object
(
Config
)
if
app
.
config
[
"RUNMODE"
]:
app
.
logger
.
debug
(
"Configuration set with RUNMODE=%s"
,
app
.
config
[
"RUNMODE"
])
# with FLASK_ENV=development
app
.
logger
.
debug
(
"Database URI : postgres://%s@%s:%s/%s"
,
app
.
config
[
"PGUSER"
],
app
.
config
[
"PGHOST"
],
app
.
config
[
"PGPORT"
],
app
.
config
[
"PGDATABASE"
],
)
app
.
logger
.
debug
(
"Database URI : %s"
,
app
.
config
[
"DATABASE_URI"
])
# **************************************************************************
# ********************** STATISTICS SERVICE ROUTES *************************
...
...
templates/ws-statistics.wadl.xml
View file @
95245814
...
...
@@ -43,25 +43,25 @@
<param
name=
"start"
style=
"query"
type=
"xsd:string"
/>
<param
name=
"end"
style=
"query"
type=
"xsd:string"
/>
<param
name=
"request"
style=
"query"
type=
"xsd:string"
>
<option
value=
"storage"
/>
<option
value=
"send"
/>
<option
value=
"country"
/>
<option
value=
"send"
/>
<option
value=
"storage"
/>
<option
value=
"timeseries"
/>
</param>
<param
name=
"country"
style=
"query"
type=
"xsd:string"
/>
<param
name=
"media"
style=
"query"
type=
"xsd:string"
>
<option
value=
"all"
/>
<option
value=
"dataselect"
/>
<option
value=
"seedlink"
/>
<option
value=
"all"
/>
</param>
<param
name=
"year"
style=
"query"
type=
"xsd:string"
/>
<param
name=
"type"
style=
"query"
type=
"xsd:string"
/>
<param
name=
"timeseries"
style=
"query"
type=
"xsd:string"
/>
<param
name=
"nodata"
style=
"query"
type=
"xsd:string"
/>
<param
name=
"format"
style=
"query"
type=
"xsd:string"
>
<option
value=
"csv"
/>
<option
value=
"text"
/>
<option
value=
"request"
/>
<option
value=
"sync"
/>
<option
value=
"text"
/>
</param>
</request>
...
...
tests/test.py
View file @
95245814
...
...
@@ -121,7 +121,6 @@ class MyTest(unittest.TestCase):
p1
[
"year"
]
=
None
p1
[
"granularity"
]
=
None
p1
[
"limit"
]
=
None
p1
[
"timeseries"
]
=
"F"
p1
[
"request"
]
=
""
p1
[
"protocol"
]
=
""
p1
[
"nodata"
]
=
"204"
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment