Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
OSUG
RESIF
ws-statistics
Commits
de0b7ff4
Commit
de0b7ff4
authored
Sep 01, 2020
by
Jerome Touvier
Browse files
Merge branch 'update' into 'master'
update See merge request
!9
parents
95b1b521
b6b9ca34
Changes
11
Hide whitespace changes
Inline
Side-by-side
README.md
View file @
de0b7ff4
...
...
@@ -26,7 +26,7 @@ docker run --rm --name ws-statistics -e RUNMODE=production ws-statistics
## RUNMODE builtin values
*
`production`
*
`production`
*
`test`
*
`local`
*
other values map to :
apps/globals.py
View file @
de0b7ff4
...
...
@@ -12,13 +12,15 @@ TIMEOUT = 120
# error message constants
DOCUMENTATION_URI
=
"http://ws.resif.fr/resifws/statistics/1/"
SERVICE
=
"resifws-statistics"
VERSION
=
"1.0.0"
class
Error
:
UNKNOWN_PARAM
=
"Unknown query parameter: "
MULTI_PARAM
=
"
Multiple entries for query parameter: "
MULTI_PARAM
=
"Multiple entries for query parameter: "
VALID_PARAM
=
"Valid parameters."
START_LATER
=
"The starttime cannot be later than the endtime: "
TOO_LONG_DURATION
=
"Too many days requested (greater than "
UNSPECIFIED
=
"Error processing your request."
NODATA
=
"Your query doesn't match any available data."
TIMEOUT
=
f
"Your query exceeds timeout (
{
TIMEOUT
}
seconds)."
...
...
apps/output.py
View file @
de0b7ff4
...
...
@@ -11,6 +11,17 @@ from apps.utils import error_request
from
apps.utils
import
tictac
SIZE_UNITS
=
[
"B"
,
"KB"
,
"MB"
,
"GB"
,
"TB"
,
"PB"
,
"PB"
]
def
human_readable_size
(
size_in_bytes
):
index
=
0
while
size_in_bytes
>=
1024
and
index
<
6
:
size_in_bytes
/=
1024
index
+=
1
return
f
"
{
round
(
size_in_bytes
,
2
)
}
{
SIZE_UNITS
[
index
]
}
"
def
is_like_or_equal
(
params
,
key
):
""" Builds the condition for the specified key in the "where" clause taking into account lists or wildcards. """
...
...
@@ -63,14 +74,15 @@ def sql_request(params):
return
s
.
replace
(
"?"
,
"_"
).
replace
(
"*"
,
"%"
)
else
:
if
params
[
"request"
]
==
"country"
:
s
=
f
"SELECT country,
count(date)
FROM
{
get_table
()
}
"
s
=
f
"SELECT country,
sum(requests)::INTEGER
FROM
{
get_table
()
}
"
elif
params
[
"request"
]
==
"send"
:
s
=
f
"SELECT sum(bytes) FROM
{
get_table
()
}
"
elif
params
[
"request"
]
==
"timeseries"
:
s
=
f
"
""
SELECT date, country, sum(bytes), sum(hll_cardinality(clients))::INTEGER FROM
{
get_table
()
}
"
""
s
=
f
"SELECT date, country, sum(bytes), sum(hll_cardinality(clients))::INTEGER FROM
{
get_table
()
}
"
s
=
sql_common_string
(
params
,
s
)
s
=
f
"""
{
s
}
AND (
{
is_like_or_equal
(
params
,
"country"
)
}
)"""
if
"all"
not
in
params
[
"country"
]:
s
=
f
"""
{
s
}
AND (
{
is_like_or_equal
(
params
,
"country"
)
}
)"""
if
"seedlink"
in
params
[
"media"
]
and
"dataselect"
in
params
[
"media"
]:
s
=
f
"
{
s
}
AND (protocol = 'seedlink' OR protocol = 'dataselect')"
...
...
@@ -107,22 +119,31 @@ def collect_data(params):
def
sum_results
(
params
,
data
):
""" Adds the results from the different media tables. """
newdata
=
list
()
if
params
[
"request"
]
==
"country"
:
result
=
dict
()
for
row
in
data
:
if
row
[
1
]:
if
row
[
0
]
in
result
:
if
row
[
0
]
in
result
:
# Is this country already there ?
result
[
row
[
0
]]
=
result
[
row
[
0
]]
+
int
(
row
[
1
])
else
:
result
[
row
[
0
]]
=
int
(
row
[
1
])
if
"all"
in
params
[
"country"
]:
result
=
{
"all"
:
sum
([
int
(
val
)
for
val
in
result
.
values
()])}
for
key
,
val
in
result
.
items
():
newdata
.
append
([
key
,
str
(
val
)])
newdata
.
append
([
key
,
"{:_}"
.
format
(
val
)])
elif
params
[
"request"
]
==
"send"
:
result
=
0
for
row
in
data
:
if
row
[
0
]:
result
=
result
+
int
(
row
[
0
])
newdata
.
append
([
str
(
result
)])
newdata
.
append
(
[
"{:_}"
.
format
(
result
)
+
f
" bytes (
{
human_readable_size
(
result
)
}
)"
]
)
return
newdata
...
...
@@ -135,11 +156,11 @@ def get_header(params):
elif
params
[
"request"
]
==
"send"
:
if
len
(
params
[
"media"
])
==
1
and
"all"
not
in
params
[
"media"
]:
if
"seedlink"
in
params
[
"media"
]:
header
=
[
"SEEDLINK
(in bytes)
"
]
header
=
[
"SEEDLINK"
]
elif
"dataselect"
in
params
[
"media"
]:
header
=
[
"DATASELECT
(in bytes)
"
]
header
=
[
"DATASELECT"
]
else
:
header
=
[
"SEEDLINK and DATASELECT
(in bytes)
"
]
header
=
[
"SEEDLINK and DATASELECT"
]
elif
params
[
"request"
]
==
"timeseries"
:
header
=
[
"time"
,
"country"
,
"bytes"
,
"clients"
]
return
header
...
...
@@ -200,14 +221,15 @@ def get_output(params):
data
=
collect_data
(
params
)
if
data
is
None
:
return
data
if
params
[
"request"
]
in
(
"send"
,
"country"
):
data
=
sum_results
(
params
,
data
)
if
not
data
:
code
=
params
[
"nodata"
]
return
error_request
(
msg
=
f
"HTTP._
{
code
}
_"
,
details
=
Error
.
NODATA
,
code
=
code
)
logging
.
info
(
f
"Number of collected rows:
{
len
(
data
)
}
"
)
if
params
[
"request"
]
in
(
"send"
,
"country"
):
data
=
sum_results
(
params
,
data
)
data
=
[[
str
(
val
)
for
val
in
row
]
for
row
in
data
]
return
get_response
(
params
,
data
)
except
Exception
as
ex
:
...
...
apps/
constant
s.py
→
apps/
parameter
s.py
View file @
de0b7ff4
VERSION
=
"1.0.0"
class
Parameters
:
def
__init__
(
self
):
self
.
network
=
None
...
...
@@ -25,6 +22,14 @@ class Parameters:
self
.
format
=
"text"
self
.
nodata
=
"204"
self
.
constraints
=
{
"alias"
:
[
(
"network"
,
"net"
),
(
"station"
,
"sta"
),
(
"location"
,
"loc"
),
(
"channel"
,
"cha"
),
(
"starttime"
,
"start"
),
(
"endtime"
,
"end"
),
],
"booleans"
:
[],
"floats"
:
[],
"not_none"
:
[
"request"
],
...
...
@@ -32,13 +37,3 @@ class Parameters:
def
todict
(
self
):
return
self
.
__dict__
ALIAS
=
[
(
"network"
,
"net"
),
(
"station"
,
"sta"
),
(
"location"
,
"loc"
),
(
"channel"
,
"cha"
),
(
"starttime"
,
"start"
),
(
"endtime"
,
"end"
),
]
apps/root.py
View file @
de0b7ff4
...
...
@@ -4,12 +4,11 @@ from multiprocessing import Process, Queue
from
flask
import
request
from
apps.constants
import
ALIAS
from
apps.constants
import
Parameters
from
apps.globals
import
Error
from
apps.globals
import
HTTP
from
apps.globals
import
TIMEOUT
from
apps.output
import
get_output
from
apps.parameters
import
Parameters
from
apps.utils
import
check_base_parameters
from
apps.utils
import
check_request
from
apps.utils
import
error_param
...
...
@@ -34,7 +33,6 @@ def check_parameters(params):
country
=
params
[
"country"
].
split
(
","
)
for
c
in
country
:
logging
.
debug
(
c
)
if
not
is_valid_country
(
c
):
return
error_param
(
params
,
Error
.
COUNTRY
+
c
)
...
...
@@ -48,9 +46,10 @@ def check_parameters(params):
# media parameter validation
if
params
[
"media"
]:
if
params
[
"media"
]
==
"all"
:
params
[
"media"
]
=
"dataselect,seedlink"
params
[
"media"
]
=
params
[
"media"
].
split
(
","
)
if
"all"
in
params
[
"media"
]:
params
[
"media"
]
=
[
"dataselect"
,
"seedlink"
]
for
ind
,
media
in
enumerate
(
params
[
"media"
]):
if
not
is_valid_media
(
media
):
return
error_param
(
params
,
Error
.
MEDIA
+
str
(
media
))
...
...
@@ -103,22 +102,9 @@ def checks_get():
params
=
Parameters
().
todict
()
# check if the parameters are unknown
(
p
,
result
)
=
check_request
(
params
,
ALIAS
)
(
p
,
result
)
=
check_request
(
params
)
if
result
[
"code"
]
!=
200
:
return
(
p
,
result
)
# determine selected features
params
[
"request"
]
=
tuple
(
request
.
args
)
for
key
,
val
in
params
.
items
():
params
[
key
]
=
request
.
args
.
get
(
key
,
val
)
for
key
,
alias
in
ALIAS
:
if
params
[
key
]
is
None
:
params
[
key
]
=
request
.
args
.
get
(
alias
,
params
[
alias
])
else
:
params
[
alias
]
=
params
[
key
]
return
check_parameters
(
params
)
...
...
apps/utils.py
View file @
de0b7ff4
...
...
@@ -4,42 +4,40 @@ import time
from
difflib
import
SequenceMatcher
from
datetime
import
datetime
,
timedelta
from
flask
import
Response
,
request
from
flask
import
request
,
Response
from
apps.
constant
s
import
VERSION
from
apps.
global
s
import
DATATYPE
from
apps.globals
import
DOCUMENTATION_URI
from
apps.globals
import
Error
from
apps.globals
import
GRANULARITY
from
apps.globals
import
HTTP
from
apps.globals
import
REQUEST
from
apps.globals
import
MEDIA
from
apps.globals
import
NODATA_CODE
from
apps.globals
import
ORDERBY
from
apps.globals
import
OUTPUT
from
apps.globals
import
REQUEST
from
apps.globals
import
SERVICE
from
apps.globals
import
STRING_FALSE
from
apps.globals
import
STRING_TRUE
from
apps.globals
import
DATATYPE
from
apps.globals
import
GRANULARITY
from
apps.globals
import
VERSION
def
is_valid_integer
(
dimension
,
mini
=
0
,
maxi
=
sys
.
maxsize
):
# by default valid for positive integers
try
:
dimension
=
int
(
dimension
)
if
mini
<=
dimension
<=
maxi
:
return
True
except
Exception
:
return
False
return
bool
(
mini
<=
dimension
<=
maxi
)
def
is_valid_float
(
dimension
,
mini
=
sys
.
float_info
.
epsilon
,
maxi
=
sys
.
float_info
.
max
):
# by default valid for strictly positive floats
try
:
dimension
=
float
(
dimension
)
if
mini
<=
dimension
<=
maxi
:
return
True
except
Exception
:
return
False
return
bool
(
mini
<=
dimension
<=
maxi
)
def
is_valid_datetime
(
date
):
...
...
@@ -89,7 +87,9 @@ def is_valid_output(output):
def
is_valid_country
(
country
):
return
re
.
match
(
"[A-Za-z*?]{1,2}$"
,
country
)
if
country
else
False
return
(
re
.
match
(
"[A-Za-z*?]{1,2}$"
,
country
)
or
country
==
"all"
if
country
else
False
)
def
is_valid_year
(
year
):
...
...
@@ -158,30 +158,38 @@ def error_500(dmesg):
return
error_request
(
msg
=
HTTP
.
_500_
,
details
=
dmesg
,
code
=
500
)
#
check
request
def
check_request
(
params
,
alias
):
keys
=
list
(
params
.
keys
())
def
check
_
request
(
params
):
# preliminary parameter checks
for
key
,
val
in
request
.
args
.
items
():
if
key
not
in
keys
:
ratios
=
[
SequenceMatcher
(
None
,
key
,
p
).
ratio
()
for
p
in
keys
]
guess
=
max
([(
v
,
keys
[
ind
])
for
ind
,
v
in
enumerate
(
ratios
)])
# stops at the first unknown parameter meet:
if
key
not
in
params
:
ratios
=
((
SequenceMatcher
(
None
,
key
,
p
).
ratio
(),
p
)
for
p
in
params
)
guess
=
max
(
ratios
)
hint
=
". Did you mean "
+
guess
[
1
]
+
" ?"
if
guess
[
0
]
>
0.7
else
""
return
error_param
(
key
s
,
Error
.
UNKNOWN_PARAM
+
key
+
hint
)
return
error_param
(
param
s
,
Error
.
UNKNOWN_PARAM
+
key
+
hint
)
# find nonword chars except :
# "," for lists "*?" for wildcards and ".:-" for date
if
re
.
search
(
r
"[^a-zA-Z0-9_,*?.:-]"
,
val
):
return
error_param
(
key
s
,
Error
.
CHAR
+
key
)
return
error_param
(
param
s
,
Error
.
CHAR
+
key
)
for
key
in
keys
:
for
key
,
val
in
params
.
items
()
:
if
len
(
request
.
args
.
getlist
(
key
))
>
1
:
return
error_param
(
keys
,
Error
.
MULTI_PARAM
+
key
)
return
error_param
(
params
,
Error
.
MULTI_PARAM
+
key
)
params
[
key
]
=
request
.
args
.
get
(
key
,
val
)
for
key
,
alias
in
params
[
"constraints"
][
"alias"
]:
if
key
in
request
.
args
and
alias
in
request
.
args
:
return
error_param
(
params
,
f
"
{
Error
.
MULTI_PARAM
}{
key
}
(and is shorthand
{
alias
}
)"
)
if
params
[
key
]
is
None
or
params
[
key
]
==
"false"
:
params
[
key
]
=
request
.
args
.
get
(
alias
,
params
[
alias
])
else
:
params
[
alias
]
=
params
[
key
]
for
key
in
alias
:
if
len
([
v
for
v
in
key
if
v
in
request
.
args
])
>
1
:
return
error_param
(
keys
,
Error
.
MULTI_PARAM
+
" and is shorthand "
.
join
(
key
))
return
(
keys
,
{
"msg"
:
HTTP
.
_200_
,
"details"
:
Error
.
VALID_PARAM
,
"code"
:
200
})
return
(
params
,
{
"msg"
:
HTTP
.
_200_
,
"details"
:
Error
.
VALID_PARAM
,
"code"
:
200
})
def
check_base_parameters
(
params
,
max_days
=
None
):
...
...
docs/USAGE_EN.md
View file @
de0b7ff4
...
...
@@ -65,7 +65,7 @@ This service provides access to the RESIF-DC statistics.
## Date and time formats
YYYY-MM-DDThh:mm:ss[.ssssss] ex. 1997-01-31T12:04:32.123
YYYY-MM-DD ex. 1997-01-31 (
a time of 00:00:00 is assumed)
YYYY-MM-DD ex. 1997-01-31 (a time of 00:00:00 is assumed)
where:
...
...
start.py
View file @
de0b7ff4
...
...
@@ -3,7 +3,7 @@ import os
from
flask
import
Flask
,
make_response
,
render_template
from
apps.
constant
s
import
VERSION
from
apps.
global
s
import
VERSION
from
apps.root
import
output
from
config
import
Config
...
...
static/resifws.css
View file @
de0b7ff4
...
...
@@ -2,7 +2,7 @@ body {
margin
:
auto
;
padding-right
:
1em
;
padding-left
:
1em
;
max-width
:
75em
;
max-width
:
75em
;
border-left
:
1px
solid
black
;
border-right
:
1px
solid
black
;
color
:
black
;
...
...
templates/doc_en.html
View file @
de0b7ff4
...
...
@@ -157,7 +157,7 @@ output-options :: [format=<csv|request|sync|text>]
</ul>
<h2
id=
"date-and-time-formats"
>
Date and time formats
</h2>
<pre><code>
YYYY-MM-DDThh:mm:ss[.ssssss] ex. 1997-01-31T12:04:32.123
YYYY-MM-DD ex. 1997-01-31 (
a time of 00:00:00 is assumed)
YYYY-MM-DD ex. 1997-01-31 (a time of 00:00:00 is assumed)
where:
...
...
tests/test.py
View file @
de0b7ff4
...
...
@@ -7,6 +7,7 @@ sys.path.append("../")
from
apps.globals
import
Error
from
apps.globals
import
HTTP
from
apps.parameters
import
Parameters
from
apps.root
import
check_parameters
from
apps.utils
import
error_param
from
apps.utils
import
is_valid_integer
...
...
@@ -104,9 +105,7 @@ class MyTest(unittest.TestCase):
self
.
assertFalse
(
is_valid_bool_string
(
"oui"
))
def
test_parameters
(
self
):
# ?network=FR&channel=HHZ&starttime=2018-02-12T03:08:02&endtime=2018-02-12T03:10:00&station=CIEL
p1
=
{}
p1
=
Parameters
().
todict
()
p1
[
"network"
]
=
"FR"
p1
[
"station"
]
=
"CIEL"
p1
[
"location"
]
=
"*"
...
...
@@ -123,12 +122,6 @@ class MyTest(unittest.TestCase):
p1
[
"limit"
]
=
None
p1
[
"request"
]
=
""
p1
[
"protocol"
]
=
""
p1
[
"nodata"
]
=
"204"
p1
[
"constraints"
]
=
{
"booleans"
:
[],
"floats"
:
[],
"not_none"
:
[
"request"
],
}
valid_param
=
{
"msg"
:
HTTP
.
_200_
,
"details"
:
Error
.
VALID_PARAM
,
"code"
:
200
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment