Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
OSUG
RESIF
ws-statistics
Commits
b212081f
Commit
b212081f
authored
Jan 06, 2021
by
Jerome Touvier
Browse files
Merge branch 'update' into 'master'
fix map and timeseries requests See merge request
!30
parents
ad673ee2
9be42246
Changes
3
Hide whitespace changes
Inline
Side-by-side
apps/globals.py
View file @
b212081f
...
...
@@ -45,6 +45,7 @@ class Error:
NODATA_CODE
=
f
"Accepted nodata values are:
{
NODATA_CODE
}
."
+
BAD_VAL
NO_SELECTION
=
"Request contains no selections."
NO_WILDCARDS
=
"Wildcards or lists are not allowed."
+
BAD_VAL
COUNTRY_ALL
=
f
"The special option country=all has no meaning in list."
+
BAD_VAL
class
HTTP
:
...
...
apps/output.py
100644 → 100755
View file @
b212081f
...
...
@@ -57,7 +57,8 @@ def sql_common_string(params):
s
=
f
"
{
s
}
AND protocol = 'dataselect'"
if
params
[
"request"
]
in
(
"send"
,
"timeseries"
):
s
=
f
"""
{
s
}
AND (
{
is_like_or_equal
(
params
,
"country"
)
}
)"""
if
"all"
not
in
params
[
"country"
]
and
params
[
"country"
]
!=
"*"
:
s
=
f
"""
{
s
}
AND (
{
is_like_or_equal
(
params
,
"country"
)
}
)"""
# starttime, endtime parameters
if
params
[
"start"
]:
...
...
@@ -88,7 +89,7 @@ def sql_request(params):
if
params
[
"request"
]
==
"country"
:
s
=
f
"SELECT 'all' AS country, sum(requests)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM
{
table
}
"
elif
params
[
"request"
]
==
"map"
:
s
=
f
"SELECT
extract(year from date)::INTEGER AS year,
country, sum(requests)::BIGINT FROM
{
table
}
"
s
=
f
"SELECT country, sum(requests)::BIGINT FROM
{
table
}
"
elif
params
[
"request"
]
==
"timeseries"
:
if
params
[
"country"
]
==
"all"
:
s
=
f
"SELECT date, 'all' as country, sum(bytes)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM
{
table
}
"
...
...
@@ -108,7 +109,7 @@ def sql_request(params):
tmp
=
tmp
+
f
""" AND (
{
is_like_or_equal
(
params
,
"country"
)
}
)"""
s
=
f
"
{
s
}
UNION
{
tmp
}
GROUP BY country ORDER BY 2 DESC"
elif
params
[
"request"
]
==
"map"
:
s
=
f
"
{
s
}
GROUP BY country
, year
"
s
=
f
"
{
s
}
GROUP BY country"
elif
params
[
"request"
]
==
"timeseries"
:
if
params
[
"country"
]
==
"all"
:
s
=
f
"
{
s
}
GROUP BY date ORDER BY date"
...
...
@@ -138,7 +139,7 @@ def collect_data(params):
def
format_results
(
params
,
data
):
data
=
list
(
map
(
list
,
data
))
if
params
[
"format"
]
!=
"csv"
:
if
params
[
"format"
]
not
in
[
"csv"
,
"json"
]
:
if
params
[
"request"
]
==
"country"
:
for
row
in
data
:
if
row
[
1
]
and
row
[
2
]:
...
...
@@ -193,35 +194,31 @@ def records_to_text(params, data, sep=" "):
return
text
def
records_to_
dictlist
(
params
,
data
):
def
records_to_
json
(
params
,
data
):
""" Create json output """
dictlist
=
list
()
header
=
get_header
(
params
)
header
=
[
h
.
lower
()
for
h
in
header
]
for
row
in
data
:
dictlist
.
append
(
dict
(
zip
(
header
,
row
)))
return
{
header
=
[
h
.
lower
()
for
h
in
get_header
(
params
)]
dictlist
=
[
dict
(
zip
(
header
,
row
))
for
row
in
data
]
data
=
{
"created"
:
datetime
.
utcnow
().
isoformat
(
timespec
=
"seconds"
)
+
"Z"
,
"datasources"
:
dictlist
,
}
return
json
.
dumps
(
data
,
sort_keys
=
False
)
def
get_response
(
params
,
data
):
tic
=
time
.
time
()
fname
=
"resifws-statistics"
headers
=
{
"Content-type"
:
"text/plain"
}
if
params
[
"format"
]
==
"text"
:
response
=
make_response
(
records_to_text
(
params
,
data
),
headers
)
response
=
make_response
(
records_to_text
(
params
,
data
))
response
.
headers
[
"Content-type"
]
=
"text/plain"
elif
params
[
"format"
]
==
"csv"
:
headers
=
{
"Content-Disposition"
:
f
"attachment; filename=
{
fname
}
.csv"
}
response
=
make_response
(
records_to_text
(
params
,
data
,
","
),
headers
)
response
.
headers
[
"Content-type"
]
=
"text/csv"
elif
params
[
"format"
]
==
"json"
:
headers
=
{
"Content-type"
:
"application/json"
}
response
=
make_response
(
json
.
dumps
(
records_to_dictlist
(
params
,
data
),
sort_keys
=
False
),
headers
)
response
=
make_response
(
records_to_json
(
params
,
data
))
response
.
headers
[
"Content-type"
]
=
"application/json"
logging
.
debug
(
f
"Response built in
{
tictac
(
tic
)
}
seconds."
)
return
response
...
...
@@ -246,18 +243,12 @@ def map_requests(params, data):
geodata
[
"requests"
]
=
[
0
for
i
in
range
(
0
,
len
(
geodata
[
"name"
]))]
if
params
[
"year"
]:
years
=
sorted
(
params
[
"year"
].
split
(
","
))
else
:
years
=
[
str
(
datetime
.
now
().
year
)]
for
year
,
code
,
requests
in
data
:
if
str
(
year
)
in
years
:
country
=
codes
.
get
(
code
)
if
country
:
for
i
,
name
in
enumerate
(
geodata
[
"name"
]):
if
country
.
lower
()
in
name
:
geodata
[
"requests"
][
i
]
+=
requests
for
code
,
requests
in
data
:
country
=
codes
.
get
(
code
)
if
country
:
for
i
,
name
in
enumerate
(
geodata
[
"name"
]):
if
country
.
lower
()
in
name
:
geodata
[
"requests"
][
i
]
+=
requests
my_palette
=
(
"#f2f2f2"
,)
+
Viridis256
# my_palette = ("#f2f2f2",) + tuple(reversed(Plasma256))
...
...
@@ -268,7 +259,7 @@ def map_requests(params, data):
TOOLS
=
"save, pan, wheel_zoom, box_zoom, reset"
plot
=
figure
(
title
=
f
"
""
Number of requests to the RESIF data center
in
{
" and "
.
join
(
years
)
}
""
"
,
title
=
f
"Number of requests to the RESIF data center"
,
tools
=
TOOLS
,
plot_width
=
1200
,
plot_height
=
800
,
...
...
apps/root.py
View file @
b212081f
...
...
@@ -74,6 +74,10 @@ def check_parameters(params):
return
error_param
(
params
,
Error
.
REQUEST
+
str
(
params
[
"request"
]))
params
[
"request"
]
=
params
[
"request"
].
lower
()
# country="all" has no meaning in list
if
"all"
in
params
[
"country"
]
and
params
[
"country"
]
!=
"all"
:
return
error_param
(
params
,
Error
.
COUNTRY_ALL
+
str
(
params
[
"country"
]))
# output parameter validation
if
not
is_valid_output
(
params
[
"format"
]):
return
error_param
(
params
,
Error
.
OUTPUT
+
str
(
params
[
"format"
]))
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment