Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
OSUG
RESIF
ws-statistics
Commits
06a90256
Commit
06a90256
authored
Feb 08, 2021
by
Jerome Touvier
Browse files
Merge branch 'update2' into 'master'
update See merge request
!37
parents
705f8cfd
7d047c41
Changes
2
Hide whitespace changes
Inline
Side-by-side
apps/output.py
View file @
06a90256
...
...
@@ -38,6 +38,12 @@ def is_like_or_equal(params, key):
return
" OR "
.
join
(
subquery
)
def
sum_request
():
return
(
"COALESCE(SUM(requests), HLL_CARDINALITY(HLL_UNION_AGG(requests_set)))::BIGINT"
)
def
sql_common_string
(
params
):
# network, station, location, channel parameters
s
=
f
"""WHERE (
{
is_like_or_equal
(
params
,
"network"
)
}
)"""
...
...
@@ -71,9 +77,9 @@ def sql_common_string(params):
def
country_sql_request
(
params
):
table
=
"sent_data_summary_weekly"
if
params
[
"country"
]
!=
"all"
:
s
=
f
"SELECT
coalesce
(country, 'n/a'), sum
(
request
s)::BIGINT, hll_cardinality(hll_union_agg
(clients))::INTEGER FROM
{
table
}
"
s
=
f
"SELECT
COALESCE
(country, 'n/a'),
{
sum
_
request
()
}
, HLL_CARDINALITY(HLL_UNION_AGG
(clients))::INTEGER FROM
{
table
}
"
else
:
s
=
f
"SELECT 'all' AS country, sum
(
request
s)::BIGINT, hll_cardinality(hll_union_agg
(clients))::INTEGER FROM
{
table
}
"
s
=
f
"SELECT 'all' AS country,
{
sum
_
request
()
}
, HLL_CARDINALITY(HLL_UNION_AGG
(clients))::INTEGER FROM
{
table
}
"
s
=
f
"
{
s
}
{
sql_common_string
(
params
)
}
"
if
params
[
"country"
]
!=
"all"
:
s
=
f
"
{
s
}
GROUP BY country ORDER BY 2 DESC"
...
...
@@ -82,7 +88,7 @@ def country_sql_request(params):
def
map_sql_request
(
params
):
table
=
"sent_data_summary_weekly"
s
=
f
"SELECT country, sum
(
request
s)::BIGINT
FROM
{
table
}
"
s
=
f
"SELECT country,
{
sum
_
request
()
}
FROM
{
table
}
"
s
=
f
"
{
s
}
{
sql_common_string
(
params
)
}
GROUP BY country"
return
s
...
...
@@ -90,17 +96,17 @@ def map_sql_request(params):
def
send_sql_request
(
params
):
table
=
"sent_data_summary_weekly"
if
params
[
"sum"
]:
s
=
f
"""SELECT '
{
params
[
"network"
]
}
' AS network, 'all' AS station,
sum
(bytes)::BIGINT AS size FROM
{
table
}
"""
s
=
f
"""SELECT '
{
params
[
"network"
]
}
' AS network, 'all' AS station,
SUM
(bytes)::BIGINT AS size FROM
{
table
}
"""
s
=
f
"
{
s
}
{
sql_common_string
(
params
)
}
"
elif
params
[
"ranklimit"
]:
s
=
f
"""SELECT network, station, size FROM
(SELECT *, DENSE_RANK() OVER (PARTITION BY network ORDER BY size DESC) AS rank FROM
(SELECT network, station,
sum
(bytes)::BIGINT AS size FROM
{
table
}
{
sql_common_string
(
params
)
}
GROUP BY network, station
(SELECT network, station,
SUM
(bytes)::BIGINT AS size FROM
{
table
}
{
sql_common_string
(
params
)
}
GROUP BY network, station
) AS table1
) AS table2
WHERE rank <=
{
params
[
"ranklimit"
]
}
;"""
else
:
s
=
f
"SELECT network, station,
sum
(bytes)::BIGINT AS size FROM
{
table
}
{
sql_common_string
(
params
)
}
"
s
=
f
"SELECT network, station,
SUM
(bytes)::BIGINT AS size FROM
{
table
}
{
sql_common_string
(
params
)
}
"
s
=
f
"
{
s
}
GROUP BY network, station ORDER BY network, size DESC, station"
return
s
...
...
@@ -108,8 +114,8 @@ def send_sql_request(params):
def
storage_sql_request
(
params
):
table
=
"dataholdings"
columns
=
"year, network, station, channel, quality"
s
=
f
"SELECT DISTINCT ON (
{
columns
}
)
{
columns
}
, date,
sum
(size)::BIGINT FROM
{
table
}
"
s
=
f
"
{
s
}
{
sql_common_string
(
params
)
}
AND channel
is not
NULL"
s
=
f
"SELECT DISTINCT ON (
{
columns
}
)
{
columns
}
, date,
SUM
(size)::BIGINT FROM
{
table
}
"
s
=
f
"
{
s
}
{
sql_common_string
(
params
)
}
AND channel
IS NOT
NULL"
if
params
[
"type"
]:
s
=
f
"""
{
s
}
AND (
{
is_like_or_equal
(
params
,
"type"
)
}
)"""
s
=
s
.
replace
(
"buffer"
,
"bud"
)
...
...
@@ -122,9 +128,9 @@ def storage_sql_request(params):
def
timeseries_sql_request
(
params
):
table
=
"sent_data_summary_weekly"
if
params
[
"country"
]
!=
"all"
:
s
=
f
"SELECT date, country,
sum
(bytes)::BIGINT,
hll_cardinality(hll_union_agg
(clients))::INTEGER FROM
{
table
}
"
s
=
f
"SELECT date, country,
SUM
(bytes)::BIGINT,
HLL_CARDINALITY(HLL_UNION_AGG
(clients))::INTEGER FROM
{
table
}
"
else
:
s
=
f
"SELECT date, 'all'
as
country,
sum
(bytes)::BIGINT,
hll_cardinality(hll_union_agg
(clients))::INTEGER FROM
{
table
}
"
s
=
f
"SELECT date, 'all'
AS
country,
SUM
(bytes)::BIGINT,
HLL_CARDINALITY(HLL_UNION_AGG
(clients))::INTEGER FROM
{
table
}
"
s
=
f
"
{
s
}
{
sql_common_string
(
params
)
}
"
if
params
[
"country"
]
!=
"all"
:
s
=
f
"
{
s
}
GROUP BY date, country ORDER BY date"
...
...
apps/utils.py
View file @
06a90256
...
...
@@ -42,7 +42,7 @@ def is_valid_float(dimension, mini=sys.float_info.epsilon, maxi=sys.float_info.m
def
is_valid_datetime
(
date
):
for
df
in
(
"%Y-%m-%d"
,
"%Y-%m-%dT%H:%M:%S"
,
"%Y-%m-%dT%H:%M:%S.%f"
):
try
:
return
datetime
.
strptime
(
date
,
df
)
return
datetime
.
strptime
(
date
.
replace
(
"Z"
,
""
)
,
df
)
except
(
ValueError
,
TypeError
):
pass
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment