Commit 06a90256 authored by Jerome Touvier's avatar Jerome Touvier
Browse files

Merge branch 'update2' into 'master'

update

See merge request !37
parents 705f8cfd 7d047c41
Pipeline #59465 passed with stage
in 39 seconds
......@@ -38,6 +38,12 @@ def is_like_or_equal(params, key):
return " OR ".join(subquery)
def sum_request():
return (
"COALESCE(SUM(requests), HLL_CARDINALITY(HLL_UNION_AGG(requests_set)))::BIGINT"
)
def sql_common_string(params):
# network, station, location, channel parameters
s = f"""WHERE ({is_like_or_equal(params, "network")})"""
......@@ -71,9 +77,9 @@ def sql_common_string(params):
def country_sql_request(params):
table = "sent_data_summary_weekly"
if params["country"] != "all":
s = f"SELECT coalesce(country, 'n/a'), sum(requests)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}"
s = f"SELECT COALESCE(country, 'n/a'), {sum_request()}, HLL_CARDINALITY(HLL_UNION_AGG(clients))::INTEGER FROM {table}"
else:
s = f"SELECT 'all' AS country, sum(requests)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}"
s = f"SELECT 'all' AS country, {sum_request()}, HLL_CARDINALITY(HLL_UNION_AGG(clients))::INTEGER FROM {table}"
s = f"{s} {sql_common_string(params)}"
if params["country"] != "all":
s = f"{s} GROUP BY country ORDER BY 2 DESC"
......@@ -82,7 +88,7 @@ def country_sql_request(params):
def map_sql_request(params):
table = "sent_data_summary_weekly"
s = f"SELECT country, sum(requests)::BIGINT FROM {table}"
s = f"SELECT country, {sum_request()} FROM {table}"
s = f"{s} {sql_common_string(params)} GROUP BY country"
return s
......@@ -90,17 +96,17 @@ def map_sql_request(params):
def send_sql_request(params):
table = "sent_data_summary_weekly"
if params["sum"]:
s = f"""SELECT '{params["network"]}' AS network, 'all' AS station, sum(bytes)::BIGINT AS size FROM {table}"""
s = f"""SELECT '{params["network"]}' AS network, 'all' AS station, SUM(bytes)::BIGINT AS size FROM {table}"""
s = f"{s} {sql_common_string(params)}"
elif params["ranklimit"]:
s = f"""SELECT network, station, size FROM
(SELECT *, DENSE_RANK() OVER (PARTITION BY network ORDER BY size DESC) AS rank FROM
(SELECT network, station, sum(bytes)::BIGINT AS size FROM {table} {sql_common_string(params)} GROUP BY network, station
(SELECT network, station, SUM(bytes)::BIGINT AS size FROM {table} {sql_common_string(params)} GROUP BY network, station
) AS table1
) AS table2
WHERE rank <= {params["ranklimit"]};"""
else:
s = f"SELECT network, station, sum(bytes)::BIGINT AS size FROM {table} {sql_common_string(params)}"
s = f"SELECT network, station, SUM(bytes)::BIGINT AS size FROM {table} {sql_common_string(params)}"
s = f"{s} GROUP BY network, station ORDER BY network, size DESC, station"
return s
......@@ -108,8 +114,8 @@ def send_sql_request(params):
def storage_sql_request(params):
table = "dataholdings"
columns = "year, network, station, channel, quality"
s = f"SELECT DISTINCT ON ({columns}) {columns}, date, sum(size)::BIGINT FROM {table}"
s = f"{s} {sql_common_string(params)} AND channel is not NULL"
s = f"SELECT DISTINCT ON ({columns}) {columns}, date, SUM(size)::BIGINT FROM {table}"
s = f"{s} {sql_common_string(params)} AND channel IS NOT NULL"
if params["type"]:
s = f"""{s} AND ({is_like_or_equal(params, "type")})"""
s = s.replace("buffer", "bud")
......@@ -122,9 +128,9 @@ def storage_sql_request(params):
def timeseries_sql_request(params):
table = "sent_data_summary_weekly"
if params["country"] != "all":
s = f"SELECT date, country, sum(bytes)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}"
s = f"SELECT date, country, SUM(bytes)::BIGINT, HLL_CARDINALITY(HLL_UNION_AGG(clients))::INTEGER FROM {table}"
else:
s = f"SELECT date, 'all' as country, sum(bytes)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}"
s = f"SELECT date, 'all' AS country, SUM(bytes)::BIGINT, HLL_CARDINALITY(HLL_UNION_AGG(clients))::INTEGER FROM {table}"
s = f"{s} {sql_common_string(params)}"
if params["country"] != "all":
s = f"{s} GROUP BY date, country ORDER BY date"
......
......@@ -42,7 +42,7 @@ def is_valid_float(dimension, mini=sys.float_info.epsilon, maxi=sys.float_info.m
def is_valid_datetime(date):
for df in ("%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%dT%H:%M:%S.%f"):
try:
return datetime.strptime(date, df)
return datetime.strptime(date.replace("Z", ""), df)
except (ValueError, TypeError):
pass
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment