Commit 8a66106f authored by Jerome Touvier's avatar Jerome Touvier
Browse files

total request number with country option

parent f14a21d6
Pipeline #54832 failed with stage
in 22 seconds
......@@ -38,10 +38,9 @@ def is_like_or_equal(params, key):
return " OR ".join(subquery)
def sql_common_string(params, begin=""):
s = begin
def sql_common_string(params):
# network, station, location, channel parameters
s = f"""{s} WHERE ({is_like_or_equal(params, "network")})"""
s = f"""WHERE ({is_like_or_equal(params, "network")})"""
if params["station"] != "*":
s = f"""{s} AND ({is_like_or_equal(params, "station")})"""
if params["location"] != "*" and params["request"] != "storage":
......@@ -49,6 +48,14 @@ def sql_common_string(params, begin=""):
if params["channel"] != "*":
s = f"""{s} AND ({is_like_or_equal(params, "channel")})"""
if params["request"] != "storage":
if "seedlink" in params["media"] and "dataselect" in params["media"]:
s = f"{s} AND (protocol = 'seedlink' OR protocol = 'dataselect')"
elif "seedlink" in params["media"]:
s = f"{s} AND protocol = 'seedlink'"
elif "dataselect" in params["media"]:
s = f"{s} AND protocol = 'dataselect'"
# starttime, endtime parameters
if params["start"]:
start = datetime.strftime(params["start"], "%Y-%m-%d")
......@@ -65,52 +72,39 @@ def sql_request(params):
if params["request"] == "storage":
columns = "year, network, station, channel, quality, type"
s = f"SELECT DISTINCT ON ({columns}) size, {columns}, date FROM dataholdings"
s = sql_common_string(params, s)
s = f"{s} AND channel is not NULL"
where = sql_common_string(params)
s = f"{s} {where} AND channel is not NULL"
if params["type"] != "all":
s = f"""{s} AND (type = '{params["type"]}')"""
if params["year"]:
s = f"""{s} AND ({is_like_or_equal(params, "year")})"""
s = f"""{s} ORDER BY {columns}, date DESC"""
return s.replace("?", "_").replace("*", "%")
s = f"{s} ORDER BY {columns}, date DESC"
else:
table = "sent_data_summary_weekly"
if params["request"] == "map":
if params["request"] == "country":
s = f"SELECT 'all' AS country, sum(requests)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}"
elif params["request"] == "map":
s = f"SELECT extract(year from date)::INTEGER as year, country, sum(requests)::BIGINT FROM {table}"
elif params["request"] == "country":
if "all" in params["country"]:
s = f"SELECT sum(requests)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}"
else:
s = f"SELECT country, sum(requests)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}"
elif params["request"] == "send":
s = f"SELECT sum(bytes)::BIGINT FROM {table}"
elif params["request"] == "timeseries":
s = f"SELECT date, country, sum(bytes)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}"
elif params["request"] == "send":
s = f"SELECT sum(bytes)::BIGINT FROM {table}"
s = sql_common_string(params, s)
if "all" not in params["country"]:
s = f"""{s} AND ({is_like_or_equal(params, "country")})"""
if "seedlink" in params["media"] and "dataselect" in params["media"]:
s = f"{s} AND (protocol = 'seedlink' OR protocol = 'dataselect')"
elif "seedlink" in params["media"]:
s = f"{s} AND protocol = 'seedlink'"
elif "dataselect" in params["media"]:
s = f"{s} AND protocol = 'dataselect'"
if params["request"] == "country" and "all" not in params["country"]:
s = f"""{s} GROUP BY country"""
where = sql_common_string(params)
s = f"{s} {where}"
if params["request"] == "country":
tmp = f"SELECT country, sum(requests)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table} {where}"
if params["country"] != "*":
tmp = f"""{tmp} AND ({is_like_or_equal(params, "country")})"""
s = f"{s} UNION {tmp} GROUP BY country ORDER BY 2 DESC" ""
elif params["request"] == "map":
s = f"""{s} GROUP BY country, year"""
s = f"{s} GROUP BY country, year"
elif params["request"] == "timeseries":
if "seedlink" in params["media"] and "dataselect" in params["media"]:
s = f"""{s} GROUP BY date, country ORDER BY date"""
s = f"{s} GROUP BY date, country ORDER BY date"
else:
s = f"""{s} GROUP BY date, country, protocol ORDER BY date"""
return s.replace("?", "_").replace("*", "%")
s = f"{s} GROUP BY date, country, protocol ORDER BY date"
return s.replace("?", "_").replace("*", "%")
def collect_data(params):
......@@ -129,18 +123,13 @@ def collect_data(params):
def format_results(params, data):
data = list(map(list, data))
if params["request"] == "country":
if "all" in params["country"]:
data = [["all", data[0][0], data[0][1]]]
if params["format"] != "csv":
if params["format"] != "csv":
if params["request"] == "country":
for row in data:
if row[1] and row[2]:
row[:] = [row[0], "{:_}".format(row[1]), "{:_}".format(row[2])]
elif params["request"] == "send":
if params["format"] != "csv":
elif params["request"] == "send":
result = data[0][0]
data = [["{:_}".format(result) + f" bytes ({human_readable_size(result)})"]]
......
......@@ -87,9 +87,7 @@ def is_valid_output(output):
def is_valid_country(country):
return (
re.match("[A-Za-z*?]{1,2}$", country) or country == "all" if country else False
)
return re.match("[A-Za-z*?]{1,2}$", country) if country else False
def is_valid_year(year):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment