Commit b212081f authored by Jerome Touvier's avatar Jerome Touvier
Browse files

Merge branch 'update' into 'master'

fix map and timeseries requests

See merge request !30
parents ad673ee2 9be42246
Pipeline #56054 passed with stage
in 40 seconds
......@@ -45,6 +45,7 @@ class Error:
NODATA_CODE = f"Accepted nodata values are: {NODATA_CODE}." + BAD_VAL
NO_SELECTION = "Request contains no selections."
NO_WILDCARDS = "Wildcards or lists are not allowed." + BAD_VAL
COUNTRY_ALL = f"The special option country=all has no meaning in list." + BAD_VAL
class HTTP:
......
......@@ -57,7 +57,8 @@ def sql_common_string(params):
s = f"{s} AND protocol = 'dataselect'"
if params["request"] in ("send", "timeseries"):
s = f"""{s} AND ({is_like_or_equal(params, "country")})"""
if "all" not in params["country"] and params["country"] != "*":
s = f"""{s} AND ({is_like_or_equal(params, "country")})"""
# starttime, endtime parameters
if params["start"]:
......@@ -88,7 +89,7 @@ def sql_request(params):
if params["request"] == "country":
s = f"SELECT 'all' AS country, sum(requests)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}"
elif params["request"] == "map":
s = f"SELECT extract(year from date)::INTEGER AS year, country, sum(requests)::BIGINT FROM {table}"
s = f"SELECT country, sum(requests)::BIGINT FROM {table}"
elif params["request"] == "timeseries":
if params["country"] == "all":
s = f"SELECT date, 'all' as country, sum(bytes)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}"
......@@ -108,7 +109,7 @@ def sql_request(params):
tmp = tmp + f""" AND ({is_like_or_equal(params, "country")})"""
s = f"{s} UNION {tmp} GROUP BY country ORDER BY 2 DESC"
elif params["request"] == "map":
s = f"{s} GROUP BY country, year"
s = f"{s} GROUP BY country"
elif params["request"] == "timeseries":
if params["country"] == "all":
s = f"{s} GROUP BY date ORDER BY date"
......@@ -138,7 +139,7 @@ def collect_data(params):
def format_results(params, data):
data = list(map(list, data))
if params["format"] != "csv":
if params["format"] not in ["csv", "json"]:
if params["request"] == "country":
for row in data:
if row[1] and row[2]:
......@@ -193,35 +194,31 @@ def records_to_text(params, data, sep=" "):
return text
def records_to_dictlist(params, data):
def records_to_json(params, data):
""" Create json output """
dictlist = list()
header = get_header(params)
header = [h.lower() for h in header]
for row in data:
dictlist.append(dict(zip(header, row)))
return {
header = [h.lower() for h in get_header(params)]
dictlist = [dict(zip(header, row)) for row in data]
data = {
"created": datetime.utcnow().isoformat(timespec="seconds") + "Z",
"datasources": dictlist,
}
return json.dumps(data, sort_keys=False)
def get_response(params, data):
tic = time.time()
fname = "resifws-statistics"
headers = {"Content-type": "text/plain"}
if params["format"] == "text":
response = make_response(records_to_text(params, data), headers)
response = make_response(records_to_text(params, data))
response.headers["Content-type"] = "text/plain"
elif params["format"] == "csv":
headers = {"Content-Disposition": f"attachment; filename={fname}.csv"}
response = make_response(records_to_text(params, data, ","), headers)
response.headers["Content-type"] = "text/csv"
elif params["format"] == "json":
headers = {"Content-type": "application/json"}
response = make_response(
json.dumps(records_to_dictlist(params, data), sort_keys=False), headers
)
response = make_response(records_to_json(params, data))
response.headers["Content-type"] = "application/json"
logging.debug(f"Response built in {tictac(tic)} seconds.")
return response
......@@ -246,18 +243,12 @@ def map_requests(params, data):
geodata["requests"] = [0 for i in range(0, len(geodata["name"]))]
if params["year"]:
years = sorted(params["year"].split(","))
else:
years = [str(datetime.now().year)]
for year, code, requests in data:
if str(year) in years:
country = codes.get(code)
if country:
for i, name in enumerate(geodata["name"]):
if country.lower() in name:
geodata["requests"][i] += requests
for code, requests in data:
country = codes.get(code)
if country:
for i, name in enumerate(geodata["name"]):
if country.lower() in name:
geodata["requests"][i] += requests
my_palette = ("#f2f2f2",) + Viridis256
# my_palette = ("#f2f2f2",) + tuple(reversed(Plasma256))
......@@ -268,7 +259,7 @@ def map_requests(params, data):
TOOLS = "save, pan, wheel_zoom, box_zoom, reset"
plot = figure(
title=f"""Number of requests to the RESIF data center in {" and ".join(years)}""",
title=f"Number of requests to the RESIF data center",
tools=TOOLS,
plot_width=1200,
plot_height=800,
......
......@@ -74,6 +74,10 @@ def check_parameters(params):
return error_param(params, Error.REQUEST + str(params["request"]))
params["request"] = params["request"].lower()
# country="all" has no meaning in list
if "all" in params["country"] and params["country"] != "all":
return error_param(params, Error.COUNTRY_ALL + str(params["country"]))
# output parameter validation
if not is_valid_output(params["format"]):
return error_param(params, Error.OUTPUT + str(params["format"]))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment