Merge branch 'update' into 'master'

fix map and timeseries requests

See merge request !30

apps/globals.py

@@ -45,6 +45,7 @@ class Error:
     NODATA_CODE = f"Accepted nodata values are: {NODATA_CODE}." + BAD_VAL
     NO_SELECTION = "Request contains no selections."
     NO_WILDCARDS = "Wildcards or lists are not allowed." + BAD_VAL
+    COUNTRY_ALL = f"The special option country=all has no meaning in list." + BAD_VAL
 
 
 class HTTP:

apps/output.py

@@ -57,7 +57,8 @@ def sql_common_string(params):
             s = f"{s} AND protocol = 'dataselect'"
 
     if params["request"] in ("send", "timeseries"):
-        s = f"""{s} AND ({is_like_or_equal(params, "country")})"""
+        if "all" not in params["country"] and params["country"] != "*":
+            s = f"""{s} AND ({is_like_or_equal(params, "country")})"""
 
     # starttime, endtime parameters
     if params["start"]:
@@ -88,7 +89,7 @@ def sql_request(params):
         if params["request"] == "country":
             s = f"SELECT 'all' AS country, sum(requests)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}"
         elif params["request"] == "map":
-            s = f"SELECT extract(year from date)::INTEGER AS year, country, sum(requests)::BIGINT FROM {table}"
+            s = f"SELECT country, sum(requests)::BIGINT FROM {table}"
         elif params["request"] == "timeseries":
             if params["country"] == "all":
                 s = f"SELECT date, 'all' as country, sum(bytes)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}"
@@ -108,7 +109,7 @@ def sql_request(params):
                 tmp = tmp + f""" AND ({is_like_or_equal(params, "country")})"""
             s = f"{s} UNION {tmp} GROUP BY country ORDER BY 2 DESC"
         elif params["request"] == "map":
-            s = f"{s} GROUP BY country, year"
+            s = f"{s} GROUP BY country"
         elif params["request"] == "timeseries":
             if params["country"] == "all":
                 s = f"{s} GROUP BY date ORDER BY date"
@@ -138,7 +139,7 @@ def collect_data(params):
 
 def format_results(params, data):
     data = list(map(list, data))
-    if params["format"] != "csv":
+    if params["format"] not in ["csv", "json"]:
         if params["request"] == "country":
             for row in data:
                 if row[1] and row[2]:
@@ -193,35 +194,31 @@ def records_to_text(params, data, sep=" "):
     return text
 
 
-def records_to_dictlist(params, data):
+def records_to_json(params, data):
     """ Create json output """
 
-    dictlist = list()
-    header = get_header(params)
-    header = [h.lower() for h in header]
-    for row in data:
-        dictlist.append(dict(zip(header, row)))
-    return {
+    header = [h.lower() for h in get_header(params)]
+    dictlist = [dict(zip(header, row)) for row in data]
+    data = {
         "created": datetime.utcnow().isoformat(timespec="seconds") + "Z",
         "datasources": dictlist,
     }
+    return json.dumps(data, sort_keys=False)
 
 
 def get_response(params, data):
     tic = time.time()
     fname = "resifws-statistics"
-    headers = {"Content-type": "text/plain"}
     if params["format"] == "text":
-        response = make_response(records_to_text(params, data), headers)
+        response = make_response(records_to_text(params, data))
+        response.headers["Content-type"] = "text/plain"
     elif params["format"] == "csv":
         headers = {"Content-Disposition": f"attachment; filename={fname}.csv"}
         response = make_response(records_to_text(params, data, ","), headers)
         response.headers["Content-type"] = "text/csv"
     elif params["format"] == "json":
-        headers = {"Content-type": "application/json"}
-        response = make_response(
-            json.dumps(records_to_dictlist(params, data), sort_keys=False), headers
-        )
+        response = make_response(records_to_json(params, data))
+        response.headers["Content-type"] = "application/json"
     logging.debug(f"Response built in {tictac(tic)} seconds.")
     return response
 
@@ -246,18 +243,12 @@ def map_requests(params, data):
 
     geodata["requests"] = [0 for i in range(0, len(geodata["name"]))]
 
-    if params["year"]:
-        years = sorted(params["year"].split(","))
-    else:
-        years = [str(datetime.now().year)]
-
-    for year, code, requests in data:
-        if str(year) in years:
-            country = codes.get(code)
-            if country:
-                for i, name in enumerate(geodata["name"]):
-                    if country.lower() in name:
-                        geodata["requests"][i] += requests
+    for code, requests in data:
+        country = codes.get(code)
+        if country:
+            for i, name in enumerate(geodata["name"]):
+                if country.lower() in name:
+                    geodata["requests"][i] += requests
 
     my_palette = ("#f2f2f2",) + Viridis256
     # my_palette = ("#f2f2f2",) + tuple(reversed(Plasma256))
@@ -268,7 +259,7 @@ def map_requests(params, data):
 
     TOOLS = "save, pan, wheel_zoom, box_zoom, reset"
     plot = figure(
-        title=f"""Number of requests to the RESIF data center in {" and ".join(years)}""",
+        title=f"Number of requests to the RESIF data center",
         tools=TOOLS,
         plot_width=1200,
         plot_height=800,

apps/root.py

@@ -74,6 +74,10 @@ def check_parameters(params):
             return error_param(params, Error.REQUEST + str(params["request"]))
         params["request"] = params["request"].lower()
 
+    # country="all" has no meaning in list
+    if "all" in params["country"] and params["country"] != "all":
+        return error_param(params, Error.COUNTRY_ALL + str(params["country"]))
+
     # output parameter validation
     if not is_valid_output(params["format"]):
         return error_param(params, Error.OUTPUT + str(params["format"]))