Merge branch 'update' into 'master'

update

See merge request !9

README.md

@@ -26,7 +26,7 @@ docker run --rm --name ws-statistics -e RUNMODE=production ws-statistics
 
 ## RUNMODE builtin values
 
-  * `production` 
+  * `production`
   * `test`
   * `local`
   * other values map to :

apps/globals.py

@@ -12,13 +12,15 @@ TIMEOUT = 120
 # error message constants
 DOCUMENTATION_URI = "http://ws.resif.fr/resifws/statistics/1/"
 SERVICE = "resifws-statistics"
+VERSION = "1.0.0"
 
 
 class Error:
     UNKNOWN_PARAM = "Unknown query parameter: "
-    MULTI_PARAM = " Multiple entries for query parameter: "
+    MULTI_PARAM = "Multiple entries for query parameter: "
     VALID_PARAM = "Valid parameters."
     START_LATER = "The starttime cannot be later than the endtime: "
+    TOO_LONG_DURATION = "Too many days requested (greater than "
     UNSPECIFIED = "Error processing your request."
     NODATA = "Your query doesn't match any available data."
     TIMEOUT = f"Your query exceeds timeout ({TIMEOUT} seconds)."

apps/output.py

@@ -11,6 +11,17 @@ from apps.utils import error_request
 from apps.utils import tictac
 
 
+SIZE_UNITS = ["B", "KB", "MB", "GB", "TB", "PB", "PB"]
+
+
+def human_readable_size(size_in_bytes):
+    index = 0
+    while size_in_bytes >= 1024 and index < 6:
+        size_in_bytes /= 1024
+        index += 1
+    return f"{round(size_in_bytes, 2)} {SIZE_UNITS[index]}"
+
+
 def is_like_or_equal(params, key):
     """ Builds the condition for the specified key in the "where" clause taking into account lists or wildcards. """
 
@@ -63,14 +74,15 @@ def sql_request(params):
         return s.replace("?", "_").replace("*", "%")
     else:
         if params["request"] == "country":
-            s = f"SELECT country, count(date) FROM {get_table()}"
+            s = f"SELECT country, sum(requests)::INTEGER FROM {get_table()}"
         elif params["request"] == "send":
             s = f"SELECT sum(bytes) FROM {get_table()}"
         elif params["request"] == "timeseries":
-            s = f"""SELECT date, country, sum(bytes), sum(hll_cardinality(clients))::INTEGER FROM {get_table()}"""
+            s = f"SELECT date, country, sum(bytes), sum(hll_cardinality(clients))::INTEGER FROM {get_table()}"
 
         s = sql_common_string(params, s)
-        s = f"""{s} AND ({is_like_or_equal(params, "country")})"""
+        if "all" not in params["country"]:
+            s = f"""{s} AND ({is_like_or_equal(params, "country")})"""
 
         if "seedlink" in params["media"] and "dataselect" in params["media"]:
             s = f"{s} AND (protocol = 'seedlink' OR protocol = 'dataselect')"
@@ -107,22 +119,31 @@ def collect_data(params):
 def sum_results(params, data):
     """ Adds the results from the different media tables. """
     newdata = list()
+
     if params["request"] == "country":
         result = dict()
         for row in data:
             if row[1]:
-                if row[0] in result:
+                if row[0] in result:  # Is this country already there ?
                     result[row[0]] = result[row[0]] + int(row[1])
                 else:
                     result[row[0]] = int(row[1])
+
+        if "all" in params["country"]:
+            result = {"all": sum([int(val) for val in result.values()])}
+
         for key, val in result.items():
-            newdata.append([key, str(val)])
+            newdata.append([key, "{:_}".format(val)])
+
     elif params["request"] == "send":
         result = 0
         for row in data:
             if row[0]:
                 result = result + int(row[0])
-        newdata.append([str(result)])
+        newdata.append(
+            ["{:_}".format(result) + f" bytes ({human_readable_size(result)})"]
+        )
+
     return newdata
 
 
@@ -135,11 +156,11 @@ def get_header(params):
     elif params["request"] == "send":
         if len(params["media"]) == 1 and "all" not in params["media"]:
             if "seedlink" in params["media"]:
-                header = ["SEEDLINK (in bytes)"]
+                header = ["SEEDLINK"]
             elif "dataselect" in params["media"]:
-                header = ["DATASELECT (in bytes)"]
+                header = ["DATASELECT"]
         else:
-            header = ["SEEDLINK and DATASELECT (in bytes)"]
+            header = ["SEEDLINK and DATASELECT"]
     elif params["request"] == "timeseries":
         header = ["time", "country", "bytes", "clients"]
     return header
@@ -200,14 +221,15 @@ def get_output(params):
         data = collect_data(params)
         if data is None:
             return data
+
+        if params["request"] in ("send", "country"):
+            data = sum_results(params, data)
+
         if not data:
             code = params["nodata"]
             return error_request(msg=f"HTTP._{code}_", details=Error.NODATA, code=code)
         logging.info(f"Number of collected rows: {len(data)}")
 
-        if params["request"] in ("send", "country"):
-            data = sum_results(params, data)
-
         data = [[str(val) for val in row] for row in data]
         return get_response(params, data)
     except Exception as ex:

apps/constants.py → apps/parameters.py

-VERSION = "1.0.0"
-
-
 class Parameters:
     def __init__(self):
         self.network = None
@@ -25,6 +22,14 @@ class Parameters:
         self.format = "text"
         self.nodata = "204"
         self.constraints = {
+            "alias": [
+                ("network", "net"),
+                ("station", "sta"),
+                ("location", "loc"),
+                ("channel", "cha"),
+                ("starttime", "start"),
+                ("endtime", "end"),
+            ],
             "booleans": [],
             "floats": [],
             "not_none": ["request"],
@@ -32,13 +37,3 @@ class Parameters:
 
     def todict(self):
         return self.__dict__
-
-
-ALIAS = [
-    ("network", "net"),
-    ("station", "sta"),
-    ("location", "loc"),
-    ("channel", "cha"),
-    ("starttime", "start"),
-    ("endtime", "end"),
-]

apps/root.py

@@ -4,12 +4,11 @@ from multiprocessing import Process, Queue
 
 from flask import request
 
-from apps.constants import ALIAS
-from apps.constants import Parameters
 from apps.globals import Error
 from apps.globals import HTTP
 from apps.globals import TIMEOUT
 from apps.output import get_output
+from apps.parameters import Parameters
 from apps.utils import check_base_parameters
 from apps.utils import check_request
 from apps.utils import error_param
@@ -34,7 +33,6 @@ def check_parameters(params):
 
     country = params["country"].split(",")
     for c in country:
-        logging.debug(c)
         if not is_valid_country(c):
             return error_param(params, Error.COUNTRY + c)
 
@@ -48,9 +46,10 @@ def check_parameters(params):
 
     # media parameter validation
     if params["media"]:
-        if params["media"] == "all":
-            params["media"] = "dataselect,seedlink"
         params["media"] = params["media"].split(",")
+        if "all" in params["media"]:
+            params["media"] = ["dataselect", "seedlink"]
+
         for ind, media in enumerate(params["media"]):
             if not is_valid_media(media):
                 return error_param(params, Error.MEDIA + str(media))
@@ -103,22 +102,9 @@ def checks_get():
     params = Parameters().todict()
 
     # check if the parameters are unknown
-    (p, result) = check_request(params, ALIAS)
+    (p, result) = check_request(params)
     if result["code"] != 200:
         return (p, result)
-
-    # determine selected features
-    params["request"] = tuple(request.args)
-
-    for key, val in params.items():
-        params[key] = request.args.get(key, val)
-
-    for key, alias in ALIAS:
-        if params[key] is None:
-            params[key] = request.args.get(alias, params[alias])
-        else:
-            params[alias] = params[key]
-
     return check_parameters(params)
 
 

apps/utils.py

@@ -4,42 +4,40 @@ import time
 from difflib import SequenceMatcher
 from datetime import datetime, timedelta
 
-from flask import Response, request
+from flask import request, Response
 
-from apps.constants import VERSION
+from apps.globals import DATATYPE
 from apps.globals import DOCUMENTATION_URI
 from apps.globals import Error
+from apps.globals import GRANULARITY
 from apps.globals import HTTP
-from apps.globals import REQUEST
 from apps.globals import MEDIA
 from apps.globals import NODATA_CODE
 from apps.globals import ORDERBY
 from apps.globals import OUTPUT
+from apps.globals import REQUEST
 from apps.globals import SERVICE
 from apps.globals import STRING_FALSE
 from apps.globals import STRING_TRUE
-from apps.globals import DATATYPE
-from apps.globals import GRANULARITY
+from apps.globals import VERSION
 
 
 def is_valid_integer(dimension, mini=0, maxi=sys.maxsize):
     # by default valid for positive integers
     try:
         dimension = int(dimension)
-        if mini <= dimension <= maxi:
-            return True
     except Exception:
         return False
+    return bool(mini <= dimension <= maxi)
 
 
 def is_valid_float(dimension, mini=sys.float_info.epsilon, maxi=sys.float_info.max):
     # by default valid for strictly positive floats
     try:
         dimension = float(dimension)
-        if mini <= dimension <= maxi:
-            return True
     except Exception:
         return False
+    return bool(mini <= dimension <= maxi)
 
 
 def is_valid_datetime(date):
@@ -89,7 +87,9 @@ def is_valid_output(output):
 
 
 def is_valid_country(country):
-    return re.match("[A-Za-z*?]{1,2}$", country) if country else False
+    return (
+        re.match("[A-Za-z*?]{1,2}$", country) or country == "all" if country else False
+    )
 
 
 def is_valid_year(year):
@@ -158,30 +158,38 @@ def error_500(dmesg):
     return error_request(msg=HTTP._500_, details=dmesg, code=500)
 
 
-# check request
-def check_request(params, alias):
-    keys = list(params.keys())
+def check_request(params):
+    # preliminary parameter checks
+
     for key, val in request.args.items():
-        if key not in keys:
-            ratios = [SequenceMatcher(None, key, p).ratio() for p in keys]
-            guess = max([(v, keys[ind]) for ind, v in enumerate(ratios)])
+        # stops at the first unknown parameter meet:
+        if key not in params:
+            ratios = ((SequenceMatcher(None, key, p).ratio(), p) for p in params)
+            guess = max(ratios)
             hint = ". Did you mean " + guess[1] + " ?" if guess[0] > 0.7 else ""
-            return error_param(keys, Error.UNKNOWN_PARAM + key + hint)
+            return error_param(params, Error.UNKNOWN_PARAM + key + hint)
 
         # find nonword chars except :
         # "," for lists "*?" for wildcards and ".:-" for date
         if re.search(r"[^a-zA-Z0-9_,*?.:-]", val):
-            return error_param(keys, Error.CHAR + key)
+            return error_param(params, Error.CHAR + key)
 
-    for key in keys:
+    for key, val in params.items():
         if len(request.args.getlist(key)) > 1:
-            return error_param(keys, Error.MULTI_PARAM + key)
+            return error_param(params, Error.MULTI_PARAM + key)
+        params[key] = request.args.get(key, val)
+
+    for key, alias in params["constraints"]["alias"]:
+        if key in request.args and alias in request.args:
+            return error_param(
+                params, f"{Error.MULTI_PARAM}{key} (and is shorthand {alias})"
+            )
+        if params[key] is None or params[key] == "false":
+            params[key] = request.args.get(alias, params[alias])
+        else:
+            params[alias] = params[key]
 
-    for key in alias:
-        if len([v for v in key if v in request.args]) > 1:
-            return error_param(keys, Error.MULTI_PARAM + " and is shorthand ".join(key))
-
-    return (keys, {"msg": HTTP._200_, "details": Error.VALID_PARAM, "code": 200})
+    return (params, {"msg": HTTP._200_, "details": Error.VALID_PARAM, "code": 200})
 
 
 def check_base_parameters(params, max_days=None):

docs/USAGE_EN.md

@@ -65,7 +65,7 @@ This service provides access to the RESIF-DC statistics.
 ## Date and time formats
 
     YYYY-MM-DDThh:mm:ss[.ssssss] ex. 1997-01-31T12:04:32.123
-    YYYY-MM-DD ex. 1997-01-31 ( a time of 00:00:00 is assumed)
+    YYYY-MM-DD ex. 1997-01-31 (a time of 00:00:00 is assumed)
 
     where:
 

start.py

@@ -3,7 +3,7 @@ import os
 
 from flask import Flask, make_response, render_template
 
-from apps.constants import VERSION
+from apps.globals import VERSION
 from apps.root import output
 from config import Config
 

static/resifws.css

@@ -2,7 +2,7 @@ body {
     margin: auto;
     padding-right: 1em;
     padding-left: 1em;
-    max-width: 75em; 
+    max-width: 75em;
     border-left: 1px solid black;
     border-right: 1px solid black;
     color: black;

templates/doc_en.html

@@ -157,7 +157,7 @@ output-options     ::  [format=<csv|request|sync|text>]
 </ul>
 <h2 id="date-and-time-formats">Date and time formats</h2>
 <pre><code>YYYY-MM-DDThh:mm:ss[.ssssss] ex. 1997-01-31T12:04:32.123
-YYYY-MM-DD ex. 1997-01-31 ( a time of 00:00:00 is assumed)
+YYYY-MM-DD ex. 1997-01-31 (a time of 00:00:00 is assumed)
 
 where:
 

tests/test.py

@@ -7,6 +7,7 @@ sys.path.append("../")
 
 from apps.globals import Error
 from apps.globals import HTTP
+from apps.parameters import Parameters
 from apps.root import check_parameters
 from apps.utils import error_param
 from apps.utils import is_valid_integer
@@ -104,9 +105,7 @@ class MyTest(unittest.TestCase):
         self.assertFalse(is_valid_bool_string("oui"))
 
     def test_parameters(self):
-
-        # ?network=FR&channel=HHZ&starttime=2018-02-12T03:08:02&endtime=2018-02-12T03:10:00&station=CIEL
-        p1 = {}
+        p1 = Parameters().todict()
         p1["network"] = "FR"
         p1["station"] = "CIEL"
         p1["location"] = "*"
@@ -123,12 +122,6 @@ class MyTest(unittest.TestCase):
         p1["limit"] = None
         p1["request"] = ""
         p1["protocol"] = ""
-        p1["nodata"] = "204"
-        p1["constraints"] = {
-            "booleans": [],
-            "floats": [],
-            "not_none": ["request"],
-        }
 
         valid_param = {"msg": HTTP._200_, "details": Error.VALID_PARAM, "code": 200}