Merge branch 'refactoring' into 'master'

code refactoring

See merge request !5

.gitlab-ci.yml

@@ -23,4 +23,4 @@ build:
   script:
     - $IMAGE_BUILD --file $DOCKERFILE $BUILD_ARG --tag $IMAGE_NAME .
     - $IMAGE_PUSH  $IMAGE_NAME $IMAGE_NAME
-
+    - echo $CI_COMMIT_SHORT_SHA > ./static/commit.txt

apps/constants.py

@@ -34,6 +34,4 @@ ALIAS = [
     ("endtime", "end"),
 ]
 
-BOOLEANS = []
-FLOATS = []
 NOT_NONE = ["request"]

apps/globals.py

@@ -5,11 +5,7 @@ DATATYPE = ("all", "bud", "validated")
 GRANULARITY = ("year", "month")
 ORDERBY = ("nslc", "time", "country", "protocol")
 OUTPUT = ("csv", "request", "sync", "text")
-MEDIA = (
-    "all",
-    "seedlink",
-    "dataselect",
-)
+MEDIA = ("all", "seedlink", "dataselect")
 NODATA_CODE = ("204", "404")
 TIMEOUT = 120
 
@@ -46,7 +42,7 @@ class Error:
     OUTPUT = f"Accepted output values are: {OUTPUT}." + BAD_VAL
     NODATA_CODE = f"Accepted nodata values are: {NODATA_CODE}." + BAD_VAL
     NO_SELECTION = "Request contains no selections."
-    NO_WILDCARDS = "Wildcards or list are not allowed." + BAD_VAL
+    NO_WILDCARDS = "Wildcards or lists are not allowed." + BAD_VAL
 
 
 class HTTP:

apps/model.py

-import logging
-
-from apps.globals import Error, HTTP
-from apps.constants import NOT_NONE
-from apps.constants import BOOLEANS
-from apps.utils import check_base_parameters
-from apps.utils import error_param
-from apps.utils import is_valid_country
-from apps.utils import is_valid_nodata
-from apps.utils import is_valid_orderby
-from apps.utils import is_valid_output
-from apps.utils import is_valid_media
-from apps.utils import is_valid_request
-from apps.utils import is_valid_datatype
-from apps.utils import is_valid_granularity
-from apps.utils import is_valid_year
-
-
-def check_parameters(params):
-
-    try:
-        # Check base parameters.
-        (params, error) = check_base_parameters(params, NOT_NONE, BOOLEANS)
-        if error["code"] != 200:
-            return (params, error)
-
-        country = params["country"].split(",")
-        for c in country:
-            logging.debug(c)
-            if not is_valid_country(c):
-                return error_param(params, Error.COUNTRY + c)
-
-        # orderby parameter validation
-        if params["orderby"] is not None:
-            if not is_valid_orderby(params["orderby"]):
-                return error_param(params, Error.ORDERBY + str(params["orderby"]))
-            params["orderby"] = params["orderby"].lower()
-        else:
-            params["orderby"] = "nslc"
-
-        # media parameter validation
-        if params["media"]:
-            if params["media"] == "all":
-                params["media"] = "dataselect,seedlink"
-            params["media"] = params["media"].split(",")
-            for ind, media in enumerate(params["media"]):
-                if not is_valid_media(media):
-                    return error_param(params, Error.MEDIA + str(media))
-                params["media"][ind] = media.lower()
-
-        # year parameter validation
-        if params["year"]:
-            years = params["year"].split(",")
-            for ind, year in enumerate(years):
-                if not is_valid_year(year):
-                    return error_param(params, Error.YEAR + str(year))
-
-        # type parameter validation
-        if params["type"]:
-            if not is_valid_datatype(params["type"]):
-                return error_param(params, Error.DATATYPE + str(params["type"]))
-            params["type"] = params["type"].lower()
-
-        # request parameter validation
-        if params["request"]:
-            if not is_valid_request(params["request"]):
-                return error_param(params, Error.REQUEST + str(params["request"]))
-            params["request"] = params["request"].lower()
-
-        # granularity parameter validation
-        if params["granularity"]:
-            if not is_valid_granularity(params["granularity"]):
-                return error_param(
-                    params, Error.GRANULARITY + str(params["granularity"])
-                )
-            params["granularity"] = params["granularity"].lower()
-
-        # output parameter validation
-        if not is_valid_output(params["format"]):
-            return error_param(params, Error.OUTPUT + str(params["format"]))
-        params["format"] = params["format"].lower()
-
-        # nodata parameter validation
-        if not is_valid_nodata(params["nodata"]):
-            return error_param(params, Error.NODATA_CODE + str(params["nodata"]))
-        params["nodata"] = params["nodata"].lower()
-
-        for key, val in params.items():
-            logging.debug(key + ": " + str(val))
-
-        return (params, error)
-
-    except Exception as e:
-        logging.exception(str(e))
-        return (params, {"msg": HTTP._500_, "details": Error.PROCESSING, "code": 500})

apps/output.py

@@ -176,7 +176,6 @@ def records_to_text(params, data, sep=" "):
 
 def get_response(params, data):
     tic = time.time()
-    data = [[str(val) for val in row] for row in data]
     fname = "resifws-statistics"
     headers = {"Content-type": "text/plain"}
     if params["format"] == "text":
@@ -193,8 +192,8 @@ def get_response(params, data):
     return response
 
 
-def get_statistics(params):
-    """Statistics output (csv, request, sync, text)
+def get_output(params):
+    """ Statistics output (csv, request, sync, text)
     :params: parameters
     :returns: text or csv with data statistics"""
 
@@ -209,11 +208,12 @@ def get_statistics(params):
             return error_request(msg=f"HTTP._{code}_", details=Error.NODATA, code=code)
 
         nrows = len(data)
-        logging.debug(f"Number of collected rows: {nrows}")
+        logging.info(f"Number of collected rows: {nrows}")
 
         if params["request"] in ("send", "country"):
             data = sum_results(params, data)
 
+        data = [[str(val) for val in row] for row in data]
         return get_response(params, data)
     except Exception as ex:
         logging.exception(str(ex))

apps/root.py

@@ -5,14 +5,97 @@ from copy import copy
 from multiprocessing import Process, Queue
 
 from apps.constants import ALIAS
+from apps.constants import NOT_NONE
 from apps.constants import parameters
-from apps.model import check_parameters
-from apps.output import get_statistics
 from apps.globals import Error
 from apps.globals import HTTP
 from apps.globals import TIMEOUT
+from apps.output import get_output
+from apps.utils import check_base_parameters
 from apps.utils import check_request
+from apps.utils import error_param
 from apps.utils import error_request
+from apps.utils import is_valid_country
+from apps.utils import is_valid_datatype
+from apps.utils import is_valid_granularity
+from apps.utils import is_valid_media
+from apps.utils import is_valid_nodata
+from apps.utils import is_valid_orderby
+from apps.utils import is_valid_output
+from apps.utils import is_valid_request
+from apps.utils import is_valid_year
+
+
+def check_parameters(params):
+
+    # check base parameters
+    (params, error) = check_base_parameters(params, not_none=NOT_NONE)
+    if error["code"] != 200:
+        return (params, error)
+
+    country = params["country"].split(",")
+    for c in country:
+        logging.debug(c)
+        if not is_valid_country(c):
+            return error_param(params, Error.COUNTRY + c)
+
+    # orderby parameter validation
+    if params["orderby"] is not None:
+        if not is_valid_orderby(params["orderby"]):
+            return error_param(params, Error.ORDERBY + str(params["orderby"]))
+        params["orderby"] = params["orderby"].lower()
+    else:
+        params["orderby"] = "nslc"
+
+    # media parameter validation
+    if params["media"]:
+        if params["media"] == "all":
+            params["media"] = "dataselect,seedlink"
+        params["media"] = params["media"].split(",")
+        for ind, media in enumerate(params["media"]):
+            if not is_valid_media(media):
+                return error_param(params, Error.MEDIA + str(media))
+            params["media"][ind] = media.lower()
+
+    # year parameter validation
+    if params["year"]:
+        years = params["year"].split(",")
+        for ind, year in enumerate(years):
+            if not is_valid_year(year):
+                return error_param(params, Error.YEAR + str(year))
+
+    # type parameter validation
+    if params["type"]:
+        if not is_valid_datatype(params["type"]):
+            return error_param(params, Error.DATATYPE + str(params["type"]))
+        params["type"] = params["type"].lower()
+
+    # request parameter validation
+    if params["request"]:
+        if not is_valid_request(params["request"]):
+            return error_param(params, Error.REQUEST + str(params["request"]))
+        params["request"] = params["request"].lower()
+
+    # granularity parameter validation
+    if params["granularity"]:
+        if not is_valid_granularity(params["granularity"]):
+            return error_param(params, Error.GRANULARITY + str(params["granularity"]))
+        params["granularity"] = params["granularity"].lower()
+
+    # output parameter validation
+    if not is_valid_output(params["format"]):
+        return error_param(params, Error.OUTPUT + str(params["format"]))
+    params["format"] = params["format"].lower()
+
+    # nodata parameter validation
+    if not is_valid_nodata(params["nodata"]):
+        return error_param(params, Error.NODATA_CODE + str(params["nodata"]))
+    params["nodata"] = params["nodata"].lower()
+
+    for key, val in params.items():
+        logging.debug(key + ": " + str(val))
+
+    return (params, {"msg": HTTP._200_, "details": Error.VALID_PARAM, "code": 200})
 
 
 def checks_get(request):
@@ -52,7 +135,7 @@ def output(request):
         if result["code"] == 200:
 
             def put_response(q):
-                q.put(get_statistics(pdic))
+                q.put(get_output(pdic))
 
             q = Queue()
             process = Process(target=put_response, args=(q,))

apps/utils.py

@@ -6,6 +6,7 @@ from datetime import datetime, timedelta
 
 from flask import Response, request
 
+from apps.constants import VERSION
 from apps.globals import Error
 from apps.globals import HTTP
 from apps.globals import REQUEST
@@ -18,8 +19,6 @@ from apps.globals import STRING_TRUE
 from apps.globals import DATATYPE
 from apps.globals import GRANULARITY
 
-from apps.constants import VERSION
-
 
 def is_valid_integer(dimension, mini=0, maxi=sys.maxsize):
     # by default valid for positive integers
@@ -77,14 +76,6 @@ def is_valid_channel(channel):
     return re.match("([A-Za-z0-9*?]{1,3})$", channel) if channel else False
 
 
-def is_valid_country(country):
-    return re.match("[A-Za-z*?]{1,2}$", country) if country else False
-
-
-def is_valid_year(year):
-    return re.match("[0-9]{4}$", year) if year else False
-
-
 def is_valid_bool_string(string):
     if string is None:
         return False
@@ -95,6 +86,14 @@ def is_valid_output(output):
     return output.lower() in OUTPUT if output else False
 
 
+def is_valid_country(country):
+    return re.match("[A-Za-z*?]{1,2}$", country) if country else False
+
+
+def is_valid_year(year):
+    return re.match("[0-9]{4}$", year) if year else False
+
+
 def is_valid_orderby(orderby):
     return orderby.lower() in ORDERBY if orderby else False
 
@@ -179,7 +178,9 @@ def check_request(request, params, alias):
     return (keys, {"msg": HTTP._200_, "details": Error.VALID_PARAM, "code": 200})
 
 
-def check_base_parameters(params, not_none=None, booleans=None, floats=None):
+def check_base_parameters(
+    params, max_days=None, not_none=None, booleans=None, floats=None
+):
 
     # Search for missing mandatory parameters
     if not_none is not None:
@@ -259,6 +260,9 @@ def check_base_parameters(params, not_none=None, booleans=None, floats=None):
             end = str(params["end"])
             return error_param(params, Error.START_LATER + start + " > " + end)
 
+        if max_days and params["end"] - params["start"] > timedelta(days=max_days):
+            return error_param(params, Error.TOO_LONG_DURATION + f"{max_days} days).")
+
     # Search for empty selection
     # if params["network"] == params["station"] == params["channel"] == "*":
     #    return error_param(params, Error.NO_SELECTION)

start.py

@@ -37,13 +37,26 @@ def statistics_wadl():
     return response
 
 
-@app.route("/version")
+@app.route("/version", strict_slashes=False)
 def version():
     response = make_response(VERSION)
     response.headers["Content-Type"] = "text/plain"
     return response
 
 
+@app.route("/commit", strict_slashes=False)
+def commit():
+    try:
+        with open("./static/commit.txt") as fc:
+            COMMIT_SHORT_SHA = fc.readline()
+    except Exception:
+        COMMIT_SHORT_SHA = "unspecified"
+        pass
+    response = make_response(COMMIT_SHORT_SHA)
+    response.headers["Content-Type"] = "text/plain"
+    return response
+
+
 @app.route("/")
 @app.route("/local=fr")
 def statistics_doc():

tests/test.py

@@ -7,7 +7,7 @@ sys.path.append("../")
 
 from apps.globals import Error
 from apps.globals import HTTP
-from apps.model import check_parameters
+from apps.root import check_parameters
 from apps.utils import error_param
 from apps.utils import is_valid_integer
 from apps.utils import is_valid_float