Merge branch 'update' into 'master'

code refactoring

See merge request OSUG/RESIF/ws-timeseries!7

apps/globals.py

@@ -7,6 +7,7 @@ DATA_MOUNT_POINT = os.getenv("DATADIR")
 USER_AGENT_TIMESERIES = "resifws-timeseries"
 USER_AGENT_TIMESERIES_INVENTORY = "resifws-timeseries_inventory"
 USER_AGENT_TIMESERIESPLOT = "resifws-timeseriesplot"
+VERSION = "1.0.0"
 
 
 # limitations
@@ -62,7 +63,7 @@ DECI_MAX = 16
 
 class Error:
     UNKNOWN_PARAM = "Unknown query parameter: "
-    MULTI_PARAM = " Multiple entries for query parameter: "
+    MULTI_PARAM = "Multiple entries for query parameter: "
     VALID_PARAM = "Valid parameters."
     START_LATER = "The starttime cannot be later than the endtime: "
     TOO_LONG_DURATION = "Too many days requested (greater than "
@@ -105,6 +106,7 @@ class Error:
     BP = "Invalid band pass filter value: "
     SCALE_DIVSCALE = "You cannot specify both scale and divscale values."
     OUTPUT_TIMESERIES = f"Accepted format values are: {OUTPUT}." + BAD_VAL
+    NODATA_CODE = f"Accepted nodata values are: {NODATA_CODE}." + BAD_VAL
     PLOTS = f"The request exceeds the limit of {MAX_PLOTS} plots. Try to reduce the number of channels requested (network, station, location and channel parameters)."
     PROCESSING = "Your request cannot be processed. Check for value consistency."
     MISSING_OUTPUT = "Missing format parameter. "

apps/timeseries/output.py

@@ -13,14 +13,14 @@ from apps.globals import FROM_CLIENT
 from apps.globals import MAX_DATA_POINTS
 from apps.globals import MAX_DATA_POINTS_PROCESSING
 from apps.globals import MAX_PLOTS
-from apps.utils import error_request
-from apps.utils import overflow_error
 from apps.globals import USER_AGENT_TIMESERIES_INVENTORY
+from apps.utils import error_request
 from apps.utils import get_bounds
 from apps.utils import get_signal
 from apps.utils import get_signal_from_client
 from apps.utils import get_periodogram
 from apps.utils import get_response
+from apps.utils import overflow_error
 from apps.utils import remove_response
 from apps.utils import static_plots
 from apps.utils import tictac
@@ -32,9 +32,9 @@ def get_processed_signal(st, params):
     tic = time.time()
     for n, tr in enumerate(st):
         logging.debug(f"Processing trace {n}...")
-        for item in params["request"]:
-            # Processings are applied only once and in the order given by the list params["request"].
-            # Do to this with boolean parameters (e.g. demean) we check if they are both true and in the request.
+        for item in params["args"]:
+            # Processings are applied only once and in the order given by the list params["args"].
+            # To do this with boolean parameters (e.g. demean) we check if they are both true and in the request.
             if params["earthunits"] and item in ("earthunits", "correct"):
                 remove_response(tr, params)
             elif params["demean"] and item == "demean":
@@ -108,7 +108,7 @@ def get_file_type(params):
     return (file_type, file_ext)
 
 
-def set_sac_header(params, st):
+def set_sac_header(st):
     for tr in st:
         stats = tr.stats
         client = Client(FDSN_CLIENT, user_agent=USER_AGENT_TIMESERIES_INVENTORY)
@@ -165,7 +165,7 @@ def get_file(params, st):
         headers = {"Content-Disposition": "attachment; filename=" + fname}
         tmp = NamedTemporaryFile(delete=True)
         if file_ext == ".sac":
-            set_sac_header(params, st)
+            set_sac_header(st)
         st.write(tmp.name, format=file_type)
         response = make_response((tmp.read(), headers))
         if file_ext == ".csv":

apps/timeseries/constants.py → apps/timeseries/parameters.py

@@ -3,8 +3,6 @@ from apps.globals import HEIGHT
 from apps.globals import PLOT_COLOR
 from apps.globals import WIDTH
 
-VERSION = "1.1.1"
-
 
 class Parameters:
     def __init__(self):
@@ -20,17 +18,17 @@ class Parameters:
         self.cha = "*"
         self.start = None
         self.end = None
-        self.spectrum = "F"
-        self.showtitle = "T"
-        self.showscale = "T"
-        self.monochrome = "F"
+        self.spectrum = "false"
+        self.showtitle = "true"
+        self.showscale = "true"
+        self.monochrome = "false"
         self.width = WIDTH
         self.height = HEIGHT
         self.dpi = DPI
         self.color = PLOT_COLOR
-        self.correct = "F"
-        self.earthunits = "F"
-        self.demean = "F"
+        self.correct = "false"
+        self.earthunits = "false"
+        self.demean = "false"
         self.waterlevel = None
         self.freqlimits = None
         self.units = "AUTO"
@@ -44,21 +42,35 @@ class Parameters:
         self.bp = None
         self.decimate = None
         self.deci = None
-        self.detrend = "F"
-        self.zerophase = "F"
-        self.envelope = "F"
-        self.diff = "F"
-        self.int = "F"
+        self.detrend = "false"
+        self.zerophase = "false"
+        self.envelope = "false"
+        self.diff = "false"
+        self.int = "false"
         self.scale = None
         self.divscale = None
         self.format = None
         self.nodata = "204"
         self.constraints = {
+            "alias": [
+                ("network", "net"),
+                ("station", "sta"),
+                ("location", "loc"),
+                ("channel", "cha"),
+                ("starttime", "start"),
+                ("endtime", "end"),
+                ("lpfilter", "lp"),
+                ("hpfilter", "hp"),
+                ("bpfilter", "bp"),
+                ("decimate", "deci"),
+                ("earthunits", "correct"),
+            ],
             "booleans": [
                 "spectrum",
                 "showtitle",
                 "showscale",
                 "monochrome",
+                "correct",
                 "earthunits",
                 "demean",
                 "detrend",
@@ -73,18 +85,3 @@ class Parameters:
 
     def todict(self):
         return self.__dict__
-
-
-ALIAS = [
-    ("network", "net"),
-    ("station", "sta"),
-    ("location", "loc"),
-    ("channel", "cha"),
-    ("starttime", "start"),
-    ("endtime", "end"),
-    ("lpfilter", "lp"),
-    ("hpfilter", "hp"),
-    ("bpfilter", "bp"),
-    ("decimate", "deci"),
-    ("earthunits", "correct"),
-]

apps/timeseries/root.py

@@ -11,16 +11,15 @@ from apps.globals import HTTP
 from apps.globals import MAX_DAYS
 from apps.globals import TAPER_WINDOWS
 from apps.globals import TIMEOUT
-from apps.timeseries.constants import ALIAS
-from apps.timeseries.constants import Parameters
+from apps.timeseries.parameters import Parameters
 from apps.timeseries.output import get_output
 from apps.utils import check_base_parameters
-from apps.utils import check_timeseriesplots_parameters
 from apps.utils import check_request
+from apps.utils import check_timeseriesplots_parameters
 from apps.utils import error_param
 from apps.utils import error_request
-from apps.utils import is_valid_integer
 from apps.utils import is_valid_bpfilter
+from apps.utils import is_valid_integer
 from apps.utils import is_valid_nodata
 from apps.utils import is_valid_output
 from apps.utils import is_valid_taper_window
@@ -95,22 +94,10 @@ def checks_get():
     params = Parameters().todict()
 
     # check if the parameters are unknown
-    (p, result) = check_request(params, ALIAS)
+    (p, result) = check_request(params)
     if result["code"] != 200:
         return (p, result)
-
-    # determine selected features
-    params["request"] = tuple(request.args)
-
-    for key, val in params.items():
-        params[key] = request.args.get(key, val)
-
-    for key, alias in ALIAS:
-        if params[key] is None:
-            params[key] = request.args.get(alias, params[alias])
-        else:
-            params[alias] = params[key]
-
+    params["args"] = tuple(request.args)
     return check_parameters(params)
 
 

apps/timeseriesplot/constants.py → apps/timeseriesplot/parameters.py

@@ -3,8 +3,6 @@ from apps.globals import HEIGHT
 from apps.globals import PLOT_COLOR
 from apps.globals import WIDTH
 
-VERSION = "1.1.1"
-
 
 class Parameters:
     def __init__(self):
@@ -20,30 +18,41 @@ class Parameters:
         self.cha = "*"
         self.start = None
         self.end = None
-        self.iplot = "F"
-        self.spectrum = "F"
-        self.showtitle = "T"
-        self.showscale = "T"
-        self.monochrome = "F"
+        self.iplot = "false"
+        self.spectrum = "false"
+        self.showtitle = "true"
+        self.showscale = "true"
+        self.monochrome = "false"
         self.width = WIDTH
         self.height = HEIGHT
         self.dpi = DPI
         self.color = PLOT_COLOR
-        self.correct = "F"
-        self.earthunits = "F"
-        self.demean = "F"
+        self.correct = "false"
+        self.earthunits = "false"
+        self.demean = "false"
         self.waterlevel = None
         self.freqlimits = None
         self.units = "AUTO"
+        self.imageformat = "png"
         self.format = "png"
         self.nodata = "204"
         self.constraints = {
+            "alias": [
+                ("network", "net"),
+                ("station", "sta"),
+                ("location", "loc"),
+                ("channel", "cha"),
+                ("starttime", "start"),
+                ("endtime", "end"),
+                ("earthunits", "correct"),
+            ],
             "booleans": [
                 "iplot",
                 "spectrum",
                 "showtitle",
                 "showscale",
                 "monochrome",
+                "correct",
                 "earthunits",
                 "demean",
             ],
@@ -53,14 +62,3 @@ class Parameters:
 
     def todict(self):
         return self.__dict__
-
-
-ALIAS = [
-    ("network", "net"),
-    ("station", "sta"),
-    ("location", "loc"),
-    ("channel", "cha"),
-    ("starttime", "start"),
-    ("endtime", "end"),
-    ("earthunits", "correct"),
-]

apps/timeseriesplot/root.py

@@ -8,12 +8,11 @@ from apps.globals import Error
 from apps.globals import HTTP
 from apps.globals import MAX_DAYS
 from apps.globals import TIMEOUT
-from apps.timeseriesplot.constants import ALIAS
-from apps.timeseriesplot.constants import Parameters
+from apps.timeseriesplot.parameters import Parameters
 from apps.timeseriesplot.output import get_output
 from apps.utils import check_base_parameters
-from apps.utils import check_timeseriesplots_parameters
 from apps.utils import check_request
+from apps.utils import check_timeseriesplots_parameters
 from apps.utils import error_param
 from apps.utils import error_request
 from apps.utils import is_valid_nodata
@@ -48,22 +47,10 @@ def checks_get():
     params = Parameters().todict()
 
     # check if the parameters are unknown
-    (p, result) = check_request(params, ALIAS)
+    (p, result) = check_request(params)
     if result["code"] != 200:
         return (p, result)
-
-    # determine selected features
-    params["request"] = tuple(request.args)
-
-    for key, val in params.items():
-        params[key] = request.args.get(key, val)
-
-    for key, alias in ALIAS:
-        if params[key] is None:
-            params[key] = request.args.get(alias, params[alias])
-        else:
-            params[alias] = params[key]
-
+    params["imageformat"] = params["format"]
     return check_parameters(params)
 
 

apps/utils.py

@@ -7,7 +7,7 @@ import time
 from difflib import SequenceMatcher
 from datetime import datetime, timedelta
 
-from flask import current_app, make_response, Response, request
+from flask import current_app, make_response, request, Response
 
 import matplotlib.pyplot as plt
 from matplotlib.ticker import FuncFormatter, MaxNLocator
@@ -40,31 +40,29 @@ from apps.globals import STRING_FALSE
 from apps.globals import STRING_TRUE
 from apps.globals import TAPER_WINDOWS
 from apps.globals import UNITS
+from apps.globals import VERSION
 from apps.globals import USER_AGENT_TIMESERIES
 from apps.globals import WIDTH_MIN, WIDTH_MAX
 from apps.globals import WL_MIN, WL_MAX
 from apps.globals import XTICKS_ROTATION
-from apps.timeseries.constants import VERSION
 
 
 def is_valid_integer(dimension, mini=0, maxi=sys.maxsize):
     # by default valid for positive integers
     try:
         dimension = int(dimension)
-        if mini <= dimension <= maxi:
-            return True
     except Exception:
         return False
+    return bool(mini <= dimension <= maxi)
 
 
 def is_valid_float(dimension, mini=sys.float_info.epsilon, maxi=sys.float_info.max):
     # by default valid for strictly positive floats
     try:
         dimension = float(dimension)
-        if mini <= dimension <= maxi:
-            return True
     except Exception:
         return False
+    return bool(mini <= dimension <= maxi)
 
 
 def is_valid_datetime(date):
@@ -162,13 +160,12 @@ def error_request(msg="", details="", code=500):
     request_date = datetime.utcnow().strftime("%Y-%b-%d %H:%M:%S UTC")
     message_error = f"""Error {code}: {msg}\n
 {details}\n
-Usage details are available from {DOCUMENTATION_URI}\n
 Request:
 {request.url}\n
 Request Submitted:
 {request_date}\n
 Service version:
-Service: {SERVICE}  version:{VERSION}"""
+version:{VERSION}"""
     return Response(message_error, status=code, mimetype="text/plain")
 
 
@@ -182,30 +179,38 @@ def error_500(dmesg):
     return error_request(msg=HTTP._500_, details=dmesg, code=500)
 
 
-# check request
-def check_request(params, alias):
-    keys = list(params.keys())
+def check_request(params):
+    # preliminary parameter checks
+
     for key, val in request.args.items():
-        if key not in keys:
-            ratios = [SequenceMatcher(None, key, p).ratio() for p in keys]
-            guess = max([(v, keys[ind]) for ind, v in enumerate(ratios)])
+        # stops at the first unknown parameter meet:
+        if key not in params:
+            ratios = ((SequenceMatcher(None, key, p).ratio(), p) for p in params)
+            guess = max(ratios)
             hint = ". Did you mean " + guess[1] + " ?" if guess[0] > 0.7 else ""
-            return error_param(keys, Error.UNKNOWN_PARAM + key + hint)
+            return error_param(params, Error.UNKNOWN_PARAM + key + hint)
 
         # find nonword chars except :
         # "," for lists "*?" for wildcards and ".:-" for date
         if re.search(r"[^a-zA-Z0-9_,*?.:-]", val):
-            return error_param(keys, Error.CHAR + key)
+            return error_param(params, Error.CHAR + key)
 
-    for key in keys:
+    for key, val in params.items():
         if len(request.args.getlist(key)) > 1:
-            return error_param(keys, Error.MULTI_PARAM + key)
-
-    for key in alias:
-        if len([v for v in key if v in request.args]) > 1:
-            return error_param(keys, Error.MULTI_PARAM + " and is shorthand ".join(key))
+            return error_param(params, Error.MULTI_PARAM + key)
+        params[key] = request.args.get(key, val)
 
-    return (keys, {"msg": HTTP._200_, "details": Error.VALID_PARAM, "code": 200})
+    for key, alias in params["constraints"]["alias"]:
+        if key in request.args and alias in request.args:
+            return error_param(
+                params, f"{Error.MULTI_PARAM}{key} (and is shorthand {alias})"
+            )
+        if params[key] is None or params[key] == "false":
+            params[key] = request.args.get(alias, params[alias])
+        else:
+            params[alias] = params[key]
+    params["spectrum"] = "false"  # disable spectrum
+    return (params, {"msg": HTTP._200_, "details": Error.VALID_PARAM, "code": 200})
 
 
 def check_base_parameters(params, max_days=None):
@@ -287,7 +292,7 @@ def check_base_parameters(params, max_days=None):
     # Search for empty selection
     if params["network"] == params["station"] == params["channel"] == "*":
         return error_param(params, Error.NO_SELECTION)
-
+    logging.debug(params)
     return (params, {"msg": HTTP._200_, "details": Error.VALID_PARAM, "code": 200})
 
 
@@ -389,7 +394,7 @@ def is_open_file(paths):
     """Embargoed files are removed from the list of paths."""
 
     if not paths:
-        return
+        return None
 
     values = ", ".join(f"""('{j}', '{k}')""" for j, k in paths)
     select = f"""SELECT path, name
@@ -426,7 +431,7 @@ def get_signal(params):
     res = extend(args=[net, sta, loc, cha, "--format=tuple"])
     logging.debug(res)
     if not res:
-        return
+        return None
 
     filenames = list()
     delta = params["end"] - params["start"]

docs/USAGE_TIMESERIESPLOT_EN.md

@@ -61,7 +61,7 @@ The four parameters (network, station, location, channel) determine channels of
 
   - Wildcards: the question mark __?__ represents any single character, while the asterisk __*__ represents zero or more characters.
 
-  - List: multiple items may also be retrieved using a comma separated list. Wildcards may be included in the list. 
+  - List: multiple items may also be retrieved using a comma separated list. Wildcards may be included in the list.
 
 For example, with channel codes: channel=EH?,BHZ
 
@@ -141,7 +141,7 @@ Notes :
 ## Date and time formats
 
     YYYY-MM-DDThh:mm:ss[.ssssss] ex. 1997-01-31T12:04:32.123
-    YYYY-MM-DD ex. 1997-01-31 ( a time of 00:00:00 is assumed)
+    YYYY-MM-DD ex. 1997-01-31 (a time of 00:00:00 is assumed)
 
     where:
 

docs/USAGE_TIMESERIES_EN.md

@@ -74,7 +74,7 @@ The four parameters (network, station, location, channel) determine channels of
 
   - Wildcards: the question mark __?__ represents any single character, while the asterisk __*__ represents zero or more characters.
 
-  - List: multiple items may also be retrieved using a comma separated list. Wildcards may be included in the list. 
+  - List: multiple items may also be retrieved using a comma separated list. Wildcards may be included in the list.
 
 For example, with channel codes: channel=EH?,BHZ
 
@@ -172,7 +172,7 @@ Notes :
 ## Date and time formats
 
     YYYY-MM-DDThh:mm:ss[.ssssss] ex. 1997-01-31T12:04:32.123
-    YYYY-MM-DD ex. 1997-01-31 ( a time of 00:00:00 is assumed)
+    YYYY-MM-DD ex. 1997-01-31 (a time of 00:00:00 is assumed)
 
     where:
 

start_timeseries.py

@@ -3,7 +3,7 @@ import os
 
 from flask import Flask, make_response, render_template
 
-from apps.timeseries.constants import VERSION
+from apps.globals import VERSION
 from apps.timeseries.root import timeseries
 
 try:

start_timeseriesplot.py

@@ -3,7 +3,7 @@ import os
 
 from flask import Flask, make_response, render_template
 
-from apps.timeseriesplot.constants import VERSION
+from apps.globals import VERSION
 from apps.timeseriesplot.root import timeseriesplot
 
 try:

static/resifws.css

@@ -2,7 +2,7 @@ body {
     margin: auto;
     padding-right: 1em;
     padding-left: 1em;
-    max-width: 75em; 
+    max-width: 75em;
     border-left: 1px solid black;
     border-right: 1px solid black;
     color: black;

templates/timeseries_doc_en.html

@@ -392,7 +392,7 @@ default values are uppercase</code></pre>
 </table>
 <h2 id="date-and-time-formats">Date and time formats</h2>
 <pre><code>YYYY-MM-DDThh:mm:ss[.ssssss] ex. 1997-01-31T12:04:32.123
-YYYY-MM-DD ex. 1997-01-31 ( a time of 00:00:00 is assumed)
+YYYY-MM-DD ex. 1997-01-31 (a time of 00:00:00 is assumed)
 
 where:
 

templates/timeseries_wadl.xml

@@ -23,7 +23,7 @@
                     <representation mediaType="application/xml"/>
                 </response>
             </method>
-        </resource>        
+        </resource>
         <resource path="query">
             <method href="#query"/>
         </resource>>

templates/timeseriesplot_doc_en.html

@@ -294,7 +294,7 @@ default values are uppercase</code></pre>
 </table>
 <h2 id="date-and-time-formats">Date and time formats</h2>
 <pre><code>YYYY-MM-DDThh:mm:ss[.ssssss] ex. 1997-01-31T12:04:32.123
-YYYY-MM-DD ex. 1997-01-31 ( a time of 00:00:00 is assumed)
+YYYY-MM-DD ex. 1997-01-31 (a time of 00:00:00 is assumed)
 
 where:
 

templates/timeseriesplot_wadl.xml

@@ -23,7 +23,7 @@
                     <representation mediaType="application/xml"/>
                 </response>
             </method>
-        </resource>        
+        </resource>
         <resource path="query">
             <method href="#query"/>
         </resource>

tests/test_timeseries.py

@@ -7,6 +7,7 @@ sys.path.append("../")
 
 from apps.globals import Error
 from apps.globals import HTTP
+from apps.timeseries.parameters import Parameters
 from apps.timeseries.root import check_parameters
 from apps.utils import error_param
 from apps.utils import is_valid_integer
@@ -111,63 +112,43 @@ class MyTest(unittest.TestCase):
         self.assertFalse(is_valid_bool_string("oui"))
 
     def test_parameters(self):
-        self.maxDiff = None
-
-        # ?network=FR&channel=HHZ&starttime=2018-02-12T03:08:02&endtime=2018-02-12T03:10:00&station=CIEL
-        p1 = {}
+        p1 = Parameters().todict()
         p1["network"] = "FR"
         p1["station"] = "CIEL"
         p1["location"] = "*"
         p1["channel"] = "HHZ"
         p1["start"] = "2018-02-12T03:08:02"
         p1["end"] = "2018-02-12T04:08:02"
-        p1["showtitle"] = "T"
-        p1["monochrome"] = "F"
-        p1["correct"] = "F"
-        p1["earthunits"] = "F"
+        p1["showtitle"] = "true"
+        p1["monochrome"] = "false"
+        p1["correct"] = "false"
+        p1["earthunits"] = "false"
         p1["imageformat"] = "png"
-        p1["spectrum"] = "T"
+        p1["spectrum"] = "true"
         p1["color"] = "B22222"
         p1["width"] = 1200
         p1["height"] = 400
         p1["dpi"] = 100
-        p1["showscale"] = "T"
+        p1["showscale"] = "true"
         p1["waterlevel"] = None
         p1["freqlimits"] = None
         p1["units"] = "AUTO"
-        p1["demean"] = "F"
-        p1["detrend"] = "F"
-        p1["zerophase"] = "F"
-        p1["envelope"] = "F"
-        p1["diff"]