Merge branch 'refactoring1' into 'master'

refactoring

See merge request OSUG/RESIF/ws-timeseries!5

apps/extender.py

@@ -33,7 +33,7 @@ def records_to_dictlist(data):
     header = ["Network", "Station", "Location", "Channel"]
     dictlist = []
     for row in data:
-        dictlist.append({k: r for k, r in zip(header, row)})
+        dictlist.append(dict(zip(header, row)))
     return {"Channel codes": dictlist}
 
 

apps/globals.py

@@ -24,7 +24,8 @@ MAX_DATA_POINTS_PROCESSING = int(MAX_POINTS_PROCESSING.replace(",", ""))
 
 # available parameter values
 IMAGE_FORMAT = ("png", "jpeg", "jpg")
-OUTPUT_TIMESERIES = ("ascii", "miniseed", "mseed", "plot", "sac", "slist", "tspair")
+OUTPUT = ("ascii", "miniseed", "mseed", "plot", "sac", "slist", "tspair")
+NODATA_CODE = ("204", "404")
 STRING_TRUE = ("yes", "true", "t", "y", "1", "")
 STRING_FALSE = ("no", "false", "f", "n", "0")
 TAPER_WINDOWS = ("HANNING", "HAMMING", "COSINE")
@@ -60,10 +61,9 @@ DECI_MAX = 16
 
 
 class Error:
-    LEN_ARGS = "Too much arguments in URL."
     UNKNOWN_PARAM = "Unknown query parameter: "
-    MULTI_PARAM = "Duplicate query parameter: "
-    VALID_PARAM = "Valid parameters. "
+    MULTI_PARAM = " Multiple entries for query parameter: "
+    VALID_PARAM = "Valid parameters."
     START_LATER = "The starttime cannot be later than the endtime: "
     TOO_LONG_DURATION = "Too many days requested (greater than "
     TOO_MUCH_DATA = f"The request exceeds the limit of {MAX_POINTS} data points."
@@ -71,14 +71,10 @@ class Error:
         f"Deconvolution is not allowed above {MAX_POINTS_PROCESSING} data points."
     )
     RESPONSE = "Deconvolution can't be performed. Instrumental response not available."
-    UNSPECIFIED = "Error.processing your request."
-    NO_CONNECTION = "No services could be discovered at http://ws.resif.fr.\n\
-                  This could be due to a temporary service outage, an invalid FDSN service address,\n\
-                  an inactive internet connection or a blocking firewall rule."
-    OK_CONNECTION = "Connection OK. "
+    UNSPECIFIED = "Error processing your request."
     NODATA = "Your query doesn't match any available data."
     TIMEOUT = f"Your query exceeds timeout ({TIMEOUT} seconds)."
-    MISSING = "Missing parameter : "
+    MISSING = "Missing parameter: "
     BAD_VAL = " Invalid value: "
     CHAR = "White space(s) or invalid string. Invalid value for: "
     EMPTY = "Empty string. Invalid value for: "
@@ -108,26 +104,22 @@ class Error:
                   Invalid taper value: "
     BP = "Invalid band pass filter value: "
     SCALE_DIVSCALE = "You cannot specify both scale and divscale values."
-    OUTPUT_TIMESERIES = f"Accepted format values are: {OUTPUT_TIMESERIES}." + BAD_VAL
+    OUTPUT_TIMESERIES = f"Accepted format values are: {OUTPUT}." + BAD_VAL
     PLOTS = f"The request exceeds the limit of {MAX_PLOTS} plots. Try to reduce the number of channels requested (network, station, location and channel parameters)."
     PROCESSING = "Your request cannot be processed. Check for value consistency."
     MISSING_OUTPUT = "Missing format parameter. "
-    INVALID_OUTPUT_PARAM = (
-        "The option 'out(put)' is no longer valid. Use 'format' instead."
-    )
-    NO_WILDCARDS = "Wildcards or lists are allowed only with plot or mseed output options (Invalid value for: "
     NO_SELECTION = "Request contains no selections."
+    NO_WILDCARDS = "Wildcards or lists are allowed only with plot or mseed output options (Invalid value for: "
 
 
 class HTTP:
     _200_ = "Successful request. "
-    _202_ = "No data matches the selection. "
+    _204_ = "No data matches the selection. "
     _400_ = "Bad request due to improper value. "
     _401_ = "Authentication is required. "
     _403_ = "Forbidden access. "
-    _404_ = "Page not found. "
+    _404_ = "No data matches the selection. "
     _408_ = "Request exceeds timeout. "
-    _409_ = "Too much data. "
     _413_ = "Request too large. "
     _414_ = "Request URI too large. "
     _500_ = "Internal server error. "

apps/timeseries/constants.py

@@ -6,89 +6,85 @@ from apps.globals import WIDTH
 VERSION = "1.1.1"
 
 
-class Args:
-    NETWORK = ("network", None)
-    STATION = ("station", None)
-    LOCATION = ("location", None)
-    CHANNEL = ("channel", None)
-    STARTTIME = ("starttime", None)
-    ENDTIME = ("endtime", None)
-    NET = ("net", "*")
-    STA = ("sta", "*")
-    LOC = ("loc", "*")
-    CHA = ("cha", "*")
-    START = ("start", None)
-    END = ("end", None)
+class Parameters:
+    def __init__(self):
+        self.network = None
+        self.station = None
+        self.location = None
+        self.channel = None
+        self.starttime = None
+        self.endtime = None
+        self.net = "*"
+        self.sta = "*"
+        self.loc = "*"
+        self.cha = "*"
+        self.start = None
+        self.end = None
+        self.spectrum = "F"
+        self.showtitle = "T"
+        self.showscale = "T"
+        self.monochrome = "F"
+        self.width = WIDTH
+        self.height = HEIGHT
+        self.dpi = DPI
+        self.color = PLOT_COLOR
+        self.correct = "F"
+        self.earthunits = "F"
+        self.demean = "F"
+        self.waterlevel = None
+        self.freqlimits = None
+        self.units = "AUTO"
+        self.imageformat = "png"
+        self.taper = None
+        self.lpfilter = None
+        self.hpfilter = None
+        self.bpfilter = None
+        self.lp = None
+        self.hp = None
+        self.bp = None
+        self.decimate = None
+        self.deci = None
+        self.detrend = "F"
+        self.zerophase = "F"
+        self.envelope = "F"
+        self.diff = "F"
+        self.int = "F"
+        self.scale = None
+        self.divscale = None
+        self.format = None
+        self.nodata = "204"
+        self.constraints = {
+            "booleans": [
+                "spectrum",
+                "showtitle",
+                "showscale",
+                "monochrome",
+                "earthunits",
+                "demean",
+                "detrend",
+                "zerophase",
+                "envelope",
+                "diff",
+                "int",
+            ],
+            "floats": ["lp", "hp", "scale", "divscale"],
+            "not_none": ["start", "end", "format"],
+        }
 
-    SPECTRUM = ("spectrum", "F")
-    SHOWTITLE = ("showtitle", "T")
-    SHOWSCALE = ("showscale", "T")
-    MONOCHROME = ("monochrome", "F")
-    WIDTH = ("width", WIDTH)
-    HEIGHT = ("height", HEIGHT)
-    DPI = ("dpi", DPI)
-    COLOR = ("color", PLOT_COLOR)
+    def todict(self):
+        return self.__dict__
 
-    CORRECT = ("correct", "F")
-    EARTHUNITS = ("earthunits", "F")
-    DEMEAN = ("demean", "F")
-    WATERLEVEL = ("waterlevel", None)
-    FREQLIMITS = ("freqlimits", None)
-    UNITS = ("units", "AUTO")
 
-    TAPER = ("taper", None)
-    LPFILTER = ("lpfilter", None)
-    HPFILTER = ("hpfilter", None)
-    BPFILTER = ("bpfilter", None)
-    LP = ("lp", None)
-    HP = ("hp", None)
-    BP = ("bp", None)
-    DECIMATE = ("decimate", None)
-    DECI = ("deci", None)
-    DETREND = ("detrend", "F")
-    ZEROPHASE = ("zerophase", "F")
-    ENV = ("envelope", "F")
-    DIFF = ("diff", "F")
-    INTEG = ("int", "F")
-    SCALE = ("scale", None)
-    DIVSCALE = ("divscale", None)
-    FORMAT = ("format", None)
-
-
-class Empty:
-    pass
-
-
-PARAMS = [v[0] for (k, v) in Args.__dict__.items() if k not in Empty.__dict__.keys()]
-MAX_PARAMS = len(PARAMS)
-
-ALIAS_PARAMS = [
-    (Args.NETWORK[0], Args.NET[0]),
-    (Args.STATION[0], Args.STA[0]),
-    (Args.LOCATION[0], Args.LOC[0]),
-    (Args.CHANNEL[0], Args.CHA[0]),
-    (Args.STARTTIME[0], Args.START[0]),
-    (Args.ENDTIME[0], Args.END[0]),
-    (Args.LPFILTER[0], Args.LP[0]),
-    (Args.HPFILTER[0], Args.HP[0]),
-    (Args.BPFILTER[0], Args.BP[0]),
-    (Args.DECIMATE[0], Args.DECI[0]),
-    (Args.EARTHUNITS[0], Args.CORRECT[0]),
-]
-
-BOOL_PARAMS = [
-    Args.SPECTRUM[0],
-    Args.SHOWTITLE[0],
-    Args.SHOWSCALE[0],
-    Args.MONOCHROME[0],
-    Args.EARTHUNITS[0],
-    Args.DEMEAN[0],
-    Args.DETREND[0],
-    Args.ZEROPHASE[0],
-    Args.ENV[0],
-    Args.DIFF[0],
-    Args.INTEG[0],
+ALIAS = [
+    ("network", "net"),
+    ("station", "sta"),
+    ("location", "loc"),
+    ("channel", "cha"),
+    ("starttime", "start"),
+    ("endtime", "end"),
+    ("lpfilter", "lp"),
+    ("hpfilter", "hp"),
+    ("bpfilter", "bp"),
+    ("decimate", "deci"),
+    ("earthunits", "correct"),
 ]
-
-NOT_NONE_PARAMS = [Args.START[0], Args.END[0], Args.FORMAT[0]]
-FLOAT_PARAMS = [Args.LP[0], Args.HP[0], Args.SCALE[0], Args.DIVSCALE[0]]

apps/timeseries/model.py

-import logging
-import re
-
-from apps.globals import DECI_MIN, DECI_MAX
-from apps.globals import Error
-from apps.globals import HTTP
-from apps.globals import MAX_DAYS
-from apps.globals import OUTPUT_TIMESERIES
-from apps.globals import TAPER_WINDOWS
-from apps.timeseries.constants import BOOL_PARAMS
-from apps.timeseries.constants import FLOAT_PARAMS
-from apps.timeseries.constants import NOT_NONE_PARAMS
-from apps.utils import check_base_parameters
-from apps.utils import check_timeseriesplots_parameters
-from apps.utils import error_param
-from apps.utils import is_valid_integer
-from apps.utils import is_valid_bpfilter
-from apps.utils import is_valid_output
-from apps.utils import is_valid_taper_window
-
-
-def check_parameters(params):
-
-    try:
-        # Check base parameters
-        (params, error) = check_base_parameters(
-            params, MAX_DAYS, NOT_NONE_PARAMS, BOOL_PARAMS, FLOAT_PARAMS
-        )
-
-        if error["code"] != 200:
-            return (params, error)
-
-        # Check timeseriesplots parameters
-        (params, error) = check_timeseriesplots_parameters(params)
-        if error["code"] != 200:
-            return (params, error)
-
-        # Scaling parameters
-        if params["scale"] is not None and params["divscale"] is not None:
-            return error_param(params, Error.SCALE_DIVSCALE)
-
-        # Decimation parameter validation
-        if params["deci"] is not None:
-            if not is_valid_integer(params["deci"], DECI_MIN, DECI_MAX):
-                return error_param(params, Error.DECI + str(params["deci"]))
-            params["deci"] = int(params["deci"])
-
-        # Filter parameter validations
-        if params["taper"] is not None:
-            params["taper"] = tuple(params["taper"].split(","))
-            if not is_valid_taper_window(params["taper"]):
-                return error_param(params, Error.TAPER + str(params["taper"]))
-            if len(params["taper"]) == 1:
-                params["taper"] = (float(params["taper"][0]), TAPER_WINDOWS[0])
-            else:
-                params["taper"] = (
-                    float(params["taper"][0]),
-                    params["taper"][1].upper(),
-                )
-
-        if params["bp"] is not None:
-            params["bp"] = tuple(re.split("[/,;-]", params["bp"]))
-            if not is_valid_bpfilter(params["bp"]):
-                return error_param(params, Error.BP + str(params["bp"]))
-            params["bp"] = (float(params["bp"][0]), float(params["bp"][1]))
-
-        # Output parameter validations
-        if not is_valid_output(params["format"], OUTPUT_TIMESERIES):
-            return error_param(params, Error.OUTPUT_TIMESERIES + str(params["format"]))
-        params["format"] = params["format"].lower()
-
-        # wildcards or list are allowed only with plot and mseed output options
-        if params["format"] not in ("plot", "mseed", "miniseed"):
-            for key in ("network", "station", "location", "channel"):
-                if re.search(r"[,*?]", params[key]):
-                    return error_param(params, Error.NO_WILDCARDS + key + ").")
-
-        for key, val in params.items():
-            logging.debug(key + ": " + str(val))
-
-        return (params, error)
-
-    except Exception as e:
-        logging.exception(str(e))
-        return (params, {"msg": HTTP._500_, "details": Error.UNSPECIFIED, "code": 500})

apps/timeseries/output.py

@@ -13,15 +13,14 @@ from apps.globals import FROM_CLIENT
 from apps.globals import MAX_DATA_POINTS
 from apps.globals import MAX_DATA_POINTS_PROCESSING
 from apps.globals import MAX_PLOTS
+from apps.utils import error_request
+from apps.utils import overflow_error
 from apps.globals import USER_AGENT_TIMESERIES_INVENTORY
-from apps.utils import error_500
 from apps.utils import get_bounds
 from apps.utils import get_signal
 from apps.utils import get_signal_from_client
 from apps.utils import get_periodogram
 from apps.utils import get_response
-from apps.utils import nodata_error
-from apps.utils import overflow_error
 from apps.utils import remove_response
 from apps.utils import static_plots
 from apps.utils import tictac
@@ -110,35 +109,31 @@ def get_file_type(params):
 
 
 def set_sac_header(params, st):
-    try:
-        for tr in st:
-            stats = tr.stats
-            client = Client(FDSN_CLIENT, user_agent=USER_AGENT_TIMESERIES_INVENTORY)
-            inventory = client.get_stations(
-                network=stats["network"],
-                station=stats["station"],
-                location=stats["location"],
-                channel=stats["channel"],
-                level="channel",
-            )
-            inv_sta = inventory[0][0]
-            inv_cha = inv_sta[0]
-            stats.sac = {}
-            if hasattr(inv_cha, "azimuth"):
-                stats.sac["cmpaz"] = inv_cha.azimuth
-            if hasattr(inv_cha, "dip"):
-                stats.sac["cmpinc"] = inv_cha.dip
-            if hasattr(inv_sta, "latitude"):
-                stats.sac["stla"] = inv_sta.latitude
-            if hasattr(inv_sta, "longitude"):
-                stats.sac["stlo"] = inv_sta.longitude
-            if hasattr(inv_sta, "depth"):
-                stats.sac["stdp"] = inv_sta.depth
-            if hasattr(inv_sta, "elevation"):
-                stats.sac["stel"] = inv_sta.elevation
-    except Exception as ex:
-        logging.exception(str(ex))
-        return error_500(Error.UNSPECIFIED)
+    for tr in st:
+        stats = tr.stats
+        client = Client(FDSN_CLIENT, user_agent=USER_AGENT_TIMESERIES_INVENTORY)
+        inventory = client.get_stations(
+            network=stats["network"],
+            station=stats["station"],
+            location=stats["location"],
+            channel=stats["channel"],
+            level="channel",
+        )
+        inv_sta = inventory[0][0]
+        inv_cha = inv_sta[0]
+        stats.sac = {}
+        if hasattr(inv_cha, "azimuth"):
+            stats.sac["cmpaz"] = inv_cha.azimuth
+        if hasattr(inv_cha, "dip"):
+            stats.sac["cmpinc"] = inv_cha.dip
+        if hasattr(inv_sta, "latitude"):
+            stats.sac["stla"] = inv_sta.latitude
+        if hasattr(inv_sta, "longitude"):
+            stats.sac["stlo"] = inv_sta.longitude
+        if hasattr(inv_sta, "depth"):
+            stats.sac["stdp"] = inv_sta.depth
+        if hasattr(inv_sta, "elevation"):
+            stats.sac["stel"] = inv_sta.elevation
 
 
 def get_file(params, st):
@@ -146,7 +141,7 @@ def get_file(params, st):
 
     The name is built according to the template :
     resifws-timeseries.2018-11-29T10_11_32.000Z.2018-11-29T23_42_56.000Z
-    :param params: Parameters object with url parameters (network, station, ...)
+    :param params: parameter dictionary with url parameters (network, station, ...)
     :param st: obspy stream
     :returns: response_class flask object containing the file
 
@@ -180,7 +175,6 @@ def get_file(params, st):
         return response
     except Exception as ex:
         logging.exception(str(ex))
-        return error_500(Error.UNSPECIFIED)
     finally:
         tmp.close()
     logging.info(f"Response with file created in {tictac(tic)} seconds.")
@@ -189,7 +183,7 @@ def get_file(params, st):
 def get_output(params):
     """Create timeseries plots.
 
-    :param params: Parameters object with url parameters (network, station, ...)
+    :param params: parameter dictionary with url parameters (network, station, ...)
     :returns: static plot(s) or data file
     :raises MemoryError raises memory exception
     :raises ValueError raises value exception
@@ -220,15 +214,19 @@ def get_output(params):
                 return overflow_error(Error.PLOTS)
 
         if params["earthunits"]:
-            tic1 = time.time()
             try:
                 st.attach_response(get_response(params))
             except Exception as ex:
                 logging.debug(str(ex))
-                return nodata_error(Error.RESPONSE)
-            logging.info(f"Attach response in {tictac(tic1)} seconds.")
+                code = params["nodata"]
+                return error_request(
+                    msg=f"HTTP._{code}_", details=Error.RESPONSE, code=code
+                )
 
         st = get_processed_signal(st, params)
+        if not st:
+            code = params["nodata"]
+            return error_request(msg=f"HTTP._{code}_", details=Error.NODATA, code=code)
 
         if params["format"] == "plot":
             response = static_plots(params, st)
@@ -240,11 +238,10 @@ def get_output(params):
         return overflow_error(Error.PROCESSING)
     except Exception as ex:
         logging.exception(str(ex))
-        return error_500(Error.UNSPECIFIED)
     finally:
         if st:
             delta = params["end"] - params["start"]
             logging.debug(f"Period of {delta} with {npoints} data points.")
             if response:
-                bytes = response.headers.get("Content-Length")
-                logging.info(f"{bytes} bytes rendered in {tictac(tic0)} seconds.")
+                nbytes = response.headers.get("Content-Length")
+                logging.info(f"{nbytes} bytes rendered in {tictac(tic0)} seconds.")

apps/timeseries/root.py

 import logging
 import queue
+import re
 from multiprocessing import Process, Queue
 
+from flask import request
+
+from apps.globals import DECI_MIN, DECI_MAX
 from apps.globals import Error
 from apps.globals import HTTP
+from apps.globals import MAX_DAYS
+from apps.globals import TAPER_WINDOWS
 from apps.globals import TIMEOUT
-from apps.timeseries.constants import ALIAS_PARAMS
-from apps.timeseries.constants import Args
-from apps.timeseries.constants import PARAMS
-from apps.timeseries.model import check_parameters
+from apps.timeseries.constants import ALIAS
+from apps.timeseries.constants import Parameters
 from apps.timeseries.output import get_output
+from apps.utils import check_base_parameters
+from apps.utils import check_timeseriesplots_parameters
 from apps.utils import check_request
+from apps.utils import error_param
 from apps.utils import error_request
+from apps.utils import is_valid_integer
+from apps.utils import is_valid_bpfilter
+from apps.utils import is_valid_nodata
+from apps.utils import is_valid_output
+from apps.utils import is_valid_taper_window
+
+
+def check_parameters(params):
+
+    # check base parameters
+    (params, error) = check_base_parameters(params, MAX_DAYS)
+    if error["code"] != 200:
+        return (params, error)
+
+    # timeseriesplots parameters
+    (params, error) = check_timeseriesplots_parameters(params)
+    if error["code"] != 200:
+        return (params, error)
+
+    # scale parameter
+    if params["scale"] is not None and params["divscale"] is not None:
+        return error_param(params, Error.SCALE_DIVSCALE)
+
+    # decimation parameter validation
+    if params["deci"] is not None:
+        if not is_valid_integer(params["deci"], DECI_MIN, DECI_MAX):
+            return error_param(params, Error.DECI + str(params["deci"]))
+        params["deci"] = int(params["deci"])
+
+    # filter parameter validations
+    if params["taper"] is not None:
+        params["taper"] = tuple(params["taper"].split(","))
+        if not is_valid_taper_window(params["taper"]):
+            return error_param(params, Error.TAPER + str(params["taper"]))
+        if len(params["taper"]) == 1:
+            params["taper"] = (float(params["taper"][0]), TAPER_WINDOWS[0])
+        else:
+            params["taper"] = (
+                float(params["taper"][0]),
+                params["taper"][1].upper(),
+            )
+
+    if params["bp"] is not None:
+        params["bp"] = tuple(re.split("[/,;-]", params["bp"]))
+        if not is_valid_bpfilter(params["bp"]):
+            return error_param(params, Error.BP + str(params["bp"]))
+        params["bp"] = (float(params["bp"][0]), float(params["bp"][1]))
+
+    # output parameter validation
+    if not is_valid_output(params["format"]):
+        return error_param(params, Error.OUTPUT_TIMESERIES + str(params["format"]))
+    params["format"] = params["format"].lower()
+
+    # nodata parameter validation
+    if not is_valid_nodata(params["nodata"]):
+        return error_param(params, Error.NODATA_CODE + str(params["nodata"]))
+    params["nodata"] = params["nodata"].lower()
+
+    # wildcards or list are allowed only with plot and mseed output options
+    if params["format"] not in ("plot", "mseed", "miniseed"):
+        for key in ("network", "station", "location", "channel"):
+            if re.search(r"[,*?]", params[key]):
+                return error_param(params, Error.NO_WILDCARDS + key + ").")
+
+    for key, val in params.items():
+        logging.debug(key + ": " + str(val))
+
+    return (params, {"msg": HTTP._200_, "details": Error.VALID_PARAM, "code": 200})
 
 
-def checks_get(request):
+def checks_get():
+
+    # get default parameters
+    params = Parameters().todict()
+
     # check if the parameters are unknown
-    (params, result) = check_request(request, PARAMS, ALIAS_PARAMS)
+    (p, result) = check_request(params, ALIAS)
     if result["code"] != 200:
-        return (params, result)
+        return (p, result)
 
     # determine selected features
-    params = {}
-    params["base_url"] = request.base_url
     params["request"] = tuple(request.args)
-    params["network"] = request.args.get(*Args.NETWORK)
-    params["station"] = request.args.get(*Args.STATION)
-    params["location"] = request.args.get(*Args.LOCATION)
-    params["channel"] = request.args.get(*Args.CHANNEL)
-    params["start"] = request.args.get(*Args.STARTTIME)
-    params["end"] = request.args.get(*Args.ENDTIME)
-
-    if params["network"] is None:
-        params["network"] = request.args.get(*Args.NET)
-    if params["station"] is None:
-        params["station"] = request.args.get(*Args.STA)
-    if params["location"] is None: