Commit e8156e52 authored by Jerome Touvier's avatar Jerome Touvier
Browse files

Merge branch 'update' into 'master'

code refactoring

See merge request OSUG/RESIF/ws-timeseries!7
parents 3503e2d1 eeda26c0
......@@ -7,6 +7,7 @@ DATA_MOUNT_POINT = os.getenv("DATADIR")
USER_AGENT_TIMESERIES = "resifws-timeseries"
USER_AGENT_TIMESERIES_INVENTORY = "resifws-timeseries_inventory"
USER_AGENT_TIMESERIESPLOT = "resifws-timeseriesplot"
VERSION = "1.0.0"
# limitations
......@@ -62,7 +63,7 @@ DECI_MAX = 16
class Error:
UNKNOWN_PARAM = "Unknown query parameter: "
MULTI_PARAM = " Multiple entries for query parameter: "
MULTI_PARAM = "Multiple entries for query parameter: "
VALID_PARAM = "Valid parameters."
START_LATER = "The starttime cannot be later than the endtime: "
TOO_LONG_DURATION = "Too many days requested (greater than "
......@@ -105,6 +106,7 @@ class Error:
BP = "Invalid band pass filter value: "
SCALE_DIVSCALE = "You cannot specify both scale and divscale values."
OUTPUT_TIMESERIES = f"Accepted format values are: {OUTPUT}." + BAD_VAL
NODATA_CODE = f"Accepted nodata values are: {NODATA_CODE}." + BAD_VAL
PLOTS = f"The request exceeds the limit of {MAX_PLOTS} plots. Try to reduce the number of channels requested (network, station, location and channel parameters)."
PROCESSING = "Your request cannot be processed. Check for value consistency."
MISSING_OUTPUT = "Missing format parameter. "
......
......@@ -13,14 +13,14 @@ from apps.globals import FROM_CLIENT
from apps.globals import MAX_DATA_POINTS
from apps.globals import MAX_DATA_POINTS_PROCESSING
from apps.globals import MAX_PLOTS
from apps.utils import error_request
from apps.utils import overflow_error
from apps.globals import USER_AGENT_TIMESERIES_INVENTORY
from apps.utils import error_request
from apps.utils import get_bounds
from apps.utils import get_signal
from apps.utils import get_signal_from_client
from apps.utils import get_periodogram
from apps.utils import get_response
from apps.utils import overflow_error
from apps.utils import remove_response
from apps.utils import static_plots
from apps.utils import tictac
......@@ -32,9 +32,9 @@ def get_processed_signal(st, params):
tic = time.time()
for n, tr in enumerate(st):
logging.debug(f"Processing trace {n}...")
for item in params["request"]:
# Processings are applied only once and in the order given by the list params["request"].
# Do to this with boolean parameters (e.g. demean) we check if they are both true and in the request.
for item in params["args"]:
# Processings are applied only once and in the order given by the list params["args"].
# To do this with boolean parameters (e.g. demean) we check if they are both true and in the request.
if params["earthunits"] and item in ("earthunits", "correct"):
remove_response(tr, params)
elif params["demean"] and item == "demean":
......@@ -108,7 +108,7 @@ def get_file_type(params):
return (file_type, file_ext)
def set_sac_header(params, st):
def set_sac_header(st):
for tr in st:
stats = tr.stats
client = Client(FDSN_CLIENT, user_agent=USER_AGENT_TIMESERIES_INVENTORY)
......@@ -165,7 +165,7 @@ def get_file(params, st):
headers = {"Content-Disposition": "attachment; filename=" + fname}
tmp = NamedTemporaryFile(delete=True)
if file_ext == ".sac":
set_sac_header(params, st)
set_sac_header(st)
st.write(tmp.name, format=file_type)
response = make_response((tmp.read(), headers))
if file_ext == ".csv":
......
......@@ -3,8 +3,6 @@ from apps.globals import HEIGHT
from apps.globals import PLOT_COLOR
from apps.globals import WIDTH
VERSION = "1.1.1"
class Parameters:
def __init__(self):
......@@ -20,17 +18,17 @@ class Parameters:
self.cha = "*"
self.start = None
self.end = None
self.spectrum = "F"
self.showtitle = "T"
self.showscale = "T"
self.monochrome = "F"
self.spectrum = "false"
self.showtitle = "true"
self.showscale = "true"
self.monochrome = "false"
self.width = WIDTH
self.height = HEIGHT
self.dpi = DPI
self.color = PLOT_COLOR
self.correct = "F"
self.earthunits = "F"
self.demean = "F"
self.correct = "false"
self.earthunits = "false"
self.demean = "false"
self.waterlevel = None
self.freqlimits = None
self.units = "AUTO"
......@@ -44,21 +42,35 @@ class Parameters:
self.bp = None
self.decimate = None
self.deci = None
self.detrend = "F"
self.zerophase = "F"
self.envelope = "F"
self.diff = "F"
self.int = "F"
self.detrend = "false"
self.zerophase = "false"
self.envelope = "false"
self.diff = "false"
self.int = "false"
self.scale = None
self.divscale = None
self.format = None
self.nodata = "204"
self.constraints = {
"alias": [
("network", "net"),
("station", "sta"),
("location", "loc"),
("channel", "cha"),
("starttime", "start"),
("endtime", "end"),
("lpfilter", "lp"),
("hpfilter", "hp"),
("bpfilter", "bp"),
("decimate", "deci"),
("earthunits", "correct"),
],
"booleans": [
"spectrum",
"showtitle",
"showscale",
"monochrome",
"correct",
"earthunits",
"demean",
"detrend",
......@@ -73,18 +85,3 @@ class Parameters:
def todict(self):
return self.__dict__
ALIAS = [
("network", "net"),
("station", "sta"),
("location", "loc"),
("channel", "cha"),
("starttime", "start"),
("endtime", "end"),
("lpfilter", "lp"),
("hpfilter", "hp"),
("bpfilter", "bp"),
("decimate", "deci"),
("earthunits", "correct"),
]
......@@ -11,16 +11,15 @@ from apps.globals import HTTP
from apps.globals import MAX_DAYS
from apps.globals import TAPER_WINDOWS
from apps.globals import TIMEOUT
from apps.timeseries.constants import ALIAS
from apps.timeseries.constants import Parameters
from apps.timeseries.parameters import Parameters
from apps.timeseries.output import get_output
from apps.utils import check_base_parameters
from apps.utils import check_timeseriesplots_parameters
from apps.utils import check_request
from apps.utils import check_timeseriesplots_parameters
from apps.utils import error_param
from apps.utils import error_request
from apps.utils import is_valid_integer
from apps.utils import is_valid_bpfilter
from apps.utils import is_valid_integer
from apps.utils import is_valid_nodata
from apps.utils import is_valid_output
from apps.utils import is_valid_taper_window
......@@ -95,22 +94,10 @@ def checks_get():
params = Parameters().todict()
# check if the parameters are unknown
(p, result) = check_request(params, ALIAS)
(p, result) = check_request(params)
if result["code"] != 200:
return (p, result)
# determine selected features
params["request"] = tuple(request.args)
for key, val in params.items():
params[key] = request.args.get(key, val)
for key, alias in ALIAS:
if params[key] is None:
params[key] = request.args.get(alias, params[alias])
else:
params[alias] = params[key]
params["args"] = tuple(request.args)
return check_parameters(params)
......
......@@ -3,8 +3,6 @@ from apps.globals import HEIGHT
from apps.globals import PLOT_COLOR
from apps.globals import WIDTH
VERSION = "1.1.1"
class Parameters:
def __init__(self):
......@@ -20,30 +18,41 @@ class Parameters:
self.cha = "*"
self.start = None
self.end = None
self.iplot = "F"
self.spectrum = "F"
self.showtitle = "T"
self.showscale = "T"
self.monochrome = "F"
self.iplot = "false"
self.spectrum = "false"
self.showtitle = "true"
self.showscale = "true"
self.monochrome = "false"
self.width = WIDTH
self.height = HEIGHT
self.dpi = DPI
self.color = PLOT_COLOR
self.correct = "F"
self.earthunits = "F"
self.demean = "F"
self.correct = "false"
self.earthunits = "false"
self.demean = "false"
self.waterlevel = None
self.freqlimits = None
self.units = "AUTO"
self.imageformat = "png"
self.format = "png"
self.nodata = "204"
self.constraints = {
"alias": [
("network", "net"),
("station", "sta"),
("location", "loc"),
("channel", "cha"),
("starttime", "start"),
("endtime", "end"),
("earthunits", "correct"),
],
"booleans": [
"iplot",
"spectrum",
"showtitle",
"showscale",
"monochrome",
"correct",
"earthunits",
"demean",
],
......@@ -53,14 +62,3 @@ class Parameters:
def todict(self):
return self.__dict__
ALIAS = [
("network", "net"),
("station", "sta"),
("location", "loc"),
("channel", "cha"),
("starttime", "start"),
("endtime", "end"),
("earthunits", "correct"),
]
......@@ -8,12 +8,11 @@ from apps.globals import Error
from apps.globals import HTTP
from apps.globals import MAX_DAYS
from apps.globals import TIMEOUT
from apps.timeseriesplot.constants import ALIAS
from apps.timeseriesplot.constants import Parameters
from apps.timeseriesplot.parameters import Parameters
from apps.timeseriesplot.output import get_output
from apps.utils import check_base_parameters
from apps.utils import check_timeseriesplots_parameters
from apps.utils import check_request
from apps.utils import check_timeseriesplots_parameters
from apps.utils import error_param
from apps.utils import error_request
from apps.utils import is_valid_nodata
......@@ -48,22 +47,10 @@ def checks_get():
params = Parameters().todict()
# check if the parameters are unknown
(p, result) = check_request(params, ALIAS)
(p, result) = check_request(params)
if result["code"] != 200:
return (p, result)
# determine selected features
params["request"] = tuple(request.args)
for key, val in params.items():
params[key] = request.args.get(key, val)
for key, alias in ALIAS:
if params[key] is None:
params[key] = request.args.get(alias, params[alias])
else:
params[alias] = params[key]
params["imageformat"] = params["format"]
return check_parameters(params)
......
......@@ -7,7 +7,7 @@ import time
from difflib import SequenceMatcher
from datetime import datetime, timedelta
from flask import current_app, make_response, Response, request
from flask import current_app, make_response, request, Response
import matplotlib.pyplot as plt
from matplotlib.ticker import FuncFormatter, MaxNLocator
......@@ -40,31 +40,29 @@ from apps.globals import STRING_FALSE
from apps.globals import STRING_TRUE
from apps.globals import TAPER_WINDOWS
from apps.globals import UNITS
from apps.globals import VERSION
from apps.globals import USER_AGENT_TIMESERIES
from apps.globals import WIDTH_MIN, WIDTH_MAX
from apps.globals import WL_MIN, WL_MAX
from apps.globals import XTICKS_ROTATION
from apps.timeseries.constants import VERSION
def is_valid_integer(dimension, mini=0, maxi=sys.maxsize):
# by default valid for positive integers
try:
dimension = int(dimension)
if mini <= dimension <= maxi:
return True
except Exception:
return False
return bool(mini <= dimension <= maxi)
def is_valid_float(dimension, mini=sys.float_info.epsilon, maxi=sys.float_info.max):
# by default valid for strictly positive floats
try:
dimension = float(dimension)
if mini <= dimension <= maxi:
return True
except Exception:
return False
return bool(mini <= dimension <= maxi)
def is_valid_datetime(date):
......@@ -162,13 +160,12 @@ def error_request(msg="", details="", code=500):
request_date = datetime.utcnow().strftime("%Y-%b-%d %H:%M:%S UTC")
message_error = f"""Error {code}: {msg}\n
{details}\n
Usage details are available from {DOCUMENTATION_URI}\n
Request:
{request.url}\n
Request Submitted:
{request_date}\n
Service version:
Service: {SERVICE} version:{VERSION}"""
version:{VERSION}"""
return Response(message_error, status=code, mimetype="text/plain")
......@@ -182,30 +179,38 @@ def error_500(dmesg):
return error_request(msg=HTTP._500_, details=dmesg, code=500)
# check request
def check_request(params, alias):
keys = list(params.keys())
def check_request(params):
# preliminary parameter checks
for key, val in request.args.items():
if key not in keys:
ratios = [SequenceMatcher(None, key, p).ratio() for p in keys]
guess = max([(v, keys[ind]) for ind, v in enumerate(ratios)])
# stops at the first unknown parameter meet:
if key not in params:
ratios = ((SequenceMatcher(None, key, p).ratio(), p) for p in params)
guess = max(ratios)
hint = ". Did you mean " + guess[1] + " ?" if guess[0] > 0.7 else ""
return error_param(keys, Error.UNKNOWN_PARAM + key + hint)
return error_param(params, Error.UNKNOWN_PARAM + key + hint)
# find nonword chars except :
# "," for lists "*?" for wildcards and ".:-" for date
if re.search(r"[^a-zA-Z0-9_,*?.:-]", val):
return error_param(keys, Error.CHAR + key)
return error_param(params, Error.CHAR + key)
for key in keys:
for key, val in params.items():
if len(request.args.getlist(key)) > 1:
return error_param(keys, Error.MULTI_PARAM + key)
for key in alias:
if len([v for v in key if v in request.args]) > 1:
return error_param(keys, Error.MULTI_PARAM + " and is shorthand ".join(key))
return error_param(params, Error.MULTI_PARAM + key)
params[key] = request.args.get(key, val)
return (keys, {"msg": HTTP._200_, "details": Error.VALID_PARAM, "code": 200})
for key, alias in params["constraints"]["alias"]:
if key in request.args and alias in request.args:
return error_param(
params, f"{Error.MULTI_PARAM}{key} (and is shorthand {alias})"
)
if params[key] is None or params[key] == "false":
params[key] = request.args.get(alias, params[alias])
else:
params[alias] = params[key]
params["spectrum"] = "false" # disable spectrum
return (params, {"msg": HTTP._200_, "details": Error.VALID_PARAM, "code": 200})
def check_base_parameters(params, max_days=None):
......@@ -287,7 +292,7 @@ def check_base_parameters(params, max_days=None):
# Search for empty selection
if params["network"] == params["station"] == params["channel"] == "*":
return error_param(params, Error.NO_SELECTION)
logging.debug(params)
return (params, {"msg": HTTP._200_, "details": Error.VALID_PARAM, "code": 200})
......@@ -389,7 +394,7 @@ def is_open_file(paths):
"""Embargoed files are removed from the list of paths."""
if not paths:
return
return None
values = ", ".join(f"""('{j}', '{k}')""" for j, k in paths)
select = f"""SELECT path, name
......@@ -426,7 +431,7 @@ def get_signal(params):
res = extend(args=[net, sta, loc, cha, "--format=tuple"])
logging.debug(res)
if not res:
return
return None
filenames = list()
delta = params["end"] - params["start"]
......
......@@ -61,7 +61,7 @@ The four parameters (network, station, location, channel) determine channels of
- Wildcards: the question mark __?__ represents any single character, while the asterisk __*__ represents zero or more characters.
- List: multiple items may also be retrieved using a comma separated list. Wildcards may be included in the list.
- List: multiple items may also be retrieved using a comma separated list. Wildcards may be included in the list.
For example, with channel codes: channel=EH?,BHZ
......@@ -141,7 +141,7 @@ Notes :
## Date and time formats
YYYY-MM-DDThh:mm:ss[.ssssss] ex. 1997-01-31T12:04:32.123
YYYY-MM-DD ex. 1997-01-31 ( a time of 00:00:00 is assumed)
YYYY-MM-DD ex. 1997-01-31 (a time of 00:00:00 is assumed)
where:
......
......@@ -74,7 +74,7 @@ The four parameters (network, station, location, channel) determine channels of
- Wildcards: the question mark __?__ represents any single character, while the asterisk __*__ represents zero or more characters.
- List: multiple items may also be retrieved using a comma separated list. Wildcards may be included in the list.
- List: multiple items may also be retrieved using a comma separated list. Wildcards may be included in the list.
For example, with channel codes: channel=EH?,BHZ
......@@ -172,7 +172,7 @@ Notes :
## Date and time formats
YYYY-MM-DDThh:mm:ss[.ssssss] ex. 1997-01-31T12:04:32.123
YYYY-MM-DD ex. 1997-01-31 ( a time of 00:00:00 is assumed)
YYYY-MM-DD ex. 1997-01-31 (a time of 00:00:00 is assumed)
where:
......
......@@ -3,7 +3,7 @@ import os
from flask import Flask, make_response, render_template
from apps.timeseries.constants import VERSION
from apps.globals import VERSION
from apps.timeseries.root import timeseries
try:
......
......@@ -3,7 +3,7 @@ import os
from flask import Flask, make_response, render_template
from apps.timeseriesplot.constants import VERSION
from apps.globals import VERSION
from apps.timeseriesplot.root import timeseriesplot
try:
......
......@@ -2,7 +2,7 @@ body {
margin: auto;
padding-right: 1em;
padding-left: 1em;
max-width: 75em;
max-width: 75em;
border-left: 1px solid black;
border-right: 1px solid black;
color: black;
......
......@@ -392,7 +392,7 @@ default values are uppercase</code></pre>
</table>
<h2 id="date-and-time-formats">Date and time formats</h2>
<pre><code>YYYY-MM-DDThh:mm:ss[.ssssss] ex. 1997-01-31T12:04:32.123
YYYY-MM-DD ex. 1997-01-31 ( a time of 00:00:00 is assumed)
YYYY-MM-DD ex. 1997-01-31 (a time of 00:00:00 is assumed)
where:
......
......@@ -23,7 +23,7 @@
<representation mediaType="application/xml"/>
</response>
</method>
</resource>
</resource>
<resource path="query">
<method href="#query"/>
</resource>>
......
......@@ -294,7 +294,7 @@ default values are uppercase</code></pre>
</table>
<h2 id="date-and-time-formats">Date and time formats</h2>
<pre><code>YYYY-MM-DDThh:mm:ss[.ssssss] ex. 1997-01-31T12:04:32.123
YYYY-MM-DD ex. 1997-01-31 ( a time of 00:00:00 is assumed)
YYYY-MM-DD ex. 1997-01-31 (a time of 00:00:00 is assumed)
where:
......
......@@ -23,7 +23,7 @@
<representation mediaType="application/xml"/>
</response>
</method>
</resource>
</resource>
<resource path="query">
<method href="#query"/>
</resource>
......
......@@ -7,6 +7,7 @@ sys.path.append("../")
from apps.globals import Error
from apps.globals import HTTP
from apps.timeseries.parameters import Parameters
from apps.timeseries.root import check_parameters
from apps.utils import error_param
from apps.utils import is_valid_integer
......@@ -111,63 +112,43 @@ class MyTest(unittest.TestCase):
self.assertFalse(is_valid_bool_string("oui"))
def test_parameters(self):
self.maxDiff = None
# ?network=FR&channel=HHZ&starttime=2018-02-12T03:08:02&endtime=2018-02-12T03:10:00&station=CIEL
p1 = {}
p1 = Parameters().todict()
p1["network"] = "FR"
p1["station"] = "CIEL"
p1["location"] = "*"
p1["channel"] = "HHZ"
p1["start"] = "2018-02-12T03:08:02"
p1["end"] = "2018-02-12T04:08:02"
p1["showtitle"] = "T"
p1["monochrome"] = "F"
p1["correct"] = "F"
p1["earthunits"] = "F"
p1["showtitle"] = "true"
p1["monochrome"] = "false"
p1["correct"] = "false"
p1["earthunits"] = "false"
p1["imageformat"] = "png"
p1["spectrum"] = "T"
p1["spectrum"] = "true"
p1["color"] = "B22222"
p1["width"] = 1200
p1["height"] = 400
p1["dpi"] = 100
p1["showscale"] = "T"
p1["showscale"] = "true"
p1["waterlevel"] = None
p1["freqlimits"] = None
p1["units"] = "AUTO"
p1["demean"] = "F"
p1["detrend"] = "F"
p1["zerophase"] = "F"
p1["envelope"] = "F"
p1["diff"]