Commit 829c2127 authored by Jerome Touvier's avatar Jerome Touvier
Browse files

Merge branch 'refactoring1' into 'master'

refactoring

See merge request OSUG/RESIF/ws-timeseries!5
parents 3f937ee1 5f289690
......@@ -33,7 +33,7 @@ def records_to_dictlist(data):
header = ["Network", "Station", "Location", "Channel"]
dictlist = []
for row in data:
dictlist.append({k: r for k, r in zip(header, row)})
dictlist.append(dict(zip(header, row)))
return {"Channel codes": dictlist}
......
......@@ -24,7 +24,8 @@ MAX_DATA_POINTS_PROCESSING = int(MAX_POINTS_PROCESSING.replace(",", ""))
# available parameter values
IMAGE_FORMAT = ("png", "jpeg", "jpg")
OUTPUT_TIMESERIES = ("ascii", "miniseed", "mseed", "plot", "sac", "slist", "tspair")
OUTPUT = ("ascii", "miniseed", "mseed", "plot", "sac", "slist", "tspair")
NODATA_CODE = ("204", "404")
STRING_TRUE = ("yes", "true", "t", "y", "1", "")
STRING_FALSE = ("no", "false", "f", "n", "0")
TAPER_WINDOWS = ("HANNING", "HAMMING", "COSINE")
......@@ -60,10 +61,9 @@ DECI_MAX = 16
class Error:
LEN_ARGS = "Too much arguments in URL."
UNKNOWN_PARAM = "Unknown query parameter: "
MULTI_PARAM = "Duplicate query parameter: "
VALID_PARAM = "Valid parameters. "
MULTI_PARAM = " Multiple entries for query parameter: "
VALID_PARAM = "Valid parameters."
START_LATER = "The starttime cannot be later than the endtime: "
TOO_LONG_DURATION = "Too many days requested (greater than "
TOO_MUCH_DATA = f"The request exceeds the limit of {MAX_POINTS} data points."
......@@ -71,14 +71,10 @@ class Error:
f"Deconvolution is not allowed above {MAX_POINTS_PROCESSING} data points."
)
RESPONSE = "Deconvolution can't be performed. Instrumental response not available."
UNSPECIFIED = "Error.processing your request."
NO_CONNECTION = "No services could be discovered at http://ws.resif.fr.\n\
This could be due to a temporary service outage, an invalid FDSN service address,\n\
an inactive internet connection or a blocking firewall rule."
OK_CONNECTION = "Connection OK. "
UNSPECIFIED = "Error processing your request."
NODATA = "Your query doesn't match any available data."
TIMEOUT = f"Your query exceeds timeout ({TIMEOUT} seconds)."
MISSING = "Missing parameter : "
MISSING = "Missing parameter: "
BAD_VAL = " Invalid value: "
CHAR = "White space(s) or invalid string. Invalid value for: "
EMPTY = "Empty string. Invalid value for: "
......@@ -108,26 +104,22 @@ class Error:
Invalid taper value: "
BP = "Invalid band pass filter value: "
SCALE_DIVSCALE = "You cannot specify both scale and divscale values."
OUTPUT_TIMESERIES = f"Accepted format values are: {OUTPUT_TIMESERIES}." + BAD_VAL
OUTPUT_TIMESERIES = f"Accepted format values are: {OUTPUT}." + BAD_VAL
PLOTS = f"The request exceeds the limit of {MAX_PLOTS} plots. Try to reduce the number of channels requested (network, station, location and channel parameters)."
PROCESSING = "Your request cannot be processed. Check for value consistency."
MISSING_OUTPUT = "Missing format parameter. "
INVALID_OUTPUT_PARAM = (
"The option 'out(put)' is no longer valid. Use 'format' instead."
)
NO_WILDCARDS = "Wildcards or lists are allowed only with plot or mseed output options (Invalid value for: "
NO_SELECTION = "Request contains no selections."
NO_WILDCARDS = "Wildcards or lists are allowed only with plot or mseed output options (Invalid value for: "
class HTTP:
_200_ = "Successful request. "
_202_ = "No data matches the selection. "
_204_ = "No data matches the selection. "
_400_ = "Bad request due to improper value. "
_401_ = "Authentication is required. "
_403_ = "Forbidden access. "
_404_ = "Page not found. "
_404_ = "No data matches the selection. "
_408_ = "Request exceeds timeout. "
_409_ = "Too much data. "
_413_ = "Request too large. "
_414_ = "Request URI too large. "
_500_ = "Internal server error. "
......
......@@ -6,89 +6,85 @@ from apps.globals import WIDTH
VERSION = "1.1.1"
class Args:
NETWORK = ("network", None)
STATION = ("station", None)
LOCATION = ("location", None)
CHANNEL = ("channel", None)
STARTTIME = ("starttime", None)
ENDTIME = ("endtime", None)
NET = ("net", "*")
STA = ("sta", "*")
LOC = ("loc", "*")
CHA = ("cha", "*")
START = ("start", None)
END = ("end", None)
class Parameters:
def __init__(self):
self.network = None
self.station = None
self.location = None
self.channel = None
self.starttime = None
self.endtime = None
self.net = "*"
self.sta = "*"
self.loc = "*"
self.cha = "*"
self.start = None
self.end = None
self.spectrum = "F"
self.showtitle = "T"
self.showscale = "T"
self.monochrome = "F"
self.width = WIDTH
self.height = HEIGHT
self.dpi = DPI
self.color = PLOT_COLOR
self.correct = "F"
self.earthunits = "F"
self.demean = "F"
self.waterlevel = None
self.freqlimits = None
self.units = "AUTO"
self.imageformat = "png"
self.taper = None
self.lpfilter = None
self.hpfilter = None
self.bpfilter = None
self.lp = None
self.hp = None
self.bp = None
self.decimate = None
self.deci = None
self.detrend = "F"
self.zerophase = "F"
self.envelope = "F"
self.diff = "F"
self.int = "F"
self.scale = None
self.divscale = None
self.format = None
self.nodata = "204"
self.constraints = {
"booleans": [
"spectrum",
"showtitle",
"showscale",
"monochrome",
"earthunits",
"demean",
"detrend",
"zerophase",
"envelope",
"diff",
"int",
],
"floats": ["lp", "hp", "scale", "divscale"],
"not_none": ["start", "end", "format"],
}
SPECTRUM = ("spectrum", "F")
SHOWTITLE = ("showtitle", "T")
SHOWSCALE = ("showscale", "T")
MONOCHROME = ("monochrome", "F")
WIDTH = ("width", WIDTH)
HEIGHT = ("height", HEIGHT)
DPI = ("dpi", DPI)
COLOR = ("color", PLOT_COLOR)
def todict(self):
return self.__dict__
CORRECT = ("correct", "F")
EARTHUNITS = ("earthunits", "F")
DEMEAN = ("demean", "F")
WATERLEVEL = ("waterlevel", None)
FREQLIMITS = ("freqlimits", None)
UNITS = ("units", "AUTO")
TAPER = ("taper", None)
LPFILTER = ("lpfilter", None)
HPFILTER = ("hpfilter", None)
BPFILTER = ("bpfilter", None)
LP = ("lp", None)
HP = ("hp", None)
BP = ("bp", None)
DECIMATE = ("decimate", None)
DECI = ("deci", None)
DETREND = ("detrend", "F")
ZEROPHASE = ("zerophase", "F")
ENV = ("envelope", "F")
DIFF = ("diff", "F")
INTEG = ("int", "F")
SCALE = ("scale", None)
DIVSCALE = ("divscale", None)
FORMAT = ("format", None)
class Empty:
pass
PARAMS = [v[0] for (k, v) in Args.__dict__.items() if k not in Empty.__dict__.keys()]
MAX_PARAMS = len(PARAMS)
ALIAS_PARAMS = [
(Args.NETWORK[0], Args.NET[0]),
(Args.STATION[0], Args.STA[0]),
(Args.LOCATION[0], Args.LOC[0]),
(Args.CHANNEL[0], Args.CHA[0]),
(Args.STARTTIME[0], Args.START[0]),
(Args.ENDTIME[0], Args.END[0]),
(Args.LPFILTER[0], Args.LP[0]),
(Args.HPFILTER[0], Args.HP[0]),
(Args.BPFILTER[0], Args.BP[0]),
(Args.DECIMATE[0], Args.DECI[0]),
(Args.EARTHUNITS[0], Args.CORRECT[0]),
]
BOOL_PARAMS = [
Args.SPECTRUM[0],
Args.SHOWTITLE[0],
Args.SHOWSCALE[0],
Args.MONOCHROME[0],
Args.EARTHUNITS[0],
Args.DEMEAN[0],
Args.DETREND[0],
Args.ZEROPHASE[0],
Args.ENV[0],
Args.DIFF[0],
Args.INTEG[0],
ALIAS = [
("network", "net"),
("station", "sta"),
("location", "loc"),
("channel", "cha"),
("starttime", "start"),
("endtime", "end"),
("lpfilter", "lp"),
("hpfilter", "hp"),
("bpfilter", "bp"),
("decimate", "deci"),
("earthunits", "correct"),
]
NOT_NONE_PARAMS = [Args.START[0], Args.END[0], Args.FORMAT[0]]
FLOAT_PARAMS = [Args.LP[0], Args.HP[0], Args.SCALE[0], Args.DIVSCALE[0]]
import logging
import re
from apps.globals import DECI_MIN, DECI_MAX
from apps.globals import Error
from apps.globals import HTTP
from apps.globals import MAX_DAYS
from apps.globals import OUTPUT_TIMESERIES
from apps.globals import TAPER_WINDOWS
from apps.timeseries.constants import BOOL_PARAMS
from apps.timeseries.constants import FLOAT_PARAMS
from apps.timeseries.constants import NOT_NONE_PARAMS
from apps.utils import check_base_parameters
from apps.utils import check_timeseriesplots_parameters
from apps.utils import error_param
from apps.utils import is_valid_integer
from apps.utils import is_valid_bpfilter
from apps.utils import is_valid_output
from apps.utils import is_valid_taper_window
def check_parameters(params):
try:
# Check base parameters
(params, error) = check_base_parameters(
params, MAX_DAYS, NOT_NONE_PARAMS, BOOL_PARAMS, FLOAT_PARAMS
)
if error["code"] != 200:
return (params, error)
# Check timeseriesplots parameters
(params, error) = check_timeseriesplots_parameters(params)
if error["code"] != 200:
return (params, error)
# Scaling parameters
if params["scale"] is not None and params["divscale"] is not None:
return error_param(params, Error.SCALE_DIVSCALE)
# Decimation parameter validation
if params["deci"] is not None:
if not is_valid_integer(params["deci"], DECI_MIN, DECI_MAX):
return error_param(params, Error.DECI + str(params["deci"]))
params["deci"] = int(params["deci"])
# Filter parameter validations
if params["taper"] is not None:
params["taper"] = tuple(params["taper"].split(","))
if not is_valid_taper_window(params["taper"]):
return error_param(params, Error.TAPER + str(params["taper"]))
if len(params["taper"]) == 1:
params["taper"] = (float(params["taper"][0]), TAPER_WINDOWS[0])
else:
params["taper"] = (
float(params["taper"][0]),
params["taper"][1].upper(),
)
if params["bp"] is not None:
params["bp"] = tuple(re.split("[/,;-]", params["bp"]))
if not is_valid_bpfilter(params["bp"]):
return error_param(params, Error.BP + str(params["bp"]))
params["bp"] = (float(params["bp"][0]), float(params["bp"][1]))
# Output parameter validations
if not is_valid_output(params["format"], OUTPUT_TIMESERIES):
return error_param(params, Error.OUTPUT_TIMESERIES + str(params["format"]))
params["format"] = params["format"].lower()
# wildcards or list are allowed only with plot and mseed output options
if params["format"] not in ("plot", "mseed", "miniseed"):
for key in ("network", "station", "location", "channel"):
if re.search(r"[,*?]", params[key]):
return error_param(params, Error.NO_WILDCARDS + key + ").")
for key, val in params.items():
logging.debug(key + ": " + str(val))
return (params, error)
except Exception as e:
logging.exception(str(e))
return (params, {"msg": HTTP._500_, "details": Error.UNSPECIFIED, "code": 500})
......@@ -13,15 +13,14 @@ from apps.globals import FROM_CLIENT
from apps.globals import MAX_DATA_POINTS
from apps.globals import MAX_DATA_POINTS_PROCESSING
from apps.globals import MAX_PLOTS
from apps.utils import error_request
from apps.utils import overflow_error
from apps.globals import USER_AGENT_TIMESERIES_INVENTORY
from apps.utils import error_500
from apps.utils import get_bounds
from apps.utils import get_signal
from apps.utils import get_signal_from_client
from apps.utils import get_periodogram
from apps.utils import get_response
from apps.utils import nodata_error
from apps.utils import overflow_error
from apps.utils import remove_response
from apps.utils import static_plots
from apps.utils import tictac
......@@ -110,35 +109,31 @@ def get_file_type(params):
def set_sac_header(params, st):
try:
for tr in st:
stats = tr.stats
client = Client(FDSN_CLIENT, user_agent=USER_AGENT_TIMESERIES_INVENTORY)
inventory = client.get_stations(
network=stats["network"],
station=stats["station"],
location=stats["location"],
channel=stats["channel"],
level="channel",
)
inv_sta = inventory[0][0]
inv_cha = inv_sta[0]
stats.sac = {}
if hasattr(inv_cha, "azimuth"):
stats.sac["cmpaz"] = inv_cha.azimuth
if hasattr(inv_cha, "dip"):
stats.sac["cmpinc"] = inv_cha.dip
if hasattr(inv_sta, "latitude"):
stats.sac["stla"] = inv_sta.latitude
if hasattr(inv_sta, "longitude"):
stats.sac["stlo"] = inv_sta.longitude
if hasattr(inv_sta, "depth"):
stats.sac["stdp"] = inv_sta.depth
if hasattr(inv_sta, "elevation"):
stats.sac["stel"] = inv_sta.elevation
except Exception as ex:
logging.exception(str(ex))
return error_500(Error.UNSPECIFIED)
for tr in st:
stats = tr.stats
client = Client(FDSN_CLIENT, user_agent=USER_AGENT_TIMESERIES_INVENTORY)
inventory = client.get_stations(
network=stats["network"],
station=stats["station"],
location=stats["location"],
channel=stats["channel"],
level="channel",
)
inv_sta = inventory[0][0]
inv_cha = inv_sta[0]
stats.sac = {}
if hasattr(inv_cha, "azimuth"):
stats.sac["cmpaz"] = inv_cha.azimuth
if hasattr(inv_cha, "dip"):
stats.sac["cmpinc"] = inv_cha.dip
if hasattr(inv_sta, "latitude"):
stats.sac["stla"] = inv_sta.latitude
if hasattr(inv_sta, "longitude"):
stats.sac["stlo"] = inv_sta.longitude
if hasattr(inv_sta, "depth"):
stats.sac["stdp"] = inv_sta.depth
if hasattr(inv_sta, "elevation"):
stats.sac["stel"] = inv_sta.elevation
def get_file(params, st):
......@@ -146,7 +141,7 @@ def get_file(params, st):
The name is built according to the template :
resifws-timeseries.2018-11-29T10_11_32.000Z.2018-11-29T23_42_56.000Z
:param params: Parameters object with url parameters (network, station, ...)
:param params: parameter dictionary with url parameters (network, station, ...)
:param st: obspy stream
:returns: response_class flask object containing the file
......@@ -180,7 +175,6 @@ def get_file(params, st):
return response
except Exception as ex:
logging.exception(str(ex))
return error_500(Error.UNSPECIFIED)
finally:
tmp.close()
logging.info(f"Response with file created in {tictac(tic)} seconds.")
......@@ -189,7 +183,7 @@ def get_file(params, st):
def get_output(params):
"""Create timeseries plots.
:param params: Parameters object with url parameters (network, station, ...)
:param params: parameter dictionary with url parameters (network, station, ...)
:returns: static plot(s) or data file
:raises MemoryError raises memory exception
:raises ValueError raises value exception
......@@ -220,15 +214,19 @@ def get_output(params):
return overflow_error(Error.PLOTS)
if params["earthunits"]:
tic1 = time.time()
try:
st.attach_response(get_response(params))
except Exception as ex:
logging.debug(str(ex))
return nodata_error(Error.RESPONSE)
logging.info(f"Attach response in {tictac(tic1)} seconds.")
code = params["nodata"]
return error_request(
msg=f"HTTP._{code}_", details=Error.RESPONSE, code=code
)
st = get_processed_signal(st, params)
if not st:
code = params["nodata"]
return error_request(msg=f"HTTP._{code}_", details=Error.NODATA, code=code)
if params["format"] == "plot":
response = static_plots(params, st)
......@@ -240,11 +238,10 @@ def get_output(params):
return overflow_error(Error.PROCESSING)
except Exception as ex:
logging.exception(str(ex))
return error_500(Error.UNSPECIFIED)
finally:
if st:
delta = params["end"] - params["start"]
logging.debug(f"Period of {delta} with {npoints} data points.")
if response:
bytes = response.headers.get("Content-Length")
logging.info(f"{bytes} bytes rendered in {tictac(tic0)} seconds.")
nbytes = response.headers.get("Content-Length")
logging.info(f"{nbytes} bytes rendered in {tictac(tic0)} seconds.")
import logging
import queue
import re
from multiprocessing import Process, Queue
from flask import request
from apps.globals import DECI_MIN, DECI_MAX
from apps.globals import Error
from apps.globals import HTTP
from apps.globals import MAX_DAYS
from apps.globals import TAPER_WINDOWS
from apps.globals import TIMEOUT
from apps.timeseries.constants import ALIAS_PARAMS
from apps.timeseries.constants import Args
from apps.timeseries.constants import PARAMS
from apps.timeseries.model import check_parameters
from apps.timeseries.constants import ALIAS
from apps.timeseries.constants import Parameters
from apps.timeseries.output import get_output
from apps.utils import check_base_parameters
from apps.utils import check_timeseriesplots_parameters
from apps.utils import check_request
from apps.utils import error_param
from apps.utils import error_request
from apps.utils import is_valid_integer
from apps.utils import is_valid_bpfilter
from apps.utils import is_valid_nodata
from apps.utils import is_valid_output
from apps.utils import is_valid_taper_window
def check_parameters(params):
# check base parameters
(params, error) = check_base_parameters(params, MAX_DAYS)
if error["code"] != 200:
return (params, error)
# timeseriesplots parameters
(params, error) = check_timeseriesplots_parameters(params)
if error["code"] != 200:
return (params, error)
# scale parameter
if params["scale"] is not None and params["divscale"] is not None:
return error_param(params, Error.SCALE_DIVSCALE)
# decimation parameter validation
if params["deci"] is not None:
if not is_valid_integer(params["deci"], DECI_MIN, DECI_MAX):
return error_param(params, Error.DECI + str(params["deci"]))
params["deci"] = int(params["deci"])
# filter parameter validations
if params["taper"] is not None:
params["taper"] = tuple(params["taper"].split(","))
if not is_valid_taper_window(params["taper"]):
return error_param(params, Error.TAPER + str(params["taper"]))
if len(params["taper"]) == 1:
params["taper"] = (float(params["taper"][0]), TAPER_WINDOWS[0])
else:
params["taper"] = (
float(params["taper"][0]),
params["taper"][1].upper(),
)
if params["bp"] is not None:
params["bp"] = tuple(re.split("[/,;-]", params["bp"]))
if not is_valid_bpfilter(params["bp"]):
return error_param(params, Error.BP + str(params["bp"]))
params["bp"] = (float(params["bp"][0]), float(params["bp"][1]))
# output parameter validation
if not is_valid_output(params["format"]):
return error_param(params, Error.OUTPUT_TIMESERIES + str(params["format"]))
params["format"] = params["format"].lower()
# nodata parameter validation
if not is_valid_nodata(params["nodata"]):
return error_param(params, Error.NODATA_CODE + str(params["nodata"]))
params["nodata"] = params["nodata"].lower()
# wildcards or list are allowed only with plot and mseed output options
if params["format"] not in ("plot", "mseed", "miniseed"):
for key in ("network", "station", "location", "channel"):
if re.search(r"[,*?]", params[key]):
return error_param(params, Error.NO_WILDCARDS + key + ").")
for key, val in params.items():
logging.debug(key + ": " + str(val))
return (params, {"msg": HTTP._200_, "details": Error.VALID_PARAM, "code": 200})
def checks_get(request):
def checks_get():
# get default parameters
params = Parameters().todict()
# check if the parameters are unknown
(params, result) = check_request(request, PARAMS, ALIAS_PARAMS)
(p, result) = check_request(params, ALIAS)
if result["code"] != 200:
return (params, result)
return (p, result)
# determine selected features
params = {}
params["base_url"] = request.base_url
params["request"] = tuple(request.args)
params["network"] = request.args.get(*Args.NETWORK)
params["station"] = request.args.get(*Args.STATION)
params["location"] = request.args.get(*Args.LOCATION)
params["channel"] = request.args.get(*Args.CHANNEL)
params["start"] = request.args.get(*Args.STARTTIME)
params["end"] = request.args.get(*Args.ENDTIME)
if params["network"] is None:
params["network"] = request.args.get(*Args.NET)
if params["station"] is None:
params["station"] = request.args.get(*Args.STA)
if params["location"] is None:
<