Vous avez reçu un message "Your GitLab account has been locked ..." ? Pas d'inquiétude : lisez cet article https://docs.gricad-pages.univ-grenoble-alpes.fr/help/unlock/

Commit de0b7ff4 authored by Jerome Touvier's avatar Jerome Touvier
Browse files

Merge branch 'update' into 'master'

update

See merge request !9
parents 95b1b521 b6b9ca34
Pipeline #48428 passed with stage
in 34 seconds
......@@ -26,7 +26,7 @@ docker run --rm --name ws-statistics -e RUNMODE=production ws-statistics
## RUNMODE builtin values
* `production`
* `production`
* `test`
* `local`
* other values map to :
......@@ -12,13 +12,15 @@ TIMEOUT = 120
# error message constants
DOCUMENTATION_URI = "http://ws.resif.fr/resifws/statistics/1/"
SERVICE = "resifws-statistics"
VERSION = "1.0.0"
class Error:
UNKNOWN_PARAM = "Unknown query parameter: "
MULTI_PARAM = " Multiple entries for query parameter: "
MULTI_PARAM = "Multiple entries for query parameter: "
VALID_PARAM = "Valid parameters."
START_LATER = "The starttime cannot be later than the endtime: "
TOO_LONG_DURATION = "Too many days requested (greater than "
UNSPECIFIED = "Error processing your request."
NODATA = "Your query doesn't match any available data."
TIMEOUT = f"Your query exceeds timeout ({TIMEOUT} seconds)."
......
......@@ -11,6 +11,17 @@ from apps.utils import error_request
from apps.utils import tictac
SIZE_UNITS = ["B", "KB", "MB", "GB", "TB", "PB", "PB"]
def human_readable_size(size_in_bytes):
index = 0
while size_in_bytes >= 1024 and index < 6:
size_in_bytes /= 1024
index += 1
return f"{round(size_in_bytes, 2)} {SIZE_UNITS[index]}"
def is_like_or_equal(params, key):
""" Builds the condition for the specified key in the "where" clause taking into account lists or wildcards. """
......@@ -63,14 +74,15 @@ def sql_request(params):
return s.replace("?", "_").replace("*", "%")
else:
if params["request"] == "country":
s = f"SELECT country, count(date) FROM {get_table()}"
s = f"SELECT country, sum(requests)::INTEGER FROM {get_table()}"
elif params["request"] == "send":
s = f"SELECT sum(bytes) FROM {get_table()}"
elif params["request"] == "timeseries":
s = f"""SELECT date, country, sum(bytes), sum(hll_cardinality(clients))::INTEGER FROM {get_table()}"""
s = f"SELECT date, country, sum(bytes), sum(hll_cardinality(clients))::INTEGER FROM {get_table()}"
s = sql_common_string(params, s)
s = f"""{s} AND ({is_like_or_equal(params, "country")})"""
if "all" not in params["country"]:
s = f"""{s} AND ({is_like_or_equal(params, "country")})"""
if "seedlink" in params["media"] and "dataselect" in params["media"]:
s = f"{s} AND (protocol = 'seedlink' OR protocol = 'dataselect')"
......@@ -107,22 +119,31 @@ def collect_data(params):
def sum_results(params, data):
""" Adds the results from the different media tables. """
newdata = list()
if params["request"] == "country":
result = dict()
for row in data:
if row[1]:
if row[0] in result:
if row[0] in result: # Is this country already there ?
result[row[0]] = result[row[0]] + int(row[1])
else:
result[row[0]] = int(row[1])
if "all" in params["country"]:
result = {"all": sum([int(val) for val in result.values()])}
for key, val in result.items():
newdata.append([key, str(val)])
newdata.append([key, "{:_}".format(val)])
elif params["request"] == "send":
result = 0
for row in data:
if row[0]:
result = result + int(row[0])
newdata.append([str(result)])
newdata.append(
["{:_}".format(result) + f" bytes ({human_readable_size(result)})"]
)
return newdata
......@@ -135,11 +156,11 @@ def get_header(params):
elif params["request"] == "send":
if len(params["media"]) == 1 and "all" not in params["media"]:
if "seedlink" in params["media"]:
header = ["SEEDLINK (in bytes)"]
header = ["SEEDLINK"]
elif "dataselect" in params["media"]:
header = ["DATASELECT (in bytes)"]
header = ["DATASELECT"]
else:
header = ["SEEDLINK and DATASELECT (in bytes)"]
header = ["SEEDLINK and DATASELECT"]
elif params["request"] == "timeseries":
header = ["time", "country", "bytes", "clients"]
return header
......@@ -200,14 +221,15 @@ def get_output(params):
data = collect_data(params)
if data is None:
return data
if params["request"] in ("send", "country"):
data = sum_results(params, data)
if not data:
code = params["nodata"]
return error_request(msg=f"HTTP._{code}_", details=Error.NODATA, code=code)
logging.info(f"Number of collected rows: {len(data)}")
if params["request"] in ("send", "country"):
data = sum_results(params, data)
data = [[str(val) for val in row] for row in data]
return get_response(params, data)
except Exception as ex:
......
VERSION = "1.0.0"
class Parameters:
def __init__(self):
self.network = None
......@@ -25,6 +22,14 @@ class Parameters:
self.format = "text"
self.nodata = "204"
self.constraints = {
"alias": [
("network", "net"),
("station", "sta"),
("location", "loc"),
("channel", "cha"),
("starttime", "start"),
("endtime", "end"),
],
"booleans": [],
"floats": [],
"not_none": ["request"],
......@@ -32,13 +37,3 @@ class Parameters:
def todict(self):
return self.__dict__
ALIAS = [
("network", "net"),
("station", "sta"),
("location", "loc"),
("channel", "cha"),
("starttime", "start"),
("endtime", "end"),
]
......@@ -4,12 +4,11 @@ from multiprocessing import Process, Queue
from flask import request
from apps.constants import ALIAS
from apps.constants import Parameters
from apps.globals import Error
from apps.globals import HTTP
from apps.globals import TIMEOUT
from apps.output import get_output
from apps.parameters import Parameters
from apps.utils import check_base_parameters
from apps.utils import check_request
from apps.utils import error_param
......@@ -34,7 +33,6 @@ def check_parameters(params):
country = params["country"].split(",")
for c in country:
logging.debug(c)
if not is_valid_country(c):
return error_param(params, Error.COUNTRY + c)
......@@ -48,9 +46,10 @@ def check_parameters(params):
# media parameter validation
if params["media"]:
if params["media"] == "all":
params["media"] = "dataselect,seedlink"
params["media"] = params["media"].split(",")
if "all" in params["media"]:
params["media"] = ["dataselect", "seedlink"]
for ind, media in enumerate(params["media"]):
if not is_valid_media(media):
return error_param(params, Error.MEDIA + str(media))
......@@ -103,22 +102,9 @@ def checks_get():
params = Parameters().todict()
# check if the parameters are unknown
(p, result) = check_request(params, ALIAS)
(p, result) = check_request(params)
if result["code"] != 200:
return (p, result)
# determine selected features
params["request"] = tuple(request.args)
for key, val in params.items():
params[key] = request.args.get(key, val)
for key, alias in ALIAS:
if params[key] is None:
params[key] = request.args.get(alias, params[alias])
else:
params[alias] = params[key]
return check_parameters(params)
......
......@@ -4,42 +4,40 @@ import time
from difflib import SequenceMatcher
from datetime import datetime, timedelta
from flask import Response, request
from flask import request, Response
from apps.constants import VERSION
from apps.globals import DATATYPE
from apps.globals import DOCUMENTATION_URI
from apps.globals import Error
from apps.globals import GRANULARITY
from apps.globals import HTTP
from apps.globals import REQUEST
from apps.globals import MEDIA
from apps.globals import NODATA_CODE
from apps.globals import ORDERBY
from apps.globals import OUTPUT
from apps.globals import REQUEST
from apps.globals import SERVICE
from apps.globals import STRING_FALSE
from apps.globals import STRING_TRUE
from apps.globals import DATATYPE
from apps.globals import GRANULARITY
from apps.globals import VERSION
def is_valid_integer(dimension, mini=0, maxi=sys.maxsize):
# by default valid for positive integers
try:
dimension = int(dimension)
if mini <= dimension <= maxi:
return True
except Exception:
return False
return bool(mini <= dimension <= maxi)
def is_valid_float(dimension, mini=sys.float_info.epsilon, maxi=sys.float_info.max):
# by default valid for strictly positive floats
try:
dimension = float(dimension)
if mini <= dimension <= maxi:
return True
except Exception:
return False
return bool(mini <= dimension <= maxi)
def is_valid_datetime(date):
......@@ -89,7 +87,9 @@ def is_valid_output(output):
def is_valid_country(country):
return re.match("[A-Za-z*?]{1,2}$", country) if country else False
return (
re.match("[A-Za-z*?]{1,2}$", country) or country == "all" if country else False
)
def is_valid_year(year):
......@@ -158,30 +158,38 @@ def error_500(dmesg):
return error_request(msg=HTTP._500_, details=dmesg, code=500)
# check request
def check_request(params, alias):
keys = list(params.keys())
def check_request(params):
# preliminary parameter checks
for key, val in request.args.items():
if key not in keys:
ratios = [SequenceMatcher(None, key, p).ratio() for p in keys]
guess = max([(v, keys[ind]) for ind, v in enumerate(ratios)])
# stops at the first unknown parameter meet:
if key not in params:
ratios = ((SequenceMatcher(None, key, p).ratio(), p) for p in params)
guess = max(ratios)
hint = ". Did you mean " + guess[1] + " ?" if guess[0] > 0.7 else ""
return error_param(keys, Error.UNKNOWN_PARAM + key + hint)
return error_param(params, Error.UNKNOWN_PARAM + key + hint)
# find nonword chars except :
# "," for lists "*?" for wildcards and ".:-" for date
if re.search(r"[^a-zA-Z0-9_,*?.:-]", val):
return error_param(keys, Error.CHAR + key)
return error_param(params, Error.CHAR + key)
for key in keys:
for key, val in params.items():
if len(request.args.getlist(key)) > 1:
return error_param(keys, Error.MULTI_PARAM + key)
return error_param(params, Error.MULTI_PARAM + key)
params[key] = request.args.get(key, val)
for key, alias in params["constraints"]["alias"]:
if key in request.args and alias in request.args:
return error_param(
params, f"{Error.MULTI_PARAM}{key} (and is shorthand {alias})"
)
if params[key] is None or params[key] == "false":
params[key] = request.args.get(alias, params[alias])
else:
params[alias] = params[key]
for key in alias:
if len([v for v in key if v in request.args]) > 1:
return error_param(keys, Error.MULTI_PARAM + " and is shorthand ".join(key))
return (keys, {"msg": HTTP._200_, "details": Error.VALID_PARAM, "code": 200})
return (params, {"msg": HTTP._200_, "details": Error.VALID_PARAM, "code": 200})
def check_base_parameters(params, max_days=None):
......
......@@ -65,7 +65,7 @@ This service provides access to the RESIF-DC statistics.
## Date and time formats
YYYY-MM-DDThh:mm:ss[.ssssss] ex. 1997-01-31T12:04:32.123
YYYY-MM-DD ex. 1997-01-31 ( a time of 00:00:00 is assumed)
YYYY-MM-DD ex. 1997-01-31 (a time of 00:00:00 is assumed)
where:
......
......@@ -3,7 +3,7 @@ import os
from flask import Flask, make_response, render_template
from apps.constants import VERSION
from apps.globals import VERSION
from apps.root import output
from config import Config
......
......@@ -2,7 +2,7 @@ body {
margin: auto;
padding-right: 1em;
padding-left: 1em;
max-width: 75em;
max-width: 75em;
border-left: 1px solid black;
border-right: 1px solid black;
color: black;
......
......@@ -157,7 +157,7 @@ output-options :: [format=&lt;csv|request|sync|text&gt;]
</ul>
<h2 id="date-and-time-formats">Date and time formats</h2>
<pre><code>YYYY-MM-DDThh:mm:ss[.ssssss] ex. 1997-01-31T12:04:32.123
YYYY-MM-DD ex. 1997-01-31 ( a time of 00:00:00 is assumed)
YYYY-MM-DD ex. 1997-01-31 (a time of 00:00:00 is assumed)
where:
......
......@@ -7,6 +7,7 @@ sys.path.append("../")
from apps.globals import Error
from apps.globals import HTTP
from apps.parameters import Parameters
from apps.root import check_parameters
from apps.utils import error_param
from apps.utils import is_valid_integer
......@@ -104,9 +105,7 @@ class MyTest(unittest.TestCase):
self.assertFalse(is_valid_bool_string("oui"))
def test_parameters(self):
# ?network=FR&channel=HHZ&starttime=2018-02-12T03:08:02&endtime=2018-02-12T03:10:00&station=CIEL
p1 = {}
p1 = Parameters().todict()
p1["network"] = "FR"
p1["station"] = "CIEL"
p1["location"] = "*"
......@@ -123,12 +122,6 @@ class MyTest(unittest.TestCase):
p1["limit"] = None
p1["request"] = ""
p1["protocol"] = ""
p1["nodata"] = "204"
p1["constraints"] = {
"booleans": [],
"floats": [],
"not_none": ["request"],
}
valid_param = {"msg": HTTP._200_, "details": Error.VALID_PARAM, "code": 200}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment