Vous avez reçu un message "Your GitLab account has been locked ..." ? Pas d'inquiétude : lisez cet article https://docs.gricad-pages.univ-grenoble-alpes.fr/help/unlock/

Commit 56b3fd67 authored by Jerome Touvier's avatar Jerome Touvier
Browse files

code refactoring

parent 502e25cb
Pipeline #46777 passed with stage
in 57 seconds
......@@ -23,4 +23,4 @@ build:
script:
- $IMAGE_BUILD --file $DOCKERFILE $BUILD_ARG --tag $IMAGE_NAME .
- $IMAGE_PUSH $IMAGE_NAME $IMAGE_NAME
- echo $CI_COMMIT_SHORT_SHA > ./static/commit.txt
......@@ -34,6 +34,4 @@ ALIAS = [
("endtime", "end"),
]
BOOLEANS = []
FLOATS = []
NOT_NONE = ["request"]
......@@ -5,11 +5,7 @@ DATATYPE = ("all", "bud", "validated")
GRANULARITY = ("year", "month")
ORDERBY = ("nslc", "time", "country", "protocol")
OUTPUT = ("csv", "request", "sync", "text")
MEDIA = (
"all",
"seedlink",
"dataselect",
)
MEDIA = ("all", "seedlink", "dataselect")
NODATA_CODE = ("204", "404")
TIMEOUT = 120
......@@ -46,7 +42,7 @@ class Error:
OUTPUT = f"Accepted output values are: {OUTPUT}." + BAD_VAL
NODATA_CODE = f"Accepted nodata values are: {NODATA_CODE}." + BAD_VAL
NO_SELECTION = "Request contains no selections."
NO_WILDCARDS = "Wildcards or list are not allowed." + BAD_VAL
NO_WILDCARDS = "Wildcards or lists are not allowed." + BAD_VAL
class HTTP:
......
import logging
from apps.globals import Error, HTTP
from apps.constants import NOT_NONE
from apps.constants import BOOLEANS
from apps.utils import check_base_parameters
from apps.utils import error_param
from apps.utils import is_valid_country
from apps.utils import is_valid_nodata
from apps.utils import is_valid_orderby
from apps.utils import is_valid_output
from apps.utils import is_valid_media
from apps.utils import is_valid_request
from apps.utils import is_valid_datatype
from apps.utils import is_valid_granularity
from apps.utils import is_valid_year
def check_parameters(params):
try:
# Check base parameters.
(params, error) = check_base_parameters(params, NOT_NONE, BOOLEANS)
if error["code"] != 200:
return (params, error)
country = params["country"].split(",")
for c in country:
logging.debug(c)
if not is_valid_country(c):
return error_param(params, Error.COUNTRY + c)
# orderby parameter validation
if params["orderby"] is not None:
if not is_valid_orderby(params["orderby"]):
return error_param(params, Error.ORDERBY + str(params["orderby"]))
params["orderby"] = params["orderby"].lower()
else:
params["orderby"] = "nslc"
# media parameter validation
if params["media"]:
if params["media"] == "all":
params["media"] = "dataselect,seedlink"
params["media"] = params["media"].split(",")
for ind, media in enumerate(params["media"]):
if not is_valid_media(media):
return error_param(params, Error.MEDIA + str(media))
params["media"][ind] = media.lower()
# year parameter validation
if params["year"]:
years = params["year"].split(",")
for ind, year in enumerate(years):
if not is_valid_year(year):
return error_param(params, Error.YEAR + str(year))
# type parameter validation
if params["type"]:
if not is_valid_datatype(params["type"]):
return error_param(params, Error.DATATYPE + str(params["type"]))
params["type"] = params["type"].lower()
# request parameter validation
if params["request"]:
if not is_valid_request(params["request"]):
return error_param(params, Error.REQUEST + str(params["request"]))
params["request"] = params["request"].lower()
# granularity parameter validation
if params["granularity"]:
if not is_valid_granularity(params["granularity"]):
return error_param(
params, Error.GRANULARITY + str(params["granularity"])
)
params["granularity"] = params["granularity"].lower()
# output parameter validation
if not is_valid_output(params["format"]):
return error_param(params, Error.OUTPUT + str(params["format"]))
params["format"] = params["format"].lower()
# nodata parameter validation
if not is_valid_nodata(params["nodata"]):
return error_param(params, Error.NODATA_CODE + str(params["nodata"]))
params["nodata"] = params["nodata"].lower()
for key, val in params.items():
logging.debug(key + ": " + str(val))
return (params, error)
except Exception as e:
logging.exception(str(e))
return (params, {"msg": HTTP._500_, "details": Error.PROCESSING, "code": 500})
......@@ -176,7 +176,6 @@ def records_to_text(params, data, sep=" "):
def get_response(params, data):
tic = time.time()
data = [[str(val) for val in row] for row in data]
fname = "resifws-statistics"
headers = {"Content-type": "text/plain"}
if params["format"] == "text":
......@@ -193,8 +192,8 @@ def get_response(params, data):
return response
def get_statistics(params):
"""Statistics output (csv, request, sync, text)
def get_output(params):
""" Statistics output (csv, request, sync, text)
:params: parameters
:returns: text or csv with data statistics"""
......@@ -209,11 +208,12 @@ def get_statistics(params):
return error_request(msg=f"HTTP._{code}_", details=Error.NODATA, code=code)
nrows = len(data)
logging.debug(f"Number of collected rows: {nrows}")
logging.info(f"Number of collected rows: {nrows}")
if params["request"] in ("send", "country"):
data = sum_results(params, data)
data = [[str(val) for val in row] for row in data]
return get_response(params, data)
except Exception as ex:
logging.exception(str(ex))
......
......@@ -5,14 +5,97 @@ from copy import copy
from multiprocessing import Process, Queue
from apps.constants import ALIAS
from apps.constants import NOT_NONE
from apps.constants import parameters
from apps.model import check_parameters
from apps.output import get_statistics
from apps.globals import Error
from apps.globals import HTTP
from apps.globals import TIMEOUT
from apps.output import get_output
from apps.utils import check_base_parameters
from apps.utils import check_request
from apps.utils import error_param
from apps.utils import error_request
from apps.utils import is_valid_country
from apps.utils import is_valid_datatype
from apps.utils import is_valid_granularity
from apps.utils import is_valid_media
from apps.utils import is_valid_nodata
from apps.utils import is_valid_orderby
from apps.utils import is_valid_output
from apps.utils import is_valid_request
from apps.utils import is_valid_year
def check_parameters(params):
# check base parameters
(params, error) = check_base_parameters(params, not_none=NOT_NONE)
if error["code"] != 200:
return (params, error)
country = params["country"].split(",")
for c in country:
logging.debug(c)
if not is_valid_country(c):
return error_param(params, Error.COUNTRY + c)
# orderby parameter validation
if params["orderby"] is not None:
if not is_valid_orderby(params["orderby"]):
return error_param(params, Error.ORDERBY + str(params["orderby"]))
params["orderby"] = params["orderby"].lower()
else:
params["orderby"] = "nslc"
# media parameter validation
if params["media"]:
if params["media"] == "all":
params["media"] = "dataselect,seedlink"
params["media"] = params["media"].split(",")
for ind, media in enumerate(params["media"]):
if not is_valid_media(media):
return error_param(params, Error.MEDIA + str(media))
params["media"][ind] = media.lower()
# year parameter validation
if params["year"]:
years = params["year"].split(",")
for ind, year in enumerate(years):
if not is_valid_year(year):
return error_param(params, Error.YEAR + str(year))
# type parameter validation
if params["type"]:
if not is_valid_datatype(params["type"]):
return error_param(params, Error.DATATYPE + str(params["type"]))
params["type"] = params["type"].lower()
# request parameter validation
if params["request"]:
if not is_valid_request(params["request"]):
return error_param(params, Error.REQUEST + str(params["request"]))
params["request"] = params["request"].lower()
# granularity parameter validation
if params["granularity"]:
if not is_valid_granularity(params["granularity"]):
return error_param(params, Error.GRANULARITY + str(params["granularity"]))
params["granularity"] = params["granularity"].lower()
# output parameter validation
if not is_valid_output(params["format"]):
return error_param(params, Error.OUTPUT + str(params["format"]))
params["format"] = params["format"].lower()
# nodata parameter validation
if not is_valid_nodata(params["nodata"]):
return error_param(params, Error.NODATA_CODE + str(params["nodata"]))
params["nodata"] = params["nodata"].lower()
for key, val in params.items():
logging.debug(key + ": " + str(val))
return (params, {"msg": HTTP._200_, "details": Error.VALID_PARAM, "code": 200})
def checks_get(request):
......@@ -52,7 +135,7 @@ def output(request):
if result["code"] == 200:
def put_response(q):
q.put(get_statistics(pdic))
q.put(get_output(pdic))
q = Queue()
process = Process(target=put_response, args=(q,))
......
......@@ -6,6 +6,7 @@ from datetime import datetime, timedelta
from flask import Response, request
from apps.constants import VERSION
from apps.globals import Error
from apps.globals import HTTP
from apps.globals import REQUEST
......@@ -18,8 +19,6 @@ from apps.globals import STRING_TRUE
from apps.globals import DATATYPE
from apps.globals import GRANULARITY
from apps.constants import VERSION
def is_valid_integer(dimension, mini=0, maxi=sys.maxsize):
# by default valid for positive integers
......@@ -77,14 +76,6 @@ def is_valid_channel(channel):
return re.match("([A-Za-z0-9*?]{1,3})$", channel) if channel else False
def is_valid_country(country):
return re.match("[A-Za-z*?]{1,2}$", country) if country else False
def is_valid_year(year):
return re.match("[0-9]{4}$", year) if year else False
def is_valid_bool_string(string):
if string is None:
return False
......@@ -95,6 +86,14 @@ def is_valid_output(output):
return output.lower() in OUTPUT if output else False
def is_valid_country(country):
return re.match("[A-Za-z*?]{1,2}$", country) if country else False
def is_valid_year(year):
return re.match("[0-9]{4}$", year) if year else False
def is_valid_orderby(orderby):
return orderby.lower() in ORDERBY if orderby else False
......@@ -179,7 +178,9 @@ def check_request(request, params, alias):
return (keys, {"msg": HTTP._200_, "details": Error.VALID_PARAM, "code": 200})
def check_base_parameters(params, not_none=None, booleans=None, floats=None):
def check_base_parameters(
params, max_days=None, not_none=None, booleans=None, floats=None
):
# Search for missing mandatory parameters
if not_none is not None:
......@@ -259,6 +260,9 @@ def check_base_parameters(params, not_none=None, booleans=None, floats=None):
end = str(params["end"])
return error_param(params, Error.START_LATER + start + " > " + end)
if max_days and params["end"] - params["start"] > timedelta(days=max_days):
return error_param(params, Error.TOO_LONG_DURATION + f"{max_days} days).")
# Search for empty selection
# if params["network"] == params["station"] == params["channel"] == "*":
# return error_param(params, Error.NO_SELECTION)
......
......@@ -37,13 +37,26 @@ def statistics_wadl():
return response
@app.route("/version")
@app.route("/version", strict_slashes=False)
def version():
response = make_response(VERSION)
response.headers["Content-Type"] = "text/plain"
return response
@app.route("/commit", strict_slashes=False)
def commit():
try:
with open("./static/commit.txt") as fc:
COMMIT_SHORT_SHA = fc.readline()
except Exception:
COMMIT_SHORT_SHA = "unspecified"
pass
response = make_response(COMMIT_SHORT_SHA)
response.headers["Content-Type"] = "text/plain"
return response
@app.route("/")
@app.route("/local=fr")
def statistics_doc():
......
......@@ -7,7 +7,7 @@ sys.path.append("../")
from apps.globals import Error
from apps.globals import HTTP
from apps.model import check_parameters
from apps.root import check_parameters
from apps.utils import error_param
from apps.utils import is_valid_integer
from apps.utils import is_valid_float
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment