Skip to content

GitLab

Commit 1b40abbe authored by Jerome Touvier's avatar Jerome Touvier
Browse files

Merge branch 'refactoring' into 'master' 

code refactoring

See merge request !5
parents 502e25cb 56b3fd67
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ 1b40abbe
......@@ -23,4 +23,4 @@ build:
script:
- $IMAGE_BUILD --file $DOCKERFILE $BUILD_ARG --tag $IMAGE_NAME .
- $IMAGE_PUSH $IMAGE_NAME $IMAGE_NAME
- echo $CI_COMMIT_SHORT_SHA > ./static/commit.txt
apps/constants.py
View file @ 1b40abbe
......@@ -34,6 +34,4 @@ ALIAS = [
("endtime", "end"),
]
BOOLEANS = []
FLOATS = []
NOT_NONE = ["request"]
apps/globals.py
View file @ 1b40abbe
......@@ -5,11 +5,7 @@ DATATYPE = ("all", "bud", "validated")
GRANULARITY = ("year", "month")
ORDERBY = ("nslc", "time", "country", "protocol")
OUTPUT = ("csv", "request", "sync", "text")
MEDIA = (
"all",
"seedlink",
"dataselect",
)
MEDIA = ("all", "seedlink", "dataselect")
NODATA_CODE = ("204", "404")
TIMEOUT = 120
......@@ -46,7 +42,7 @@ class Error:
OUTPUT = f"Accepted output values are: {OUTPUT}." + BAD_VAL
NODATA_CODE = f"Accepted nodata values are: {NODATA_CODE}." + BAD_VAL
NO_SELECTION = "Request contains no selections."
NO_WILDCARDS = "Wildcards or list are not allowed." + BAD_VAL
NO_WILDCARDS = "Wildcards or lists are not allowed." + BAD_VAL
class HTTP:
......
apps/model.py deleted 100644 → 0
View file @ 502e25cb
import logging
from apps.globals import Error, HTTP
from apps.constants import NOT_NONE
from apps.constants import BOOLEANS
from apps.utils import check_base_parameters
from apps.utils import error_param
from apps.utils import is_valid_country
from apps.utils import is_valid_nodata
from apps.utils import is_valid_orderby
from apps.utils import is_valid_output
from apps.utils import is_valid_media
from apps.utils import is_valid_request
from apps.utils import is_valid_datatype
from apps.utils import is_valid_granularity
from apps.utils import is_valid_year
def check_parameters(params):
try:
# Check base parameters.
(params, error) = check_base_parameters(params, NOT_NONE, BOOLEANS)
if error["code"] != 200:
return (params, error)
country = params["country"].split(",")
for c in country:
logging.debug(c)
if not is_valid_country(c):
return error_param(params, Error.COUNTRY + c)
# orderby parameter validation
if params["orderby"] is not None:
if not is_valid_orderby(params["orderby"]):
return error_param(params, Error.ORDERBY + str(params["orderby"]))
params["orderby"] = params["orderby"].lower()
else:
params["orderby"] = "nslc"
# media parameter validation
if params["media"]:
if params["media"] == "all":
params["media"] = "dataselect,seedlink"
params["media"] = params["media"].split(",")
for ind, media in enumerate(params["media"]):
if not is_valid_media(media):
return error_param(params, Error.MEDIA + str(media))
params["media"][ind] = media.lower()
# year parameter validation
if params["year"]:
years = params["year"].split(",")
for ind, year in enumerate(years):
if not is_valid_year(year):
return error_param(params, Error.YEAR + str(year))
# type parameter validation
if params["type"]:
if not is_valid_datatype(params["type"]):
return error_param(params, Error.DATATYPE + str(params["type"]))
params["type"] = params["type"].lower()
# request parameter validation
if params["request"]:
if not is_valid_request(params["request"]):
return error_param(params, Error.REQUEST + str(params["request"]))
params["request"] = params["request"].lower()
# granularity parameter validation
if params["granularity"]:
if not is_valid_granularity(params["granularity"]):
return error_param(
params, Error.GRANULARITY + str(params["granularity"])
)
params["granularity"] = params["granularity"].lower()
# output parameter validation
if not is_valid_output(params["format"]):
return error_param(params, Error.OUTPUT + str(params["format"]))
params["format"] = params["format"].lower()
# nodata parameter validation
if not is_valid_nodata(params["nodata"]):
return error_param(params, Error.NODATA_CODE + str(params["nodata"]))
params["nodata"] = params["nodata"].lower()
for key, val in params.items():
logging.debug(key + ": " + str(val))
return (params, error)
except Exception as e:
logging.exception(str(e))
return (params, {"msg": HTTP._500_, "details": Error.PROCESSING, "code": 500})
apps/output.py
View file @ 1b40abbe
......@@ -176,7 +176,6 @@ def records_to_text(params, data, sep=" "):
def get_response(params, data):
tic = time.time()
data = [[str(val) for val in row] for row in data]
fname = "resifws-statistics"
headers = {"Content-type": "text/plain"}
if params["format"] == "text":
......@@ -193,8 +192,8 @@ def get_response(params, data):
return response
def get_statistics(params):
"""Statistics output (csv, request, sync, text)
def get_output(params):
""" Statistics output (csv, request, sync, text)
:params: parameters
:returns: text or csv with data statistics"""
......@@ -209,11 +208,12 @@ def get_statistics(params):
return error_request(msg=f"HTTP._{code}_", details=Error.NODATA, code=code)
nrows = len(data)
logging.debug(f"Number of collected rows: {nrows}")
logging.info(f"Number of collected rows: {nrows}")
if params["request"] in ("send", "country"):
data = sum_results(params, data)
data = [[str(val) for val in row] for row in data]
return get_response(params, data)
except Exception as ex:
logging.exception(str(ex))
......
apps/root.py
View file @ 1b40abbe
......@@ -5,14 +5,97 @@ from copy import copy
from multiprocessing import Process, Queue
from apps.constants import ALIAS
from apps.constants import NOT_NONE
from apps.constants import parameters
from apps.model import check_parameters
from apps.output import get_statistics
from apps.globals import Error
from apps.globals import HTTP
from apps.globals import TIMEOUT
from apps.output import get_output
from apps.utils import check_base_parameters
from apps.utils import check_request
from apps.utils import error_param
from apps.utils import error_request
from apps.utils import is_valid_country
from apps.utils import is_valid_datatype
from apps.utils import is_valid_granularity
from apps.utils import is_valid_media
from apps.utils import is_valid_nodata
from apps.utils import is_valid_orderby
from apps.utils import is_valid_output
from apps.utils import is_valid_request
from apps.utils import is_valid_year
def check_parameters(params):
# check base parameters
(params, error) = check_base_parameters(params, not_none=NOT_NONE)
if error["code"] != 200:
return (params, error)
country = params["country"].split(",")
for c in country:
logging.debug(c)
if not is_valid_country(c):
return error_param(params, Error.COUNTRY + c)
# orderby parameter validation
if params["orderby"] is not None:
if not is_valid_orderby(params["orderby"]):
return error_param(params, Error.ORDERBY + str(params["orderby"]))
params["orderby"] = params["orderby"].lower()
else:
params["orderby"] = "nslc"
# media parameter validation
if params["media"]:
if params["media"] == "all":
params["media"] = "dataselect,seedlink"
params["media"] = params["media"].split(",")
for ind, media in enumerate(params["media"]):
if not is_valid_media(media):
return error_param(params, Error.MEDIA + str(media))
params["media"][ind] = media.lower()
# year parameter validation
if params["year"]:
years = params["year"].split(",")
for ind, year in enumerate(years):
if not is_valid_year(year):
return error_param(params, Error.YEAR + str(year))
# type parameter validation
if params["type"]:
if not is_valid_datatype(params["type"]):
return error_param(params, Error.DATATYPE + str(params["type"]))
params["type"] = params["type"].lower()
# request parameter validation
if params["request"]:
if not is_valid_request(params["request"]):
return error_param(params, Error.REQUEST + str(params["request"]))
params["request"] = params["request"].lower()
# granularity parameter validation
if params["granularity"]:
if not is_valid_granularity(params["granularity"]):
return error_param(params, Error.GRANULARITY + str(params["granularity"]))
params["granularity"] = params["granularity"].lower()
# output parameter validation
if not is_valid_output(params["format"]):
return error_param(params, Error.OUTPUT + str(params["format"]))
params["format"] = params["format"].lower()
# nodata parameter validation
if not is_valid_nodata(params["nodata"]):
return error_param(params, Error.NODATA_CODE + str(params["nodata"]))
params["nodata"] = params["nodata"].lower()
for key, val in params.items():
logging.debug(key + ": " + str(val))
return (params, {"msg": HTTP._200_, "details": Error.VALID_PARAM, "code": 200})
def checks_get(request):
......@@ -52,7 +135,7 @@ def output(request):
if result["code"] == 200:
def put_response(q):
q.put(get_statistics(pdic))
q.put(get_output(pdic))
q = Queue()
process = Process(target=put_response, args=(q,))
......
apps/utils.py
View file @ 1b40abbe
......@@ -6,6 +6,7 @@ from datetime import datetime, timedelta
from flask import Response, request
from apps.constants import VERSION
from apps.globals import Error
from apps.globals import HTTP
from apps.globals import REQUEST
......@@ -18,8 +19,6 @@ from apps.globals import STRING_TRUE
from apps.globals import DATATYPE
from apps.globals import GRANULARITY
from apps.constants import VERSION
def is_valid_integer(dimension, mini=0, maxi=sys.maxsize):
# by default valid for positive integers
......@@ -77,14 +76,6 @@ def is_valid_channel(channel):
return re.match("([A-Za-z0-9*?]{1,3})$", channel) if channel else False
def is_valid_country(country):
return re.match("[A-Za-z*?]{1,2}$", country) if country else False
def is_valid_year(year):
return re.match("[0-9]{4}$", year) if year else False
def is_valid_bool_string(string):
if string is None:
return False
......@@ -95,6 +86,14 @@ def is_valid_output(output):
return output.lower() in OUTPUT if output else False
def is_valid_country(country):
return re.match("[A-Za-z*?]{1,2}$", country) if country else False
def is_valid_year(year):
return re.match("[0-9]{4}$", year) if year else False
def is_valid_orderby(orderby):
return orderby.lower() in ORDERBY if orderby else False
......@@ -179,7 +178,9 @@ def check_request(request, params, alias):
return (keys, {"msg": HTTP._200_, "details": Error.VALID_PARAM, "code": 200})
def check_base_parameters(params, not_none=None, booleans=None, floats=None):
def check_base_parameters(
params, max_days=None, not_none=None, booleans=None, floats=None
):
# Search for missing mandatory parameters
if not_none is not None:
......@@ -259,6 +260,9 @@ def check_base_parameters(params, not_none=None, booleans=None, floats=None):
end = str(params["end"])
return error_param(params, Error.START_LATER + start + " > " + end)
if max_days and params["end"] - params["start"] > timedelta(days=max_days):
return error_param(params, Error.TOO_LONG_DURATION + f"{max_days} days).")
# Search for empty selection
# if params["network"] == params["station"] == params["channel"] == "*":
# return error_param(params, Error.NO_SELECTION)
......
start.py
View file @ 1b40abbe
......@@ -37,13 +37,26 @@ def statistics_wadl():
return response
@app.route("/version")
@app.route("/version", strict_slashes=False)
def version():
response = make_response(VERSION)
response.headers["Content-Type"] = "text/plain"
return response
@app.route("/commit", strict_slashes=False)
def commit():
try:
with open("./static/commit.txt") as fc:
COMMIT_SHORT_SHA = fc.readline()
except Exception:
COMMIT_SHORT_SHA = "unspecified"
pass
response = make_response(COMMIT_SHORT_SHA)
response.headers["Content-Type"] = "text/plain"
return response
@app.route("/")
@app.route("/local=fr")
def statistics_doc():
......
tests/test.py
View file @ 1b40abbe
......@@ -7,7 +7,7 @@ sys.path.append("../")
from apps.globals import Error
from apps.globals import HTTP
from apps.model import check_parameters
from apps.root import check_parameters
from apps.utils import error_param
from apps.utils import is_valid_integer
from apps.utils import is_valid_float
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment