Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
OSUG
RESIF
ws-statistics
Commits
1b40abbe
Commit
1b40abbe
authored
Jul 08, 2020
by
Jerome Touvier
Browse files
Merge branch 'refactoring' into 'master'
code refactoring See merge request
!5
parents
502e25cb
56b3fd67
Changes
11
Hide whitespace changes
Inline
Side-by-side
.gitlab-ci.yml
View file @
1b40abbe
...
...
@@ -23,4 +23,4 @@ build:
script
:
-
$IMAGE_BUILD --file $DOCKERFILE $BUILD_ARG --tag $IMAGE_NAME .
-
$IMAGE_PUSH $IMAGE_NAME $IMAGE_NAME
-
echo $CI_COMMIT_SHORT_SHA > ./static/commit.txt
apps/constants.py
View file @
1b40abbe
...
...
@@ -34,6 +34,4 @@ ALIAS = [
(
"endtime"
,
"end"
),
]
BOOLEANS
=
[]
FLOATS
=
[]
NOT_NONE
=
[
"request"
]
apps/globals.py
View file @
1b40abbe
...
...
@@ -5,11 +5,7 @@ DATATYPE = ("all", "bud", "validated")
GRANULARITY
=
(
"year"
,
"month"
)
ORDERBY
=
(
"nslc"
,
"time"
,
"country"
,
"protocol"
)
OUTPUT
=
(
"csv"
,
"request"
,
"sync"
,
"text"
)
MEDIA
=
(
"all"
,
"seedlink"
,
"dataselect"
,
)
MEDIA
=
(
"all"
,
"seedlink"
,
"dataselect"
)
NODATA_CODE
=
(
"204"
,
"404"
)
TIMEOUT
=
120
...
...
@@ -46,7 +42,7 @@ class Error:
OUTPUT
=
f
"Accepted output values are:
{
OUTPUT
}
."
+
BAD_VAL
NODATA_CODE
=
f
"Accepted nodata values are:
{
NODATA_CODE
}
."
+
BAD_VAL
NO_SELECTION
=
"Request contains no selections."
NO_WILDCARDS
=
"Wildcards or list are not allowed."
+
BAD_VAL
NO_WILDCARDS
=
"Wildcards or list
s
are not allowed."
+
BAD_VAL
class
HTTP
:
...
...
apps/model.py
deleted
100644 → 0
View file @
502e25cb
import
logging
from
apps.globals
import
Error
,
HTTP
from
apps.constants
import
NOT_NONE
from
apps.constants
import
BOOLEANS
from
apps.utils
import
check_base_parameters
from
apps.utils
import
error_param
from
apps.utils
import
is_valid_country
from
apps.utils
import
is_valid_nodata
from
apps.utils
import
is_valid_orderby
from
apps.utils
import
is_valid_output
from
apps.utils
import
is_valid_media
from
apps.utils
import
is_valid_request
from
apps.utils
import
is_valid_datatype
from
apps.utils
import
is_valid_granularity
from
apps.utils
import
is_valid_year
def
check_parameters
(
params
):
try
:
# Check base parameters.
(
params
,
error
)
=
check_base_parameters
(
params
,
NOT_NONE
,
BOOLEANS
)
if
error
[
"code"
]
!=
200
:
return
(
params
,
error
)
country
=
params
[
"country"
].
split
(
","
)
for
c
in
country
:
logging
.
debug
(
c
)
if
not
is_valid_country
(
c
):
return
error_param
(
params
,
Error
.
COUNTRY
+
c
)
# orderby parameter validation
if
params
[
"orderby"
]
is
not
None
:
if
not
is_valid_orderby
(
params
[
"orderby"
]):
return
error_param
(
params
,
Error
.
ORDERBY
+
str
(
params
[
"orderby"
]))
params
[
"orderby"
]
=
params
[
"orderby"
].
lower
()
else
:
params
[
"orderby"
]
=
"nslc"
# media parameter validation
if
params
[
"media"
]:
if
params
[
"media"
]
==
"all"
:
params
[
"media"
]
=
"dataselect,seedlink"
params
[
"media"
]
=
params
[
"media"
].
split
(
","
)
for
ind
,
media
in
enumerate
(
params
[
"media"
]):
if
not
is_valid_media
(
media
):
return
error_param
(
params
,
Error
.
MEDIA
+
str
(
media
))
params
[
"media"
][
ind
]
=
media
.
lower
()
# year parameter validation
if
params
[
"year"
]:
years
=
params
[
"year"
].
split
(
","
)
for
ind
,
year
in
enumerate
(
years
):
if
not
is_valid_year
(
year
):
return
error_param
(
params
,
Error
.
YEAR
+
str
(
year
))
# type parameter validation
if
params
[
"type"
]:
if
not
is_valid_datatype
(
params
[
"type"
]):
return
error_param
(
params
,
Error
.
DATATYPE
+
str
(
params
[
"type"
]))
params
[
"type"
]
=
params
[
"type"
].
lower
()
# request parameter validation
if
params
[
"request"
]:
if
not
is_valid_request
(
params
[
"request"
]):
return
error_param
(
params
,
Error
.
REQUEST
+
str
(
params
[
"request"
]))
params
[
"request"
]
=
params
[
"request"
].
lower
()
# granularity parameter validation
if
params
[
"granularity"
]:
if
not
is_valid_granularity
(
params
[
"granularity"
]):
return
error_param
(
params
,
Error
.
GRANULARITY
+
str
(
params
[
"granularity"
])
)
params
[
"granularity"
]
=
params
[
"granularity"
].
lower
()
# output parameter validation
if
not
is_valid_output
(
params
[
"format"
]):
return
error_param
(
params
,
Error
.
OUTPUT
+
str
(
params
[
"format"
]))
params
[
"format"
]
=
params
[
"format"
].
lower
()
# nodata parameter validation
if
not
is_valid_nodata
(
params
[
"nodata"
]):
return
error_param
(
params
,
Error
.
NODATA_CODE
+
str
(
params
[
"nodata"
]))
params
[
"nodata"
]
=
params
[
"nodata"
].
lower
()
for
key
,
val
in
params
.
items
():
logging
.
debug
(
key
+
": "
+
str
(
val
))
return
(
params
,
error
)
except
Exception
as
e
:
logging
.
exception
(
str
(
e
))
return
(
params
,
{
"msg"
:
HTTP
.
_500_
,
"details"
:
Error
.
PROCESSING
,
"code"
:
500
})
apps/output.py
View file @
1b40abbe
...
...
@@ -176,7 +176,6 @@ def records_to_text(params, data, sep=" "):
def
get_response
(
params
,
data
):
tic
=
time
.
time
()
data
=
[[
str
(
val
)
for
val
in
row
]
for
row
in
data
]
fname
=
"resifws-statistics"
headers
=
{
"Content-type"
:
"text/plain"
}
if
params
[
"format"
]
==
"text"
:
...
...
@@ -193,8 +192,8 @@ def get_response(params, data):
return
response
def
get_
statistics
(
params
):
"""Statistics output (csv, request, sync, text)
def
get_
output
(
params
):
"""
Statistics output (csv, request, sync, text)
:params: parameters
:returns: text or csv with data statistics"""
...
...
@@ -209,11 +208,12 @@ def get_statistics(params):
return
error_request
(
msg
=
f
"HTTP._
{
code
}
_"
,
details
=
Error
.
NODATA
,
code
=
code
)
nrows
=
len
(
data
)
logging
.
debug
(
f
"Number of collected rows:
{
nrows
}
"
)
logging
.
info
(
f
"Number of collected rows:
{
nrows
}
"
)
if
params
[
"request"
]
in
(
"send"
,
"country"
):
data
=
sum_results
(
params
,
data
)
data
=
[[
str
(
val
)
for
val
in
row
]
for
row
in
data
]
return
get_response
(
params
,
data
)
except
Exception
as
ex
:
logging
.
exception
(
str
(
ex
))
...
...
apps/root.py
View file @
1b40abbe
...
...
@@ -5,14 +5,97 @@ from copy import copy
from
multiprocessing
import
Process
,
Queue
from
apps.constants
import
ALIAS
from
apps.constants
import
NOT_NONE
from
apps.constants
import
parameters
from
apps.model
import
check_parameters
from
apps.output
import
get_statistics
from
apps.globals
import
Error
from
apps.globals
import
HTTP
from
apps.globals
import
TIMEOUT
from
apps.output
import
get_output
from
apps.utils
import
check_base_parameters
from
apps.utils
import
check_request
from
apps.utils
import
error_param
from
apps.utils
import
error_request
from
apps.utils
import
is_valid_country
from
apps.utils
import
is_valid_datatype
from
apps.utils
import
is_valid_granularity
from
apps.utils
import
is_valid_media
from
apps.utils
import
is_valid_nodata
from
apps.utils
import
is_valid_orderby
from
apps.utils
import
is_valid_output
from
apps.utils
import
is_valid_request
from
apps.utils
import
is_valid_year
def
check_parameters
(
params
):
# check base parameters
(
params
,
error
)
=
check_base_parameters
(
params
,
not_none
=
NOT_NONE
)
if
error
[
"code"
]
!=
200
:
return
(
params
,
error
)
country
=
params
[
"country"
].
split
(
","
)
for
c
in
country
:
logging
.
debug
(
c
)
if
not
is_valid_country
(
c
):
return
error_param
(
params
,
Error
.
COUNTRY
+
c
)
# orderby parameter validation
if
params
[
"orderby"
]
is
not
None
:
if
not
is_valid_orderby
(
params
[
"orderby"
]):
return
error_param
(
params
,
Error
.
ORDERBY
+
str
(
params
[
"orderby"
]))
params
[
"orderby"
]
=
params
[
"orderby"
].
lower
()
else
:
params
[
"orderby"
]
=
"nslc"
# media parameter validation
if
params
[
"media"
]:
if
params
[
"media"
]
==
"all"
:
params
[
"media"
]
=
"dataselect,seedlink"
params
[
"media"
]
=
params
[
"media"
].
split
(
","
)
for
ind
,
media
in
enumerate
(
params
[
"media"
]):
if
not
is_valid_media
(
media
):
return
error_param
(
params
,
Error
.
MEDIA
+
str
(
media
))
params
[
"media"
][
ind
]
=
media
.
lower
()
# year parameter validation
if
params
[
"year"
]:
years
=
params
[
"year"
].
split
(
","
)
for
ind
,
year
in
enumerate
(
years
):
if
not
is_valid_year
(
year
):
return
error_param
(
params
,
Error
.
YEAR
+
str
(
year
))
# type parameter validation
if
params
[
"type"
]:
if
not
is_valid_datatype
(
params
[
"type"
]):
return
error_param
(
params
,
Error
.
DATATYPE
+
str
(
params
[
"type"
]))
params
[
"type"
]
=
params
[
"type"
].
lower
()
# request parameter validation
if
params
[
"request"
]:
if
not
is_valid_request
(
params
[
"request"
]):
return
error_param
(
params
,
Error
.
REQUEST
+
str
(
params
[
"request"
]))
params
[
"request"
]
=
params
[
"request"
].
lower
()
# granularity parameter validation
if
params
[
"granularity"
]:
if
not
is_valid_granularity
(
params
[
"granularity"
]):
return
error_param
(
params
,
Error
.
GRANULARITY
+
str
(
params
[
"granularity"
]))
params
[
"granularity"
]
=
params
[
"granularity"
].
lower
()
# output parameter validation
if
not
is_valid_output
(
params
[
"format"
]):
return
error_param
(
params
,
Error
.
OUTPUT
+
str
(
params
[
"format"
]))
params
[
"format"
]
=
params
[
"format"
].
lower
()
# nodata parameter validation
if
not
is_valid_nodata
(
params
[
"nodata"
]):
return
error_param
(
params
,
Error
.
NODATA_CODE
+
str
(
params
[
"nodata"
]))
params
[
"nodata"
]
=
params
[
"nodata"
].
lower
()
for
key
,
val
in
params
.
items
():
logging
.
debug
(
key
+
": "
+
str
(
val
))
return
(
params
,
{
"msg"
:
HTTP
.
_200_
,
"details"
:
Error
.
VALID_PARAM
,
"code"
:
200
})
def
checks_get
(
request
):
...
...
@@ -52,7 +135,7 @@ def output(request):
if
result
[
"code"
]
==
200
:
def
put_response
(
q
):
q
.
put
(
get_
statistics
(
pdic
))
q
.
put
(
get_
output
(
pdic
))
q
=
Queue
()
process
=
Process
(
target
=
put_response
,
args
=
(
q
,))
...
...
apps/utils.py
View file @
1b40abbe
...
...
@@ -6,6 +6,7 @@ from datetime import datetime, timedelta
from
flask
import
Response
,
request
from
apps.constants
import
VERSION
from
apps.globals
import
Error
from
apps.globals
import
HTTP
from
apps.globals
import
REQUEST
...
...
@@ -18,8 +19,6 @@ from apps.globals import STRING_TRUE
from
apps.globals
import
DATATYPE
from
apps.globals
import
GRANULARITY
from
apps.constants
import
VERSION
def
is_valid_integer
(
dimension
,
mini
=
0
,
maxi
=
sys
.
maxsize
):
# by default valid for positive integers
...
...
@@ -77,14 +76,6 @@ def is_valid_channel(channel):
return
re
.
match
(
"([A-Za-z0-9*?]{1,3})$"
,
channel
)
if
channel
else
False
def
is_valid_country
(
country
):
return
re
.
match
(
"[A-Za-z*?]{1,2}$"
,
country
)
if
country
else
False
def
is_valid_year
(
year
):
return
re
.
match
(
"[0-9]{4}$"
,
year
)
if
year
else
False
def
is_valid_bool_string
(
string
):
if
string
is
None
:
return
False
...
...
@@ -95,6 +86,14 @@ def is_valid_output(output):
return
output
.
lower
()
in
OUTPUT
if
output
else
False
def
is_valid_country
(
country
):
return
re
.
match
(
"[A-Za-z*?]{1,2}$"
,
country
)
if
country
else
False
def
is_valid_year
(
year
):
return
re
.
match
(
"[0-9]{4}$"
,
year
)
if
year
else
False
def
is_valid_orderby
(
orderby
):
return
orderby
.
lower
()
in
ORDERBY
if
orderby
else
False
...
...
@@ -179,7 +178,9 @@ def check_request(request, params, alias):
return
(
keys
,
{
"msg"
:
HTTP
.
_200_
,
"details"
:
Error
.
VALID_PARAM
,
"code"
:
200
})
def
check_base_parameters
(
params
,
not_none
=
None
,
booleans
=
None
,
floats
=
None
):
def
check_base_parameters
(
params
,
max_days
=
None
,
not_none
=
None
,
booleans
=
None
,
floats
=
None
):
# Search for missing mandatory parameters
if
not_none
is
not
None
:
...
...
@@ -259,6 +260,9 @@ def check_base_parameters(params, not_none=None, booleans=None, floats=None):
end
=
str
(
params
[
"end"
])
return
error_param
(
params
,
Error
.
START_LATER
+
start
+
" > "
+
end
)
if
max_days
and
params
[
"end"
]
-
params
[
"start"
]
>
timedelta
(
days
=
max_days
):
return
error_param
(
params
,
Error
.
TOO_LONG_DURATION
+
f
"
{
max_days
}
days)."
)
# Search for empty selection
# if params["network"] == params["station"] == params["channel"] == "*":
# return error_param(params, Error.NO_SELECTION)
...
...
apps/READM
E_EN.md
→
docs/USAG
E_EN.md
View file @
1b40abbe
File moved
apps/README
.md
→
docs/USAGE_FR
.md
View file @
1b40abbe
File moved
start.py
View file @
1b40abbe
...
...
@@ -37,13 +37,26 @@ def statistics_wadl():
return
response
@
app
.
route
(
"/version"
)
@
app
.
route
(
"/version"
,
strict_slashes
=
False
)
def
version
():
response
=
make_response
(
VERSION
)
response
.
headers
[
"Content-Type"
]
=
"text/plain"
return
response
@
app
.
route
(
"/commit"
,
strict_slashes
=
False
)
def
commit
():
try
:
with
open
(
"./static/commit.txt"
)
as
fc
:
COMMIT_SHORT_SHA
=
fc
.
readline
()
except
Exception
:
COMMIT_SHORT_SHA
=
"unspecified"
pass
response
=
make_response
(
COMMIT_SHORT_SHA
)
response
.
headers
[
"Content-Type"
]
=
"text/plain"
return
response
@
app
.
route
(
"/"
)
@
app
.
route
(
"/local=fr"
)
def
statistics_doc
():
...
...
tests/test.py
View file @
1b40abbe
...
...
@@ -7,7 +7,7 @@ sys.path.append("../")
from
apps.globals
import
Error
from
apps.globals
import
HTTP
from
apps.
model
import
check_parameters
from
apps.
root
import
check_parameters
from
apps.utils
import
error_param
from
apps.utils
import
is_valid_integer
from
apps.utils
import
is_valid_float
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment