Commit 3130409c authored by Jerome Touvier's avatar Jerome Touvier
Browse files

add new statistics

parent e534c7be
Pipeline #55646 passed with stage
in 41 seconds
......@@ -19,7 +19,7 @@ from apps.utils import error_request
from apps.utils import tictac
def human_readable_size(size_in_bytes):
def humanize_size(size_in_bytes):
index = 0
size_units = ["B", "KB", "MB", "GB", "TB", "PB", "PB"]
while size_in_bytes >= 1024 and index < 6:
......@@ -56,6 +56,13 @@ def sql_common_string(params):
elif "dataselect" in params["media"]:
s = f"{s} AND protocol = 'dataselect'"
if (
params["request"] != "storage"
and params["request"] != "country"
and params["country"] != "all"
):
s = f"""{s} AND ({is_like_or_equal(params, "country")})"""
# starttime, endtime parameters
if params["start"]:
start = datetime.strftime(params["start"], "%Y-%m-%d")
......@@ -84,26 +91,36 @@ def sql_request(params):
if params["request"] == "country":
s = f"SELECT 'all' AS country, sum(requests)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}"
elif params["request"] == "map":
s = f"SELECT extract(year from date)::INTEGER as year, country, sum(requests)::BIGINT FROM {table}"
s = f"SELECT extract(year from date)::INTEGER AS year, country, sum(requests)::BIGINT FROM {table}"
elif params["request"] == "timeseries":
s = f"SELECT date, country, sum(bytes)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}"
if params["country"] == "all":
s = f"SELECT date, 'all' as country, sum(bytes)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}"
else:
s = f"SELECT date, country, sum(bytes)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}"
elif params["request"] == "send":
s = f"SELECT sum(bytes)::BIGINT FROM {table}"
if params["network"] == "*":
s = f"SELECT 'all' AS network, 'all' AS station, sum(bytes)::BIGINT AS size FROM {table}"
else:
s = f"SELECT network, station, sum(bytes)::BIGINT AS size FROM {table}"
where = sql_common_string(params)
s = f"{s} {where}"
if params["request"] == "country":
tmp = f"SELECT country, sum(requests)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table} {where}"
if params["country"] != "*":
tmp = f"""{tmp} AND ({is_like_or_equal(params, "country")})"""
s = f"{s} UNION {tmp} GROUP BY country ORDER BY 2 DESC" ""
tmp = tmp + f""" AND ({is_like_or_equal(params, "country")})"""
s = f"{s} UNION {tmp} GROUP BY country ORDER BY 2 DESC"
elif params["request"] == "map":
s = f"{s} GROUP BY country, year"
elif params["request"] == "timeseries":
if "seedlink" in params["media"] and "dataselect" in params["media"]:
s = f"{s} GROUP BY date, country ORDER BY date"
if params["country"] == "all":
s = f"{s} GROUP BY date ORDER BY date"
else:
s = f"{s} GROUP BY date, country, protocol ORDER BY date"
s = f"{s} GROUP BY date, country ORDER BY date"
elif params["request"] == "send":
if params["network"] != "*":
s = f"{s} GROUP BY network, station ORDER BY network, size DESC, station"
return s.replace("?", "_").replace("*", "%")
......@@ -130,8 +147,8 @@ def format_results(params, data):
if row[1] and row[2]:
row[:] = [row[0], "{:_}".format(row[1]), "{:_}".format(row[2])]
elif params["request"] == "send":
result = data[0][0]
data = [["{:_}".format(result) + f" bytes ({human_readable_size(result)})"]]
for row in data:
row[2] = "{:_}".format(row[2]) + f" ({humanize_size(row[2])})"
data = [[str(val) for val in row] for row in data]
return data
......@@ -146,13 +163,7 @@ def get_header(params):
elif params["request"] == "map":
header = ["country", "requests"]
elif params["request"] == "send":
if len(params["media"]) == 1 and "all" not in params["media"]:
if "seedlink" in params["media"]:
header = ["SEEDLINK"]
elif "dataselect" in params["media"]:
header = ["DATASELECT"]
else:
header = ["SEEDLINK and DATASELECT"]
header = ["network", "station", "bytes"]
elif params["request"] == "timeseries":
header = ["time", "country", "bytes", "clients"]
return header
......
......@@ -17,7 +17,6 @@ class Parameters:
self.type = "all"
self.request = None
self.media = "all"
self.granularity = None
self.orderby = "nslc"
self.format = "text"
self.nodata = "204"
......
......@@ -74,12 +74,6 @@ def check_parameters(params):
return error_param(params, Error.REQUEST + str(params["request"]))
params["request"] = params["request"].lower()
# granularity parameter validation
if params["granularity"]:
if not is_valid_granularity(params["granularity"]):
return error_param(params, Error.GRANULARITY + str(params["granularity"]))
params["granularity"] = params["granularity"].lower()
# output parameter validation
if not is_valid_output(params["format"]):
return error_param(params, Error.OUTPUT + str(params["format"]))
......
......@@ -87,7 +87,9 @@ def is_valid_output(output):
def is_valid_country(country):
return re.match("[A-Za-z*?]{1,2}$", country) if country else False
return (
re.match("[A-Za-z*?]{1,2}$", country) or country == "all" if country else False
)
def is_valid_year(year):
......
......@@ -21,15 +21,24 @@ This service provides access to the RESIF-DC statistics.
## Sample queries
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2020">http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2020</a>\
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2018&type=validated">http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2018&type=validated</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=WI&country=IT">http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=WI&country=IT</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=FR">http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=FR</a>\
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=FR&country=IT">http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=FR&country=IT</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=dataselect&net=RA&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=country&media=dataselect&net=RA&country=IT,US,FR</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND">http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND</a>\
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND&country=all">http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND&country=all</a>\
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND&country=IT,US,FR</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=seedlink&net=RA&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=country&media=seedlink&net=RA&starttime=2020-01-01</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01</a>\
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01&net=CL">http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01&net=CL</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=map&media=dataselect&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=map&media=dataselect&starttime=2020-01-01</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&country=all&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&country=all&starttime=2020-01-01</a>\
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&country=all&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&country=all&starttime=2020-01-01</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&starttime=2020-01-01</a>\
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&starttime=2020-01-01</a>
## Detailed descriptions of each query parameter
......
......@@ -20,13 +20,26 @@ Ce service donne accès aux statistiques de RESIF-DC.
## Exemples de requêtes
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2020">http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2020</a>\
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2018&type=validated">http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2018&type=validated</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=WI&country=IT">http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=WI&country=IT</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=FR">http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=FR</a>\
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=FR&country=IT">http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=FR&country=IT</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND">http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND</a>\
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND&country=all">http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND&country=all</a>\
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND&country=IT,US,FR</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01</a>\
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01&net=CL">http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01&net=CL</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&country=all&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&country=all&starttime=2020-01-01</a>\
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&country=all&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&country=all&starttime=2020-01-01</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=dataselect&net=RA&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=country&media=dataselect&net=RA&country=IT,US,FR</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&starttime=2020-01-01</a>\
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&starttime=2020-01-01</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=map&media=dataselect&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=map&media=dataselect&starttime=2020-01-01</a>
## Descriptions détaillées de chaque paramètre de la requête
......
......@@ -35,10 +35,19 @@ output-options :: [format=&lt;csv|text&gt;]
(..) requis
[..] optionnel</code></pre>
<h2 id="exemples-de-requêtes">Exemples de requêtes</h2>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2018&type=validated">http://ws.resif.fr/resifws/statistics/1/query?request=storage&amp;net=FR&amp;year=2018&amp;type=validated</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=WI&country=IT">http://ws.resif.fr/resifws/statistics/1/query?request=send&amp;media=dataselect&amp;net=WI&amp;country=IT</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=dataselect&net=RA&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=country&amp;media=dataselect&amp;net=RA&amp;country=IT,US,FR</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=map&media=dataselect&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=map&amp;media=dataselect&amp;starttime=2020-01-01</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2020">http://ws.resif.fr/resifws/statistics/1/query?request=storage&amp;net=FR&amp;year=2020</a><br />
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2018&type=validated">http://ws.resif.fr/resifws/statistics/1/query?request=storage&amp;net=FR&amp;year=2018&amp;type=validated</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=FR">http://ws.resif.fr/resifws/statistics/1/query?request=send&amp;media=dataselect&amp;net=FR</a><br />
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=FR&country=IT">http://ws.resif.fr/resifws/statistics/1/query?request=send&amp;media=dataselect&amp;net=FR&amp;country=IT</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND">http://ws.resif.fr/resifws/statistics/1/query?request=country&amp;net=RA,ND</a><br />
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND&country=all">http://ws.resif.fr/resifws/statistics/1/query?request=country&amp;net=RA,ND&amp;country=all</a><br />
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=country&amp;net=RA,ND&amp;country=IT,US,FR</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=map&amp;starttime=2020-01-01</a><br />
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01&net=CL">http://ws.resif.fr/resifws/statistics/1/query?request=map&amp;starttime=2020-01-01&amp;net=CL</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&country=all&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&amp;country=all&amp;starttime=2020-01-01</a><br />
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&country=all&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&amp;media=dataselect&amp;country=all&amp;starttime=2020-01-01</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&amp;starttime=2020-01-01</a><br />
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&amp;media=dataselect&amp;starttime=2020-01-01</a></p>
<h2 id="descriptions-détaillées-de-chaque-paramètre-de-la-requête">Descriptions détaillées de chaque paramètre de la requête</h2>
<table>
<colgroup>
......
......@@ -35,11 +35,19 @@ output-options :: [format=&lt;csv|text&gt;]
(..) required
[..] optional</code></pre>
<h2 id="sample-queries">Sample queries</h2>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2018&type=validated">http://ws.resif.fr/resifws/statistics/1/query?request=storage&amp;net=FR&amp;year=2018&amp;type=validated</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=WI&country=IT">http://ws.resif.fr/resifws/statistics/1/query?request=send&amp;media=dataselect&amp;net=WI&amp;country=IT</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=dataselect&net=RA&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=country&amp;media=dataselect&amp;net=RA&amp;country=IT,US,FR</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=seedlink&net=RA&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=country&amp;media=seedlink&amp;net=RA&amp;starttime=2020-01-01</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=map&media=dataselect&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=map&amp;media=dataselect&amp;starttime=2020-01-01</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2020">http://ws.resif.fr/resifws/statistics/1/query?request=storage&amp;net=FR&amp;year=2020</a><br />
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2018&type=validated">http://ws.resif.fr/resifws/statistics/1/query?request=storage&amp;net=FR&amp;year=2018&amp;type=validated</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=FR">http://ws.resif.fr/resifws/statistics/1/query?request=send&amp;media=dataselect&amp;net=FR</a><br />
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=FR&country=IT">http://ws.resif.fr/resifws/statistics/1/query?request=send&amp;media=dataselect&amp;net=FR&amp;country=IT</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND">http://ws.resif.fr/resifws/statistics/1/query?request=country&amp;net=RA,ND</a><br />
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND&country=all">http://ws.resif.fr/resifws/statistics/1/query?request=country&amp;net=RA,ND&amp;country=all</a><br />
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=country&amp;net=RA,ND&amp;country=IT,US,FR</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=map&amp;starttime=2020-01-01</a><br />
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01&net=CL">http://ws.resif.fr/resifws/statistics/1/query?request=map&amp;starttime=2020-01-01&amp;net=CL</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&country=all&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&amp;country=all&amp;starttime=2020-01-01</a><br />
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&country=all&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&amp;media=dataselect&amp;country=all&amp;starttime=2020-01-01</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&amp;starttime=2020-01-01</a><br />
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&amp;media=dataselect&amp;starttime=2020-01-01</a></p>
<h2 id="detailed-descriptions-of-each-query-parameter">Detailed descriptions of each query parameter</h2>
<table>
<colgroup>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment