diff --git a/apps/output.py b/apps/output.py index c5ee95776ab2d6f2ded678416888831f67a69c2d..e16305d2906ff3762643f7e05835668ae3793a22 100644 --- a/apps/output.py +++ b/apps/output.py @@ -19,7 +19,7 @@ from apps.utils import error_request from apps.utils import tictac -def human_readable_size(size_in_bytes): +def humanize_size(size_in_bytes): index = 0 size_units = ["B", "KB", "MB", "GB", "TB", "PB", "PB"] while size_in_bytes >= 1024 and index < 6: @@ -56,6 +56,13 @@ def sql_common_string(params): elif "dataselect" in params["media"]: s = f"{s} AND protocol = 'dataselect'" + if ( + params["request"] != "storage" + and params["request"] != "country" + and params["country"] != "all" + ): + s = f"""{s} AND ({is_like_or_equal(params, "country")})""" + # starttime, endtime parameters if params["start"]: start = datetime.strftime(params["start"], "%Y-%m-%d") @@ -84,26 +91,36 @@ def sql_request(params): if params["request"] == "country": s = f"SELECT 'all' AS country, sum(requests)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}" elif params["request"] == "map": - s = f"SELECT extract(year from date)::INTEGER as year, country, sum(requests)::BIGINT FROM {table}" + s = f"SELECT extract(year from date)::INTEGER AS year, country, sum(requests)::BIGINT FROM {table}" elif params["request"] == "timeseries": - s = f"SELECT date, country, sum(bytes)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}" + if params["country"] == "all": + s = f"SELECT date, 'all' as country, sum(bytes)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}" + else: + s = f"SELECT date, country, sum(bytes)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}" elif params["request"] == "send": - s = f"SELECT sum(bytes)::BIGINT FROM {table}" + if params["network"] == "*": + s = f"SELECT 'all' AS network, 'all' AS station, sum(bytes)::BIGINT AS size FROM {table}" + else: + s = f"SELECT network, station, sum(bytes)::BIGINT AS size FROM {table}" where = sql_common_string(params) s = f"{s} {where}" if params["request"] == "country": tmp = f"SELECT country, sum(requests)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table} {where}" if params["country"] != "*": - tmp = f"""{tmp} AND ({is_like_or_equal(params, "country")})""" - s = f"{s} UNION {tmp} GROUP BY country ORDER BY 2 DESC" "" + tmp = tmp + f""" AND ({is_like_or_equal(params, "country")})""" + s = f"{s} UNION {tmp} GROUP BY country ORDER BY 2 DESC" elif params["request"] == "map": s = f"{s} GROUP BY country, year" elif params["request"] == "timeseries": - if "seedlink" in params["media"] and "dataselect" in params["media"]: - s = f"{s} GROUP BY date, country ORDER BY date" + if params["country"] == "all": + s = f"{s} GROUP BY date ORDER BY date" else: - s = f"{s} GROUP BY date, country, protocol ORDER BY date" + s = f"{s} GROUP BY date, country ORDER BY date" + elif params["request"] == "send": + if params["network"] != "*": + s = f"{s} GROUP BY network, station ORDER BY network, size DESC, station" + return s.replace("?", "_").replace("*", "%") @@ -130,8 +147,8 @@ def format_results(params, data): if row[1] and row[2]: row[:] = [row[0], "{:_}".format(row[1]), "{:_}".format(row[2])] elif params["request"] == "send": - result = data[0][0] - data = [["{:_}".format(result) + f" bytes ({human_readable_size(result)})"]] + for row in data: + row[2] = "{:_}".format(row[2]) + f" ({humanize_size(row[2])})" data = [[str(val) for val in row] for row in data] return data @@ -146,13 +163,7 @@ def get_header(params): elif params["request"] == "map": header = ["country", "requests"] elif params["request"] == "send": - if len(params["media"]) == 1 and "all" not in params["media"]: - if "seedlink" in params["media"]: - header = ["SEEDLINK"] - elif "dataselect" in params["media"]: - header = ["DATASELECT"] - else: - header = ["SEEDLINK and DATASELECT"] + header = ["network", "station", "bytes"] elif params["request"] == "timeseries": header = ["time", "country", "bytes", "clients"] return header diff --git a/apps/parameters.py b/apps/parameters.py index c271379298838e23d8a800377e19bd1bc7405324..11a5c83545decc596a42f3adaf5f6c8139fd0397 100644 --- a/apps/parameters.py +++ b/apps/parameters.py @@ -17,7 +17,6 @@ class Parameters: self.type = "all" self.request = None self.media = "all" - self.granularity = None self.orderby = "nslc" self.format = "text" self.nodata = "204" diff --git a/apps/root.py b/apps/root.py index 1e147056640e8e97bb144c021aacfede2831fe75..ba276b7792819966f784ceecf0a5467d3db690c1 100644 --- a/apps/root.py +++ b/apps/root.py @@ -74,12 +74,6 @@ def check_parameters(params): return error_param(params, Error.REQUEST + str(params["request"])) params["request"] = params["request"].lower() - # granularity parameter validation - if params["granularity"]: - if not is_valid_granularity(params["granularity"]): - return error_param(params, Error.GRANULARITY + str(params["granularity"])) - params["granularity"] = params["granularity"].lower() - # output parameter validation if not is_valid_output(params["format"]): return error_param(params, Error.OUTPUT + str(params["format"])) diff --git a/apps/utils.py b/apps/utils.py index c4d55e137a2d81c4f7f4987258ada4592837499d..d953f218a0782a32e3f5762aebee793a6c17066b 100644 --- a/apps/utils.py +++ b/apps/utils.py @@ -87,7 +87,9 @@ def is_valid_output(output): def is_valid_country(country): - return re.match("[A-Za-z*?]{1,2}$", country) if country else False + return ( + re.match("[A-Za-z*?]{1,2}$", country) or country == "all" if country else False + ) def is_valid_year(year): diff --git a/docs/USAGE_EN.md b/docs/USAGE_EN.md index f7ebd0c4aca1a8eb44842f24b6abe4e3dc8a0cc4..0107f4bd16f4c07df06080fb6615ddf94a167304 100644 --- a/docs/USAGE_EN.md +++ b/docs/USAGE_EN.md @@ -21,15 +21,24 @@ This service provides access to the RESIF-DC statistics. ## Sample queries +http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2020\ http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2018&type=validated -http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=WI&country=IT +http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=FR\ +http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=FR&country=IT -http://ws.resif.fr/resifws/statistics/1/query?request=country&media=dataselect&net=RA&country=IT,US,FR +http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND\ +http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND&country=all\ +http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND&country=IT,US,FR -http://ws.resif.fr/resifws/statistics/1/query?request=country&media=seedlink&net=RA&starttime=2020-01-01 +http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01\ +http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01&net=CL -http://ws.resif.fr/resifws/statistics/1/query?request=map&media=dataselect&starttime=2020-01-01 +http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&country=all&starttime=2020-01-01\ +http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&country=all&starttime=2020-01-01 + +http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&starttime=2020-01-01\ +http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&starttime=2020-01-01 ## Detailed descriptions of each query parameter diff --git a/docs/USAGE_FR.md b/docs/USAGE_FR.md index 3aa35cbdb596f29d40e0f9f4b4b44b9a1037cfa5..63a804809b372dcf91921c24338688fdec58b9e1 100644 --- a/docs/USAGE_FR.md +++ b/docs/USAGE_FR.md @@ -20,13 +20,26 @@ Ce service donne accès aux statistiques de RESIF-DC. ## Exemples de requêtes + +http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2020\ http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2018&type=validated -http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=WI&country=IT +http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=FR\ +http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=FR&country=IT + +http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND\ +http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND&country=all\ +http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND&country=IT,US,FR + +http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01\ +http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01&net=CL + +http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&country=all&starttime=2020-01-01\ +http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&country=all&starttime=2020-01-01 -http://ws.resif.fr/resifws/statistics/1/query?request=country&media=dataselect&net=RA&country=IT,US,FR +http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&starttime=2020-01-01\ +http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&starttime=2020-01-01 -http://ws.resif.fr/resifws/statistics/1/query?request=map&media=dataselect&starttime=2020-01-01 ## Descriptions détaillées de chaque paramètre de la requête diff --git a/templates/doc.html b/templates/doc.html index ba5b09206f17eb47e9b72613796b3690af61e146..70f039bb681fd91474681ac6b391d5e546d1aa23 100644 --- a/templates/doc.html +++ b/templates/doc.html @@ -35,10 +35,19 @@ output-options :: [format=<csv|text>] (..) requis [..] optionnel
http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2018&type=validated
-http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=WI&country=IT
- -http://ws.resif.fr/resifws/statistics/1/query?request=map&media=dataselect&starttime=2020-01-01
+http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2020
+http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2018&type=validated
http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=FR
+http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=FR&country=IT
http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND
+http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND&country=all
+http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND&country=IT,US,FR
http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01
+http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01&net=CL
http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&country=all&starttime=2020-01-01
+http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&country=all&starttime=2020-01-01
http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&starttime=2020-01-01
+http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&starttime=2020-01-01