diff --git a/apps/globals.py b/apps/globals.py index 9012c204970d1151d10c4cfb0d686ec5e5f27648..9748094d838b2efb59a188cb4e0e5550553056bf 100644 --- a/apps/globals.py +++ b/apps/globals.py @@ -1,10 +1,10 @@ STRING_TRUE = ("yes", "true", "t", "y", "1", "") STRING_FALSE = ("no", "false", "f", "n", "0") REQUEST = ("storage", "send", "country", "timeseries", "map") -DATATYPE = ("all", "bud", "validated") +DATATYPE = ("all", "buffer", "validated") GRANULARITY = ("year", "month") ORDERBY = ("nslc", "time", "country", "protocol") -OUTPUT = ("csv", "text") +OUTPUT = ("csv", "json", "text") MEDIA = ("all", "seedlink", "dataselect") NODATA_CODE = ("204", "404") TIMEOUT = 120 diff --git a/apps/output.py b/apps/output.py index e16305d2906ff3762643f7e05835668ae3793a22..a7fec245e2dd094cd33845e88c816d3f4a565528 100644 --- a/apps/output.py +++ b/apps/output.py @@ -56,11 +56,7 @@ def sql_common_string(params): elif "dataselect" in params["media"]: s = f"{s} AND protocol = 'dataselect'" - if ( - params["request"] != "storage" - and params["request"] != "country" - and params["country"] != "all" - ): + if params["request"] in ("send", "timeseries"): s = f"""{s} AND ({is_like_or_equal(params, "country")})""" # starttime, endtime parameters @@ -77,15 +73,16 @@ def sql_request(params): """ Builds the PostgreSQL request """ if params["request"] == "storage": - columns = "year, network, station, channel, quality, type" - s = f"SELECT DISTINCT ON ({columns}) size, {columns}, date FROM dataholdings" + columns = "year, network, station, channel, quality" + s = f"SELECT DISTINCT ON ({columns}) sum(size)::BIGINT, {columns}, date FROM dataholdings" where = sql_common_string(params) s = f"{s} {where} AND channel is not NULL" if params["type"] != "all": s = f"""{s} AND (type = '{params["type"]}')""" if params["year"]: s = f"""{s} AND ({is_like_or_equal(params, "year")})""" - s = f"{s} ORDER BY {columns}, date DESC" + s = f"{s} GROUP BY {columns}, date ORDER BY {columns}, date DESC" + s = s.replace("buffer", "bud") else: table = "sent_data_summary_weekly" if params["request"] == "country": @@ -106,7 +103,7 @@ def sql_request(params): where = sql_common_string(params) s = f"{s} {where}" if params["request"] == "country": - tmp = f"SELECT country, sum(requests)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table} {where}" + tmp = f"SELECT coalesce(country, 'n/a'), sum(requests)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table} {where}" if params["country"] != "*": tmp = tmp + f""" AND ({is_like_or_equal(params, "country")})""" s = f"{s} UNION {tmp} GROUP BY country ORDER BY 2 DESC" @@ -156,7 +153,7 @@ def format_results(params, data): def get_header(params): if params["request"] == "storage": - header = ["size", "year", "network", "station", "channel", "quality", "type"] + header = ["size", "year", "network", "station", "channel", "quality"] header.append("lastupdated") elif params["request"] == "country": header = ["country", "requests", "clients"] @@ -196,27 +193,42 @@ def records_to_text(params, data, sep=" "): return text +def records_to_dictlist(params, data): + """ Create json output """ + + dictlist = list() + header = get_header(params) + header = [h.lower() for h in header] + for row in data: + dictlist.append(dict(zip(header, row))) + return { + "created": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "datasources": dictlist, + } + + def get_response(params, data): tic = time.time() fname = "resifws-statistics" headers = {"Content-type": "text/plain"} if params["format"] == "text": response = make_response(records_to_text(params, data), headers) - elif params["format"] == "request": - response = make_response(records_to_text(params, data), headers) - elif params["format"] == "sync": - response = make_response(records_to_text(params, data, "|"), headers) elif params["format"] == "csv": headers = {"Content-Disposition": f"attachment; filename={fname}.csv"} response = make_response(records_to_text(params, data, ","), headers) response.headers["Content-type"] = "text/csv" + elif params["format"] == "json": + headers = {"Content-type": "application/json"} + response = make_response( + json.dumps(records_to_dictlist(params, data), sort_keys=False), headers + ) logging.debug(f"Response built in {tictac(tic)} seconds.") return response def map_requests(params, data): - with open("countries.geo.json") as json_file: + with open("./static/countries.geo.json") as json_file: geojson = json.load(json_file) geodata = dict(x=[], y=[], name=[], requests=[]) diff --git a/docs/USAGE_EN.md b/docs/USAGE_EN.md index 0107f4bd16f4c07df06080fb6615ddf94a167304..e20f9c2bf742aa7ad68be11f13b95d8aba534cb8 100644 --- a/docs/USAGE_EN.md +++ b/docs/USAGE_EN.md @@ -11,9 +11,9 @@ This service provides access to the RESIF-DC statistics. request-options :: (request=) channel-options :: [net= & sta= & loc= & cha=] date-options :: [starttime=] & [endtime=] - storage-options :: [year=] & [type=] + storage-options :: [year=] & [type=] send-options :: [country=] & [media=] - output-options :: [format=] + output-options :: [format=] (..) required [..] optional @@ -34,11 +34,9 @@ This service provides access to the RESIF-DC statistics. http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01\ http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01&net=CL -http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&country=all&starttime=2020-01-01\ -http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&country=all&starttime=2020-01-01 - http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&starttime=2020-01-01\ -http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&starttime=2020-01-01 +http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&starttime=2020-01-01&country=all\ +http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&starttime=2020-01-01&country=all&media=dataselect ## Detailed descriptions of each query parameter @@ -51,8 +49,8 @@ This service provides access to the RESIF-DC statistics. | cha[nnel] | HHZ | Channel Code. This parameter accepts wildcards. | | start[time] | 2010-01-10T00:00:00 | Specify the responses that cover the time period after (and including) this time. | | end[time] | 2011-02-11T01:00:00 | Specify responses that cover the time period prior to this time. | -| format | json | The output format. Accepted values are text (the default) and csv. | -| request | send | Type of statistic requested. Allowed values :
- __country__ : the number of requests from each country
- __send__ : the volume of distributed data by protocol
- __storage__ : the volume of data stored
- __timeseries__ : returns a timeseries with the country, the number and the volume of requests | +| format | text | The output format. Accepted values are text (the default), json, and csv. | +| request | send | Type of statistic requested. Allowed values :
- __country__ : the number of requests from each country
- __map__ : a color coded map with the number of requests from each country
- __send__ : the volume of distributed data by protocol
- __storage__ : the volume of data stored
- __timeseries__ : returns a timeseries with the country, the number and the volume of requests | ### Additional parameters for country, send, and timeseries request options | Parameter | Example | Discussion | @@ -63,7 +61,7 @@ This service provides access to the RESIF-DC statistics. ### Additional parameters for storage request option | Parameter | Example | Discussion | | :--------- | :------- | :------------------------------------------------------------------------------ | -| type | validated | Type of data. Allowed values : all (the default), bud, and validated | +| type | validated | Choose validated data or not. Allowed values : all (the default), buffer, and validated | | years | 2019,2020 | List of years for which the volume of data is requested. | diff --git a/docs/USAGE_FR.md b/docs/USAGE_FR.md index 63a804809b372dcf91921c24338688fdec58b9e1..31f74323a75da86d209ce67d4a7b6e6cfa3ed5e0 100644 --- a/docs/USAGE_FR.md +++ b/docs/USAGE_FR.md @@ -11,9 +11,9 @@ Ce service donne accès aux statistiques de RESIF-DC. request-options :: (request=) channel-options :: [net= & sta= & loc= & cha=] date-options :: [starttime=] & [endtime=] - storage-options :: [year=] & [type=] + storage-options :: [year=] & [type=] send-options :: [country=] & [media=] - output-options :: [format=] + output-options :: [format=] (..) requis [..] optionnel @@ -34,11 +34,9 @@ Ce service donne accès aux statistiques de RESIF-DC. http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01\ http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01&net=CL -http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&country=all&starttime=2020-01-01\ -http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&country=all&starttime=2020-01-01 - http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&starttime=2020-01-01\ -http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&starttime=2020-01-01 +http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&starttime=2020-01-01&country=all\ +http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&starttime=2020-01-01&country=all&media=dataselect ## Descriptions détaillées de chaque paramètre de la requête @@ -51,8 +49,8 @@ Ce service donne accès aux statistiques de RESIF-DC. | cha[nnel] | HHZ | Code de canal. Ce paramètre accepte les jokers. | | start[time] | 2010-01-10T00:00:00 | Donne les réponses qui couvrent la période après la date donnée incluse. | | end[time] | 2011-02-11T01:00:00 | Donne les réponses qui couvrent la période avant la date donnée incluse. | -| format | txt | Format de sortie. Valeurs autorisées : text et csv. | -| request | send | Type de la statistique demandée. Valeurs autorisées :
- __country__ : nombre de requêtes passées par pays
- __send__ : volume de données distribuées par protocoles
- __storage__ : quantité de données stockées brutes et validées
- __timeseries__ : produit une série temporelle avec en colonne le pays, le nombre et la taille cumulés des requêtes | +| format | text | Format de sortie. Valeurs autorisées : text, json et csv. | +| request | send | Type de la statistique demandée. Valeurs autorisées :
- __country__ : nombre de requêtes passées par pays
- __map__ : une carte avec le nombre de requêtes provenant de chaque pays
- __send__ : volume de données distribuées par protocoles
- __storage__ : quantité de données stockées brutes et validées
- __timeseries__ : produit une série temporelle avec en colonne le pays, le nombre et la taille cumulés des requêtes | ### Options supplémentaires spécifiques aux requêtes country, send et timeseries | Paramètre | Exemple | Discussion | @@ -63,7 +61,7 @@ Ce service donne accès aux statistiques de RESIF-DC. ### Options supplémentaires spécifiques à la requête storage | Paramètre | Exemple | Discussion | | :--------- | :------ | :------------------------------------------------------------------------------ | -| type | validated | Choix entre les données stockées brutes et validées. Valeurs autorisées : all (par défaut), bud et validated | +| type | validated | Choix entre les données stockées validées ou non. Valeurs autorisées : all (par défaut), buffer et validated | | years | 2019,2020 | Liste des années pour lesquelles on demande la quantité de données. | diff --git a/countries.geo.json b/static/countries.geo.json similarity index 100% rename from countries.geo.json rename to static/countries.geo.json diff --git a/templates/doc.html b/templates/doc.html index 70f039bb681fd91474681ac6b391d5e546d1aa23..5d068fdb18159a42a3eba66deb2be3c2a1bcbb63 100644 --- a/templates/doc.html +++ b/templates/doc.html @@ -28,9 +28,9 @@ où : request-options :: (request=<country|map|send|storage|timeseries>) channel-options :: [net=<network> & sta=<station> & loc=<location> & cha=<channel>] date-options :: [starttime=<date>] & [endtime=<date>] -storage-options :: [year=<year>] & [type=<all|bud|validated>] +storage-options :: [year=<year>] & [type=<all|buffer|validated>] send-options :: [country=<country_code>] & [media=<all|dataselect|seedlink>] -output-options :: [format=<csv|text>] +output-options :: [format=<csv|json|text>] (..) requis [..] optionnel @@ -44,10 +44,9 @@ output-options :: [format=<csv|text>] http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND&country=IT,US,FR

http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01
http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01&net=CL

-

http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&country=all&starttime=2020-01-01
-http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&country=all&starttime=2020-01-01

http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&starttime=2020-01-01
-http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&starttime=2020-01-01

+http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&starttime=2020-01-01&country=all
+http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&starttime=2020-01-01&country=all&media=dataselect

Descriptions détaillées de chaque paramètre de la requête

@@ -95,13 +94,13 @@ output-options :: [format=<csv|text>] - - + + - +
formattxtFormat de sortie. Valeurs autorisées : text et csv.textFormat de sortie. Valeurs autorisées : text, json et csv.
request sendType de la statistique demandée. Valeurs autorisées :
- country : nombre de requêtes passées par pays
- send : volume de données distribuées par protocoles
- storage : quantité de données stockées brutes et validées
- timeseries : produit une série temporelle avec en colonne le pays, le nombre et la taille cumulés des requêtes		Type de la statistique demandée. Valeurs autorisées :
- country : nombre de requêtes passées par pays
- map : une carte avec le nombre de requêtes provenant de chaque pays
- send : volume de données distribuées par protocoles
- storage : quantité de données stockées brutes et validées
- timeseries : produit une série temporelle avec en colonne le pays, le nombre et la taille cumulés des requêtes
@@ -150,7 +149,7 @@ output-options :: [format=<csv|text>] type validated -Choix entre les données stockées brutes et validées. Valeurs autorisées : all (par défaut), bud et validated +Choix entre les données stockées validées ou non. Valeurs autorisées : all (par défaut), buffer et validated years diff --git a/templates/doc_en.html b/templates/doc_en.html index 3374db7028066d9afe64feb0e44fdb5b09d52f24..dd70725fc5d4cab456acd086cc4e5f691b23e32c 100644 --- a/templates/doc_en.html +++ b/templates/doc_en.html @@ -28,9 +28,9 @@ where : request-options :: (request=<country|map|send|storage|timeseries>) channel-options :: [net=<network> & sta=<station> & loc=<location> & cha=<channel>] date-options :: [starttime=<date>] & [endtime=<date>] -storage-options :: [year=<year>] & [type=<all|bud|validated>] +storage-options :: [year=<year>] & [type=<all|buffer|validated>] send-options :: [country=<country_code>] & [media=<all|dataselect|seedlink>] -output-options :: [format=<csv|text>] +output-options :: [format=<csv|json|text>] (..) required [..] optional @@ -44,10 +44,9 @@ output-options :: [format=<csv|text>] http://ws.resif.fr/resifws/statistics/1/query?request=country&net=RA,ND&country=IT,US,FR

http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01
http://ws.resif.fr/resifws/statistics/1/query?request=map&starttime=2020-01-01&net=CL

-

http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&country=all&starttime=2020-01-01
-http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&country=all&starttime=2020-01-01

http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&starttime=2020-01-01
-http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&media=dataselect&starttime=2020-01-01

+http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&starttime=2020-01-01&country=all
+http://ws.resif.fr/resifws/statistics/1/query?request=timeseries&starttime=2020-01-01&country=all&media=dataselect

Detailed descriptions of each query parameter

@@ -95,13 +94,13 @@ output-options :: [format=<csv|text>] - - + + - +
formatjsonThe output format. Accepted values are text (the default) and csv.textThe output format. Accepted values are text (the default), json, and csv.
request sendType of statistic requested. Allowed values :
- country : the number of requests from each country
- send : the volume of distributed data by protocol
- storage : the volume of data stored
- timeseries : returns a timeseries with the country, the number and the volume of requests		Type of statistic requested. Allowed values :
- country : the number of requests from each country
- map : a color coded map with the number of requests from each country
- send : the volume of distributed data by protocol
- storage : the volume of data stored
- timeseries : returns a timeseries with the country, the number and the volume of requests
@@ -150,7 +149,7 @@ output-options :: [format=<csv|text>] type validated -Type of data. Allowed values : all (the default), bud, and validated +Choose validated data or not. Allowed values : all (the default), buffer, and validated years diff --git a/templates/wadl.xml b/templates/wadl.xml index b6b552d2706c0be48cfb3b56dd1bc5b7a9ccdea4..4ba6db15001f93da3f434312b836901f2aa83489 100644 --- a/templates/wadl.xml +++ b/templates/wadl.xml @@ -44,6 +44,7 @@