Commit 3bb84bd9 authored by Jerome Touvier's avatar Jerome Touvier
Browse files

pre fix css

parent ad8657ce
This diff is collapsed.
# Webservice statistics
## Go on with Flask
FLASK_ENV=development RUNMODE=test python start.py
## Play around with docker
```
docker build -t ws-statistics .
docker run --rm -e RUNMODE=test -p 8000:8000 --name ws-statistics ws-statistics:latest
```
Then :
```
wget -O - http://localhost:8000/1/application.wadl
```
Run it in debug mode with flask :
```
docker build -t ws-statistics .
docker run --rm --name ws-statistics -e RUNMODE=production ws-statistics
```
## RUNMODE builtin values
* `production`
* `test`
* `local`
* other values map to :
......@@ -8,12 +8,12 @@ Ce service donne accès aux statistiques de RESIF-DC.
où :
request-options :: (request=<storage|send|lands>)
request-options :: (request=<storage|send|country>)
channel-options :: [net=<network> & sta=<station> & loc=<location> & cha=<channel>]
date-options :: [starttime=<date>] & [endtime=<date>]
storage-options :: [year=<year>] & [type=<type>]
send-options :: [country=<country_code>] & [media=<all|dataselect|seedlink>]
output-options :: [format=<csv|request|sync|text>]
output-options :: [timeseries=<true|false>] & [format=<csv|request|sync|text>]
(..) requis
[..] optionnel
......@@ -24,6 +24,6 @@ Ce service donne accès aux statistiques de RESIF-DC.
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=WI&country=IT">http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=WI&country=IT</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=dataselect&net=RA&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=dataselect&net=RA&country=IT,US,FR</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=dataselect&net=RA&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=country&media=dataselect&net=RA&country=IT,US,FR</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=seedlink&net=RA&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=seedlink&net=RA&starttime=2020-01-01</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=seedlink&net=RA&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=country&media=seedlink&net=RA&starttime=2020-01-01</a>
......@@ -8,12 +8,12 @@ This service provides access to the RESIF-DC statistics.
where :
request-options :: (request=<storage|send|lands>)
request-options :: (request=<storage|send|country>)
channel-options :: [net=<network> & sta=<station> & loc=<location> & cha=<channel>]
date-options :: [starttime=<date>] & [endtime=<date>]
storage-options :: [year=<year>] & [type=<type>]
send-options :: [country=<country_code>] & [media=<all|dataselect|seedlink>]
output-options :: [format=<csv|request|sync|text>]
output-options :: [timeseries=<true|false>] & [format=<csv|request|sync|text>]
(..) required
[..] optional
......@@ -25,6 +25,6 @@ This service provides access to the RESIF-DC statistics.
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=WI&country=IT">http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=WI&country=IT</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=dataselect&net=RA&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=dataselect&net=RA&country=IT,US,FR</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=dataselect&net=RA&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=country&media=dataselect&net=RA&country=IT,US,FR</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=seedlink&net=RA&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=seedlink&net=RA&starttime=2020-01-01</a>
<a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=seedlink&net=RA&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=country&media=seedlink&net=RA&starttime=2020-01-01</a>
......@@ -17,6 +17,7 @@ parameters = {
"year": None,
"type": "all",
"request": None,
"timeseries": "F",
"media": "all",
"granularity": None,
"orderby": "nslc",
......@@ -34,6 +35,6 @@ ALIAS = [
("endtime", "end"),
]
BOOLEANS = []
BOOLEANS = ["timeseries"]
FLOATS = []
NOT_NONE = ["request"]
STRING_TRUE = ("yes", "true", "t", "y", "1", "")
STRING_FALSE = ("no", "false", "f", "n", "0")
REQUEST = ("storage", "send", "lands")
REQUEST = ("storage", "send", "country")
DATATYPE = ("all", "bud", "validated")
GRANULARITY = ("year", "month")
ORDERBY = ("nslc", "time", "country", "protocol")
......
......@@ -2,6 +2,7 @@ import logging
from apps.globals import Error, HTTP
from apps.constants import NOT_NONE
from apps.constants import BOOLEANS
from apps.utils import check_base_parameters
from apps.utils import error_param
from apps.utils import is_valid_country
......@@ -20,7 +21,7 @@ def check_parameters(params):
try:
# Check base parameters.
(params, error) = check_base_parameters(params, NOT_NONE)
(params, error) = check_base_parameters(params, NOT_NONE, BOOLEANS)
if error["code"] != 200:
return (params, error)
......
# request liste des requêtes parmi :
# storage : quantité de données stockées au nœud B, brute et validée
# Option years : liste des années pour lesquelles on demande la quantité de données
# send : volume de données distribuées par protocoles (accepte les options media et country)
# lands : liste des pays avec le nombre de requêtes passées depuis ces pays (accepte les options media et country)
# granularity : soit par année (year), soit par mois (month) (ie en interne, time(52w) et time (4w)) TODO.
# Pour les paramètres acceptant des listes le séparateur utilisé est la virgule. Les options de code réseau network, station, location et channel acceptent les jokers et les listes comme les autres WS FDSN. Pour les jokers, ? représente n’importe quel caractère unique, alors que * représente zéro caractère ou plus.
# Options supplémentaires :
# media : protocoles utilisés pour la requête (par défaut, tous)
# dataselect : pour les activités par webservice dataselect
# seedlink : pour les activités par ringserver (Pour l'instant ne peut être obtenu que indépendament.)
# all : somme de tous les types d'activité (TODO. Pour l'instant signifie tous les dataselect.)
# country : filtre les réponses avec une liste des pays d'origine des requêtes.
import logging
import os
import re
......@@ -86,13 +68,15 @@ def sql_request(params):
select = list()
for media in params["media"]:
date_s = "time" if media == "seedlink" else "date"
if params["request"] == "lands":
if params["request"] == "country":
s = f"SELECT country, count({date_s}) FROM {get_table(media)}"
if params["request"] == "send":
s = f"SELECT sum(bytes) FROM {get_table(media)}"
if params["timeseries"]:
s = f"SELECT date, network, station, location, channel, bytes, country FROM {get_table(media)}"
s = sql_common_string(params, s, date_s)
if params["request"] == "lands":
if params["request"] == "country":
s = f"""{s} AND ({is_like_or_equal(params, "country")}) GROUP BY country"""
if params["request"] == "send":
s = f"""{s} AND ({is_like_or_equal(params, "country")})"""
......@@ -109,8 +93,8 @@ def collect_data(params):
""" Connect to the PostgreSQL RESIF database """
tic = time.time()
data = list()
logging.debug("Start collecting data...")
print(os.getenv("PG_DBURI"))
with psycopg2.connect(os.getenv("PG_DBURI")) as conn:
logging.debug(conn.get_dsn_parameters())
logging.debug(f"Postgres version : {conn.server_version}")
......@@ -119,29 +103,29 @@ def collect_data(params):
curs.execute(SQL_SELECT)
logging.debug(f"{SQL_SELECT}")
logging.debug(curs.statusmessage)
for row in curs.fetchall():
data.append([str(val) for val in row])
return curs.fetchall()
logging.debug(f"Get data in {tictac(tic)} seconds.")
return data
def sum_results(params, data):
""" Adds the results from the different media tables. """
newdata = list()
if params["request"] == "lands":
if params["request"] == "country":
result = dict()
for row in data:
if row[0] in result:
result[row[0]] = result[row[0]] + int(row[1])
else:
result[row[0]] = int(row[1])
if row[1]:
if row[0] in result:
result[row[0]] = result[row[0]] + int(row[1])
else:
result[row[0]] = int(row[1])
for key, val in result.items():
newdata.append([key, str(val)])
elif params["request"] == "send":
print(data)
result = 0
for row in data:
result = result + int(row[0])
if row[0]:
result = result + int(row[0])
newdata.append([str(result)])
return newdata
......@@ -150,7 +134,7 @@ def get_header(params):
if params["request"] == "storage":
header = ["size", "year", "network", "station", "channel", "quality", "type"]
header.append("lastupdated")
elif params["request"] == "lands":
elif params["request"] == "country":
header = ["country", "requests"]
elif params["request"] == "send":
if len(params["media"]) == 1 and "all" not in params["media"]:
......@@ -158,6 +142,16 @@ def get_header(params):
header = ["SEEDLINK (in bytes)"]
elif "dataselect" in params["media"]:
header = ["DATASELECT (in bytes)"]
elif params["timeseries"]:
header = [
"date",
"network",
"station",
"location",
"channel",
"bytes",
"country",
]
else:
header = ["SEEDLINK and DATASELECT (in bytes)"]
return header
......@@ -192,6 +186,7 @@ def records_to_text(params, data, sep=" "):
def get_response(params, data):
tic = time.time()
data = [[str(val) for val in row] for row in data]
fname = "resifws-statistics"
headers = {"Content-type": "text/plain"}
if params["format"] == "text":
......@@ -226,7 +221,7 @@ def get_statistics(params):
nrows = len(data)
logging.debug(f"Number of collected rows: {nrows}")
if params["request"] != "storage":
if params["request"] != "storage" and not params["timeseries"]:
data = sum_results(params, data)
return get_response(params, data)
......
import logging
import os
from flask import Flask, make_response, render_template, redirect, request
from flask import Flask, make_response, render_template, request
from apps.constants import VERSION
from apps.root import output
......@@ -31,20 +31,13 @@ app.logger.debug(
# ********************** STATISTICS SERVICE ROUTES *************************
# **************************************************************************
BASE_URL = "/" + VERSION.split(".")[0]
@app.route("/", strict_slashes=False)
def root():
return redirect(BASE_URL, code=301)
@app.route(BASE_URL + "/query", methods=["GET"])
@app.route("/query", methods=["GET"])
def statistics_root_get():
return output(request)
@app.route(BASE_URL + "/application.wadl")
@app.route("/application.wadl")
def statistics_wadl():
template = render_template("ws-statistics.wadl.xml")
response = make_response(template)
......@@ -52,24 +45,24 @@ def statistics_wadl():
return response
@app.route(BASE_URL + "/version")
@app.route("/version")
def version():
response = make_response(VERSION)
response.headers["Content-Type"] = "text/plain"
return response
@app.route(BASE_URL, strict_slashes=False)
@app.route(BASE_URL + "/local=fr")
@app.route("/")
@app.route("/local=fr")
def statistics_doc():
return render_template("statistics_doc.html")
@app.route(BASE_URL + "/local=en")
@app.route("/local=en")
def statistics_doc_en():
return render_template("statistics_doc_en.html")
# **** MAIN ****
if __name__ == "__main__":
app.run(host="0.0.0.0", debug=True)
app.run()
......@@ -14,11 +14,11 @@
<!--[if lt IE 9]>
<script src="//cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv-printshiv.min.js"></script>
<![endif]-->
<link rel="stylesheet" href="../static/resifws.css"/>
<a href="local=en"> <img src="../static/images/en.png" alt="Français - Anglais"> </a>
<link rel="stylesheet" href="./static/resifws.css"/>
<a href="./local=en"> <img src="./static/images/en.png" alt="Français - Anglais"> </a>
</head>
<body>
<p style="text-align:center;"> <img src="../static/images/logoresif.png" alt="logoresif" width="341" height="95"></p>
<p style="text-align:center;"> <img src="./static/images/logoresif.png" alt="logoresif" width="341" height="95"></p>
<h1 id="webservice-fdsn-ws-statistics">Webservice FDSN ws-statistics</h1>
<p>Ce service donne accès aux statistiques de RESIF-DC.</p>
<h2 id="utilisation-de-la-requête">Utilisation de la requête</h2>
......@@ -26,7 +26,7 @@
où :
request-options :: (request=&lt;storage|send|lands&gt;)
request-options :: (request=&lt;storage|send|country&gt;)
channel-options :: [net=&lt;network&gt; &amp; sta=&lt;station&gt; &amp; loc=&lt;location&gt; &amp; cha=&lt;channel&gt;]
date-options :: [starttime=&lt;date&gt;] &amp; [endtime=&lt;date&gt;]
storage-options :: [year=&lt;year&gt;] &amp; [type=&lt;type&gt;]
......@@ -38,7 +38,7 @@ output-options :: [format=&lt;csv|request|sync|text&gt;]
<h2 id="exemples-de-requêtes">Exemples de requêtes</h2>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2018&type=validated">http://ws.resif.fr/resifws/statistics/1/query?request=storage&amp;net=FR&amp;year=2018&amp;type=validated</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=WI&country=IT">http://ws.resif.fr/resifws/statistics/1/query?request=send&amp;media=dataselect&amp;net=WI&amp;country=IT</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=dataselect&net=RA&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=lands&amp;media=dataselect&amp;net=RA&amp;country=IT,US,FR</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=seedlink&net=RA&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=lands&amp;media=seedlink&amp;net=RA&amp;starttime=2020-01-01</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=dataselect&net=RA&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=country&amp;media=dataselect&amp;net=RA&amp;country=IT,US,FR</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=seedlink&net=RA&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=country&amp;media=seedlink&amp;net=RA&amp;starttime=2020-01-01</a></p>
</body>
</html>
......@@ -14,11 +14,11 @@
<!--[if lt IE 9]>
<script src="//cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv-printshiv.min.js"></script>
<![endif]-->
<link rel="stylesheet" href="../static/resifws.css"/>
<a href="local=fr"> <img src="../static/images/fr.png" alt="Anglais - Français"> </a>
<link rel="stylesheet" href="./static/resifws.css"/>
<a href="./local=fr"> <img src="./static/images/fr.png" alt="Anglais - Français"> </a>
</head>
<body>
<p style="text-align:center;"> <img src="../static/images/logoresif.png" alt="logoresif" width="341" height="95"></p>
<p style="text-align:center;"> <img src="./static/images/logoresif.png" alt="logoresif" width="341" height="95"></p>
<h1 id="webservice-fdsn-ws-statistics">Webservice FDSN ws-statistics</h1>
<p>This service provides access to the RESIF-DC statistics.</p>
<h2 id="query-usage">Query usage</h2>
......@@ -26,7 +26,7 @@
where :
request-options :: (request=&lt;storage|send|lands&gt;)
request-options :: (request=&lt;storage|send|country&gt;)
channel-options :: [net=&lt;network&gt; &amp; sta=&lt;station&gt; &amp; loc=&lt;location&gt; &amp; cha=&lt;channel&gt;]
date-options :: [starttime=&lt;date&gt;] &amp; [endtime=&lt;date&gt;]
storage-options :: [year=&lt;year&gt;] &amp; [type=&lt;type&gt;]
......@@ -38,7 +38,7 @@ output-options :: [format=&lt;csv|request|sync|text&gt;]
<h2 id="sample-queries">Sample queries</h2>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2018&type=validated">http://ws.resif.fr/resifws/statistics/1/query?request=storage&amp;net=FR&amp;year=2018&amp;type=validated</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=WI&country=IT">http://ws.resif.fr/resifws/statistics/1/query?request=send&amp;media=dataselect&amp;net=WI&amp;country=IT</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=dataselect&net=RA&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=lands&amp;media=dataselect&amp;net=RA&amp;country=IT,US,FR</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=seedlink&net=RA&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=lands&amp;media=seedlink&amp;net=RA&amp;starttime=2020-01-01</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=dataselect&net=RA&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=country&amp;media=dataselect&amp;net=RA&amp;country=IT,US,FR</a></p>
<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=seedlink&net=RA&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=country&amp;media=seedlink&amp;net=RA&amp;starttime=2020-01-01</a></p>
</body>
</html>
......@@ -45,7 +45,7 @@
<param name="request" style="query" type="xsd:string">
<option value="storage"/>
<option value="send"/>
<option value="lands"/>
<option value="country"/>
</param>
<param name="country" style="query" type="xsd:string"/>
<param name="media" style="query" type="xsd:string">
......@@ -55,6 +55,7 @@
</param>
<param name="year" style="query" type="xsd:string"/>
<param name="type" style="query" type="xsd:string"/>
<param name="timeseries" style="query" type="xsd:string"/>
<param name="nodata" style="query" type="xsd:string"/>
<param name="format" style="query" type="xsd:string">
<option value="csv"/>
......
......@@ -121,6 +121,7 @@ class MyTest(unittest.TestCase):
p1["year"] = None
p1["granularity"] = None
p1["limit"] = None
p1["timeseries"] = "F"
p1["request"] = ""
p1["protocol"] = ""
p1["nodata"] = "204"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment