pre fix css

3bb84bd9 · Jerome Touvier · ad8657ce · 3bb84bd9 · 3bb84bd9 · 3bb84bd9
Commit 3bb84bd9 authored Jun 22, 2020 by Jerome Touvier
--- a/LICENSE
+++ b/LICENSE
--- a/README.md
+++ b/README.md
+# Webservice statistics
+
+## Go on with Flask
+
+    FLASK_ENV=development RUNMODE=test python start.py
+
+## Play around with docker
+
+```
+docker build -t ws-statistics .
+docker run --rm -e RUNMODE=test -p 8000:8000 --name ws-statistics ws-statistics:latest
+```
+
+Then :
+
+```
+wget -O - http://localhost:8000/1/application.wadl
+```
+
+Run it in debug mode with flask :
+
+```
+docker build -t ws-statistics .
+docker run --rm --name ws-statistics -e RUNMODE=production ws-statistics
+```
+
+## RUNMODE builtin values
+
+  * `production` 
+  * `test`
+  * `local`
+  * other values map to :
--- a/apps/README.md
+++ b/apps/README.md
@@ -8,12 +8,12 @@ Ce service donne accès aux statistiques de RESIF-DC.

    où :

-    request-options    ::  (request=<storage|send|lands>)
+    request-options    ::  (request=<storage|send|country>)
    channel-options    ::  [net=<network> & sta=<station> & loc=<location> & cha=<channel>]
    date-options       ::  [starttime=<date>] & [endtime=<date>]
    storage-options    ::  [year=<year>] & [type=<type>]
    send-options       ::  [country=<country_code>] & [media=<all|dataselect|seedlink>]
-    output-options     ::  [format=<csv|request|sync|text>]
+    output-options     ::  [timeseries=<true|false>] & [format=<csv|request|sync|text>]

    (..) requis
    [..] optionnel
@@ -24,6 +24,6 @@ Ce service donne accès aux statistiques de RESIF-DC.

 <a href="http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=WI&country=IT">http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=WI&country=IT</a>

-<a href="http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=dataselect&net=RA&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=dataselect&net=RA&country=IT,US,FR</a>
+<a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=dataselect&net=RA&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=country&media=dataselect&net=RA&country=IT,US,FR</a>

-<a href="http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=seedlink&net=RA&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=seedlink&net=RA&starttime=2020-01-01</a>
+<a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=seedlink&net=RA&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=country&media=seedlink&net=RA&starttime=2020-01-01</a>
--- a/apps/README_EN.md
+++ b/apps/README_EN.md
@@ -8,12 +8,12 @@ This service provides access to the RESIF-DC statistics.

    where :

-    request-options    ::  (request=<storage|send|lands>)
+    request-options    ::  (request=<storage|send|country>)
    channel-options    ::  [net=<network> & sta=<station> & loc=<location> & cha=<channel>]
    date-options       ::  [starttime=<date>] & [endtime=<date>]
    storage-options    ::  [year=<year>] & [type=<type>]
    send-options       ::  [country=<country_code>] & [media=<all|dataselect|seedlink>]
-    output-options     ::  [format=<csv|request|sync|text>]
+    output-options     ::  [timeseries=<true|false>] & [format=<csv|request|sync|text>]

    (..) required
    [..] optional
@@ -25,6 +25,6 @@ This service provides access to the RESIF-DC statistics.

 <a href="http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=WI&country=IT">http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=WI&country=IT</a>

-<a href="http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=dataselect&net=RA&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=dataselect&net=RA&country=IT,US,FR</a>
+<a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=dataselect&net=RA&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=country&media=dataselect&net=RA&country=IT,US,FR</a>

-<a href="http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=seedlink&net=RA&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=seedlink&net=RA&starttime=2020-01-01</a>
+<a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=seedlink&net=RA&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=country&media=seedlink&net=RA&starttime=2020-01-01</a>
--- a/apps/constants.py
+++ b/apps/constants.py
@@ -17,6 +17,7 @@ parameters = {
    "year": None,
    "type": "all",
    "request": None,
+    "timeseries": "F",
    "media": "all",
    "granularity": None,
    "orderby": "nslc",
@@ -34,6 +35,6 @@ ALIAS = [
    ("endtime", "end"),
 ]

-BOOLEANS = []
+BOOLEANS = ["timeseries"]
 FLOATS = []
 NOT_NONE = ["request"]
--- a/apps/globals.py
+++ b/apps/globals.py
 STRING_TRUE = ("yes", "true", "t", "y", "1", "")
 STRING_FALSE = ("no", "false", "f", "n", "0")
-REQUEST = ("storage", "send", "lands")
+REQUEST = ("storage", "send", "country")
 DATATYPE = ("all", "bud", "validated")
 GRANULARITY = ("year", "month")
 ORDERBY = ("nslc", "time", "country", "protocol")

--- a/apps/model.py
+++ b/apps/model.py
@@ -2,6 +2,7 @@ import logging

 from apps.globals import Error, HTTP
 from apps.constants import NOT_NONE
+from apps.constants import BOOLEANS
 from apps.utils import check_base_parameters
 from apps.utils import error_param
 from apps.utils import is_valid_country
@@ -20,7 +21,7 @@ def check_parameters(params):

    try:
        # Check base parameters.
-        (params, error) = check_base_parameters(params, NOT_NONE)
+        (params, error) = check_base_parameters(params, NOT_NONE, BOOLEANS)
        if error["code"] != 200:
            return (params, error)


--- a/apps/output.py
+++ b/apps/output.py
-#    request liste des requêtes parmi :
-#        storage : quantité de données stockées au nœud B, brute et validée
-#            Option years : liste des années pour lesquelles on demande la quantité de données
-#        send : volume de données distribuées par protocoles (accepte les options media et country)
-#        lands : liste des pays avec le nombre de requêtes passées depuis ces pays (accepte les options media et country)
-#    granularity : soit par année (year), soit par mois (month) (ie en interne, time(52w) et time (4w)) TODO.
-
-# Pour les paramètres acceptant des listes le séparateur utilisé est la virgule. Les options de code réseau network, station, location et channel acceptent les jokers et les listes comme les autres WS FDSN. Pour les jokers, ? représente n’importe quel caractère unique, alors que * représente zéro caractère ou plus.
-
-# Options supplémentaires :
-
-#    media : protocoles utilisés pour la requête (par défaut, tous)
-#        dataselect : pour les activités par webservice dataselect
-#        seedlink : pour les activités par ringserver (Pour l'instant ne peut être obtenu que indépendament.)
-#        all : somme de tous les types d'activité (TODO. Pour l'instant signifie tous les dataselect.)
-#    country : filtre les réponses avec une liste des pays d'origine des requêtes.
-
-
 import logging
 import os
 import re
@@ -86,13 +68,15 @@ def sql_request(params):
        select = list()
        for media in params["media"]:
            date_s = "time" if media == "seedlink" else "date"
-            if params["request"] == "lands":
+            if params["request"] == "country":
                s = f"SELECT country, count({date_s}) FROM {get_table(media)}"
            if params["request"] == "send":
                s = f"SELECT sum(bytes) FROM {get_table(media)}"
+                if params["timeseries"]:
+                    s = f"SELECT date, network, station, location, channel, bytes, country FROM {get_table(media)}"
            s = sql_common_string(params, s, date_s)

-            if params["request"] == "lands":
+            if params["request"] == "country":
                s = f"""{s} AND ({is_like_or_equal(params, "country")}) GROUP BY country"""
            if params["request"] == "send":
                s = f"""{s} AND ({is_like_or_equal(params, "country")})"""
@@ -109,8 +93,8 @@ def collect_data(params):
    """ Connect to the PostgreSQL RESIF database """

    tic = time.time()
-    data = list()
    logging.debug("Start collecting data...")
+    print(os.getenv("PG_DBURI"))
    with psycopg2.connect(os.getenv("PG_DBURI")) as conn:
        logging.debug(conn.get_dsn_parameters())
        logging.debug(f"Postgres version : {conn.server_version}")
@@ -119,29 +103,29 @@ def collect_data(params):
            curs.execute(SQL_SELECT)
            logging.debug(f"{SQL_SELECT}")
            logging.debug(curs.statusmessage)
-            for row in curs.fetchall():
-                data.append([str(val) for val in row])
+            return curs.fetchall()
            logging.debug(f"Get data in {tictac(tic)} seconds.")
-    return data


 def sum_results(params, data):
    """ Adds the results from the different media tables. """
    newdata = list()
-    if params["request"] == "lands":
+    if params["request"] == "country":
        result = dict()
        for row in data:
-            if row[0] in result:
-                result[row[0]] = result[row[0]] + int(row[1])
-            else:
-                result[row[0]] = int(row[1])
+            if row[1]:
+                if row[0] in result:
+                    result[row[0]] = result[row[0]] + int(row[1])
+                else:
+                    result[row[0]] = int(row[1])
        for key, val in result.items():
            newdata.append([key, str(val)])
    elif params["request"] == "send":
        print(data)
        result = 0
        for row in data:
-            result = result + int(row[0])
+            if row[0]:
+                result = result + int(row[0])
        newdata.append([str(result)])
    return newdata

@@ -150,7 +134,7 @@ def get_header(params):
    if params["request"] == "storage":
        header = ["size", "year", "network", "station", "channel", "quality", "type"]
        header.append("lastupdated")
-    elif params["request"] == "lands":
+    elif params["request"] == "country":
        header = ["country", "requests"]
    elif params["request"] == "send":
        if len(params["media"]) == 1 and "all" not in params["media"]:
@@ -158,6 +142,16 @@ def get_header(params):
                header = ["SEEDLINK (in bytes)"]
            elif "dataselect" in params["media"]:
                header = ["DATASELECT (in bytes)"]
+        elif params["timeseries"]:
+            header = [
+                "date",
+                "network",
+                "station",
+                "location",
+                "channel",
+                "bytes",
+                "country",
+            ]
        else:
            header = ["SEEDLINK and DATASELECT (in bytes)"]
    return header
@@ -192,6 +186,7 @@ def records_to_text(params, data, sep=" "):

 def get_response(params, data):
    tic = time.time()
+    data = [[str(val) for val in row] for row in data]
    fname = "resifws-statistics"
    headers = {"Content-type": "text/plain"}
    if params["format"] == "text":
@@ -226,7 +221,7 @@ def get_statistics(params):
        nrows = len(data)
        logging.debug(f"Number of collected rows: {nrows}")

-        if params["request"] != "storage":
+        if params["request"] != "storage" and not params["timeseries"]:
            data = sum_results(params, data)

        return get_response(params, data)

--- a/start.py
+++ b/start.py
 import logging
 import os

-from flask import Flask, make_response, render_template, redirect, request
+from flask import Flask, make_response, render_template, request

 from apps.constants import VERSION
 from apps.root import output
@@ -31,20 +31,13 @@ app.logger.debug(
 # ********************** STATISTICS SERVICE ROUTES *************************
 # **************************************************************************

-BASE_URL = "/" + VERSION.split(".")[0]

-
-@app.route("/", strict_slashes=False)
-def root():
-    return redirect(BASE_URL, code=301)
-
-
-@app.route(BASE_URL + "/query", methods=["GET"])
+@app.route("/query", methods=["GET"])
 def statistics_root_get():
    return output(request)


-@app.route(BASE_URL + "/application.wadl")
+@app.route("/application.wadl")
 def statistics_wadl():
    template = render_template("ws-statistics.wadl.xml")
    response = make_response(template)
@@ -52,24 +45,24 @@ def statistics_wadl():
    return response


-@app.route(BASE_URL + "/version")
+@app.route("/version")
 def version():
    response = make_response(VERSION)
    response.headers["Content-Type"] = "text/plain"
    return response


-@app.route(BASE_URL, strict_slashes=False)
-@app.route(BASE_URL + "/local=fr")
+@app.route("/")
+@app.route("/local=fr")
 def statistics_doc():
    return render_template("statistics_doc.html")


-@app.route(BASE_URL + "/local=en")
+@app.route("/local=en")
 def statistics_doc_en():
    return render_template("statistics_doc_en.html")


 # **** MAIN ****
 if __name__ == "__main__":
-    app.run(host="0.0.0.0", debug=True)
+    app.run()
--- a/templates/statistics_doc.html
+++ b/templates/statistics_doc.html
@@ -14,11 +14,11 @@
  <!--[if lt IE 9]>
    <script src="//cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv-printshiv.min.js"></script>
  <![endif]-->
-<link rel="stylesheet" href="../static/resifws.css"/>
-<a href="local=en"> <img src="../static/images/en.png" alt="Français - Anglais"> </a>
+<link rel="stylesheet" href="./static/resifws.css"/>
+<a href="./local=en"> <img src="./static/images/en.png" alt="Français - Anglais"> </a>
 </head>
 <body>
-<p style="text-align:center;"> <img src="../static/images/logoresif.png" alt="logoresif" width="341" height="95"></p>
+<p style="text-align:center;"> <img src="./static/images/logoresif.png" alt="logoresif" width="341" height="95"></p>
 <h1 id="webservice-fdsn-ws-statistics">Webservice FDSN ws-statistics</h1>
 <p>Ce service donne accès aux statistiques de RESIF-DC.</p>
 <h2 id="utilisation-de-la-requête">Utilisation de la requête</h2>
@@ -26,7 +26,7 @@

 où :

-request-options    ::  (request=&lt;storage|send|lands&gt;)
+request-options    ::  (request=&lt;storage|send|country&gt;)
 channel-options    ::  [net=&lt;network&gt; &amp; sta=&lt;station&gt; &amp; loc=&lt;location&gt; &amp; cha=&lt;channel&gt;]
 date-options       ::  [starttime=&lt;date&gt;] &amp; [endtime=&lt;date&gt;]
 storage-options    ::  [year=&lt;year&gt;] &amp; [type=&lt;type&gt;]
@@ -38,7 +38,7 @@ output-options     ::  [format=&lt;csv|request|sync|text&gt;]
 <h2 id="exemples-de-requêtes">Exemples de requêtes</h2>
 <p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2018&type=validated">http://ws.resif.fr/resifws/statistics/1/query?request=storage&amp;net=FR&amp;year=2018&amp;type=validated</a></p>
 <p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=WI&country=IT">http://ws.resif.fr/resifws/statistics/1/query?request=send&amp;media=dataselect&amp;net=WI&amp;country=IT</a></p>
-<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=dataselect&net=RA&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=lands&amp;media=dataselect&amp;net=RA&amp;country=IT,US,FR</a></p>
-<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=seedlink&net=RA&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=lands&amp;media=seedlink&amp;net=RA&amp;starttime=2020-01-01</a></p>
+<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=dataselect&net=RA&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=country&amp;media=dataselect&amp;net=RA&amp;country=IT,US,FR</a></p>
+<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=seedlink&net=RA&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=country&amp;media=seedlink&amp;net=RA&amp;starttime=2020-01-01</a></p>
 </body>
 </html>
--- a/templates/statistics_doc_en.html
+++ b/templates/statistics_doc_en.html
@@ -14,11 +14,11 @@
  <!--[if lt IE 9]>
    <script src="//cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv-printshiv.min.js"></script>
  <![endif]-->
-<link rel="stylesheet" href="../static/resifws.css"/>
-<a href="local=fr"> <img src="../static/images/fr.png" alt="Anglais - Français"> </a>
+<link rel="stylesheet" href="./static/resifws.css"/>
+<a href="./local=fr"> <img src="./static/images/fr.png" alt="Anglais - Français"> </a>
 </head>
 <body>
-<p style="text-align:center;"> <img src="../static/images/logoresif.png" alt="logoresif" width="341" height="95"></p>
+<p style="text-align:center;"> <img src="./static/images/logoresif.png" alt="logoresif" width="341" height="95"></p>
 <h1 id="webservice-fdsn-ws-statistics">Webservice FDSN ws-statistics</h1>
 <p>This service provides access to the RESIF-DC statistics.</p>
 <h2 id="query-usage">Query usage</h2>
@@ -26,7 +26,7 @@

 where :

-request-options    ::  (request=&lt;storage|send|lands&gt;)
+request-options    ::  (request=&lt;storage|send|country&gt;)
 channel-options    ::  [net=&lt;network&gt; &amp; sta=&lt;station&gt; &amp; loc=&lt;location&gt; &amp; cha=&lt;channel&gt;]
 date-options       ::  [starttime=&lt;date&gt;] &amp; [endtime=&lt;date&gt;]
 storage-options    ::  [year=&lt;year&gt;] &amp; [type=&lt;type&gt;]
@@ -38,7 +38,7 @@ output-options     ::  [format=&lt;csv|request|sync|text&gt;]
 <h2 id="sample-queries">Sample queries</h2>
 <p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=storage&net=FR&year=2018&type=validated">http://ws.resif.fr/resifws/statistics/1/query?request=storage&amp;net=FR&amp;year=2018&amp;type=validated</a></p>
 <p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=send&media=dataselect&net=WI&country=IT">http://ws.resif.fr/resifws/statistics/1/query?request=send&amp;media=dataselect&amp;net=WI&amp;country=IT</a></p>
-<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=dataselect&net=RA&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=lands&amp;media=dataselect&amp;net=RA&amp;country=IT,US,FR</a></p>
-<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=lands&media=seedlink&net=RA&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=lands&amp;media=seedlink&amp;net=RA&amp;starttime=2020-01-01</a></p>
+<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=dataselect&net=RA&country=IT,US,FR">http://ws.resif.fr/resifws/statistics/1/query?request=country&amp;media=dataselect&amp;net=RA&amp;country=IT,US,FR</a></p>
+<p><a href="http://ws.resif.fr/resifws/statistics/1/query?request=country&media=seedlink&net=RA&starttime=2020-01-01">http://ws.resif.fr/resifws/statistics/1/query?request=country&amp;media=seedlink&amp;net=RA&amp;starttime=2020-01-01</a></p>
 </body>
 </html>
--- a/templates/ws-statistics.wadl.xml
+++ b/templates/ws-statistics.wadl.xml
@@ -45,7 +45,7 @@
            <param name="request" style="query" type="xsd:string">
                <option value="storage"/>
                <option value="send"/>
-                <option value="lands"/>
+                <option value="country"/>
            </param>
            <param name="country" style="query" type="xsd:string"/>
            <param name="media" style="query" type="xsd:string">
@@ -55,6 +55,7 @@
            </param>
            <param name="year" style="query" type="xsd:string"/>
            <param name="type" style="query" type="xsd:string"/>
+            <param name="timeseries" style="query" type="xsd:string"/>
            <param name="nodata" style="query" type="xsd:string"/>
            <param name="format" style="query" type="xsd:string">
                <option value="csv"/>

--- a/tests/test.py
+++ b/tests/test.py
@@ -121,6 +121,7 @@ class MyTest(unittest.TestCase):
        p1["year"] = None
        p1["granularity"] = None
        p1["limit"] = None
+        p1["timeseries"] = "F"
        p1["request"] = ""
        p1["protocol"] = ""
        p1["nodata"] = "204"