Merge branch 'upadate' into 'master'

add timeseries

See merge request !2

apps/README.md

@@ -8,12 +8,12 @@ Ce service donne accès aux statistiques de RESIF-DC.
 
     où :
 
-    request-options    ::  (request=<storage|send|country>)
+    request-options    ::  (request=<country|send|storage|timeseries>)
     channel-options    ::  [net=<network> & sta=<station> & loc=<location> & cha=<channel>]
     date-options       ::  [starttime=<date>] & [endtime=<date>]
     storage-options    ::  [year=<year>] & [type=<type>]
     send-options       ::  [country=<country_code>] & [media=<all|dataselect|seedlink>]
-    output-options     ::  [timeseries=<true|false>] & [format=<csv|request|sync|text>]
+    output-options     ::  [format=<csv|request|sync|text>]
 
     (..) requis
     [..] optionnel

apps/README_EN.md

@@ -8,12 +8,12 @@ This service provides access to the RESIF-DC statistics.
 
     where :
 
-    request-options    ::  (request=<storage|send|country>)
+    request-options    ::  (request=<country|send|storage|timeseries>)
     channel-options    ::  [net=<network> & sta=<station> & loc=<location> & cha=<channel>]
     date-options       ::  [starttime=<date>] & [endtime=<date>]
     storage-options    ::  [year=<year>] & [type=<type>]
     send-options       ::  [country=<country_code>] & [media=<all|dataselect|seedlink>]
-    output-options     ::  [timeseries=<true|false>] & [format=<csv|request|sync|text>]
+    output-options     ::  [format=<csv|request|sync|text>]
 
     (..) required
     [..] optional

apps/constants.py

@@ -17,7 +17,6 @@ parameters = {
     "year": None,
     "type": "all",
     "request": None,
-    "timeseries": "F",
     "media": "all",
     "granularity": None,
     "orderby": "nslc",
@@ -35,6 +34,6 @@ ALIAS = [
     ("endtime", "end"),
 ]
 
-BOOLEANS = ["timeseries"]
+BOOLEANS = []
 FLOATS = []
 NOT_NONE = ["request"]

apps/globals.py

 STRING_TRUE = ("yes", "true", "t", "y", "1", "")
 STRING_FALSE = ("no", "false", "f", "n", "0")
-REQUEST = ("storage", "send", "country")
+REQUEST = ("storage", "send", "country", "timeseries")
 DATATYPE = ("all", "bud", "validated")
 GRANULARITY = ("year", "month")
 ORDERBY = ("nslc", "time", "country", "protocol")

apps/model.py

@@ -40,6 +40,8 @@ def check_parameters(params):
 
         # media parameter validation
         if params["media"]:
+            if params["media"] == "all":
+                params["media"] = "dataselect,seedlink"
             params["media"] = params["media"].split(",")
             for ind, media in enumerate(params["media"]):
                 if not is_valid_media(media):

apps/output.py

@@ -5,7 +5,7 @@ import time
 from datetime import datetime
 
 import psycopg2
-from flask import make_response
+from flask import current_app, make_response
 
 from apps.globals import Error
 from apps.utils import error_request
@@ -22,13 +22,11 @@ def is_like_or_equal(params, key):
     return " OR ".join(subquery)
 
 
-def get_table(media):
-    if media == "seedlink":
-        return "ringserver_events"
-    return "dataselectvol"
+def get_table():
+    return "sent_data_summary_weekly"
 
 
-def sql_common_string(params, begin="", date_s="date"):
+def sql_common_string(params, begin=""):
     s = begin
     # network, station, location, channel parameters
     s = f"""{s} WHERE ({is_like_or_equal(params, "network")})"""
@@ -42,10 +40,10 @@ def sql_common_string(params, begin="", date_s="date"):
     # starttime, endtime parameters
     if params["start"]:
         start = datetime.strftime(params["start"], "%Y-%m-%d")
-        s = f"""{s} AND {date_s} >= '{start}'"""
+        s = f"""{s} AND date >= '{start}'"""
     if params["end"]:
         end = datetime.strftime(params["end"], "%Y-%m-%d")
-        s = f"""{s} AND {date_s} <= '{end}'"""
+        s = f"""{s} AND date <= '{end}'"""
     return s
 
 
@@ -65,42 +63,45 @@ def sql_request(params):
         s = f"""{s} ORDER BY {columns}, date DESC"""
         return s.replace("?", "_").replace("*", "%")
     else:
-        select = list()
-        for media in params["media"]:
-            date_s = "time" if media == "seedlink" else "date"
-            if params["request"] == "country":
-                s = f"SELECT country, count({date_s}) FROM {get_table(media)}"
-            if params["request"] == "send":
-                s = f"SELECT sum(bytes) FROM {get_table(media)}"
-                if params["timeseries"]:
-                    s = f"SELECT date, network, station, location, channel, bytes, country FROM {get_table(media)}"
-            s = sql_common_string(params, s, date_s)
-
-            if params["request"] == "country":
-                s = f"""{s} AND ({is_like_or_equal(params, "country")}) GROUP BY country"""
-            if params["request"] == "send":
-                s = f"""{s} AND ({is_like_or_equal(params, "country")})"""
-
-            #            if params["granularity"]:
-            #                granularity = "52w" if params["granularity"] == "year" else "4w"
-            #                s = f"""{s} , time({granularity})"""
-            select.append(s)
-        select = " UNION ".join(select)
-        return select.replace("?", "_").replace("*", "%")
+        if params["request"] == "country":
+            s = f"SELECT country, count(date) FROM {get_table()}"
+        elif params["request"] == "send":
+            s = f"SELECT sum(bytes) FROM {get_table()}"
+        elif params["request"] == "timeseries":
+            s = f"""SELECT date, country, sum(bytes), sum(hll_cardinality(clients))::INTEGER FROM {get_table()}"""
+
+        s = sql_common_string(params, s)
+        s = f"""{s} AND ({is_like_or_equal(params, "country")})"""
+
+        if "seedlink" in params["media"] and "dataselect" in params["media"]:
+            s = f"{s} AND (protocol = 'seedlink' OR protocol = 'dataselect')"
+        elif "seedlink" in params["media"]:
+            s = f"{s} AND protocol = 'seedlink'"
+        elif "dataselect" in params["media"]:
+            s = f"{s} AND protocol = 'dataselect'"
+
+        if params["request"] == "country":
+            s = f"""{s} GROUP BY country"""
+        elif params["request"] == "timeseries":
+            if "seedlink" in params["media"] and "dataselect" in params["media"]:
+                s = f"""{s} GROUP BY date, country ORDER BY date"""
+            else:
+                s = f"""{s} GROUP BY date, country, protocol ORDER BY date"""
+        return s.replace("?", "_").replace("*", "%")
 
 
 def collect_data(params):
-    """ Connect to the PostgreSQL RESIF database """
+    """ Get the result of the SQL query. """
 
     tic = time.time()
     logging.debug("Start collecting data...")
-    with psycopg2.connect(os.getenv("PG_DBURI")) as conn:
+    with psycopg2.connect(current_app.config["DATABASE_URI"]) as conn:
         logging.debug(conn.get_dsn_parameters())
         logging.debug(f"Postgres version : {conn.server_version}")
         with conn.cursor() as curs:
-            SQL_SELECT = sql_request(params)
-            curs.execute(SQL_SELECT)
-            logging.debug(f"{SQL_SELECT}")
+            select = sql_request(params)
+            logging.debug(select)
+            curs.execute(select)
             logging.debug(curs.statusmessage)
             return curs.fetchall()
             logging.debug(f"Get data in {tictac(tic)} seconds.")
@@ -140,18 +141,10 @@ def get_header(params):
                 header = ["SEEDLINK (in bytes)"]
             elif "dataselect" in params["media"]:
                 header = ["DATASELECT (in bytes)"]
-        elif params["timeseries"]:
-            header = [
-                "date",
-                "network",
-                "station",
-                "location",
-                "channel",
-                "bytes",
-                "country",
-            ]
         else:
             header = ["SEEDLINK and DATASELECT (in bytes)"]
+    elif params["request"] == "timeseries":
+        header = ["time", "country", "bytes", "clients"]
     return header
 
 
@@ -219,7 +212,7 @@ def get_statistics(params):
         nrows = len(data)
         logging.debug(f"Number of collected rows: {nrows}")
 
-        if params["request"] != "storage" and not params["timeseries"]:
+        if params["request"] in ("send", "country"):
             data = sum_results(params, data)
 
         return get_response(params, data)

config.py

@@ -9,7 +9,6 @@ class Config:
     Each parameter can be overriden directly by an environment variable.
     """
 
-    PGHOST = PGUSER = PGPORT = PGDATABASE = ""
     RUNMODE = os.environ.get("RUNMODE")
     if RUNMODE == "production":
         PGHOST = "resif-pgprod.u-ga.fr"
@@ -27,10 +26,9 @@ class Config:
         PGUSER = os.environ.get("PGUSER") or PGUSER
         PGPORT = os.environ.get("PGPORT") or PGPORT
         PGDATABASE = os.environ.get("PGDATABASE") or PGDATABASE
-        os.environ["PG_DBURI"] = f"postgresql://{PGUSER}@{PGHOST}:{PGPORT}/{PGDATABASE}"
-    except NameError as ne:
-        print(ne)
+    except NameError:
         print(
             "Missing environment variables. Either RUNMODE=(test|production) or PGHOST, PGUSER, PGPORT and PGDATABASE should be set."
         )
         raise
+    DATABASE_URI = f"postgresql://{PGUSER}@{PGHOST}:{PGPORT}/{PGDATABASE}"

start.py

@@ -17,15 +17,7 @@ logging.basicConfig(format=fmt, level=loglevel)
 app.config.from_object(Config)
 if app.config["RUNMODE"]:
     app.logger.debug("Configuration set with RUNMODE=%s", app.config["RUNMODE"])
-
-# with FLASK_ENV=development
-app.logger.debug(
-    "Database URI : postgres://%s@%s:%s/%s",
-    app.config["PGUSER"],
-    app.config["PGHOST"],
-    app.config["PGPORT"],
-    app.config["PGDATABASE"],
-)
+app.logger.debug("Database URI : %s", app.config["DATABASE_URI"])
 
 # **************************************************************************
 # ********************** STATISTICS SERVICE ROUTES *************************

templates/ws-statistics.wadl.xml

@@ -43,25 +43,25 @@
             <param name="start" style="query" type="xsd:string"/>
             <param name="end" style="query" type="xsd:string"/>
             <param name="request" style="query" type="xsd:string">
-                <option value="storage"/>
-                <option value="send"/>
                 <option value="country"/>
+                <option value="send"/>
+                <option value="storage"/>
+                <option value="timeseries"/>
             </param>
             <param name="country" style="query" type="xsd:string"/>
             <param name="media" style="query" type="xsd:string">
+                <option value="all"/>
                 <option value="dataselect"/>
                 <option value="seedlink"/>
-                <option value="all"/>
             </param>
             <param name="year" style="query" type="xsd:string"/>
             <param name="type" style="query" type="xsd:string"/>
-            <param name="timeseries" style="query" type="xsd:string"/>
             <param name="nodata" style="query" type="xsd:string"/>
             <param name="format" style="query" type="xsd:string">
                 <option value="csv"/>
-                <option value="text"/>
                 <option value="request"/>
                 <option value="sync"/>
+                <option value="text"/>
             </param>
         </request>
 

tests/test.py

@@ -121,7 +121,6 @@ class MyTest(unittest.TestCase):
         p1["year"] = None
         p1["granularity"] = None
         p1["limit"] = None
-        p1["timeseries"] = "F"
         p1["request"] = ""
         p1["protocol"] = ""
         p1["nodata"] = "204"