Vous avez reçu un message "Your GitLab account has been locked ..." ? Pas d'inquiétude : lisez cet article https://docs.gricad-pages.univ-grenoble-alpes.fr/help/unlock/

Commit 61617353 authored by Jerome Touvier's avatar Jerome Touvier
Browse files

add timeseries

parent 8a638fde
Pipeline #46430 passed with stage
in 52 seconds
...@@ -8,12 +8,12 @@ Ce service donne accès aux statistiques de RESIF-DC. ...@@ -8,12 +8,12 @@ Ce service donne accès aux statistiques de RESIF-DC.
où : où :
request-options :: (request=<storage|send|country>) request-options :: (request=<country|send|storage|timeseries>)
channel-options :: [net=<network> & sta=<station> & loc=<location> & cha=<channel>] channel-options :: [net=<network> & sta=<station> & loc=<location> & cha=<channel>]
date-options :: [starttime=<date>] & [endtime=<date>] date-options :: [starttime=<date>] & [endtime=<date>]
storage-options :: [year=<year>] & [type=<type>] storage-options :: [year=<year>] & [type=<type>]
send-options :: [country=<country_code>] & [media=<all|dataselect|seedlink>] send-options :: [country=<country_code>] & [media=<all|dataselect|seedlink>]
output-options :: [timeseries=<true|false>] & [format=<csv|request|sync|text>] output-options :: [format=<csv|request|sync|text>]
(..) requis (..) requis
[..] optionnel [..] optionnel
......
...@@ -8,12 +8,12 @@ This service provides access to the RESIF-DC statistics. ...@@ -8,12 +8,12 @@ This service provides access to the RESIF-DC statistics.
where : where :
request-options :: (request=<storage|send|country>) request-options :: (request=<country|send|storage|timeseries>)
channel-options :: [net=<network> & sta=<station> & loc=<location> & cha=<channel>] channel-options :: [net=<network> & sta=<station> & loc=<location> & cha=<channel>]
date-options :: [starttime=<date>] & [endtime=<date>] date-options :: [starttime=<date>] & [endtime=<date>]
storage-options :: [year=<year>] & [type=<type>] storage-options :: [year=<year>] & [type=<type>]
send-options :: [country=<country_code>] & [media=<all|dataselect|seedlink>] send-options :: [country=<country_code>] & [media=<all|dataselect|seedlink>]
output-options :: [timeseries=<true|false>] & [format=<csv|request|sync|text>] output-options :: [format=<csv|request|sync|text>]
(..) required (..) required
[..] optional [..] optional
......
...@@ -17,7 +17,6 @@ parameters = { ...@@ -17,7 +17,6 @@ parameters = {
"year": None, "year": None,
"type": "all", "type": "all",
"request": None, "request": None,
"timeseries": "F",
"media": "all", "media": "all",
"granularity": None, "granularity": None,
"orderby": "nslc", "orderby": "nslc",
...@@ -35,6 +34,6 @@ ALIAS = [ ...@@ -35,6 +34,6 @@ ALIAS = [
("endtime", "end"), ("endtime", "end"),
] ]
BOOLEANS = ["timeseries"] BOOLEANS = []
FLOATS = [] FLOATS = []
NOT_NONE = ["request"] NOT_NONE = ["request"]
STRING_TRUE = ("yes", "true", "t", "y", "1", "") STRING_TRUE = ("yes", "true", "t", "y", "1", "")
STRING_FALSE = ("no", "false", "f", "n", "0") STRING_FALSE = ("no", "false", "f", "n", "0")
REQUEST = ("storage", "send", "country") REQUEST = ("storage", "send", "country", "timeseries")
DATATYPE = ("all", "bud", "validated") DATATYPE = ("all", "bud", "validated")
GRANULARITY = ("year", "month") GRANULARITY = ("year", "month")
ORDERBY = ("nslc", "time", "country", "protocol") ORDERBY = ("nslc", "time", "country", "protocol")
......
...@@ -40,6 +40,8 @@ def check_parameters(params): ...@@ -40,6 +40,8 @@ def check_parameters(params):
# media parameter validation # media parameter validation
if params["media"]: if params["media"]:
if params["media"] == "all":
params["media"] = "dataselect,seedlink"
params["media"] = params["media"].split(",") params["media"] = params["media"].split(",")
for ind, media in enumerate(params["media"]): for ind, media in enumerate(params["media"]):
if not is_valid_media(media): if not is_valid_media(media):
......
...@@ -5,7 +5,7 @@ import time ...@@ -5,7 +5,7 @@ import time
from datetime import datetime from datetime import datetime
import psycopg2 import psycopg2
from flask import make_response from flask import current_app, make_response
from apps.globals import Error from apps.globals import Error
from apps.utils import error_request from apps.utils import error_request
...@@ -22,13 +22,11 @@ def is_like_or_equal(params, key): ...@@ -22,13 +22,11 @@ def is_like_or_equal(params, key):
return " OR ".join(subquery) return " OR ".join(subquery)
def get_table(media): def get_table():
if media == "seedlink": return "sent_data_summary_weekly"
return "ringserver_events"
return "dataselectvol"
def sql_common_string(params, begin="", date_s="date"): def sql_common_string(params, begin=""):
s = begin s = begin
# network, station, location, channel parameters # network, station, location, channel parameters
s = f"""{s} WHERE ({is_like_or_equal(params, "network")})""" s = f"""{s} WHERE ({is_like_or_equal(params, "network")})"""
...@@ -42,10 +40,10 @@ def sql_common_string(params, begin="", date_s="date"): ...@@ -42,10 +40,10 @@ def sql_common_string(params, begin="", date_s="date"):
# starttime, endtime parameters # starttime, endtime parameters
if params["start"]: if params["start"]:
start = datetime.strftime(params["start"], "%Y-%m-%d") start = datetime.strftime(params["start"], "%Y-%m-%d")
s = f"""{s} AND {date_s} >= '{start}'""" s = f"""{s} AND date >= '{start}'"""
if params["end"]: if params["end"]:
end = datetime.strftime(params["end"], "%Y-%m-%d") end = datetime.strftime(params["end"], "%Y-%m-%d")
s = f"""{s} AND {date_s} <= '{end}'""" s = f"""{s} AND date <= '{end}'"""
return s return s
...@@ -65,42 +63,45 @@ def sql_request(params): ...@@ -65,42 +63,45 @@ def sql_request(params):
s = f"""{s} ORDER BY {columns}, date DESC""" s = f"""{s} ORDER BY {columns}, date DESC"""
return s.replace("?", "_").replace("*", "%") return s.replace("?", "_").replace("*", "%")
else: else:
select = list() if params["request"] == "country":
for media in params["media"]: s = f"SELECT country, count(date) FROM {get_table()}"
date_s = "time" if media == "seedlink" else "date" elif params["request"] == "send":
if params["request"] == "country": s = f"SELECT sum(bytes) FROM {get_table()}"
s = f"SELECT country, count({date_s}) FROM {get_table(media)}" elif params["request"] == "timeseries":
if params["request"] == "send": s = f"""SELECT date, country, sum(bytes), sum(hll_cardinality(clients))::INTEGER FROM {get_table()}"""
s = f"SELECT sum(bytes) FROM {get_table(media)}"
if params["timeseries"]: s = sql_common_string(params, s)
s = f"SELECT date, network, station, location, channel, bytes, country FROM {get_table(media)}" s = f"""{s} AND ({is_like_or_equal(params, "country")})"""
s = sql_common_string(params, s, date_s)
if "seedlink" in params["media"] and "dataselect" in params["media"]:
if params["request"] == "country": s = f"{s} AND (protocol = 'seedlink' OR protocol = 'dataselect')"
s = f"""{s} AND ({is_like_or_equal(params, "country")}) GROUP BY country""" elif "seedlink" in params["media"]:
if params["request"] == "send": s = f"{s} AND protocol = 'seedlink'"
s = f"""{s} AND ({is_like_or_equal(params, "country")})""" elif "dataselect" in params["media"]:
s = f"{s} AND protocol = 'dataselect'"
# if params["granularity"]:
# granularity = "52w" if params["granularity"] == "year" else "4w" if params["request"] == "country":
# s = f"""{s} , time({granularity})""" s = f"""{s} GROUP BY country"""
select.append(s) elif params["request"] == "timeseries":
select = " UNION ".join(select) if "seedlink" in params["media"] and "dataselect" in params["media"]:
return select.replace("?", "_").replace("*", "%") s = f"""{s} GROUP BY date, country ORDER BY date"""
else:
s = f"""{s} GROUP BY date, country, protocol ORDER BY date"""
return s.replace("?", "_").replace("*", "%")
def collect_data(params): def collect_data(params):
""" Connect to the PostgreSQL RESIF database """ """ Get the result of the SQL query. """
tic = time.time() tic = time.time()
logging.debug("Start collecting data...") logging.debug("Start collecting data...")
with psycopg2.connect(os.getenv("PG_DBURI")) as conn: with psycopg2.connect(current_app.config["DATABASE_URI"]) as conn:
logging.debug(conn.get_dsn_parameters()) logging.debug(conn.get_dsn_parameters())
logging.debug(f"Postgres version : {conn.server_version}") logging.debug(f"Postgres version : {conn.server_version}")
with conn.cursor() as curs: with conn.cursor() as curs:
SQL_SELECT = sql_request(params) select = sql_request(params)
curs.execute(SQL_SELECT) logging.debug(select)
logging.debug(f"{SQL_SELECT}") curs.execute(select)
logging.debug(curs.statusmessage) logging.debug(curs.statusmessage)
return curs.fetchall() return curs.fetchall()
logging.debug(f"Get data in {tictac(tic)} seconds.") logging.debug(f"Get data in {tictac(tic)} seconds.")
...@@ -140,18 +141,10 @@ def get_header(params): ...@@ -140,18 +141,10 @@ def get_header(params):
header = ["SEEDLINK (in bytes)"] header = ["SEEDLINK (in bytes)"]
elif "dataselect" in params["media"]: elif "dataselect" in params["media"]:
header = ["DATASELECT (in bytes)"] header = ["DATASELECT (in bytes)"]
elif params["timeseries"]:
header = [
"date",
"network",
"station",
"location",
"channel",
"bytes",
"country",
]
else: else:
header = ["SEEDLINK and DATASELECT (in bytes)"] header = ["SEEDLINK and DATASELECT (in bytes)"]
elif params["request"] == "timeseries":
header = ["time", "country", "bytes", "clients"]
return header return header
...@@ -219,7 +212,7 @@ def get_statistics(params): ...@@ -219,7 +212,7 @@ def get_statistics(params):
nrows = len(data) nrows = len(data)
logging.debug(f"Number of collected rows: {nrows}") logging.debug(f"Number of collected rows: {nrows}")
if params["request"] != "storage" and not params["timeseries"]: if params["request"] in ("send", "country"):
data = sum_results(params, data) data = sum_results(params, data)
return get_response(params, data) return get_response(params, data)
......
...@@ -9,7 +9,6 @@ class Config: ...@@ -9,7 +9,6 @@ class Config:
Each parameter can be overriden directly by an environment variable. Each parameter can be overriden directly by an environment variable.
""" """
PGHOST = PGUSER = PGPORT = PGDATABASE = ""
RUNMODE = os.environ.get("RUNMODE") RUNMODE = os.environ.get("RUNMODE")
if RUNMODE == "production": if RUNMODE == "production":
PGHOST = "resif-pgprod.u-ga.fr" PGHOST = "resif-pgprod.u-ga.fr"
...@@ -27,10 +26,9 @@ class Config: ...@@ -27,10 +26,9 @@ class Config:
PGUSER = os.environ.get("PGUSER") or PGUSER PGUSER = os.environ.get("PGUSER") or PGUSER
PGPORT = os.environ.get("PGPORT") or PGPORT PGPORT = os.environ.get("PGPORT") or PGPORT
PGDATABASE = os.environ.get("PGDATABASE") or PGDATABASE PGDATABASE = os.environ.get("PGDATABASE") or PGDATABASE
os.environ["PG_DBURI"] = f"postgresql://{PGUSER}@{PGHOST}:{PGPORT}/{PGDATABASE}" except NameError:
except NameError as ne:
print(ne)
print( print(
"Missing environment variables. Either RUNMODE=(test|production) or PGHOST, PGUSER, PGPORT and PGDATABASE should be set." "Missing environment variables. Either RUNMODE=(test|production) or PGHOST, PGUSER, PGPORT and PGDATABASE should be set."
) )
raise raise
DATABASE_URI = f"postgresql://{PGUSER}@{PGHOST}:{PGPORT}/{PGDATABASE}"
...@@ -17,15 +17,7 @@ logging.basicConfig(format=fmt, level=loglevel) ...@@ -17,15 +17,7 @@ logging.basicConfig(format=fmt, level=loglevel)
app.config.from_object(Config) app.config.from_object(Config)
if app.config["RUNMODE"]: if app.config["RUNMODE"]:
app.logger.debug("Configuration set with RUNMODE=%s", app.config["RUNMODE"]) app.logger.debug("Configuration set with RUNMODE=%s", app.config["RUNMODE"])
app.logger.debug("Database URI : %s", app.config["DATABASE_URI"])
# with FLASK_ENV=development
app.logger.debug(
"Database URI : postgres://%s@%s:%s/%s",
app.config["PGUSER"],
app.config["PGHOST"],
app.config["PGPORT"],
app.config["PGDATABASE"],
)
# ************************************************************************** # **************************************************************************
# ********************** STATISTICS SERVICE ROUTES ************************* # ********************** STATISTICS SERVICE ROUTES *************************
......
...@@ -43,25 +43,25 @@ ...@@ -43,25 +43,25 @@
<param name="start" style="query" type="xsd:string"/> <param name="start" style="query" type="xsd:string"/>
<param name="end" style="query" type="xsd:string"/> <param name="end" style="query" type="xsd:string"/>
<param name="request" style="query" type="xsd:string"> <param name="request" style="query" type="xsd:string">
<option value="storage"/>
<option value="send"/>
<option value="country"/> <option value="country"/>
<option value="send"/>
<option value="storage"/>
<option value="timeseries"/>
</param> </param>
<param name="country" style="query" type="xsd:string"/> <param name="country" style="query" type="xsd:string"/>
<param name="media" style="query" type="xsd:string"> <param name="media" style="query" type="xsd:string">
<option value="all"/>
<option value="dataselect"/> <option value="dataselect"/>
<option value="seedlink"/> <option value="seedlink"/>
<option value="all"/>
</param> </param>
<param name="year" style="query" type="xsd:string"/> <param name="year" style="query" type="xsd:string"/>
<param name="type" style="query" type="xsd:string"/> <param name="type" style="query" type="xsd:string"/>
<param name="timeseries" style="query" type="xsd:string"/>
<param name="nodata" style="query" type="xsd:string"/> <param name="nodata" style="query" type="xsd:string"/>
<param name="format" style="query" type="xsd:string"> <param name="format" style="query" type="xsd:string">
<option value="csv"/> <option value="csv"/>
<option value="text"/>
<option value="request"/> <option value="request"/>
<option value="sync"/> <option value="sync"/>
<option value="text"/>
</param> </param>
</request> </request>
......
...@@ -121,7 +121,6 @@ class MyTest(unittest.TestCase): ...@@ -121,7 +121,6 @@ class MyTest(unittest.TestCase):
p1["year"] = None p1["year"] = None
p1["granularity"] = None p1["granularity"] = None
p1["limit"] = None p1["limit"] = None
p1["timeseries"] = "F"
p1["request"] = "" p1["request"] = ""
p1["protocol"] = "" p1["protocol"] = ""
p1["nodata"] = "204" p1["nodata"] = "204"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment