Skip to content

GitLab

Vous avez reçu un message "Your GitLab account has been locked ..." ? Pas d'inquiétude : lisez cet article https://docs.gricad-pages.univ-grenoble-alpes.fr/help/unlock/

Commit 95245814 authored by Jerome Touvier's avatar Jerome Touvier
Browse files

Merge branch 'upadate' into 'master' 

add timeseries

See merge request !2
parents 8a638fde 61617353
Pipeline #46432 passed with stage
in 50 seconds
Hide whitespace changes
Inline Side-by-side
Showing with 55 additions and 72 deletions
apps/README.md
View file @ 95245814
......@@ -8,12 +8,12 @@ Ce service donne accès aux statistiques de RESIF-DC.
où :
request-options :: (request=<storage|send|country>)
request-options :: (request=<country|send|storage|timeseries>)
channel-options :: [net=<network> & sta=<station> & loc=<location> & cha=<channel>]
date-options :: [starttime=<date>] & [endtime=<date>]
storage-options :: [year=<year>] & [type=<type>]
send-options :: [country=<country_code>] & [media=<all|dataselect|seedlink>]
output-options :: [timeseries=<true|false>] & [format=<csv|request|sync|text>]
output-options :: [format=<csv|request|sync|text>]
(..) requis
[..] optionnel
......
apps/README_EN.md
View file @ 95245814
......@@ -8,12 +8,12 @@ This service provides access to the RESIF-DC statistics.
where :
request-options :: (request=<storage|send|country>)
request-options :: (request=<country|send|storage|timeseries>)
channel-options :: [net=<network> & sta=<station> & loc=<location> & cha=<channel>]
date-options :: [starttime=<date>] & [endtime=<date>]
storage-options :: [year=<year>] & [type=<type>]
send-options :: [country=<country_code>] & [media=<all|dataselect|seedlink>]
output-options :: [timeseries=<true|false>] & [format=<csv|request|sync|text>]
output-options :: [format=<csv|request|sync|text>]
(..) required
[..] optional
......
apps/constants.py
View file @ 95245814
......@@ -17,7 +17,6 @@ parameters = {
"year": None,
"type": "all",
"request": None,
"timeseries": "F",
"media": "all",
"granularity": None,
"orderby": "nslc",
......@@ -35,6 +34,6 @@ ALIAS = [
("endtime", "end"),
]
BOOLEANS = ["timeseries"]
BOOLEANS = []
FLOATS = []
NOT_NONE = ["request"]
apps/globals.py
View file @ 95245814
STRING_TRUE = ("yes", "true", "t", "y", "1", "")
STRING_FALSE = ("no", "false", "f", "n", "0")
REQUEST = ("storage", "send", "country")
REQUEST = ("storage", "send", "country", "timeseries")
DATATYPE = ("all", "bud", "validated")
GRANULARITY = ("year", "month")
ORDERBY = ("nslc", "time", "country", "protocol")
......
apps/model.py
View file @ 95245814
......@@ -40,6 +40,8 @@ def check_parameters(params):
# media parameter validation
if params["media"]:
if params["media"] == "all":
params["media"] = "dataselect,seedlink"
params["media"] = params["media"].split(",")
for ind, media in enumerate(params["media"]):
if not is_valid_media(media):
......
apps/output.py
View file @ 95245814
......@@ -5,7 +5,7 @@ import time
from datetime import datetime
import psycopg2
from flask import make_response
from flask import current_app, make_response
from apps.globals import Error
from apps.utils import error_request
......@@ -22,13 +22,11 @@ def is_like_or_equal(params, key):
return " OR ".join(subquery)
def get_table(media):
if media == "seedlink":
return "ringserver_events"
return "dataselectvol"
def get_table():
return "sent_data_summary_weekly"
def sql_common_string(params, begin="", date_s="date"):
def sql_common_string(params, begin=""):
s = begin
# network, station, location, channel parameters
s = f"""{s} WHERE ({is_like_or_equal(params, "network")})"""
......@@ -42,10 +40,10 @@ def sql_common_string(params, begin="", date_s="date"):
# starttime, endtime parameters
if params["start"]:
start = datetime.strftime(params["start"], "%Y-%m-%d")
s = f"""{s} AND {date_s} >= '{start}'"""
s = f"""{s} AND date >= '{start}'"""
if params["end"]:
end = datetime.strftime(params["end"], "%Y-%m-%d")
s = f"""{s} AND {date_s} <= '{end}'"""
s = f"""{s} AND date <= '{end}'"""
return s
......@@ -65,42 +63,45 @@ def sql_request(params):
s = f"""{s} ORDER BY {columns}, date DESC"""
return s.replace("?", "_").replace("*", "%")
else:
select = list()
for media in params["media"]:
date_s = "time" if media == "seedlink" else "date"
if params["request"] == "country":
s = f"SELECT country, count({date_s}) FROM {get_table(media)}"
if params["request"] == "send":
s = f"SELECT sum(bytes) FROM {get_table(media)}"
if params["timeseries"]:
s = f"SELECT date, network, station, location, channel, bytes, country FROM {get_table(media)}"
s = sql_common_string(params, s, date_s)
if params["request"] == "country":
s = f"""{s} AND ({is_like_or_equal(params, "country")}) GROUP BY country"""
if params["request"] == "send":
s = f"""{s} AND ({is_like_or_equal(params, "country")})"""
# if params["granularity"]:
# granularity = "52w" if params["granularity"] == "year" else "4w"
# s = f"""{s} , time({granularity})"""
select.append(s)
select = " UNION ".join(select)
return select.replace("?", "_").replace("*", "%")
if params["request"] == "country":
s = f"SELECT country, count(date) FROM {get_table()}"
elif params["request"] == "send":
s = f"SELECT sum(bytes) FROM {get_table()}"
elif params["request"] == "timeseries":
s = f"""SELECT date, country, sum(bytes), sum(hll_cardinality(clients))::INTEGER FROM {get_table()}"""
s = sql_common_string(params, s)
s = f"""{s} AND ({is_like_or_equal(params, "country")})"""
if "seedlink" in params["media"] and "dataselect" in params["media"]:
s = f"{s} AND (protocol = 'seedlink' OR protocol = 'dataselect')"
elif "seedlink" in params["media"]:
s = f"{s} AND protocol = 'seedlink'"
elif "dataselect" in params["media"]:
s = f"{s} AND protocol = 'dataselect'"
if params["request"] == "country":
s = f"""{s} GROUP BY country"""
elif params["request"] == "timeseries":
if "seedlink" in params["media"] and "dataselect" in params["media"]:
s = f"""{s} GROUP BY date, country ORDER BY date"""
else:
s = f"""{s} GROUP BY date, country, protocol ORDER BY date"""
return s.replace("?", "_").replace("*", "%")
def collect_data(params):
""" Connect to the PostgreSQL RESIF database """
""" Get the result of the SQL query. """
tic = time.time()
logging.debug("Start collecting data...")
with psycopg2.connect(os.getenv("PG_DBURI")) as conn:
with psycopg2.connect(current_app.config["DATABASE_URI"]) as conn:
logging.debug(conn.get_dsn_parameters())
logging.debug(f"Postgres version : {conn.server_version}")
with conn.cursor() as curs:
SQL_SELECT = sql_request(params)
curs.execute(SQL_SELECT)
logging.debug(f"{SQL_SELECT}")
select = sql_request(params)
logging.debug(select)
curs.execute(select)
logging.debug(curs.statusmessage)
return curs.fetchall()
logging.debug(f"Get data in {tictac(tic)} seconds.")
......@@ -140,18 +141,10 @@ def get_header(params):
header = ["SEEDLINK (in bytes)"]
elif "dataselect" in params["media"]:
header = ["DATASELECT (in bytes)"]
elif params["timeseries"]:
header = [
"date",
"network",
"station",
"location",
"channel",
"bytes",
"country",
]
else:
header = ["SEEDLINK and DATASELECT (in bytes)"]
elif params["request"] == "timeseries":
header = ["time", "country", "bytes", "clients"]
return header
......@@ -219,7 +212,7 @@ def get_statistics(params):
nrows = len(data)
logging.debug(f"Number of collected rows: {nrows}")
if params["request"] != "storage" and not params["timeseries"]:
if params["request"] in ("send", "country"):
data = sum_results(params, data)
return get_response(params, data)
......
config.py
View file @ 95245814
......@@ -9,7 +9,6 @@ class Config:
Each parameter can be overriden directly by an environment variable.
"""
PGHOST = PGUSER = PGPORT = PGDATABASE = ""
RUNMODE = os.environ.get("RUNMODE")
if RUNMODE == "production":
PGHOST = "resif-pgprod.u-ga.fr"
......@@ -27,10 +26,9 @@ class Config:
PGUSER = os.environ.get("PGUSER") or PGUSER
PGPORT = os.environ.get("PGPORT") or PGPORT
PGDATABASE = os.environ.get("PGDATABASE") or PGDATABASE
os.environ["PG_DBURI"] = f"postgresql://{PGUSER}@{PGHOST}:{PGPORT}/{PGDATABASE}"
except NameError as ne:
print(ne)
except NameError:
print(
"Missing environment variables. Either RUNMODE=(test|production) or PGHOST, PGUSER, PGPORT and PGDATABASE should be set."
)
raise
DATABASE_URI = f"postgresql://{PGUSER}@{PGHOST}:{PGPORT}/{PGDATABASE}"
start.py
View file @ 95245814
......@@ -17,15 +17,7 @@ logging.basicConfig(format=fmt, level=loglevel)
app.config.from_object(Config)
if app.config["RUNMODE"]:
app.logger.debug("Configuration set with RUNMODE=%s", app.config["RUNMODE"])
# with FLASK_ENV=development
app.logger.debug(
"Database URI : postgres://%s@%s:%s/%s",
app.config["PGUSER"],
app.config["PGHOST"],
app.config["PGPORT"],
app.config["PGDATABASE"],
)
app.logger.debug("Database URI : %s", app.config["DATABASE_URI"])
# **************************************************************************
# ********************** STATISTICS SERVICE ROUTES *************************
......
templates/ws-statistics.wadl.xml
View file @ 95245814
......@@ -43,25 +43,25 @@
<param name="start" style="query" type="xsd:string"/>
<param name="end" style="query" type="xsd:string"/>
<param name="request" style="query" type="xsd:string">
<option value="storage"/>
<option value="send"/>
<option value="country"/>
<option value="send"/>
<option value="storage"/>
<option value="timeseries"/>
</param>
<param name="country" style="query" type="xsd:string"/>
<param name="media" style="query" type="xsd:string">
<option value="all"/>
<option value="dataselect"/>
<option value="seedlink"/>
<option value="all"/>
</param>
<param name="year" style="query" type="xsd:string"/>
<param name="type" style="query" type="xsd:string"/>
<param name="timeseries" style="query" type="xsd:string"/>
<param name="nodata" style="query" type="xsd:string"/>
<param name="format" style="query" type="xsd:string">
<option value="csv"/>
<option value="text"/>
<option value="request"/>
<option value="sync"/>
<option value="text"/>
</param>
</request>
......
tests/test.py
View file @ 95245814
......@@ -121,7 +121,6 @@ class MyTest(unittest.TestCase):
p1["year"] = None
p1["granularity"] = None
p1["limit"] = None
p1["timeseries"] = "F"
p1["request"] = ""
p1["protocol"] = ""
p1["nodata"] = "204"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment