Merge branch 'update2' into 'master'

update

See merge request !37

apps/output.py

@@ -38,6 +38,12 @@ def is_like_or_equal(params, key):
     return " OR ".join(subquery)
 
 
+def sum_request():
+    return (
+        "COALESCE(SUM(requests), HLL_CARDINALITY(HLL_UNION_AGG(requests_set)))::BIGINT"
+    )
+
+
 def sql_common_string(params):
     # network, station, location, channel parameters
     s = f"""WHERE ({is_like_or_equal(params, "network")})"""
@@ -71,9 +77,9 @@ def sql_common_string(params):
 def country_sql_request(params):
     table = "sent_data_summary_weekly"
     if params["country"] != "all":
-        s = f"SELECT coalesce(country, 'n/a'), sum(requests)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}"
+        s = f"SELECT COALESCE(country, 'n/a'), {sum_request()}, HLL_CARDINALITY(HLL_UNION_AGG(clients))::INTEGER FROM {table}"
     else:
-        s = f"SELECT 'all' AS country, sum(requests)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}"
+        s = f"SELECT 'all' AS country, {sum_request()}, HLL_CARDINALITY(HLL_UNION_AGG(clients))::INTEGER FROM {table}"
     s = f"{s} {sql_common_string(params)}"
     if params["country"] != "all":
         s = f"{s} GROUP BY country ORDER BY 2 DESC"
@@ -82,7 +88,7 @@ def country_sql_request(params):
 
 def map_sql_request(params):
     table = "sent_data_summary_weekly"
-    s = f"SELECT country, sum(requests)::BIGINT FROM {table}"
+    s = f"SELECT country, {sum_request()} FROM {table}"
     s = f"{s} {sql_common_string(params)} GROUP BY country"
     return s
 
@@ -90,17 +96,17 @@ def map_sql_request(params):
 def send_sql_request(params):
     table = "sent_data_summary_weekly"
     if params["sum"]:
-        s = f"""SELECT '{params["network"]}' AS network, 'all' AS station, sum(bytes)::BIGINT AS size FROM {table}"""
+        s = f"""SELECT '{params["network"]}' AS network, 'all' AS station, SUM(bytes)::BIGINT AS size FROM {table}"""
         s = f"{s} {sql_common_string(params)}"
     elif params["ranklimit"]:
         s = f"""SELECT network, station, size FROM
         (SELECT *, DENSE_RANK() OVER (PARTITION BY network ORDER BY size DESC) AS rank FROM
-          (SELECT network, station, sum(bytes)::BIGINT AS size FROM {table} {sql_common_string(params)} GROUP BY network, station
+          (SELECT network, station, SUM(bytes)::BIGINT AS size FROM {table} {sql_common_string(params)} GROUP BY network, station
           ) AS table1
         ) AS table2
         WHERE rank <= {params["ranklimit"]};"""
     else:
-        s = f"SELECT network, station, sum(bytes)::BIGINT AS size FROM {table} {sql_common_string(params)}"
+        s = f"SELECT network, station, SUM(bytes)::BIGINT AS size FROM {table} {sql_common_string(params)}"
         s = f"{s} GROUP BY network, station ORDER BY network, size DESC, station"
     return s
 
@@ -108,8 +114,8 @@ def send_sql_request(params):
 def storage_sql_request(params):
     table = "dataholdings"
     columns = "year, network, station, channel, quality"
-    s = f"SELECT DISTINCT ON ({columns}) {columns}, date, sum(size)::BIGINT FROM {table}"
-    s = f"{s} {sql_common_string(params)} AND channel is not NULL"
+    s = f"SELECT DISTINCT ON ({columns}) {columns}, date, SUM(size)::BIGINT FROM {table}"
+    s = f"{s} {sql_common_string(params)} AND channel IS NOT NULL"
     if params["type"]:
         s = f"""{s} AND ({is_like_or_equal(params, "type")})"""
         s = s.replace("buffer", "bud")
@@ -122,9 +128,9 @@ def storage_sql_request(params):
 def timeseries_sql_request(params):
     table = "sent_data_summary_weekly"
     if params["country"] != "all":
-        s = f"SELECT date, country, sum(bytes)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}"
+        s = f"SELECT date, country, SUM(bytes)::BIGINT, HLL_CARDINALITY(HLL_UNION_AGG(clients))::INTEGER FROM {table}"
     else:
-        s = f"SELECT date, 'all' as country, sum(bytes)::BIGINT, hll_cardinality(hll_union_agg(clients))::INTEGER FROM {table}"
+        s = f"SELECT date, 'all' AS country, SUM(bytes)::BIGINT, HLL_CARDINALITY(HLL_UNION_AGG(clients))::INTEGER FROM {table}"
     s = f"{s} {sql_common_string(params)}"
     if params["country"] != "all":
         s = f"{s} GROUP BY date, country ORDER BY date"

apps/utils.py

@@ -42,7 +42,7 @@ def is_valid_float(dimension, mini=sys.float_info.epsilon, maxi=sys.float_info.m
 def is_valid_datetime(date):
     for df in ("%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%dT%H:%M:%S.%f"):
         try:
-            return datetime.strptime(date, df)
+            return datetime.strptime(date.replace("Z", ""), df)
         except (ValueError, TypeError):
             pass