Vous avez reçu un message "Your GitLab account has been locked ..." ? Pas d'inquiétude : lisez cet article https://docs.gricad-pages.univ-grenoble-alpes.fr/help/unlock/

Commit d9bf4066 authored by Jerome Touvier's avatar Jerome Touvier
Browse files

ad map

parent ea2a1da9
Pipeline #49947 failed with stage
in 0 seconds
STRING_TRUE = ("yes", "true", "t", "y", "1", "")
STRING_FALSE = ("no", "false", "f", "n", "0")
REQUEST = ("storage", "send", "country", "timeseries")
REQUEST = ("storage", "send", "country", "timeseries", "map")
DATATYPE = ("all", "bud", "validated")
GRANULARITY = ("year", "month")
ORDERBY = ("nslc", "time", "country", "protocol")
......
codes = {
"AF": "AFGHANISTAN",
"AX": "ÅLAND ISLANDS",
"AL": "ALBANIA",
"DZ": "ALGERIA",
"AS": "AMERICAN SAMOA",
"AD": "ANDORRA",
"AO": "ANGOLA",
"AI": "ANGUILLA",
"AQ": "ANTARCTICA",
"AG": "ANTIGUA AND BARBUDA",
"AR": "ARGENTINA",
"AM": "ARMENIA",
"AW": "ARUBA",
"AU": "AUSTRALIA",
"AT": "AUSTRIA",
"AZ": "AZERBAIJAN",
"BS": "BAHAMAS",
"BH": "BAHRAIN",
"BD": "BANGLADESH",
"BB": "BARBADOS",
"BY": "BELARUS",
"BE": "BELGIUM",
"BZ": "BELIZE",
"BJ": "BENIN",
"BM": "BERMUDA",
"BT": "BHUTAN",
"BO": "BOLIVIA, PLURINATIONAL STATE OF",
"BQ": "BONAIRE, SINT EUSTATIUS AND SABA",
"BA": "BOSNIA AND HERZEGOVINA",
"BW": "BOTSWANA",
"BV": "BOUVET ISLAND",
"BR": "BRAZIL",
"IO": "BRITISH INDIAN OCEAN TERRITORY",
"BN": "BRUNEI DARUSSALAM",
"BG": "BULGARIA",
"BF": "BURKINA FASO",
"BI": "BURUNDI",
"KH": "CAMBODIA",
"CM": "CAMEROON",
"CA": "CANADA",
"CV": "CAPE VERDE",
"KY": "CAYMAN ISLANDS",
"CF": "CENTRAL AFRICAN REPUBLIC",
"TD": "CHAD",
"CL": "CHILE",
"CN": "CHINA",
"CX": "CHRISTMAS ISLAND",
"CC": "COCOS (KEELING) ISLANDS",
"CO": "COLOMBIA",
"KM": "COMOROS",
"CG": "CONGO",
"CD": "CONGO, THE DEMOCRATIC REPUBLIC OF THE",
"CK": "COOK ISLANDS",
"CR": "COSTA RICA",
"CI": "CÔTE D'IVOIRE",
"HR": "CROATIA",
"CU": "CUBA",
"CW": "CURAÇAO",
"CY": "CYPRUS",
"CZ": "CZECH REPUBLIC",
"DK": "DENMARK",
"DJ": "DJIBOUTI",
"DM": "DOMINICA",
"DO": "DOMINICAN REPUBLIC",
"EC": "ECUADOR",
"EG": "EGYPT",
"SV": "EL SALVADOR",
"GQ": "EQUATORIAL GUINEA",
"ER": "ERITREA",
"EE": "ESTONIA",
"ET": "ETHIOPIA",
"FK": "FALKLAND ISLANDS (MALVINAS)",
"FO": "FAROE ISLANDS",
"FJ": "FIJI",
"FI": "FINLAND",
"FR": "FRANCE",
"GF": "FRENCH GUIANA",
"PF": "FRENCH POLYNESIA",
"TF": "FRENCH SOUTHERN TERRITORIES",
"GA": "GABON",
"GM": "GAMBIA",
"GE": "GEORGIA",
"DE": "GERMANY",
"GH": "GHANA",
"GI": "GIBRALTAR",
"GR": "GREECE",
"GL": "GREENLAND",
"GD": "GRENADA",
"GP": "GUADELOUPE",
"GU": "GUAM",
"GT": "GUATEMALA",
"GG": "GUERNSEY",
"GN": "GUINEA",
"GW": "GUINEA-BISSAU",
"GY": "GUYANA",
"HT": "HAITI",
"HM": "HEARD ISLAND AND MCDONALD ISLANDS",
"VA": "HOLY SEE (VATICAN CITY STATE)",
"HN": "HONDURAS",
"HK": "HONG KONG",
"HU": "HUNGARY",
"IS": "ICELAND",
"IN": "INDIA",
"ID": "INDONESIA",
"IR": "IRAN, ISLAMIC REPUBLIC OF",
"IQ": "IRAQ",
"IE": "IRELAND",
"IM": "ISLE OF MAN",
"IL": "ISRAEL",
"IT": "ITALY",
"JM": "JAMAICA",
"JP": "JAPAN",
"JE": "JERSEY",
"JO": "JORDAN",
"KZ": "KAZAKHSTAN",
"KE": "KENYA",
"KI": "KIRIBATI",
"KP": "KOREA, DEMOCRATIC PEOPLE'S REPUBLIC OF",
"KR": "KOREA, REPUBLIC OF",
"KW": "KUWAIT",
"KG": "KYRGYZSTAN",
"LA": "LAO PEOPLE'S DEMOCRATIC REPUBLIC",
"LV": "LATVIA",
"LB": "LEBANON",
"LS": "LESOTHO",
"LR": "LIBERIA",
"LY": "LIBYA",
"LI": "LIECHTENSTEIN",
"LT": "LITHUANIA",
"LU": "LUXEMBOURG",
"MO": "MACAO",
"MK": "MACEDONIA, THE FORMER YUGOSLAV REPUBLIC OF",
"MG": "MADAGASCAR",
"MW": "MALAWI",
"MY": "MALAYSIA",
"MV": "MALDIVES",
"ML": "MALI",
"MT": "MALTA",
"MH": "MARSHALL ISLANDS",
"MQ": "MARTINIQUE",
"MR": "MAURITANIA",
"MU": "MAURITIUS",
"YT": "MAYOTTE",
"MX": "MEXICO",
"FM": "MICRONESIA, FEDERATED STATES OF",
"MD": "MOLDOVA, REPUBLIC OF",
"MC": "MONACO",
"MN": "MONGOLIA",
"ME": "MONTENEGRO",
"MS": "MONTSERRAT",
"MA": "MOROCCO",
"MZ": "MOZAMBIQUE",
"MM": "MYANMAR",
"NA": "NAMIBIA",
"NR": "NAURU",
"NP": "NEPAL",
"NL": "NETHERLANDS",
"NC": "NEW CALEDONIA",
"NZ": "NEW ZEALAND",
"NI": "NICARAGUA",
"NE": "NIGER",
"NG": "NIGERIA",
"NU": "NIUE",
"NF": "NORFOLK ISLAND",
"MP": "NORTHERN MARIANA ISLANDS",
"NO": "NORWAY",
"OM": "OMAN",
"PK": "PAKISTAN",
"PW": "PALAU",
"PS": "PALESTINE, STATE OF",
"PA": "PANAMA",
"PG": "PAPUA NEW GUINEA",
"PY": "PARAGUAY",
"PE": "PERU",
"PH": "PHILIPPINES",
"PN": "PITCAIRN",
"PL": "POLAND",
"PT": "PORTUGAL",
"PR": "PUERTO RICO",
"QA": "QATAR",
"RE": "RÉUNION",
"RO": "ROMANIA",
"RU": "RUSSIAN FEDERATION",
"RW": "RWANDA",
"BL": "SAINT BARTHÉLEMY",
"SH": "SAINT HELENA, ASCENSION AND TRISTAN DA CUNHA",
"KN": "SAINT KITTS AND NEVIS",
"LC": "SAINT LUCIA",
"MF": "SAINT MARTIN (FRENCH PART)",
"PM": "SAINT PIERRE AND MIQUELON",
"VC": "SAINT VINCENT AND THE GRENADINES",
"WS": "SAMOA",
"SM": "SAN MARINO",
"ST": "SAO TOME AND PRINCIPE",
"SA": "SAUDI ARABIA",
"SN": "SENEGAL",
"RS": "SERBIA",
"SC": "SEYCHELLES",
"SL": "SIERRA LEONE",
"SG": "SINGAPORE",
"SX": "SINT MAARTEN (DUTCH PART)",
"SK": "SLOVAKIA",
"SI": "SLOVENIA",
"SB": "SOLOMON ISLANDS",
"SO": "SOMALIA",
"ZA": "SOUTH AFRICA",
"GS": "SOUTH GEORGIA AND THE SOUTH SANDWICH ISLANDS",
"SS": "SOUTH SUDAN",
"ES": "SPAIN",
"LK": "SRI LANKA",
"SD": "SUDAN",
"SR": "SURINAME",
"SJ": "SVALBARD AND JAN MAYEN",
"SZ": "SWAZILAND",
"SE": "SWEDEN",
"CH": "SWITZERLAND",
"SY": "SYRIAN ARAB REPUBLIC",
"TW": "TAIWAN, PROVINCE OF CHINA",
"TJ": "TAJIKISTAN",
"TZ": "TANZANIA, UNITED REPUBLIC OF",
"TH": "THAILAND",
"TL": "TIMOR-LESTE",
"TG": "TOGO",
"TK": "TOKELAU",
"TO": "TONGA",
"TT": "TRINIDAD AND TOBAGO",
"TN": "TUNISIA",
"TR": "TURKEY",
"TM": "TURKMENISTAN",
"TC": "TURKS AND CAICOS ISLANDS",
"TV": "TUVALU",
"UG": "UGANDA",
"UA": "UKRAINE",
"AE": "UNITED ARAB EMIRATES",
"GB": "UNITED KINGDOM",
"US": "UNITED STATES",
"UM": "UNITED STATES MINOR OUTLYING ISLANDS",
"UY": "URUGUAY",
"UZ": "UZBEKISTAN",
"VU": "VANUATU",
"VE": "VENEZUELA, BOLIVARIAN REPUBLIC OF",
"VN": "VIET NAM",
"VG": "VIRGIN ISLANDS, BRITISH",
"VI": "VIRGIN ISLANDS, U.S.",
"WF": "WALLIS AND FUTUNA",
"EH": "WESTERN SAHARA",
"YE": "YEMEN",
"ZM": "ZAMBIA",
"ZW": "ZIMBABWE",
}
import json
import logging
import re
import time
from datetime import datetime
import psycopg2
from bokeh.embed import file_html
from bokeh.models import ColorBar, LogColorMapper, LogTicker, NumeralTickFormatter
from bokeh.palettes import Plasma256, Viridis256
from bokeh.plotting import figure
from bokeh.resources import CDN
from flask import current_app, make_response
import psycopg2
from apps.globals import Error
from apps.isoalpha2 import codes
from apps.utils import error_request
from apps.utils import tictac
SIZE_UNITS = ["B", "KB", "MB", "GB", "TB", "PB", "PB"]
def human_readable_size(size_in_bytes):
index = 0
size_units = ["B", "KB", "MB", "GB", "TB", "PB", "PB"]
while size_in_bytes >= 1024 and index < 6:
size_in_bytes /= 1024
index += 1
return f"{round(size_in_bytes, 2)} {SIZE_UNITS[index]}"
return f"{round(size_in_bytes, 2)} {size_units[index]}"
def is_like_or_equal(params, key):
......@@ -71,7 +77,10 @@ def sql_request(params):
else:
table = "sent_data_summary_weekly"
if params["request"] == "country":
if params["request"] == "map":
s = f"SELECT extract(year from date)::INTEGER as year, country, sum(requests)::BIGINT FROM {table}"
elif params["request"] == "country":
if "all" in params["country"]:
s = f"SELECT sum(requests)::BIGINT, sum(hll_cardinality(clients))::INTEGER FROM {table}"
else:
......@@ -94,6 +103,8 @@ def sql_request(params):
if params["request"] == "country" and "all" not in params["country"]:
s = f"""{s} GROUP BY country"""
elif params["request"] == "map":
s = f"""{s} GROUP BY country, year"""
elif params["request"] == "timeseries":
if "seedlink" in params["media"] and "dataselect" in params["media"]:
s = f"""{s} GROUP BY date, country ORDER BY date"""
......@@ -140,6 +151,8 @@ def get_header(params):
header.append("lastupdated")
elif params["request"] == "country":
header = ["country", "requests", "clients"]
elif params["request"] == "map":
header = ["country", "requests"]
elif params["request"] == "send":
if len(params["media"]) == 1 and "all" not in params["media"]:
if "seedlink" in params["media"]:
......@@ -198,6 +211,93 @@ def get_response(params, data):
return response
def map_requests(params, data):
with open("countries.geo.json") as json_file:
geojson = json.load(json_file)
geodata = dict(x=[], y=[], name=[], requests=[])
for country in geojson["features"]:
if country["geometry"]["type"] not in ["Polygon", "MultiPolygon"]:
break
if country["geometry"]["type"] == "Polygon":
country["geometry"]["coordinates"] = [country["geometry"]["coordinates"]]
for polygon in country["geometry"]["coordinates"]:
geodata["x"].append([x[0] for x in polygon[0]])
geodata["y"].append([x[1] for x in polygon[0]])
geodata["name"].append(country["properties"]["name"].lower())
geodata["requests"] = [0 for i in range(0, len(geodata["name"]))]
if params["year"]:
years = sorted(params["year"].split(","))
else:
years = [str(datetime.now().year)]
for year, code, requests in data:
if str(year) in years:
country = codes.get(code)
if country:
for i, name in enumerate(geodata["name"]):
if country.lower() in name:
geodata["requests"][i] += requests
# my_palette = ("#f2f2f2",) + Viridis256
# my_palette = ("#f2f2f2",) + tuple(reversed(Plasma256))
my_palette = ("#fee5d9", "#fcbba1", "#fc9272", "#fb6a4a", "#de2d26")
color_mapper = LogColorMapper(palette=("#f2f2f2",) + my_palette, low=0.1)
if sum(geodata["requests"]) == 0:
color_mapper = None
TOOLS = "save, pan, wheel_zoom, box_zoom, reset"
plot = figure(
title=f"""Number of requests to the RESIF data center in {" and ".join(years)}""",
tools=TOOLS,
plot_width=1200,
plot_height=800,
x_axis_location=None,
y_axis_location=None,
tooltips=[
("Name", "@name"),
("Requests", "@requests"),
("(Long, Lat)", "($x, $y)"),
],
active_drag="box_zoom",
active_scroll="wheel_zoom",
# sizing_mode='stretch_both'
)
plot.toolbar.logo = None
plot.grid.grid_line_color = None
plot.hover.point_policy = "follow_mouse"
plot.patches(
"x",
"y",
source=geodata,
fill_color={"field": "requests", "transform": color_mapper},
line_width=0.2,
fill_alpha=0.8,
# line_color="white",
)
color_bar = ColorBar(
color_mapper=color_mapper,
ticker=LogTicker(),
formatter=NumeralTickFormatter(),
label_standoff=5,
width=500,
height=10,
location=(0, 0),
orientation="horizontal",
)
plot.add_layout(color_bar, "below")
return make_response(file_html(plot, CDN, "resifws-statistics"))
def get_output(params):
"""Statistics output (csv, request, sync, text)
:params: parameters
......@@ -214,6 +314,8 @@ def get_output(params):
return error_request(msg=f"HTTP._{code}_", details=Error.NODATA, code=code)
logging.info(f"Number of collected rows: {len(data)}")
if params["request"] == "map":
return map_requests(params, data)
data = format_results(params, data)
return get_response(params, data)
except Exception as ex:
......
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment