Commit ca6d15d2 authored by bourgesl's avatar bourgesl
Browse files

Merge branch 'release/2020_05_04'

parents 9e286d81 56b2a60c
Pipeline #42798 passed with stage
in 2 minutes and 51 seconds
......@@ -6,7 +6,7 @@ TH=$1
export LANG=C
# production
#HOST="http://obs.jmmc.fr"
#HOST="http://obs-preprod.jmmc.fr"
# dev
HOST="http://localhost:6543"
......
......@@ -71,4 +71,3 @@ search.votable?ra=291.6330691666667&dec=11.851068&
search.votable?ra=279.15259000000003&dec=12.987611&
search.votable?ra=270.7129108333333&dec=-24.282468&
search.votable?ra=176.77937708333334&dec=-35.906862&
......@@ -133,7 +133,7 @@ class Exposure(Base):
Field(votable, name="target_id", datatype="unicodeChar", arraysize="*", ucd="meta.id"),
Field(votable, name="target_name", datatype="unicodeChar", arraysize="*", ucd="meta.id;meta.main"),
Field(votable, name="target_ra", datatype="double", ucd="pos.eq.ra;meta.main", unit="deg"),
Field(votable, name="target_dec", datatype="double", ucd="pos.eq.dec;meta.main", unit="deg"),
Field(votable, name="target_dec", datatype="double", ucd="pos.eq.dec;meta.main", unit="deg")
]
def votable_array(self):
......@@ -169,9 +169,119 @@ class Exposure(Base):
self.observation.target_id or '',
self.observation.target.name or '',
self.observation.target.ra,
self.observation.target.dec,
self.observation.target.dec
)
# Optimized VOTable ################################################################################################
@staticmethod
def write_votable_header(fd, status, last_mod_date):
fd.write(("""<?xml version="1.0" encoding="utf-8"?>
<!-- Produced with astropy.io.votable version 4.0
http://www.astropy.org/ -->
<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.4" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://www.ivoa.net/xml/VOTable/v1.4">
<RESOURCE type="results">
<PARAM ID="status" arraysize="*" datatype="unicodeChar" name="status" value="%s"/>
<PARAM ID="last_mod_date" arraysize="*" datatype="unicodeChar" name="last_mod_date" value="%s"/>
<TABLE>
<FIELD ID="exp_id" arraysize="*" datatype="unicodeChar" name="exp_id" ucd="meta.id"/>
<FIELD ID="exp_header_id" arraysize="*" datatype="unicodeChar" name="exp_header_id" ucd="meta.id"/>
<FIELD ID="exp_mjd_start" datatype="double" name="exp_mjd_start" ucd="time.start;obs.exposure"/>
<FIELD ID="exp_mjd_end" datatype="double" name="exp_mjd_end" ucd="time.end;obs.exposure"/>
<FIELD ID="exp_projected_baselines" arraysize="*" datatype="unicodeChar" name="exp_projected_baselines" ucd="instr.baseline"/>
<FIELD ID="exp_validation_level" datatype="int" name="exp_validation_level" ucd="meta.code.qual"/>
<FIELD ID="exp_validation_log" arraysize="*" datatype="unicodeChar" name="exp_validation_log"/>
<FIELD ID="exp_tau0" datatype="double" name="exp_tau0" ucd="time.processing"/>
<FIELD ID="exp_temperature" datatype="double" name="exp_temperature" ucd="phys.temperature"/>
<FIELD ID="exp_seeing" datatype="double" name="exp_seeing" ucd="instr.obsty.seeing"/>
<FIELD ID="exp_date_updated" arraysize="*" datatype="unicodeChar" name="exp_date_updated" ucd="time.processing"/>
<FIELD ID="obs_id" arraysize="*" datatype="unicodeChar" name="obs_id" ucd="meta.id"/>
<FIELD ID="obs_type" arraysize="*" datatype="unicodeChar" name="obs_type" ucd="meta.id;class"/>
<FIELD ID="obs_program" arraysize="*" datatype="unicodeChar" name="obs_program" ucd="meta.id"/>
<FIELD ID="interferometer_name" arraysize="*" datatype="unicodeChar" name="interferometer_name" ucd="meta.id"/>
<FIELD ID="interferometer_stations" arraysize="*" datatype="unicodeChar" name="interferometer_stations"/>
<FIELD ID="instrument_name" arraysize="*" datatype="unicodeChar" name="instrument_name" ucd="meta.id;instr"/>
<FIELD ID="instrument_mode" arraysize="*" datatype="unicodeChar" name="instrument_mode" ucd="meta.id;instr.setup"/>
<FIELD ID="instrument_submode" arraysize="*" datatype="unicodeChar" name="instrument_submode" ucd="meta.id;instr.setup"/>
<FIELD ID="target_id" arraysize="*" datatype="unicodeChar" name="target_id" ucd="meta.id"/>
<FIELD ID="target_name" arraysize="*" datatype="unicodeChar" name="target_name" ucd="meta.id;meta.main"/>
<FIELD ID="target_ra" datatype="double" name="target_ra" ucd="pos.eq.ra;meta.main" unit="deg"/>
<FIELD ID="target_dec" datatype="double" name="target_dec" ucd="pos.eq.dec;meta.main" unit="deg"/>
<DATA>
<TABLEDATA>""" % (status, last_mod_date)).encode())
@staticmethod
def write_votable_footer(fd):
fd.write("""
</TABLEDATA>
</DATA>
</TABLE>
</RESOURCE>
</VOTABLE>
""".encode())
def write_votable_row(self, fd):
# note: python gives 'nan' not 'NaN': TO BE FIXED
fd.write(("""
<TR>
<TD>%s</TD>
<TD>%s</TD>
<TD>%s</TD>
<TD>%s</TD>
<TD>%s</TD>
<TD>%d</TD>
<TD>%s</TD>
<TD>%s</TD>
<TD>%s</TD>
<TD>%s</TD>
<TD>%s</TD>
<TD>%s</TD>
<TD>%s</TD>
<TD>%s</TD>
<TD>%s</TD>
<TD>%s</TD>
<TD>%s</TD>
<TD>%s</TD>
<TD>%s</TD>
<TD>%s</TD>
<TD>%s</TD>
<TD>%s</TD>
<TD>%s</TD>
</TR>""" % (
# EXPOSURE
self.id,
self.header_id,
'%.15g' % self.mjd_start if self.mjd_start else '',
'%.15g' % self.mjd_end if self.mjd_end else '',
self.projected_baselines or '',
self.validation_level or 0,
self.header.validation_log or '',
'%.15g' % self.tau0 if self.tau0 else '',
'%.15g' % self.temperature if self.temperature else '',
'%.15g' % self.seeing if self.seeing else '',
str(self.date_updated),
# OBSERVATION
self.observation_id,
self.observation.category,
self.observation.program_id or '',
# INTERFEROMETER
self.observation.instrument.interferometer_id or '',
self.observation.stations or '',
# INSTRUMENT
self.observation.instrument_id or '',
self.observation.instrument_mode.name if self.observation.instrument_mode else '',
self.observation.instrument_submode or '',
# TARGET
self.observation.target_id or '',
self.observation.target.name or '',
'%.15g' % self.observation.target.ra if self.observation.target.ra else '',
'%.15g' % self.observation.target.dec if self.observation.target.dec else ''
)).encode())
# SEARCH ###########################################################################################################
@classmethod
......@@ -192,7 +302,8 @@ class Exposure(Base):
order_by_field=None,
order_asc=True,
use_limit=True,
load_validation_log=False):
load_validation_log=False,
as_query=False):
# Build query params
statement_params = {}
......@@ -293,6 +404,9 @@ class Exposure(Base):
# Execute query
logger.debug(f"Search(Exposure): {query}")
if as_query:
return query.params(statement_params).yield_per(100) # process by chunks
results = query.params(statement_params).all()
logger.debug(f"Search(Exposure): {len(results)} results")
return results
......@@ -4,6 +4,8 @@ import uuid
import os
from gzip import GzipFile
from io import BytesIO
from tempfile import NamedTemporaryFile
from pyramid.request import Request
from pyramid.response import FileResponse, Response
from sqlalchemy.orm import Session
......@@ -97,14 +99,19 @@ class ViewVOTableAbstract(ViewAbstract):
return votable
def _render_votable(self, votable, compressed=True):
# return self._render(votable, compressed)
return self._render_stream(votable, compressed)
# deprecated
def _render(self, votable, compressed=False):
def _render(self, votable, compressed):
"""
Render a VOTable as FileResponse
:param votable: VOTableFile
:param compressed: bool
:return: FileResponse
"""
logger.info('RENDER')
# Define Content Type
content_type = 'text/xml' #'application/x-votable+xml'
......@@ -118,25 +125,25 @@ class ViewVOTableAbstract(ViewAbstract):
# Generate a temporary filename
tmp_file_path = os.path.join(self.settings.get('obsportal.paths.tmp', '/tmp'), "%s" % uuid.uuid4())
# Now write the whole thing to a file.
votable.to_xml(tmp_file_path, compressed)
with NamedTemporaryFile(mode='wb', prefix=tmp_file_path,
suffix='.votable.gz', delete=True) as tmp:
# Build response
response = FileResponse(tmp_file_path, self.request, cache_max_age=0,
content_type=content_type, content_encoding=content_encoding)
# Now write the whole thing to a file.
votable.to_xml(tmp, compressed)
# Clean temporary file
try:
os.remove(tmp_file_path)
except Exception:
# TODO
pass
tmp.flush()
# Return response
return response
# Build response
response = FileResponse(os.path.abspath(tmp.name), self.request, cache_max_age=0,
content_type=content_type, content_encoding=content_encoding)
logger.info('VOTABLE: done')
# Return response
return response
@staticmethod
def _render_stream(votable, compressed=True):
def _render_stream(votable, compressed):
"""
Render a VOTable as FileResponse
:param votable: VOTableFile
......@@ -148,13 +155,14 @@ class ViewVOTableAbstract(ViewAbstract):
with BytesIO() as buffer:
if compressed:
content_encoding = 'gzip'
with GzipFile(fileobj=buffer, mode='wb', compresslevel=5) as buffer_gzip:
with GzipFile(fileobj=buffer, mode='wb') as buffer_gzip:
votable.to_xml(buffer_gzip)
else:
content_encoding = None
votable.to_xml(buffer)
buffer.seek(0)
response = Response(body=buffer.getvalue(), content_type='text/xml', content_encoding=content_encoding)
logger.info('VOTABLE: done')
......
# coding=utf-8
import logging
from gzip import GzipFile
from io import BytesIO
from pyramid.response import Response
from pyramid.view import view_config
from pyramid.httpexceptions import HTTPNotFound
from . import ViewAbstract, ViewVOTableAbstract
from ..models import Header
from ..models.exposure import Exposure
from ..forms.search import FormSearch
logger = logging.getLogger(__name__)
use_votable_fast = True
if use_votable_fast:
logger.info("use_votable_fast: enabled.")
class ViewVOTableExposureAbstract(ViewVOTableAbstract):
def render_votable(self, exposures):
if use_votable_fast:
return self.render_votable_fast(exposures)
logger.info(f'BUILD VOTABLE: {len(exposures)}')
votable = self._build_votable(Exposure, exposures)
# Render
return self._render_votable(votable)
def render_votable_fast(self, exposures):
logger.info('RENDER')
count = 0 # size not known (query streaming)
with BytesIO() as buffer:
content_encoding = 'gzip'
with GzipFile(fileobj=buffer, mode='wb') as buffer_gzip:
# ... with parameters
status = 'SYNC' if self.synchronize_running else 'OK'
last_mod_date = Header.last_modification_date(self.db_session)
Exposure.write_votable_header(buffer_gzip, status, str(last_mod_date))
for exp in exposures:
exp.write_votable_row(buffer_gzip)
count += 1
Exposure.write_votable_footer(buffer_gzip)
buffer.seek(0)
response = Response(body=buffer.getvalue(), content_type='text/xml', content_encoding=content_encoding)
logger.info(f'VOTABLE: done ({count} exposures)')
# Return response
return response
@view_config(route_name='detail',
match_param='type=exposure',
......@@ -33,7 +84,8 @@ class ViewExposureList(ViewAbstract):
def __call__(self):
# Get list anyway
# NOTE: use limit to not exceed server memory
exposures = Exposure.search(self.db_session, order_by_field=Exposure.date_updated, order_asc=False, use_limit=True)
exposures = Exposure.search(self.db_session, order_by_field=Exposure.date_updated, order_asc=False,
use_limit=True)
# Render
return {'exposures': exposures, 'updating': self.synchronize_running}
......@@ -41,19 +93,17 @@ class ViewExposureList(ViewAbstract):
@view_config(route_name='list_votable',
match_param='type=exposure')
class ViewExposureListVotable(ViewVOTableAbstract):
class ViewExposureListVotable(ViewVOTableExposureAbstract):
def __call__(self):
# Get list anyway
# NOTE: use limit to not exceed server memory
exposures = Exposure.search(self.db_session, order_by_field=Exposure.date_updated, use_limit=True, load_validation_log=True)
exposures = Exposure.search(self.db_session, order_by_field=Exposure.date_updated,
use_limit=True, load_validation_log=True,
as_query=use_votable_fast)
# Build VOTable
logger.info(f'BUILD VOTABLE: {len(exposures)}')
votable = self._build_votable(Exposure, exposures)
# Render
return self._render_stream(votable)
return self.render_votable(exposures)
@view_config(route_name='search', renderer='../templates/views/exposure/search.jinja2')
......@@ -87,7 +137,7 @@ class ViewExposureSearch(ViewAbstract):
@view_config(route_name='search', request_param="format=votable")
@view_config(route_name='search_votable')
class ViewExposureSearchVotable(ViewVOTableAbstract):
class ViewExposureSearchVotable(ViewVOTableExposureAbstract):
def __call__(self):
# Search
......@@ -110,11 +160,7 @@ class ViewExposureSearchVotable(ViewVOTableAbstract):
self.request.params.get('mjd_to'),
valid_level=valid_level,
use_limit=False, # no limit
load_validation_log=True)
load_validation_log=True,
as_query=use_votable_fast)
# Build VOTable
logger.info(f'BUILD VOTABLE: {len(exposures)}')
votable = self._build_votable(Exposure, exposures)
# Render
return self._render_stream(votable)
return self.render_votable(exposures)
......@@ -57,7 +57,7 @@ class ViewHeaderDetailVotable(ViewVOTableAbstract):
votable = self._build_votable(Header, header)
# Render
return ViewVOTableAbstract._render_stream(votable)
return self._render_votable(votable)
@view_config(route_name='list',
......@@ -86,4 +86,4 @@ class ViewHeaderListVotable(ViewVOTableAbstract):
votable = self._build_votable(Header, headers)
# Render
return ViewVOTableAbstract._render_stream(votable)
return self._render_votable(votable)
......@@ -38,7 +38,7 @@ class ViewObservationDetailVotable(ViewVOTableAbstract):
votable = self._build_votable(Observation, observation)
# Render
return ViewVOTableAbstract._render_stream(votable)
return self._render_votable(votable)
@view_config(route_name='list',
......@@ -74,4 +74,4 @@ class ViewObservationListVotable(ViewVOTableAbstract):
votable = self._build_votable(Observation, observations)
# Render
return ViewVOTableAbstract._render_stream(votable)
return self._render_votable(votable)
......@@ -55,4 +55,4 @@ class ViewTargetListVotable(ViewVOTableAbstract):
votable = self._build_votable(Target, targets)
# Render
return ViewVOTableAbstract._render_stream(votable)
return self._render_votable(votable)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment