Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
OSUG
JMMC
jmmc-obsportal
Commits
d129ae4f
Commit
d129ae4f
authored
Jan 11, 2021
by
bourgesl
Browse files
Merge branch 'release/2021_01_11'
parents
37c5df74
c18da8b0
Pipeline
#56509
skipped with stage
Changes
6
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
doc/db_er.png
View replaced file @
37c5df74
View file @
d129ae4f
62.1 KB
|
W:
|
H:
82.3 KB
|
W:
|
H:
2-up
Swipe
Onion skin
doc/db_uml.png
View replaced file @
37c5df74
View file @
d129ae4f
170 KB
|
W:
|
H:
264 KB
|
W:
|
H:
2-up
Swipe
Onion skin
obsportal/models/database_tag.py
View file @
d129ae4f
# coding: utf-8
from
datetime
import
datetime
import
logging
from
sqlalchemy
import
(
...
...
@@ -10,8 +11,13 @@ from .meta import Base
logger
=
logging
.
getLogger
(
__name__
)
# Possible tags:
DatabaseTagVersion
=
'database_version'
SynchronizeStart
=
'synchronize_start'
SynchronizeEnd
=
'synchronize_end'
# Current last database model version
# Changing that value means re-processing of all headers is needed:
CurrentDatabaseTagVersion
=
'obsportal_2020_06_05'
...
...
@@ -26,6 +32,8 @@ class DatabaseTag(Base):
# value
value
=
Column
(
Unicode
,
index
=
True
)
# Get / Set db version #############################################################################################
@
classmethod
def
get_db_version
(
cls
,
db_session
):
return
DatabaseTag
.
getValue
(
db_session
,
DatabaseTagVersion
)
...
...
@@ -47,6 +55,17 @@ class DatabaseTag(Base):
return
False
return
True
# Set synchronize dates ############################################################################################
@
classmethod
def
set_synchronize_start_now
(
cls
,
db_session
):
DatabaseTag
.
setValue
(
db_session
,
SynchronizeStart
,
str
(
datetime
.
now
()))
@
classmethod
def
set_synchronize_end_now
(
cls
,
db_session
):
DatabaseTag
.
setValue
(
db_session
,
SynchronizeEnd
,
str
(
datetime
.
now
()))
# SEARCH ###########################################################################################################
@
classmethod
...
...
@@ -59,10 +78,13 @@ class DatabaseTag(Base):
@
classmethod
def
list
(
cls
,
db_session
):
try
:
return
db_session
.
query
(
cls
).
all
()
# always sort by id to ensure stable sort:
return
db_session
.
query
(
cls
).
order_by
(
DatabaseTag
.
id
.
asc
()).
all
()
except
:
return
None
# Get / Set value ##################################################################################################
@
classmethod
def
getValue
(
cls
,
db_session
,
oid
):
tag
=
DatabaseTag
.
get
(
db_session
,
oid
)
...
...
@@ -78,6 +100,6 @@ class DatabaseTag(Base):
tag
.
id
=
oid
if
value
!=
tag
.
value
:
# set value
logger
.
info
(
f
"
setValue():
id=
{
oid
}
v
alue
=
{
value
}
"
)
logger
.
info
(
f
"
DatabaseTag(
id=
{
oid
}
).setV
alue
('
{
value
}
')
"
)
tag
.
value
=
value
db_session
.
add
(
tag
)
obsportal/models/exposure.py
View file @
d129ae4f
...
...
@@ -12,7 +12,7 @@ from sqlalchemy import (
Float
,
SmallInteger
,
ForeignKey
,
text
text
,
func
)
from
sqlalchemy.orm
import
relationship
,
joinedload
,
contains_eager
from
sqlalchemy.sql.expression
import
bindparam
...
...
@@ -577,8 +577,8 @@ class Exposure(Base):
query
=
query
.
filter
(
XMatch
(
Target
.
ra
,
Target
.
dec
))
if
target_name
:
statement_params
[
'target_name'
]
=
target_name
query
=
query
.
filter
(
Target
.
name
==
bindparam
(
'target_name'
))
statement_params
[
'target_name'
]
=
f
"%
{
target_name
.
lower
()
}
%"
query
=
query
.
filter
(
func
.
lower
(
Target
.
name
).
like
(
func
.
lower
(
bindparam
(
'target_name'
))
))
if
instrument
:
statement_params
[
'instrument'
]
=
instrument
...
...
obsportal/services/eso/__init__.py
View file @
d129ae4f
...
...
@@ -228,15 +228,19 @@ class ServiceEso(ServiceAbstract):
excluded_instruments
=
[
'GRAVITY_FT'
]
instruments
=
[]
# TEST ONLY:
# if False:
# excluded_instruments = ['GRAVITY_FT', 'AMBER', 'MIDI', 'PIONIER']
# limit = 5000
# Get all instruments of VLTI to synchronize:
# update database tags:
db_session
=
self
.
get_session
()
try
:
# check database version:
if
not
DatabaseTag
.
is_current_db_version
(
db_session
):
force_update
=
True
# TEST LBO
force_update
=
False
logger
.
info
(
f
"synchronize(): force_update =
{
force_update
}
"
)
# query VLTI instruments:
...
...
@@ -247,6 +251,15 @@ class ServiceEso(ServiceAbstract):
instruments
.
append
(
instrument
.
id
)
else
:
logger
.
warning
(
"eso:synchronize() Unable to find the VLTI interferometer in database."
)
# Start transaction
ServiceEso
.
begin
(
db_session
)
DatabaseTag
.
set_synchronize_start_now
(
db_session
)
# Commit transaction
ServiceEso
.
commit
(
db_session
)
except
BaseException
as
e
:
ServiceEso
.
rollback
(
db_session
)
logger
.
warning
(
"eso:synchronize() failure"
)
...
...
@@ -254,11 +267,11 @@ class ServiceEso(ServiceAbstract):
finally
:
ServiceEso
.
close
(
db_session
)
# Synchronize headers for each instrument
# Synchronize headers for each instrument
:
for
instrument
in
instruments
:
self
.
_synchronize_instrument
(
instrument
,
limit
=
limit
,
force_update
=
force_update
,
debug
=
debug
)
# Compute stats
# Compute stats
:
db_session
=
self
.
get_session
()
try
:
# Start transaction
...
...
@@ -276,25 +289,27 @@ class ServiceEso(ServiceAbstract):
finally
:
ServiceEso
.
close
(
db_session
)
if
force_update
:
# Update database version to current
db_session
=
self
.
get_session
()
try
:
# Start transaction
ServiceEso
.
begin
(
db_session
)
# finally update database tags:
db_session
=
self
.
get_session
()
try
:
# Start transaction
ServiceEso
.
begin
(
db_session
)
# set database version:
if
force_update
:
# Update database version to current
DatabaseTag
.
set_current_db_version
(
db_session
)
# Commit transaction
ServiceEso
.
commit
(
db_session
)
DatabaseTag
.
set_synchronize_end_now
(
db_session
)
except
BaseException
as
e
:
ServiceEso
.
rollback
(
db_session
)
logger
.
warning
(
"eso:synchronize() failure"
)
raise
e
finally
:
ServiceEso
.
close
(
db_session
)
# Commit transaction
ServiceEso
.
commit
(
db_session
)
except
BaseException
as
e
:
ServiceEso
.
rollback
(
db_session
)
logger
.
warning
(
"eso:synchronize() failure"
)
raise
e
finally
:
ServiceEso
.
close
(
db_session
)
def
_synchronize_instrument
(
self
,
instrument
,
limit
=
None
,
force_update
=
False
,
debug
=
False
):
"""
...
...
obsportal/views/health.py
View file @
d129ae4f
# coding=utf-8
import
logging
import
os
from
astropy.time
import
Time
from
sqlalchemy
import
func
,
text
...
...
@@ -9,6 +10,8 @@ from . import ViewAbstract
from
..models
import
Observation
,
Header
,
Target
,
DatabaseTag
from
..models.exposure
import
Exposure
logger
=
logging
.
getLogger
(
__name__
)
@
view_config
(
route_name
=
'health'
,
renderer
=
'../templates/views/health.jinja2'
)
...
...
@@ -103,6 +106,18 @@ class ViewHealth(ViewAbstract):
# 5: Tmp path
services
[
'tmp directory'
]
=
os
.
path
.
exists
(
self
.
settings
.
get
(
'obsportal.paths.tmp'
))
# Final status on all services:
state
=
True
for
status
in
services
.
values
():
if
not
status
:
state
=
False
break
if
not
state
:
logger
.
warning
(
f
"Health check failed:
{
services
}
"
)
# Set HTTP 500 to let kubernetes liveliness fail
self
.
request
.
response
.
status
=
500
return
{
'settings'
:
self
.
settings
,
'services'
:
services
,
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment