Commit 1dc01120 authored by bourgesl's avatar bourgesl
Browse files

Merge branch 'release/2020_04_22_b1'

parents 8ded7679 065198c6
Pipeline #42145 passed with stage
in 2 minutes and 40 seconds
......@@ -11,7 +11,6 @@ dist/
.tox/
nosetests.xml
env*/
tests/
tmp/
Data.fs*
*.sublime-project
......
......@@ -22,5 +22,5 @@ build obs:
<<: *docker_build
variables:
IMAGE: obs
IMAGE_TAG: "2020_04_16_b1"
IMAGE_TAG: $CI_COMMIT_TAG
......@@ -34,9 +34,6 @@ RUN set -eux ; \
true
ARG SHAREDCONFFILE=shared.ini-dist
ADD $SHAREDCONFFILE /app/shared.ini
ARG APPCONFFILE=production.ini-dist
ADD $APPCONFFILE /app/docker.ini
......
......@@ -23,26 +23,30 @@ Application setup
pip3 install -e .
- Configure the shared database parameters
cp shared.ini-dist shared.ini
nano shared.ini
- And configure this parameter regarding to your deployment environment:
- sqlalchemy.url : database connection string. See https://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql
- With <application_settings_file> = production.ini OR development.ini
cp <application_settings_file>-dist <application_settings_file>
nano <application_settings_file>
- And configure these parameters regarding to your deployment environment:
- sqlalchemy.url : database connection string. See https://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql
- global.paths.data : top-directory storing all data retrieved by the application
- global.paths.logs : top-directory storing all logs generated by the application
- global.paths.tmp : top-directory storing all temporary files generated by the application
- obsportal.paths.data : top-directory storing all data retrieved by the application
- obsportal.paths.logs : top-directory storing all logs generated by the application
- obsportal.paths.tmp : top-directory storing all temporary files generated by the application
- obsportal.variant : blue/green variant using Docker/Kubernetes deployment
- Or declare environment variables in place of hard-coded values in the INI file:
- use UPCASED keys with spaces replaced by underscores (_)
- always prefix an ENV key by 'OBSPORTAL_' only if the INI key doesn't begin by 'obsportal.' :
- sqlalchemy.url => OBSPORTAL_SQLALCHEMY_URL
- obsportal.variant => OBSPORTAL_VARIANT
- export OBSPORTAL_VARIANT=blue
- export OBSPORTAL_SQLALCHEMY_URL=postgresql://user:password@host/dbname?client_encoding=utf8
- PLEASE DO NOT USE AND MODIFY *.ini-dist FILES, USE LOCAL *ini FILES INSTEAD
......@@ -66,6 +70,27 @@ Database setup
obsportal-cli database init --debug --settings=<application_settings_file>
Database (Alembic) migrations:
------------------------------
- Only once:
Stamp the first revision (= DB structure at this time will be the top reference for Alembic versionning)
DONE
- Each time, after initialization:
After each modification on DB structure, generate a new Alembic revision.
obsportal-cli database revision --autogenerate --settings=client.ini
Review the SQL code generated by the revision. Add missing non-detected DB structure changes. Add DB content upgrade code.
- Apply the revision (already included in shell scripts):
obsportal-cli database upgrade --settings=client.ini
Database feeding from ESO
-------------------------
......@@ -86,8 +111,8 @@ Build Docker image:
* For production:
docker build -t "obsportal:latest" .
* For development:
docker build -t "obsportal:latest" --build-arg SHAREDCONFFILE=shared.ini --build-arg APPCONFFILE=development.ini .
docker run -p 6543:6543 --rm --name d_obsportal "obsportal:latest"
docker build -t "obsportal:latest" --build-arg APPCONFFILE=development.ini --build-arg CLICONFFILE=client.ini .
docker run -p 6543:6543 -v /data:/data:rw --rm --name d_obsportal "obsportal:latest"
* Initialize existing postgres DB:
(see db/ scripts to initialize manually the postgres + pgsphere database)
......@@ -127,5 +152,19 @@ Publish release:
git push
*******************
Launch unit tests
*******************
_ Install the project with 'testing' dependencies:
pip3 install -e ".[testing]"
- Copy testing.ini-dist to testing.ini and configure the parameters (see above)
- Ensure that database is running and feeded
- Run : obsportal-cli test
That's All, folks !
#!/bin/bash
N=2 # max 5 threads in dev
# TRAP: Do not leave children jobs running if the shell has been cancelled
cleanup_trap() {
CHILDREN_PIDS=$(jobs -p)
if [ -n "$CHILDREN_PIDS" ]
then
trap - EXIT
echo -e "SHELL cancelled, stopping $CHILDREN_PIDS"
# we may try to send only TERM before a pause and a last loop with KILL signal ?
kill $CHILDREN_PIDS
echo -e "SHELL cancelled, waiting on $CHILDREN_PIDS"
# wait for all pids
for pid in $CHILDREN_PIDS; do
wait $pid
done
CHILDREN_PIDS=$(jobs -p)
if [ -n "$CHILDREN_PIDS" ]
then
echo -e "SHELL cancelled, killing $CHILDREN_PIDS"
kill -9 $CHILDREN_PIDS
fi
fi
}
trap cleanup_trap EXIT
CMD="./get-loop-votable-queries.sh"
DIR="./tests"
mkdir $DIR
logFile="$DIR/get_votable_"
logExt=txt
echo "Started @ `date`"
for i in $(seq $N)
do
TH=$((i-1))
LOG=${logFile}${i}.${logExt}
echo "Spawn process ${i}), logs in $LOG"
nohup $CMD $TH < /dev/null > ${logFile}${i}.${logExt} 2>&1 &
done
echo "Waiting ..."
# Wait for runner to end properly:
PID=`ps -ef | grep "$CMD" | grep -v "grep" | grep -v "-bench_votable-queries.sh" | awk '{print $2}'`
N_PID=`echo "${PID}" | wc -w`
while [ "${N_PID}" -ne "0" ]; do
sleep 1
PID=`ps -ef | grep "$CMD" | grep -v "grep" | grep -v "-bench_votable-queries.sh" | awk '{print $2}'`
N_PID=`echo "${PID}" | wc -w`
done
echo "All Processes terminated."
echo "Stopped @ `date`"
......@@ -3,40 +3,50 @@
# https://pastedeploy.readthedocs.io/en/latest/#global-configuration
#################################################
[DEFAULT]
global.paths.data = /data
global.paths.logs = /logs
global.paths.tmp = /tmp
obsportal.environment = client
obsportal.variant = local
global.environment = client
obsportal.paths.data = /data/cache
obsportal.paths.logs = /logs
obsportal.paths.tmp = /tmp
obsportal.locks.synchronize = %(obsportal.paths.tmp)s/synchronize.lock
#################################################
# Application configuration
# https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html
#################################################
[app:main]
use = config:shared.ini#shared
use = egg:obsportal
#######################
## Database parameters
#######################
# Using PostgreSQL (pg_sphere required)
# See https://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql
sqlalchemy.url = postgresql://user:password@host/dbname?client_encoding=utf8
# Connection Pool settings:
sqlalchemy.pool_size = 5
sqlalchemy.max_overflow = 5
# recycle opened connection after 1h idle:
sqlalchemy.pool_recycle = 3600
#############################
## Framework debug parameters
#############################
#########################
# Pyramid configuration
#########################
pyramid.reload_templates = false
pyramid.debug_authorization = false
pyramid.debug_notfound = false
pyramid.debug_routematch = false
pyramid.includes =
pyramid.default_locale_name = en
# See https://docs.pylonsproject.org/projects/pyramid-retry/en/latest/
retry.attempts = 3
pyramid.includes =
########################
## Jinja2 custom filters
......@@ -120,7 +130,7 @@ formatter = generic
[handler_file]
class = FileHandler
args=(__import__("datetime").datetime.now().strftime('%(global.paths.logs)s/%(global.environment)s_%%Y-%%m-%%d_%%H-%%M-%%S.log'), 'a')
args=(__import__("datetime").datetime.now().strftime('%(obsportal.paths.logs)s/%(obsportal.environment)s_%%Y-%%m-%%d_%%H-%%M-%%S.log'), 'a')
level = NOTSET
formatter = generic
......
......@@ -3,46 +3,56 @@
# https://pastedeploy.readthedocs.io/en/latest/#global-configuration
#################################################
[DEFAULT]
global.paths.data = /data
global.paths.logs = /logs
global.paths.tmp = /tmp
obsportal.environment = development
obsportal.variant = local
global.environment = development
obsportal.paths.data = /data
obsportal.paths.logs = /logs
obsportal.paths.tmp = /tmp
obsportal.locks.synchronize = %(obsportal.paths.tmp)s/synchronize.lock
#################################################
# Application configuration
# https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html
#################################################
[app:main]
use = config:shared.ini#shared
use = egg:obsportal
#######################
## Database parameters
#######################
# Using PostgreSQL (pg_sphere required)
# See https://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql
sqlalchemy.url = postgresql://user:password@host/dbname?client_encoding=utf8
# Connection Pool settings:
sqlalchemy.pool_size = 5
sqlalchemy.max_overflow = 5
# recycle opened connection after 1h idle:
sqlalchemy.pool_recycle = 3600
#############################
## Framework debug parameters
#############################
#########################
# Pyramid configuration
#########################
pyramid.reload_templates = true
pyramid.debug_authorization = false
pyramid.debug_notfound = false
pyramid.debug_routematch = false
pyramid.default_locale_name = en
# See https://docs.pylonsproject.org/projects/pyramid-retry/en/latest/
retry.attempts = 3
pyramid.includes =
# pyramid_debugtoolbar
pyramid_debugtoolbar
# By default, the toolbar only appears for clients from IP addresses
# '127.0.0.1' and '::1'.
# debugtoolbar.hosts = 127.0.0.1 ::1
########################
## Jinja2 custom filters
########################
......@@ -125,7 +135,7 @@ formatter = generic
[handler_file]
class = FileHandler
args = ('%(global.paths.logs)s/%(global.environment)s.log','a')
args = ('%(obsportal.paths.logs)s/%(obsportal.environment)s.log','a')
level = NOTSET
formatter = generic
......
#!/bin/bash
N=10
TH=$1
export LANG=C
for i in $(seq $N)
do
echo "Loop: ${i}/$N ..."
echo "-------------------"
time ./get-votable-queries.sh $TH
echo "-------------------"
sleep 1
done
echo "Loops done"
#!/bin/bash
N=10
TH=$1
export LANG=C
# production
#HOST="http://obs.jmmc.fr"
# dev
HOST="http://localhost:6543"
echo "HOST: $HOST"
DIR="./tests"
mkdir $DIR
cd $DIR
i=0
cat "../http_votable_queries.txt" | grep -v "#" | while read line
do
URL="${HOST}/${line}"
echo "Get '$URL'"
wget -q -O "votable_${TH}_${line}.vot.gz" $URL
i=$((i + 1))
done
echo "End: $i requests"
search.votable?ra=170.6319752350479&dec=-53.369848897307776&
search.votable?ra=237.300425&dec=-35.65142222222222&
search.votable?ra=277.19942485763295&dec=0.1444235118386111&
search.votable?ra=285.77809157917875&dec=-37.21380093975917&
search.votable?ra=207.9077333333333&dec=-61.65208888888889&
search.votable?ra=275.0947905&dec=-10.187118416666667&
search.votable?ra=285.286&dec=-36.955799999999996&
search.votable?ra=239.03823574999998&dec=-37.93503313888889&
search.votable?ra=291.49478941511836&dec=21.20870367244639&
search.votable?ra=180.02119602028793&dec=-78.19293312643083&
search.votable?ra=193.32167848874&dec=-77.11964657277139&
search.votable?ra=237.49061864327584&dec=-3.921206673083889&
search.votable?ra=239.16675955898208&dec=-22.027779124985834&
search.votable?ra=239.17629581170252&dec=-37.820964852544165&
search.votable?ra=250.074678625&dec=-23.89588297222222&
search.votable?ra=241.74147094454665&dec=-27.719377226827223&
search.votable?ra=252.19013481649623&dec=-14.276624984416667&
search.votable?ra=252.31376481159168&dec=-14.369067226606388&
search.votable?ra=257.5338491666667&dec=-27.2552225&
search.votable?ra=269.0887009119171&dec=-21.956075650911668&
search.votable?ra=285.47368759494753&dec=-36.952262644436665&
search.votable?ra=287.7968919648308&dec=15.787675614647222&
search.votable?ra=236.303612566705&dec=-34.29184636696639&
search.votable?ra=246.48402478525253&dec=-24.346733130509723&
search.votable?ra=242.88059583333333&dec=-18.640555555555554&
search.votable?ra=246.59871300696457&dec=-24.720524088134166&
search.votable?ra=246.6003733616908&dec=-24.270404633786665&
search.votable?ra=238.30972791666665&dec=-34.881881&
search.votable?ra=252.68401708333332&dec=-8.732641&
search.votable?ra=276.07116291666665&dec=0.456107&
search.votable?ra=249.16113083333332&dec=-25.330679&
search.votable?ra=243.6514658333333&dec=-23.215068&
search.votable?ra=205.26658583333335&dec=-62.776003&
search.votable?ra=238.29278791666667&dec=-38.352351&
search.votable?ra=171.26787916666666&dec=-53.022053&
search.votable?ra=207.90078&dec=-61.648955&
search.votable?ra=260.341025&dec=-11.947016&
search.votable?ra=289.841195&dec=-11.898609&
search.votable?ra=285.2858&dec=-36.9556&
search.votable?ra=268.86333083333335&dec=-37.803877&
search.votable?ra=296.5065441666667&dec=-40.927519&
search.votable?ra=178.07595583333332&dec=-51.314768&
search.votable?ra=287.897135&dec=-0.070237&
search.votable?ra=226.03183416666667&dec=-62.976367&
search.votable?ra=226.06866&dec=-63.202877&
search.votable?ra=230.93212409784542&dec=-1.022385115025&
search.votable?ra=253.03200791666669&dec=-1.369657&
search.votable?ra=284.93603083333335&dec=-34.471054&
search.votable?ra=299.55565&dec=-36.686926&
search.votable?ra=291.17176708333335&dec=19.786938&
search.votable?ra=288.26346083333334&dec=16.502817&
search.votable?ra=298.33685291666666&dec=22.616982&
search.votable?ra=241.07424500000002&dec=-33.213718&
search.votable?ra=258.50842083333333&dec=-27.796569&
search.votable?ra=249.6193861036075&dec=-18.220475660670832&
search.votable?ra=249.40487583333334&dec=-16.849827&
search.votable?ra=275.0947541666667&dec=-10.18710861111111&
search.votable?ra=275.094825&dec=-10.18682138888889&
search.votable?ra=253.68687406692413&dec=-36.88848911498945&
search.votable?ra=276.91469427398755&dec=-3.831149035247778&
search.votable?ra=277.35704791681206&dec=-6.077024977952222&
search.votable?ra=236.40620083333334&dec=-36.67481&
search.votable?ra=240.40672208333334&dec=-34.069574&
search.votable?ra=238.87084791666666&dec=-33.401242&
search.votable?ra=267.93566500000003&dec=-45.600703&
search.votable?ra=311.30466916666666&dec=-27.24728&
search.votable?ra=189.38566666666665&dec=-66.11136111111111&
search.votable?ra=253.86643708333335&dec=-32.503581&
search.votable?ra=245.39401791666666&dec=-44.944742&
search.votable?ra=291.6330691666667&dec=11.851068&
search.votable?ra=279.15259000000003&dec=12.987611&
search.votable?ra=270.7129108333333&dec=-24.282468&
search.votable?ra=176.77937708333334&dec=-35.906862&
......@@ -16,8 +16,14 @@ then
fi
# Move log before starting new process
mv $logFile ${logFile}.$(date +"%F_%T")
mv $logFile ${logFile}.$(date +"%F_%T")
# Upgrade the database structure
CMD="obsportal-cli database upgrade --settings=client.ini"
echo "Running: $CMD"
$CMD
# Launch the synchronization
CMD="obsportal-cli synchronize --settings=client.ini"
echo "Running: $CMD"
$CMD
......
......@@ -4,12 +4,17 @@ import os
from datetime import datetime
from pyramid.config import Configurator
from setuptools_scm import get_version
from obsportal.tools.settings import update_settings_with_envvars
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
with Configurator(settings=settings) as config:
app_settings = global_config.copy()
app_settings.update(settings)
app_settings = update_settings_with_envvars(app_settings, 'obsportal')
with Configurator(settings=app_settings) as config:
config.include('.models')
config.include('pyramid_jinja2')
config.include('.routes')
......
"""Pyramid bootstrap environment. """
from alembic import context
from pyramid.paster import get_appsettings, setup_logging
from pyramid.paster import setup_logging
from sqlalchemy import engine_from_config
from obsportal.models.meta import Base
from obsportal.tools.settings import load_settings
config = context.config
setup_logging(config.config_file_name)
settings = get_appsettings(config.config_file_name)
settings = load_settings(config.config_file_name)
target_metadata = Base.metadata
......
......@@ -7,6 +7,9 @@ Create Date: ${create_date}
"""
from alembic import op
import sqlalchemy as sa
import sqlalchemy_utils
import sqlalchemy_utc
import sqlalchemy_jsonfield
${imports if imports else ""}
# revision identifiers, used by Alembic.
......
"""First release (ObsPortal2020_04_16)
Revision ID: 5a7cecf8ad60
Revises:
Create Date: 2020-04-16 11:06:58.244786
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy_utils import UUIDType, ChoiceType
from sqlalchemy_utc import UtcDateTime
from sqlalchemy_jsonfield import JSONField
from obsportal.models.enums.observation import EnumObservationCategory
from obsportal.models.enums.program import EnumProgramType
# revision identifiers, used by Alembic.
revision = '5a7cecf8ad60'
down_revision = None
branch_labels = None
depends_on = None
def upgrade():
op.execute("CREATE SCHEMA IF NOT EXISTS obsportal")
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('interferometer',
sa.Column('id', sa.Unicode(), nullable=False),
sa.Column('name', sa.Unicode(), nullable=False),
sa.Column('latitude', sa.Float(), nullable=True),
sa.Column('longitude', sa.Float(), nullable=True),
sa.Column('altitude', sa.Float(), nullable=True),
sa.Column('headers_count', sa.Integer(), server_default=sa.text('0'), nullable=True),
sa.Column('observations_count', sa.Integer(), server_default=sa.text('0'), nullable=True),
sa.Column('exposures_count', sa.Integer(), server_default=sa.text('0'), nullable=True),
sa.PrimaryKeyConstraint('id', name=op.f('pk_interferometer')),
sa.UniqueConstraint('name', name=op.f('uq_interferometer_name')),
schema='obsportal'
)
op.create_table('program',
sa.Column('id', UUIDType(), nullable=False),
sa.Column('identifier', sa.Unicode(), nullable=True),
sa.Column('alternative_identifier', sa.Unicode(), nullable=True),
sa.Column('period', sa.Integer(), nullable=True),
sa.Column('name', sa.Text(), nullable=True),
sa.Column('type', ChoiceType(EnumProgramType), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_program')),
schema='obsportal'
)
op.create_table('target',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.Unicode(), nullable=True),
sa.Column('ra', sa.Float(), nullable=True),
sa.Column('dec', sa.Float(), nullable=True),
sa.Column('observations_count', sa.Integer(), server_default=sa.text('0'), nullable=True),
sa.Column('exposures_count', sa.Integer(), server_default=sa.text('0'), nullable=True),
sa.PrimaryKeyConstraint('id', name=op.f('pk_target')),
sa.UniqueConstraint('name', 'ra', 'dec', name='uq_target_name_ra_dec'),
schema='obsportal'
)
op.create_index(op.f('ix_obsportal_target_dec'), 'target', ['dec'], unique=False, schema='obsportal')
op.create_index(op.f('ix_obsportal_target_name'), 'target', ['name'], unique=False, schema='obsportal')
op.create_table('instrument',
sa.Column('id', sa.Unicode(), nullable=False),
sa.Column('interferometer_id', sa.Unicode(), nullable=False),
sa.Column('name', sa.Unicode(), nullable=False),
sa.Column('headers_count', sa.Integer(), server_default=sa.text('0'), nullable=True),
sa.Column('observations_count', sa.Integer(), server_default=sa.text('0'), nullable=True),
sa.Column('exposures_count', sa.Integer(), server_default=sa.text('0'), nullable=True),
sa.ForeignKeyConstraint(['interferometer_id'], ['obsportal.interferometer.id'], name=op.f('fk_instrument_interferometer_id_interferometer'), onupdate='CASCADE', ondelete='RESTRICT'),
sa.PrimaryKeyConstraint('id', name=op.f('pk_instrument')),
sa.UniqueConstraint('name', name=op.f('uq_instrument_name')),
schema='obsportal'
)
op.create_index(op.f('ix_obsportal_instrument_interferometer_id'), 'instrument', ['interferometer_id'], unique=False, schema='obsportal')
op.create_table('program_run',
sa.Column('id', UUIDType(), nullable=False),
sa.Column('program_id', UUIDType(), nullable=True),
sa.Column('identifier', sa.Unicode(), nullable=True),
sa.Column('alternative_identifier', sa.Unicode(), nullable=True),
sa.ForeignKeyConstraint(['program_id'], ['obsportal.program.id'], name=op.f('fk_program_run_program_id_program'), onupdate='CASCADE', ondelete='RESTRICT'),
sa.PrimaryKeyConstraint('id', name=op.f('pk_program_run')),
schema='obsportal'
)