Commit 688274dd authored by bourgesl's avatar bourgesl
Browse files

Merge branch 'release/2020_04_16'

parents aa52da8f 52bd39b0
Pipeline #41614 skipped with stage
[run]
source = obsportal
omit = obsportal/test*
Dockerfile
*-dev.sh
*.egg
*.egg-info
*.pyc
*$py.class
*~
.coverage*
coverage.xml
pyproject.toml
pytest.ini
test
.env
.idea
*.local
*.sqlite
*.tar.gz
.python-version
*.egg
*.egg-info
*.pyc
*$py.class
*~
.coverage
coverage.xml
build/
dist/
.tox/
nosetests.xml
env*/
tests/
tmp/
Data.fs*
*.sublime-project
*.sublime-workspace
.*.sw?
.sw?
.DS_Store
coverage
test
.env
.idea
*.local
*.sqlite
*.ini
*.tar.gz
*.votable.gz
image : docker
variables:
DOCKER_DRIVER: overlay2
stages :
- build
.docker_build_template: &docker_build
tags:
- dind
stage : build
when: manual
services:
- docker:dind
script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker build -t $CI_REGISTRY_IMAGE/$IMAGE:$IMAGE_TAG .
- docker push $CI_REGISTRY_IMAGE/$IMAGE:$IMAGE_TAG
build obs:
<<: *docker_build
variables:
IMAGE: obs
IMAGE_TAG: "latest"
0.0
---
- Initial version.
FROM python:3.7-slim
###################################
#
# Usual maintenance
#
# ALWAYS RUN apt-get update && apt-get install -y --no-install-recommends
# Install debian packages:
RUN set -eux ; \
apt-get update && apt-get upgrade -y && \
apt-get install -y --no-install-recommends vim python3-pip python3-dev postgresql-common postgresql-client libpq-dev git && \
apt-get clean && \
true
# Pre-install important pip packages (for speedup during the application setup)
RUN set -eux ; \
pip3 install --upgrade pip setuptools wheel setuptools_scm && \
pip3 install lxml cryptography click pyramid pyramid_jinja2 wtforms alembic waitress SQLAlchemy transaction psycopg2_binary numpy astropy astroquery pyvo && \
true
# TODO: perform cleanup in install folder: never include .git tree !
ADD . /app
WORKDIR /app
RUN set -eux ; \
# Install application:
echo "install Obsportal with dependencies ..." && \
mkdir -p /data /logs && \
pip3 install -e . && \
true
ARG SHAREDCONFFILE=shared.ini-dist
ADD $SHAREDCONFFILE /app/shared.ini
ARG APPCONFFILE=production.ini-dist
ADD $APPCONFFILE /app/docker.ini
ARG CLICONFFILE=client.ini-dist
ADD $CLICONFFILE /app/client.ini
CMD ["sh", "run-obsportal.sh", "docker.ini" ]
EXPOSE 6543
ObsPortal
=========
Application setup
-----------------
- Init a Python 3 virtualenv
cd <directory containing this file>
python3 -m venv .env
- Activate the virtualenv
.env/bin/activate
- Now, we're assuming you are inside <directory containing this file> in the activated virtualenv!
- Upgrade packaging tools.
pip3 install --upgrade pip setuptools wheel setuptools_scm
- Install the project in editable mode
pip3 install -e .
- Configure the shared database parameters
cp shared.ini-dist shared.ini
nano shared.ini
- And configure this parameter regarding to your deployment environment:
- sqlalchemy.url : database connection string. See https://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql
- With <application_settings_file> = production.ini OR development.ini
cp <application_settings_file>-dist <application_settings_file>
nano <application_settings_file>
- And configure these parameters regarding to your deployment environment:
- global.paths.data : top-directory storing all data retrieved by the application
- global.paths.logs : top-directory storing all logs generated by the application
- global.paths.tmp : top-directory storing all temporary files generated by the application
- PLEASE DO NOT USE AND MODIFY *.ini-dist FILES, USE LOCAL *ini FILES INSTEAD
- PLEASE DO NOT COMMIT LOCAL *.ini FILES, KEEP IT FOR YOUR OWN USAGE ONLY
Database setup
--------------
- If your postgresql role has admin powers:
obsportal-cli database create --structure --debug --settings=<application_settings_file>
- Else, if your database already exists
obsportal-cli database create-structure --debug --settings=<application_settings_file>
- Finally, load initial data:
obsportal-cli database init --debug --settings=<application_settings_file>
Database feeding from ESO
-------------------------
- Synchronize content from ESO/CHARA:
obsportal-cli synchronize --debug --settings=<application_settings_file>
Launch the application
----------------------
pserve <application_settings_file> --reload
*******************
Build Docker image:
*******************
* For production:
docker build -t "obsportal:latest" .
* For development:
docker build -t "obsportal:latest" --build-arg SHAREDCONFFILE=shared.ini --build-arg APPCONFFILE=development.ini .
docker run -p 6543:6543 --rm --name d_obsportal "obsportal:latest"
* Initialize existing postgres DB:
(see db/ scripts to initialize manually the postgres + pgsphere database)
docker exec -it d_obsportal obsportal-cli database create-structure --skip_exist --debug --settings=client.ini
docker exec -it d_obsportal obsportal-cli database init --debug --settings=client.ini
* Import Data (Ctrl-C to stop):
docker exec -it d_obsportal-cli synchronize --debug --settings=client.ini
* Drop database content (BE CAREFUL):
docker exec -it d_obsportal-cli database drop-structure --debug --settings=client.ini
OR
docker exec -it d_obsportal obsportal-cli database run-sql-script db/_clear_db.sql --settings=client.ini
* Use bash in container:
docker exec -it d_obsportal bash
#!/bin/bash
docker exec -it d_obsportal bash
#################################################
# Global configuration
# https://pastedeploy.readthedocs.io/en/latest/#global-configuration
#################################################
[DEFAULT]
global.paths.data = /data
global.paths.logs = /logs
global.paths.tmp = /tmp
global.environment = client
#################################################
# Application configuration
# https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html
#################################################
[app:main]
use = config:shared.ini#shared
#######################
## Database parameters
#######################
# Connection Pool settings:
sqlalchemy.pool_size = 5
sqlalchemy.max_overflow = 5
# recycle opened connection after 1h idle:
sqlalchemy.pool_recycle = 3600
#############################
## Framework debug parameters
#############################
pyramid.reload_templates = false
pyramid.debug_authorization = false
pyramid.debug_notfound = false
pyramid.debug_routematch = false
pyramid.includes =
########################
## Jinja2 custom filters
########################
jinja2.filters =
datetime = obsportal.filters:filter_datetime
#################################################
## Alembic configuration
#################################################
[alembic]
# path to migration scripts
script_location = obsportal/alembic
file_template = %%(year)d%%(month).2d%%(day).2d_%%(rev)s
# file_template = %%(rev)s_%%(slug)s
#################################################
## Framework shell parameters
#################################################
[pshell]
setup = obsportal.pshell.setup
#################################################
## WSGI server configuration
#################################################
[server:main]
use = egg:waitress#main
listen =
#################################################
## Logging configuration
##https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html
#################################################
[loggers]
keys = root, obsportal, waitress, sqlalchemy, sqlalchemy_pool, urllib
[handlers]
keys = console, file
[formatters]
keys = generic
[logger_root]
level = INFO
handlers = console, file
[logger_obsportal]
level = INFO
handlers =
qualname = obsportal
[logger_sqlalchemy]
level = WARN
handlers = console
qualname = sqlalchemy.engine
# "level = INFO" logs SQL queries.
# "level = DEBUG" logs SQL queries and results.
# "level = WARN" logs neither. (Recommended for production systems.)
[logger_sqlalchemy_pool]
level = INFO
handlers =
qualname = sqlalchemy.pool
[logger_waitress]
level = INFO
handlers =
qualname = waitress
[logger_urllib]
level = INFO
handlers =
qualname = urllib3
[handler_console]
class = StreamHandler
args = (sys.stdout,)
level = NOTSET
formatter = generic
[handler_file]
class = FileHandler
args=(__import__("datetime").datetime.now().strftime('%(global.paths.logs)s/%(global.environment)s_%%Y-%%m-%%d_%%H-%%M-%%S.log'), 'a')
level = NOTSET
formatter = generic
[formatter_generic]
format = %(asctime)s %(levelname)-5.5s [%(name)s:%(lineno)s][%(threadName)s] %(message)s
DROP SCHEMA obsportal CASCADE;
This diff is collapsed.
#!/bin/bash
psql -U postgres -c 'CREATE EXTENSION pg_sphere;' obsdb
-- defines PgSphere spatial index on target table :
CREATE INDEX ix_target_ra_dec_spatial ON obsportal.target USING GIST(spoint(radians(ra), radians(dec)));
-- obsolete script (2020.02.17):
DROP INDEX obsportal.obs_target_spatial;
-- drop table created by sql alchemy:
DROP TABLE obsportal.obs_target_uniq_obs;
DROP MATERIALIZED VIEW obsportal.obs_target_uniq_obs;
DROP MATERIALIZED VIEW obsportal.obs_target_uniq;
DROP MATERIALIZED VIEW obsportal.obs_target_bad_matches;
DROP MATERIALIZED VIEW obsportal.obs_target_all_matches;
DROP MATERIALIZED VIEW obsportal.obs_target;
-- VIEW obs_target
CREATE MATERIALIZED VIEW obsportal.obs_target AS SELECT DISTINCT target_name, target_ra, target_dec, DENSE_RANK() OVER(ORDER BY target_name, target_dec, target_ra) AS row_number FROM obsportal.observation ORDER BY target_name, row_number;
CREATE INDEX obs_target_spatial ON obsportal.obs_target USING GIST(spoint(radians(target_ra), radians(target_dec)));
-- 10 arcsec = 10 / 3600 = 0.002777777777777778
-- VIEW obs_target_all_matches
-- exclude same row (only extra matches on the right side):
CREATE MATERIALIZED VIEW obsportal.obs_target_all_matches AS SELECT
degrees( spoint(radians(o1.target_ra), radians(o1.target_dec)) <-> spoint(radians(o2.target_ra), radians(o2.target_dec)) ) * 3600.0 AS dist_as,
o1.row_number as rowNum1, o1.target_name as name1, o1.target_ra as ra1, o1.target_dec as dec1,
o2.row_number as rowNum2, o2.target_name as name2, o2.target_ra as ra2, o2.target_dec as dec2
FROM obsportal.obs_target o1, obsportal.obs_target o2
WHERE
spoint(radians(o2.target_ra), radians(o2.target_dec)) @ scircle(spoint(radians(o1.target_ra), radians(o1.target_dec)), radians(0.002777777777777778))
AND (o2.row_number > o1.row_number)
ORDER BY o1.row_number, dist_as, o2.row_number;
-- VIEW obs_target_bad_matches
CREATE MATERIALIZED VIEW obsportal.obs_target_bad_matches AS SELECT DISTINCT(rowNum2) as bad_num
FROM obsportal.obs_target_all_matches order by rowNum2;
-- Uniq ids not in obs_target_bad_matches:
CREATE MATERIALIZED VIEW obsportal.obs_target_uniq AS SELECT *
FROM obsportal.obs_target ot
WHERE ot.row_number NOT IN (SELECT bad_num FROM obsportal.obs_target_bad_matches) ORDER BY ot.target_name, ot.row_number;
CREATE MATERIALIZED VIEW obsportal.obs_target_uniq_obs AS SELECT ot.*,
(SELECT COUNT(*) FROM obsportal.observation o WHERE (spoint(radians(o.target_ra), radians(o.target_dec)) @ scircle(spoint(radians(ot.target_ra), radians(ot.target_dec)), radians(0.002777777777777778)) ) OR o.target_name = ot.target_name ) as obs_count
FROM obsportal.obs_target_uniq ot;
-- slower:
-- add OR ot.target_name = o.target_name
-- defines ObsDB group with users :
CREATE ROLE obsdb_group WITH NOSUPERUSER NOCREATEDB NOCREATEROLE NOLOGIN;
ALTER ROLE obsdb_group SET client_encoding = 'UTF8';
CREATE ROLE obsdb_admin WITH LOGIN PASSWORD 'admin' INHERIT IN ROLE obsdb_group;
-- creates ObsDB database owned by osug_doi_super :
CREATE DATABASE obsdb WITH OWNER = obsdb_admin;
This diff is collapsed.
This diff is collapsed.
# night
2018-09-20
2018-09-21
2018-09-22
2018-09-24
2018-09-25
2018-09-27
2018-09-28
2018-09-29
2018-09-30
2018-10-01
2018-10-02
2018-10-03
2018-10-04
2018-10-05
2018-10-06
2018-10-07
2018-10-08
2018-10-09
2018-10-10
2018-10-11
2018-10-12
2018-10-13
2018-10-15
2018-10-16
2018-10-17
2018-10-18
2018-10-19
2018-10-20
2018-10-21
2018-10-22
2018-10-23
2018-10-24
2018-10-25
2018-10-26
2018-10-27
2018-10-28
2018-10-29
2018-10-30
2018-10-31
2018-11-01
2018-11-02
2018-11-03
2018-11-04
2018-11-05
2018-11-06
2018-11-07
2018-11-08
2018-11-09
2018-11-10
2018-11-11
2018-11-12
2018-11-13
2018-11-14
2018-11-15
2018-11-16
2018-11-17
2018-11-18
2018-11-19
2018-11-20
2018-11-21
2018-11-22
2018-11-23
2018-11-24
2018-11-25
2018-11-26
2018-11-27
2018-11-28
2018-11-29
2018-12-01
2018-12-02
2018-12-03
2018-12-04
2018-12-05
2018-12-06
2018-12-07
2018-12-08
2018-12-09
2018-12-10
2018-12-11
2018-12-12
2018-12-13
2018-12-14
2018-12-15
2018-12-16
2018-12-17
2018-12-18
2018-12-19
2018-12-20
2018-12-21
2018-12-22
2018-12-23
2018-12-24
2018-12-25
2018-12-26
2019-01-01
2019-01-08
2019-01-09
2019-01-10
2019-01-11
2019-01-12
2019-01-13
2019-01-14
2019-01-15
2019-01-16
2019-01-17
2019-01-18
2019-01-19
2019-01-20
2019-01-21
2019-01-22
2019-01-23
2019-01-24
2019-01-25
2019-01-26
2019-01-27
2019-01-28
2019-01-29
2019-01-30
2019-01-31
2019-02-01
2019-02-02
2019-02-03
2019-02-04
2019-02-05
2019-02-06
2019-02-07
2019-02-08
2019-02-09
2019-02-10
2019-02-11
2019-02-12
2019-02-13
2019-02-14
2019-02-15
2019-02-16
2019-02-17
2019-02-18
2019-02-19
2019-02-20
2019-02-21
2019-02-22
2019-02-23
2019-02-24
2019-02-25
2019-02-26
2019-02-27
2019-02-28
2019-03-01
2019-03-02
2019-03-03
2019-03-04
2019-03-05
2019-03-06
2019-03-07
2019-03-08
2019-03-09
2019-03-10
2019-03-11
2019-03-12
2019-03-13
2019-03-14
2019-03-15
2019-03-16
2019-03-17
2019-03-18
2019-03-19
2019-03-20
2019-03-21
2019-03-22