...
 
Commits (13)
IMAGE_TAG=0.22-beta
IMAGE_TAG=0.23-beta
from pyramid.session import SignedCookieSessionFactory
from pyramid.config import Configurator
from pyramid.session import SignedCookieSessionFactory
def main(global_config, **settings):
......
......@@ -5,9 +5,8 @@ Revises:
Create Date: 2019-07-24 12:04:47.744393
"""
from alembic import op
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = '59e01862a009'
......
......@@ -5,9 +5,8 @@ Revises: e969bef706d9
Create Date: 2019-07-26 15:48:45.120586
"""
from alembic import op
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = '0197b06cd69d'
......
......@@ -5,9 +5,8 @@ Revises: 59e01862a009
Create Date: 2019-07-26 14:34:34.781225
"""
from alembic import op
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = 'aaee0e29c9ad'
......
......@@ -5,9 +5,8 @@ Revises: aaee0e29c9ad
Create Date: 2019-07-26 15:44:30.034370
"""
from alembic import op
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = 'e969bef706d9'
......
"""add star_info.sinpe
Revision ID: e891b3cc374d
Revises: 0197b06cd69d
Create Date: 2020-01-17 12:25:22.100925
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = 'e891b3cc374d'
down_revision = '0197b06cd69d'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('star_info', sa.Column('sinpe', sa.Boolean(), nullable=True))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('star_info', 'sinpe')
# ### end Alembic commands ###
from wtforms.form import Form
from wtforms import TextAreaField, StringField, FileField
from wtforms import validators
from wtforms.form import Form
class FormSubmitResults(Form):
email = StringField('Email', [validators.required()])
......
import zope.sqlalchemy
from sqlalchemy import engine_from_config
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm import configure_mappers
import zope.sqlalchemy
from sqlalchemy.orm import sessionmaker
# import or define all models here to ensure they are attached to the
# Base.metadata prior to any initialization routines
from .star_data import Catalog, Star, StarName
from .submission import StarInfoEntry, Submission
from .star_info import StarInfo
from .submission import StarInfoEntry, Submission
# run configure_mappers after defining all of the models to ensure
# all relationships can be setup
......
......@@ -5,14 +5,13 @@ import urllib.parse
import requests
from psycopg2.errors import UniqueViolation
from sqlalchemy import (Boolean, Column, Float, ForeignKey, Index, Integer,
from sqlalchemy import (Boolean, Column, ForeignKey, Integer,
String)
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.orm import relationship
from sqlalchemy.orm.collections import attribute_mapped_collection
from .meta import Base
#
# utility functions
#
......
import re
from sqlalchemy import Column, Float, Index, Integer, String
from sqlalchemy import Column, Float, Integer, String, Boolean
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.inspection import inspect
from sqlalchemy.orm import relationship
......@@ -8,6 +6,7 @@ from sqlalchemy.orm import relationship
from .meta import Base
from .star_data import Star, StarName, Band
#---------------------------------------------------------------------------
#-- II/345 JMDC : JMMC Measured Stellar Diameters Catalogue (Duvert, 2016)
#--------------------------------------------------------------------------
......@@ -81,7 +80,8 @@ class StarInfo(Base):
'NOTES': { 'field': 'notes', 'must': False },
'BIBCODE': { 'field': 'bibcode', 'must': True, 'check': 'check_bibcode', 'alternate': 'REFERENCE'},
'COM': { 'field': 'com', 'must': False, 'alternate': 'REFERENCE'},
'REFERENCE': { 'must': False, 'parse': 'parse_reference' }
'REFERENCE': { 'must': False, 'parse': 'parse_reference' },
'SINPE': {'field': 'sinpe', 'must': False, 'check': 'check_bool', 'parse': 'parse_sinpe'}, # SIMBAD Id Not Precise Enough
},
'checks': [
# here we should have names of methods to call that check object coherency
......@@ -131,6 +131,7 @@ class StarInfo(Base):
notes = Column(String(255))
bibcode = Column(String(19))
com = Column(String(255))
sinpe = Column(Boolean)
star_info_entry = relationship('StarInfoEntry', back_populates='star_info')
submission = association_proxy('star_info_entry', 'submission')
......@@ -257,7 +258,7 @@ class StarInfo(Base):
# print("setting value '%s' to object variable '%s'"%(str(value), field_name))
setattr(self, field_name, value)
def check_star(self, field_name, field_info, value, accept_empty_value=False):
def check_star(self, field_name, field_info, value, accept_empty_value=False, accept_invalid_star_id=False):
# print("Checking star '%s' for field '%s'"%(value, field_name))
if not value:
if not accept_empty_value :
......@@ -266,7 +267,10 @@ class StarInfo(Base):
try:
star = self.query_starname(value)
except StarDoesNotExist:
self.append_error(field_name, "invalid star identifier '%s'"%(value))
if accept_invalid_star_id:
self.append_warning(field_name, "invalid star identifier '%s'"%(value))
else:
self.append_error(field_name, "invalid star identifier '%s'" % (value))
else:
print("found star for field '%s' :"%(field_name), star, star.id)
field_var = field_info.get('field', None)
......@@ -275,7 +279,7 @@ class StarInfo(Base):
self.set_field_value(field_var, value)
def check_star2(self, field_name, field_info, value):
self.check_star(field_name, field_info, value, True)
self.check_star(field_name, field_info, value, True, self.sinpe)
def check_float(self, field_name, field_info, value):
......@@ -292,6 +296,21 @@ class StarInfo(Base):
if field_var:
self.set_field_value(field_var, value)
def check_bool(self, field_name, field_info, value):
if isinstance(value, str):
if not value:
value = None
else:
try:
value = value.lower() in ("true", "t", "1")
except ValueError:
self.append_error(field_name, "invalid value '%s'"%(value))
return
field_var = field_info.get('field', None)
if field_var:
self.set_field_value(field_var, value)
def check_band(self, field_name, field_info, value):
field_var = field_info.get('field', None)
if field_var:
......@@ -400,6 +419,10 @@ class StarInfo(Base):
if comment:
self.set_field_value('com', comment)
# force parsing before checking since starname check depends on this field
def parse_sinpe(self, field_name, field_info, value):
self.check_bool(field_name, field_info, value)
#
# object level checks
#
......@@ -473,7 +496,7 @@ class StarInfo(Base):
elif bc:
self.append_error('band', 'calculated band_code invalid \'%s\''%(repr(bc)))
#
#
# header parsing function
#
......
from sqlalchemy import Boolean, Column, ForeignKey, Index, Integer, TIMESTAMP, Text
from sqlalchemy import Boolean, Column, ForeignKey, Integer, TIMESTAMP, Text
from sqlalchemy.orm import relationship
from .meta import Base
from .star_info import StarInfo
class StarInfoEntry(Base):
__tablename__ = 'star_info_entry'
......
......@@ -17,24 +17,26 @@
<th>notes</th>
<th>bibcode</th>
<th>com</th>
<th>sinpe</th>
</tr>
</thead>
<tbody>
{% for star in stars %}
{% if star.star_info_entry.0.validated%}
<tr>
<td>{{star.id1}}</td>
<td>{{star.id2}}</td>
<td>{{star.ud_diam}}</td>
<td>{{star.ld_diam}}</td>
<td>{{star.e_ld_diam}}</td>
<td>{{star.band}}</td>
<td>{{star.mu_lambda}}</td>
<td>{{star.method}}</td>
<td>{{star.band_code}}</td>
<td>{{star.notes}}</td>
<td>{{star.bibcode}}</td>
<td>{{star.com}}</td>
<td>{{star.id1 or '' }}</td>
<td>{{star.id2 or '' }}</td>
<td>{{star.ud_diam or '' }}</td>
<td>{{star.ld_diam or '' }}</td>
<td>{{star.e_ld_diam or '' }}</td>
<td>{{star.band or '' }}</td>
<td>{{star.mu_lambda or '' }}</td>
<td>{{star.method or '' }}</td>
<td>{{star.band_code or '' }}</td>
<td>{{star.notes or '' }}</td>
<td>{{star.bibcode or '' }}</td>
<td>{{star.com or '' }}</td>
<td>{{star.sinpe or '' }}</td>
</tr>
{% endif %}
{% endfor %}
......
id1,id2,ud_diam,ld_diam,e_ld_diam,band,mu_lambda,method,band_code,notes,bibcode,com{% for star in stars %}{% if star.star_info_entry.0.validated%}{{star.id1}},{{star.id2}},{{star.ud_diam}},{{star.ld_diam}},{{star.e_ld_diam}},{{star.band}},{{star.mu_lambda}},{{star.method}},{{star.band_code}},"{{star.notes}}",{{star.bibcode}},"{{star.com}}"{% endif %}
id1,id2,ud_diam,ld_diam,e_ld_diam,band,mu_lambda,method,band_code,notes,bibcode,com{% for star in stars %}{% if star.star_info_entry.0.validated%}{{star.id1 or ''}},{{star.id2 or ''}},{{star.ud_diam or ''}},{{star.ld_diam or ''}},{{star.e_ld_diam or ''}},{{star.band or ''}},{{star.mu_lambda or ''}},{{star.method or ''}},{{star.band_code or ''}},"{{star.notes or ''}}",{{star.bibcode or ''}},"{{star.com or ''}}","{{star.sinpe or ''}}"{% endif %}
{% endfor %}
......@@ -22,29 +22,31 @@
<th>notes</th>
<th>bibcode</th>
<th>com</th>
<th>sinpe</th>
</tr>
</thead>
<tbody>
{% for entry in entries %}
<tr>
<td><input type="checkbox" name="valid[{{entry.star_info.id}}]" value="{{entry.star_info.id}}" {% if entry.validated %}checked="on"{% endif %}/></td>
<td>{{ entry.star_info.id }}</td>
<td><a href="http://simbad.u-strasbg.fr/simbad/sim-id?Ident={{ entry.star_info.id1 }}">{{ entry.star_info.id1 }}</a></td>
<td>{{ entry.star_info.id2}}</td>
<td>{{ entry.star_info.ud_diam}}</td>
<td>{{ entry.star_info.ld_diam}}</td>
<td>{{ entry.star_info.e_ld_diam}}</td>
<td>{{ entry.star_info.band}}</td>
<td>{{ entry.star_info.mu_lambda}}</td>
<td>{{ entry.star_info.method}}</td>
<td>{{ entry.star_info.band_code}}</td>
<td>{{ entry.star_info.notes}}</td>
<td><a href="https://ui.adsabs.harvard.edu/abs/{{ entry.star_info.bibcode}}/abstract">{{ entry.star_info.bibcode}}</a></td>
<td>{{ entry.star_info.com}}</td>
<td>{{ entry.star_info.id or '' }}</td>
<td><a href="http://simbad.u-strasbg.fr/simbad/sim-id?Ident={{ entry.star_info.id1 or '' }}">{{ entry.star_info.id1 or '' }}</a></td>
<td>{{ entry.star_info.id2 or '' }}</td>
<td>{{ entry.star_info.ud_diam or '' }}</td>
<td>{{ entry.star_info.ld_diam or '' }}</td>
<td>{{ entry.star_info.e_ld_diam or '' }}</td>
<td>{{ entry.star_info.band or '' }}</td>
<td>{{ entry.star_info.mu_lambda or '' }}</td>
<td>{{ entry.star_info.method or '' }}</td>
<td>{{ entry.star_info.band_code or '' }}</td>
<td>{{ entry.star_info.notes or '' }}</td>
<td><a href="https://ui.adsabs.harvard.edu/abs/{{ entry.star_info.bibcode or '' }}/abstract">{{ entry.star_info.bibcode or '' }}</a></td>
<td>{{ entry.star_info.com or '' }}</td>
<td>{{ entry.star_info.sinpe or '' }}</td>
</tr>
<tr>
<td>&nbsp;</td>
<td colspan="13"><input type="text" name="comment[{{entry.star_info.id}}]" value="{% if entry.comment %}{{entry.comment}}{% endif %}"></input></td>
<td colspan="13"><input type="text" name="comment[{{entry.star_info.id or '' }}]" value="{% if entry.comment %}{{entry.comment or '' }}{% endif %}"></input></td>
</tr>
{% endfor %}
</tbody>
......
import unittest
from pyramid import testing
import transaction
from pyramid import testing
def dummy_request(dbsession):
......
from pyramid.view import view_config
from pyramid.response import Response
from sqlalchemy.exc import DBAPIError
from .. import models
......
from pyramid.response import Response
from pyramid.view import view_config
from sqlalchemy.exc import DBAPIError
@view_config(route_name='jmdc_about', renderer='../templates/jmdc_about.jinja2')
def jmdc_about(request):
......
#from pyramid.response import Response
from pyramid.view import view_config
#from sqlalchemy.exc import DBAPIError
from jmdc_app.models.star_info import StarInfo
from jmdc_app.models.submission import StarInfoEntry
@view_config(route_name='jmdc_catalog', renderer='../templates/jmdc_catalog.jinja2')
def jmdc_home(request):
session = request.dbsession
# order by id (creation order / look during import )
stars = session.query(StarInfo).order_by(StarInfo.id).all()
# order by id (creation order / look during import ) and check for validated ones
stars = session.query(StarInfo).filter(StarInfo.star_info_entry.any(StarInfoEntry.validated==True)).order_by(
StarInfo.id).all()
return { 'stars': stars }
\ No newline at end of file
from pyramid.response import Response
from pyramid.view import view_config
from sqlalchemy.exc import DBAPIError
@view_config(route_name='jmdc_doc', renderer='../templates/jmdc_doc.jinja2')
def jmdc_doc(request):
......
from pyramid.response import Response
from pyramid.view import view_config
from sqlalchemy.exc import DBAPIError
from jmdc_app.models.star_info import StarInfo
from jmdc_app.models.submission import StarInfoEntry
@view_config(route_name='jmdc_export_csv', renderer='../templates/jmdc_export_csv.jinja2')
def jmdc_export_csv(request):
......@@ -10,6 +9,8 @@ def jmdc_export_csv(request):
session = request.dbsession
request.response.content_type = 'text/plain'
stars = session.query(StarInfo).all()
# order by id (creation order / look during import ) and check for validated ones
stars = session.query(StarInfo).filter(StarInfo.star_info_entry.any(StarInfoEntry.validated == True)).order_by(
StarInfo.id).all()
return { 'stars': stars }
import re
from pyramid.response import Response
from pyramid.view import view_config
from sqlalchemy.exc import DBAPIError
from jmdc_app.models.submission import StarInfoEntry, Submission
......
from pyramid.view import notfound_view_config
# @notfound_view_config(renderer='../templates/404.jinja2')
# def notfound_view(request):
# request.response.status = 404
......
......@@ -2,21 +2,16 @@ import codecs
import csv
import datetime
import json
import secrets
from pyramid.config import Configurator
import secrets
from pyramid.httpexceptions import HTTPFound
from pyramid.response import Response
from pyramid.view import view_config
from pyramid_mailer.message import Message
from sqlalchemy.exc import DBAPIError
from jmdc_app.forms.submit import FormImportCsv, FormSubmitResults
from jmdc_app.models.star_info import StarInfo
from jmdc_app.models.submission import StarInfoEntry, Submission
from .. import models
class import_new_data():
......
import json
import threading
import time
import traceback
from multiprocessing import Process
import pyramid.httpexceptions as exc
from pyramid.request import Request
from pyramid.response import Response
from pyramid.renderers import render_to_response
from pyramid.view import view_config
from pyramid.threadlocal import get_current_registry
from pyramid_mailer.mailer import Mailer
from pyramid_mailer.message import Message
from sqlalchemy import create_engine
from sqlalchemy.exc import DBAPIError
from sqlalchemy.orm import sessionmaker
from jmdc_app.models.star_info import StarInfo
......@@ -210,8 +205,8 @@ def moderate_submission(request):
for k in v.keys():
if k not in problem_fields:
problem_fields.append(k)
context['has_problem'] = True
problem_fields.sort()
context['has_problem'] = True
entry['problem_fields'] = problem_fields
entries.append({'line': e.line_number-1, 'entry': entry})
context['entries'] = entries
......