Commit 7a4db14a authored by Guillaume Mella's avatar Guillaume Mella

Handle duplicates resolution : propose remove or change values to the user

parent 74cc484e
......@@ -2,6 +2,7 @@ from sqlalchemy import Column, Float, Integer, String, Boolean
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.inspection import inspect
from sqlalchemy.orm import relationship
from sqlalchemy import or_
from .meta import Base
from .star_data import Star, StarName, Band
......@@ -50,7 +51,9 @@ from .star_data import Star, StarName, Band
# Exx.y : Floating point in exponential form
# xx : total length
# y : digits to right of decimal point
# Ixx : Integer length xx
# Ixx : Integer length xx
DUPLICATED_ERROR_MSG = 'star entry already present in the catalog (same id1, ud_diam, ld_diam and bibcode)'
class StarDoesNotExist(Exception):
pass
......@@ -202,8 +205,11 @@ class StarInfo(Base):
star_info_entry = relationship('StarInfoEntry', back_populates='star_info')
submission = association_proxy('star_info_entry', 'submission')
# temp to check uniq accross same submission lines
submission_id = -1
# initialize from a row of data, presumably from parsing a CSV
def __init__(self, session, header, data):
def __init__(self, session, header, data, submission_id=-1):
self.session = session
self.srcdata = data
self.problem_fields = None
......@@ -211,10 +217,12 @@ class StarInfo(Base):
self.errors = None
self.star_id_1 = None
self.star_id_2 = None
self.submission_id = submission_id
# self.parse_format_dump(data)
self.parse(header, data)
self.unique = self.is_unique()
print(self.warnings, self.errors)
# print(self.warnings, self.errors)
#
# error handling
......@@ -265,19 +273,21 @@ class StarInfo(Base):
return msg
return None
#
# tests on the star entry object state
#
def is_unique(self):
# same id1, mesurements, bibcodes
#StarInfo.submission.any(id=self.submission)
def check_unique(self):
print ("sub_id : %s "%self.submission_id)
# same id1, mesurements, bibcodes (compared to a validated or not validated)
q = self.session.query(StarInfo).filter_by(id1=self.id1, ud_diam=self.ud_diam, ld_diam=self.ld_diam,
e_ld_diam=self.e_ld_diam, bibcode=self.bibcode).count()
e_ld_diam=self.e_ld_diam, bibcode=self.bibcode).filter(
or_(
StarInfo.star_info_entry.any(validated=True),
StarInfo.star_info_entry.any(submission_id=self.submission_id),
)
).count()
print("is_unique query returns : %s" % q)
if q > 0:
self.append_entry_error(
'star entry already present in the catalog (same id1, ud_diam, ld_diam and bibcode)')
return not q
self.append_entry_error(DUPLICATED_ERROR_MSG)
#
# field parsing functions
......@@ -585,8 +595,8 @@ class StarInfo(Base):
all_mandatory = True
for f in fields.keys():
field = fields[f]
print(f, end=' - ')
print(fields, end=' - ')
# print(f, end=' - ')
# print(fields, end=' - ')
if field.get('must', None):
if f not in header:
# print('%s not in header'%(f), end=' - ')
......@@ -695,3 +705,7 @@ class StarInfo(Base):
if check_func:
if callable(check_func):
check_func()
# and finally retest for uniqueness
self.check_unique()
from sqlalchemy import Boolean, Column, ForeignKey, Integer, TIMESTAMP, Text
from sqlalchemy.orm import relationship
from . import StarInfo
from .meta import Base
from .star_info import DUPLICATED_ERROR_MSG
class StarInfoEntry(Base):
......@@ -33,6 +35,10 @@ class StarInfoEntry(Base):
star_info = relationship('StarInfo', back_populates='star_info_entry')
star = relationship('Star', back_populates='star_info_entries')
def is_unique(self):
# print("is_unique(): %s : %s " % (self.error_data, DUPLICATED_ERROR_MSG in self.error_data))
return DUPLICATED_ERROR_MSG in self.error_data
class Submission(Base):
__tablename__ = 'submission'
......
......@@ -48,6 +48,31 @@ function submit_line_form(event) {
ajax.send(formdata);
}
function delete_line(event) {
event.preventDefault();
let button = event.target;
let line = button.closest(".entry-log");
let line_no = parseInt(line.querySelector('.lineno').innerHTML);
let do_rm = confirm("Please confirm request to delete enty with line "+line_no);
if ( do_rm == null || do_rm == "") {
return ;
}
let ajax = new XMLHttpRequest();
ajax.addEventListener("error", (event) => {
alert("Sorry, an error occured trying to delete line "+ line_no+"\n"+line_no);
console.log(event);
});
ajax.addEventListener("load", (event) => {
location.reload();
});
ajax.open('DELETE', document.location+"/"+line_no);
ajax.send();
}
function update_line(event, form) {
if (event.target.status!=200) {
console.log("error");
......@@ -59,14 +84,19 @@ function update_line(event, form) {
template.innerHTML = event.target.response.trim();
var new_entry = template.content.firstChild;
new_entry = new_entry.querySelector('.entry-log');
let error_lines = new_entry.getElementsByClassName("error");
if (error_lines.length>0) {
let lineno = new_entry.querySelector('.lineno');
lineno.classList.add('button');
lineno.addEventListener("click", edit_line);
console.log(lineno);
if ( new_entry ) {
let error_lines = new_entry.getElementsByClassName("error");
if (error_lines.length>0) {
let lineno = new_entry.querySelector('.lineno');
lineno.classList.add('button');
lineno.addEventListener("click", edit_line);
console.log(lineno);
}
entry.replaceWith(new_entry);
}else{
entry.remove();
}
entry.replaceWith(new_entry);
}
function edit_line(event) {
......@@ -86,6 +116,11 @@ function initialize_form() {
lineno.addEventListener("click", edit_line);
}
}
var deletelinebuttons = document.getElementsByClassName("deletelinebutton");
for ( deletelinebutton of deletelinebuttons ) {
deletelinebutton.addEventListener("click", delete_line);
}
}
};
......
<p>Delete done</p>
\ No newline at end of file
<form class="input-form">
{% for field_name in headers %}{% if field_name %}
<div class="input-group"><label for="{{ field_name }}">{{ field_name }}</label><input type="text" name="{{ field_name }}" value="{{ values[loop.index0] }}"/></div>
{% endif %}{% endfor %}
<button type="button" name="action" value="update">update line</button>
{% for field_name in headers %}{% if field_name %}
<div class="input-group"><label for="{{ field_name }}">{{ field_name }}</label><input type="text" name="{{ field_name }}" value="{{ values[loop.index0] }}"/></div>
{% endif %}{% endfor %}
<button type="button" name="action" value="update">update line</button>
</form>
\ No newline at end of file
......@@ -15,6 +15,7 @@
<div class="line">
<div class="lineno">{{entry.line+1}}</div>
<div class="messages">
<div class="field-info">{{entry.entry.srcdata}}</div>
{% if entry.entry.problem_fields %}
{% for field in entry.entry.problem_fields %}
<div class="field-info">
......@@ -34,13 +35,11 @@
</div>
{% endfor %}
{% endif %}
{% if not entry.entry.unique %}
<div class="field-info">
<div class="field"></div>
<div class="msglist"><div class="error msg">entry is not unique</div></div>
</div>
{% endif %}
<div class="field-info">{{entry.entry.srcdata}}</div>
{% if entry.entry.duplicated %}
<div class="error msg">Your entry already is in your submission or in the catalog. Please modify your entry or delete using next button.
<button class="deletelinebutton" type="button" name="action" value="delete">delete line</button>
</div>
{% endif %}
</div>
</div>
</div>
......
......@@ -87,6 +87,9 @@ class import_new_data():
self.entries = []
self.entries.append({'line': self.line, 'entry': si})
# TODO check next line of code...
# is_unique can duplicate errors for non-uniqueness
# NOTE:
# this does only check if the entry is not YET in the database.
# entries for the current batch is NOT checked for uniqueness
......
......@@ -4,6 +4,7 @@ import traceback
from multiprocessing import Process
import pyramid.httpexceptions as exc
from pyramid.httpexceptions import HTTPFound
from pyramid.renderers import render_to_response
from pyramid.view import view_config
from pyramid_mailer.mailer import Mailer
......@@ -79,7 +80,7 @@ class ImportProcess(threading.Thread):
print(i_session)
i_entry = i_session.query(StarInfoEntry).filter_by(submission_id=self.sub.id,line_number=entry.line_number).first()
print(i_entry)
si = StarInfo(i_session, headers, line_data)
si = StarInfo(i_session, headers, line_data, self.sub_id)
si_msg = si.all_messages()
if si_msg and si_msg.get('errors', None) is not None:
errors_present = True
......@@ -188,7 +189,8 @@ def moderate_submission(request):
sie = session.query(StarInfoEntry).filter_by(submission_id=submission.id).order_by(StarInfoEntry.line_number).all()
for e in sie:
entry = {
'srcdata': e.line_data
'srcdata': e.line_data,
'duplicated' : e.is_unique()
}
ed = e.error_data
# print(type(ed), ed)
......@@ -200,7 +202,7 @@ def moderate_submission(request):
error_data = json.loads(ed)
if error_data:
entry.update(error_data)
entry['unique'] = True
# why ? entry['unique'] = True
problem_fields = []
for v in entry.values():
if type(v) is dict:
......@@ -228,11 +230,11 @@ def get_line_form(request):
submission = session.query(Submission).filter_by(upload_ticket=token).first()
if not submission:
raise exc.HTTPForbidden()
line = session.query(StarInfoEntry).filter_by(submission_id=submission.id, line_number=line).first()
if not line:
sie = session.query(StarInfoEntry).filter_by(submission_id=submission.id, line_number=line).first()
if not sie:
raise exc.HTTPNotFound()
print(token, line)
print(token, sie)
print(submission.headers)
context = {
......@@ -240,10 +242,18 @@ def get_line_form(request):
headers = json.loads(submission.headers)
if request.method == 'DELETE':
session.delete(sie.star_info)
session.delete(sie)
# mark processing to restart since duplicate errors can change
# TODO set to True only if submissiion's entries contains duplicates
submission.processing=True
renderer = '../templates/line_delete.jinja2'
if request.method == 'GET':
renderer = '../templates/line_form.jinja2'
context['headers'] = headers
context['values'] = json.loads(line.line_data)
context['values'] = json.loads(sie.line_data)
print(context)
if request.method == 'POST':
......@@ -257,9 +267,9 @@ def get_line_form(request):
if len(values)==1:
value = values[0]
line_data.append(value)
line.line_data = json.dumps(line_data)
sie.line_data = json.dumps(line_data)
print(line_data)
si = line.star_info
si = sie.star_info
si.session = session
si.warnings = None
si.errors = None
......@@ -270,7 +280,7 @@ def get_line_form(request):
si_msg = si.all_messages()
if si_msg and si_msg.get('errors', None) is not None:
errors_present = True
line.error_data = json.dumps(si_msg)
sie.error_data = json.dumps(si_msg)
if si_msg:
print('--------------------------------------------------')
print(si_msg)
......@@ -278,8 +288,8 @@ def get_line_form(request):
entries = []
entry = {
'srcdata': line.line_data,
'unique': True
'srcdata': sie.line_data,
'unique': sie.is_unique()
}
if si_msg:
entry.update(si_msg)
......@@ -291,7 +301,7 @@ def get_line_form(request):
problem_fields.append(k)
problem_fields.sort()
entry['problem_fields'] = problem_fields
entries.append({'line': line.line_number-1, 'entry': entry})
entries.append({'line': sie.line_number-1, 'entry': entry})
context['entries'] = entries
print(context)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment