star_info.py 28.4 KB
Newer Older
Guillaume Mella's avatar
Guillaume Mella committed
1
from sqlalchemy import Column, Float, Integer, String, Boolean
2
from sqlalchemy.ext.associationproxy import association_proxy
3
from sqlalchemy.inspection import inspect
4
from sqlalchemy.orm import relationship
5
from sqlalchemy import or_
6
7

from .meta import Base
Raphael Jacquot's avatar
Raphael Jacquot committed
8
from .star_data import Star, StarName, Band
9

Guillaume Mella's avatar
Guillaume Mella committed
10

11
12
13
14
15
# ---------------------------------------------------------------------------
# -- II/345 JMDC : JMMC Measured Stellar Diameters Catalogue (Duvert, 2016)
# --------------------------------------------------------------------------
# ---Table: II/345/jmdc.dat JMMC Measured stellar Diameters Catalog (as part of Chelli et al., 2016A&A...589A.112C)  (1554 records)
# -------------------------------------------------------------------------------
16
#     Label Format Unit  Explanations
17
# -------------------------------------------------------------------------------
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
#       ID1 A23    ---   Normalised star name (preferably HD) (ID1)
#       ID2 A20    ---   Name used in the original publication (ID2)
#    UDdiam E13.6  mas   ?=-1 Uniform Disk Diameter (UD_MEAS)
#    LDdiam E13.6  mas   ?=-1 Limb-Darkened Disk diameter (LD_MEAS)
#  e_LDdiam E13.6  mas   ?=-1 Error on UDdiam and LDdiam (E_LD_MEAS) (1)
#      Band A10    ---   Text describing the wavelength or band
#                         of the measurement (BAND) (2)
# mu-lambda E13.6  ---   ?=-1 When possible, value of the conversion
#                         UDD-to-LDD user by the author (ORIG_MU_LAMBDA)
#    Method I1     ---   Integer code of the method used (METHOD) (3)
#  BandCode I3     ---   ?=- Integer code of the band used
#                         (BANDCODE) (4)
#     Notes A224   ---   Note about the star and/or measurement (NOTES)
#   BibCode A19    ---   BibCode
#       Com A59    ---   Author's name and comments (REFERENCE)
33
34
# -------------------------------------------------------------------------------
# Note (1): In general, quotes the published error on LDD or UDD, which are
35
36
37
# essentially equivalent. The blanking value of -1 can be due to an absence of
# published error, or has been fixed thus when the measurement is
# (retrospectively) in doubt, in which case an explanation lies in the Notes.
38
# Note (2): A loosely defined string representing the band (UBVRIJHKLMNQ) or the
39
40
# central wavelength, in microns if not otherwise precised, of the observation.
# A stands for Angstroem, nm for nanometer.
41
# Note (3): Code for the observational method used as follows:
42
43
44
#    1 = optical interferometry
#    2 = Lunar occultation
#    3 = intensity interferometry
45
# Note (4): Index from 1 (band U) and up through bands B,V,R,I,J,H,K,L,M,N,Q.
46
47
48
49
50
51
52
53


# format notes :
# the formats above are fortran format codes
# Axx   : Ascii length xx
# Exx.y : Floating point in exponential form
#         xx : total length
#         y  : digits to right of decimal point
54
55
56
# Ixx   : Integer length xx
DUPLICATED_ERROR_MSG = 'star entry already present in the catalog (same id1, ud_diam, ld_diam and bibcode)'

57

58
59
60
61
class StarDoesNotExist(Exception):
    pass


62
63
64
class StarInfo(Base):
    __tablename__ = 'star_info'

65
66
    F_DUMP = 0
    F_USER = 1
67
68
    MAINFIELDS = ['ID1', 'ID2', 'UD_DIAM', 'LD_DIAM', 'E_LD_DIAM', 'BAND', 'MU_LAMBDA', 'METHOD', 'BANDCODE', 'NOTES',
                  'BIBCODE', 'SINPE']
69
70
71
72

    FIELDS_DUMP = {
        'mode_id': F_DUMP,
        'fields': {
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
            'ID1': {
                'field': 'id1',
                'must': True,
                'check': 'check_star',
                "doc": "normalised star name (preferably HD)"},
            'ID2': {
                'field': 'id2',
                'must': True,
                'check': 'check_star2',
                'doc': "Name used in the original publication"},
            'UD_DIAM': {
                'field': 'ud_diam',
                'must': True,
                'check': 'check_float',
                'alternate': 'UD_MEAS',
                'doc': "Uniform Disk Diameter",
                'unit': "mas"},
            'UD_MEAS': {
                'field': 'ud_diam',
                'check': 'check_float'},
            'LD_DIAM': {
                'field': 'ld_diam',
                'must': True,
                'check': 'check_float',
                'alternate': 'LD_MEAS',
                'doc': 'Limb-Darkened Disk diameter',
                'unit': 'mas'},
            'LD_MEAS': {
                'field': 'ld_diam',
                'check': 'check_float'},
            'E_LD_DIAM': {
                'field': 'e_ld_diam',
                'must': True,
                'check': 'check_float',
                'alternate': 'E_LD_MEAS',
                'doc': 'Error on UDdiam and LDdiam',
                'unit': 'mas',
                'note': 'In general, quotes the published error on LDD or UDD, which are essentially equivalent. The blanking value of -1 can be due to an absence of published error, or has been fixed thus when the measurement is (retrospectively) in doubt, in which case an explanation lies in the Notes'},
            'E_LD_MEAS': {
                'field': 'e_ld_diam',
                'check': 'check_float'},
            'BAND': {
                'field': 'band',
                'must': True,
                'check': 'check_band',
                'doc': 'Text describing the wavelength or band of the measurement',
                'note': 'A loosely defined string representing the band (UBVRIJHKLMNQ) or the central wavelength, in microns if not otherwise precised, of the observation. A stands for Angstroem, nm for nanometer.', },
            'MU_LAMBDA': {
                'field': 'mu_lambda',
                'must': False,
                'check': 'check_float',
                'doc': 'When possible, value of the conversion UDD-to-LDD user by the author'},
            'METHOD': {
                'field': 'method',
                'must': False,
                'check': 'check_method',
                'doc': "Integer code of the method used",
                'note': "Code for the observational method used as follows:    1 = optical interferometry, 2 = Lunar occultation, 3 = intensity interferometry"},
            'BANDCODE': {
                'field': 'band_code',
                'must': False,
                'check': 'check_band_code',
                'doc': 'Integer code of the band used'},
            'NOTES': {
                'field': 'notes',
                'must': False,
                'doc': 'Note about the star and/or measurement'},
            'BIBCODE': {
                'field': 'bibcode',
                'must': True,
                'check': 'check_bibcode',
                'doc': 'BibCode',
                'alternate': 'REFERENCE'},
            'REFERENCE': {
                'must': False,
                'parse': 'parse_reference'},
            'SINPE': {
                'field': 'sinpe',
                'must': False,
                'check': 'check_bool',
                'parse': 'parse_sinpe',
                'doc': "Flag that SIMBAD is not precise enough for the measured star ID"}
155
        },
156
        'checks': [
Raphael Jacquot's avatar
Raphael Jacquot committed
157
            # here we should have names of methods to call that check object coherency
158
            'check_star_ids',
Raphael Jacquot's avatar
Raphael Jacquot committed
159
            'check_diameter_data',
Raphael Jacquot's avatar
Raphael Jacquot committed
160
            'set_band_code'
161
        ]
162
163
164
165
166
    }

    FIELDS_USER = {
        'mode_id': F_USER,
        'fields': {
167
168
169
170
171
172
173
174
175
176
            'ID': {'field': 'starname', 'must': True, 'check': 'check_star', 'alternate': 'STARNAME'},
            'STARNAME': {'field': 'starname', 'check': 'check_star'},
            'UD_MEAS': {'field': 'ud_diam', 'must': True, 'check': 'check_float'},
            'LD_MEAS': {'field': 'ld_diam', 'must': True, 'check': 'check_float'},
            'E_LD_MEAS': {'field': 'e_ld_diam', 'must': True, 'check': 'check_float'},
            'BAND': {'field': 'band', 'must': True, 'check': 'check_band'},
            'MU_LAMBDA': {'field': 'mu_lambda', 'must': False},
            'METHOD': {'field': 'method', 'must': False, 'check': 'check_method'},
            'NOTES': {'field': 'notes', 'must': False, 'check': 'check_notes'},
            'REFERENCE': {'must': True, 'parse': 'parse_reference'}
Raphael Jacquot's avatar
Raphael Jacquot committed
177
        },
178
179
        'checks': [
            'set_star_ids',
Raphael Jacquot's avatar
Raphael Jacquot committed
180
            'check_diameter_data',
Raphael Jacquot's avatar
Raphael Jacquot committed
181
            'set_band_code'
182
        ]
183
184
    }

185
    FIELDS = [FIELDS_DUMP, FIELDS_USER]
Raphael Jacquot's avatar
Raphael Jacquot committed
186

187
    # 123456789012
Raphael Jacquot's avatar
Raphael Jacquot committed
188

189
    PREFERED_CATALOGS = ['HD', 'HIP', '2MASS', 'BD']
Raphael Jacquot's avatar
Raphael Jacquot committed
190

191
    id = Column(Integer, primary_key=True)
192
193
    id1 = Column(String(32))
    id2 = Column(String(32))
194
195
196
197
198
199
200
    ud_diam = Column(Float)
    ld_diam = Column(Float)
    e_ld_diam = Column(Float)
    band = Column(String(10))
    mu_lambda = Column(Float)
    method = Column(Integer)
    band_code = Column(Integer)
201
    notes = Column(String(255))
202
    bibcode = Column(String(19))
Guillaume Mella's avatar
Guillaume Mella committed
203
    sinpe = Column(Boolean)
204

205
206
    star_info_entry = relationship('StarInfoEntry', back_populates='star_info')
    submission = association_proxy('star_info_entry', 'submission')
207

208
209
210
    # temp to check uniq accross same submission lines
    submission_id = -1

211
    # initialize from a row of data, presumably from parsing a CSV
212
    def __init__(self, session, header, data, submission_id=-1):
213
        self.session = session
214
215
216
217
        self.srcdata = data
        self.problem_fields = None
        self.warnings = None
        self.errors = None
Raphael Jacquot's avatar
Raphael Jacquot committed
218
219
        self.star_id_1 = None
        self.star_id_2 = None
220
        self.submission_id = submission_id
221
        # self.parse_format_dump(data)
222
        self.parse(header, data)
223
224
225

#        print(self.warnings, self.errors)

226
227
228
229
230
231
232
233

    #
    # error handling
    #

    def append_problem_field(self, field):
        if not self.problem_fields:
            self.problem_fields = []
Raphael Jacquot's avatar
Raphael Jacquot committed
234
235
        if field not in self.problem_fields:
            self.problem_fields.append(field)
236

237
238
239
    def append_warning(self, field, msg):
        self.append_problem_field(field)
        if not self.warnings:
240
            self.warnings = {}
241
242
243
244
245
246
        f_warn = self.warnings.get(field, None)
        if not f_warn:
            f_warn = []
            self.warnings[field] = f_warn
        f_warn.append(msg)

Raphael Jacquot's avatar
Raphael Jacquot committed
247
248
249
    def append_entry_warning(self, msg):
        self.append_warning('_', msg)

250
251
252
    def append_error(self, field, msg):
        self.append_problem_field(field)
        if not self.errors:
253
            self.errors = {}
254
255
256
257
258
259
        f_err = self.errors.get(field, None)
        if not f_err:
            f_err = []
            self.errors[field] = f_err
        f_err.append(msg)

Guillaume Mella's avatar
Guillaume Mella committed
260
261
262
    def has_problem(self, field):
        return self.problem_fields and field in self.problem_fields

263
264
265
    def append_entry_error(self, msg):
        self.append_error('_', msg)

266
267
268
269
270
271
272
273
274
    def all_messages(self):
        msg = {}
        if self.errors:
            msg['errors'] = self.errors
        if self.warnings:
            msg['warnings'] = self.warnings
        if msg:
            return msg
        return None
275

276
277
278
279
#StarInfo.submission.any(id=self.submission)
    def check_unique(self):
        print ("sub_id : %s "%self.submission_id)
        # same id1, mesurements, bibcodes (compared to a validated or not validated)
280
        q = self.session.query(StarInfo).filter_by(id1=self.id1, ud_diam=self.ud_diam, ld_diam=self.ld_diam,
281
282
283
284
285
286
                                                   e_ld_diam=self.e_ld_diam, bibcode=self.bibcode).filter(
             or_(
                StarInfo.star_info_entry.any(validated=True),
                StarInfo.star_info_entry.any(submission_id=self.submission_id),
             )
            ).count()
287
        print("is_unique query returns : %s" % q)
Guillaume Mella's avatar
Guillaume Mella committed
288
        if q > 0:
289
290
            self.append_entry_error(DUPLICATED_ERROR_MSG)

291
292
293
294
295

    #
    # field parsing functions
    #

296
297
298
299
    def query_starname(self, name):
        """
        looks for starname, returns Star.id
        """
Raphael Jacquot's avatar
Raphael Jacquot committed
300
301
302
        # lookup the star in the cache
        identifier = StarName.lookup_name(self.session, name)
        if identifier:
303
            # self.append_entry_warning(repr(identifier))
Raphael Jacquot's avatar
Raphael Jacquot committed
304
305
306
307
            print("query_starname found star in cache, exiting early", identifier)
            obj_id, canon_name = identifier
            return self.session.query(Star).filter_by(id=obj_id).first()

308
        print("name '%s' not found in cache" % (name))
Raphael Jacquot's avatar
Raphael Jacquot committed
309
        # time to query the CDS
310
        identifier = StarName.cleanup_name(name)
311
312
        if not identifier:
            raise StarDoesNotExist()
Raphael Jacquot's avatar
Raphael Jacquot committed
313

Guillaume Mella's avatar
Guillaume Mella committed
314
        # self.append_entry_warning(repr(identifier))
315
        obj_id, canon_name = identifier
316
        print("'%d' - '%s'" % (obj_id, canon_name))
317
318
319

        # search for star with object id.
        # if does not exist, add star
320
        # print("searching for star with object id %d"%(obj_id))
Raphael Jacquot's avatar
Raphael Jacquot committed
321
        star = self.session.query(Star).filter_by(id=obj_id).first()
322
        # print("obtained the info")
Raphael Jacquot's avatar
Raphael Jacquot committed
323
        if not star:
324
            # print("star %d-'%s' not found, adding"%(obj_id, canon_name))
325
326
            star = Star(self.session, obj_id, canon_name)
            self.session.add(star)
Raphael Jacquot's avatar
Raphael Jacquot committed
327
        else:
328
329
            # there's one star with that id (as it should)
            star.session = self.session
Raphael Jacquot's avatar
Raphael Jacquot committed
330
            print("WARNING: updating star names")
331
332
            star.update_names()
        return star
333

Raphael Jacquot's avatar
Raphael Jacquot committed
334
335
336
337
    # 
    # field handling methods
    # 

338
    def set_field_value(self, field_name, value):
339
        # print("setting value '%s' to object variable '%s'"%(str(value), field_name))
340
        setattr(self, field_name, value)
Raphael Jacquot's avatar
Raphael Jacquot committed
341

Guillaume Mella's avatar
Guillaume Mella committed
342
    def check_star(self, field_name, field_info, value, accept_empty_value=False, accept_invalid_star_id=False):
343
        # print("Checking star '%s' for field '%s'"%(value, field_name))
344
        if not value:
345
            if not accept_empty_value:
Guillaume Mella's avatar
Guillaume Mella committed
346
                self.append_error(field_name, 'star identifier not set')
347
            return
Raphael Jacquot's avatar
Raphael Jacquot committed
348
        try:
349
            star = self.query_starname(value)
Raphael Jacquot's avatar
Raphael Jacquot committed
350
        except StarDoesNotExist:
Guillaume Mella's avatar
Guillaume Mella committed
351
            if accept_invalid_star_id:
352
                self.append_warning(field_name, "invalid star identifier '%s'" % (value))
Guillaume Mella's avatar
Guillaume Mella committed
353
354
            else:
                self.append_error(field_name, "invalid star identifier '%s'" % (value))
Raphael Jacquot's avatar
Raphael Jacquot committed
355
        else:
356
            print("found star for field '%s' :" % (field_name), star, star.id)
357
358
            field_var = field_info.get('field', None)
            if field_var:
359
                self.set_field_value(field_var + '_id', star.id)
360
                self.set_field_value(field_var, value)
Raphael Jacquot's avatar
Raphael Jacquot committed
361

Guillaume Mella's avatar
Guillaume Mella committed
362
    def check_star2(self, field_name, field_info, value):
Guillaume Mella's avatar
Guillaume Mella committed
363
        self.check_star(field_name, field_info, value, True, self.sinpe)
Guillaume Mella's avatar
Guillaume Mella committed
364

Raphael Jacquot's avatar
Raphael Jacquot committed
365
366
367
368
369
370
371
372
    def check_float(self, field_name, field_info, value):
        if isinstance(value, str):
            if not value:
                value = None
            else:
                try:
                    value = float(value)
                except ValueError:
373
                    self.append_error(field_name, "invalid value '%s'" % (value))
Raphael Jacquot's avatar
Raphael Jacquot committed
374
375
376
                    return
        field_var = field_info.get('field', None)
        if field_var:
377
            self.set_field_value(field_var, value)
Raphael Jacquot's avatar
Raphael Jacquot committed
378

Guillaume Mella's avatar
Guillaume Mella committed
379
380
381
382
383
384
385
386
    def check_bool(self, field_name, field_info, value):
        if isinstance(value, str):
            if not value:
                value = None
            else:
                try:
                    value = value.lower() in ("true", "t", "1")
                except ValueError:
387
                    self.append_error(field_name, "invalid value '%s'" % (value))
Guillaume Mella's avatar
Guillaume Mella committed
388
389
390
391
392
                    return
        field_var = field_info.get('field', None)
        if field_var:
            self.set_field_value(field_var, value)

Raphael Jacquot's avatar
Raphael Jacquot committed
393
394
395
    def check_band(self, field_name, field_info, value):
        field_var = field_info.get('field', None)
        if field_var:
396
            self.set_field_value(field_var, value.strip())
Raphael Jacquot's avatar
Raphael Jacquot committed
397
398

    def check_method(self, field_name, field_info, value):
Raphael Jacquot's avatar
Raphael Jacquot committed
399
400
401
402
        if value:
            if value.isdecimal():
                value = int(value)

403
            if value not in [1, 2, 3]:
Raphael Jacquot's avatar
Raphael Jacquot committed
404
                self.append_error(field_name, 'Invalid value, should be 1, 2 or 3')
405
                return
Raphael Jacquot's avatar
Raphael Jacquot committed
406

Raphael Jacquot's avatar
Raphael Jacquot committed
407
408
409
410
411
        field_var = field_info.get('field', None)
        if field_var:
            self.set_field_value(field_var, value)

    def check_band_code(self, field_name, field_info, value):
Raphael Jacquot's avatar
Raphael Jacquot committed
412
413
        band_values = Band.gen_numeric_band_code_list()
        band_codes = Band.gen_letter_band_code_list()
Raphael Jacquot's avatar
Raphael Jacquot committed
414
415
        # we assume a string
        # check if we have a number
Raphael Jacquot's avatar
Raphael Jacquot committed
416
417
        if value.isdecimal():
            code = int(value)
Raphael Jacquot's avatar
Raphael Jacquot committed
418
            if code not in band_values:
419
420
                self.append_error(field_name,
                                  "numerical value should be between %d and %d" % (band_values[0], band_values[-1]))
Raphael Jacquot's avatar
Raphael Jacquot committed
421
422
423
424
        else:
            if not value:
                # value is empty
                # should try to guess from band ?
425
                code = None
426
            elif (len(value) != 1) or (value not in band_codes):
427
428
                self.append_error(field_name,
                                  "character value should be one character long and in the set '%s'" % (band_codes))
Raphael Jacquot's avatar
Raphael Jacquot committed
429
430
                return
            else:
431
432
433
434
                code = band_codes.index(value) + 1
        if code is None:
            # check if we have band_code set
            pass
Raphael Jacquot's avatar
Raphael Jacquot committed
435
436
437
438
        field_var = field_info.get('field', None)
        if field_var:
            self.set_field_value(field_var, code)

439
440
    def check_notes(self, field_name, field_info, value):
        field_var = field_info.get('field', None)
441

442
443
444
445
446
447
        field_type = inspect(self).mapper.columns[field_var].type
        if field_type.python_type is not str:
            # error, this must be str / can't happen
            raise TypeError
        if value:
            if len(value) > field_type.length:
448
                self.append_error(field_name, "string too long, maximum length %d" % (field_type.length))
449
450
451
452
                return
        if field_var:
            self.set_field_value(field_var, value)

Raphael Jacquot's avatar
Raphael Jacquot committed
453
    def is_bibcode_valid(self, bibcode):
454
        if len(bibcode) != 19:
Raphael Jacquot's avatar
Raphael Jacquot committed
455
            return False
456
        bibcode_year = bibcode[0:4].strip('.')
Raphael Jacquot's avatar
Raphael Jacquot committed
457
        bibcode_journal = bibcode[4:9].strip('.')
458
        bibcode_volume = bibcode[9:13].strip('.')
Raphael Jacquot's avatar
Raphael Jacquot committed
459
        bibcode_section = bibcode[13].strip('.')
460
461
462
463
        bibcode_page = bibcode[14:18].strip('.')
        bibcode_author = bibcode[18].strip('.')
        print("|%s|%s|%s|%s|%s|%s|" % (
        bibcode_year, bibcode_journal, bibcode_volume, bibcode_section, bibcode_page, bibcode_author), end=' ')
Raphael Jacquot's avatar
Raphael Jacquot committed
464
465
        if bibcode_year and bibcode_journal and bibcode_volume and bibcode_page and bibcode_author:
            print("bc check")
Raphael Jacquot's avatar
Raphael Jacquot committed
466
467
468
469
470
471
472
473
            return True
        print("bc invalid ?")
        return None

    def check_bibcode(self, field_name, field_info, value):
        bc_valid = self.is_bibcode_valid(value)
        if bc_valid is False:
            self.append_error(field_name, "invalid value")
Raphael Jacquot's avatar
Raphael Jacquot committed
474
        else:
Raphael Jacquot's avatar
Raphael Jacquot committed
475
476
477
478
            if bc_valid is None:
                self.append_warning(field_name, "incompletely specified")
            field_var = field_info.get('field', None)
            if field_var:
Guillaume Mella's avatar
bugfix    
Guillaume Mella committed
479
                self.set_field_value('bibcode', value)
Raphael Jacquot's avatar
Raphael Jacquot committed
480
481

    def parse_reference(self, field_name, field_info, value):
482
        print("reference: '%s'" % (value))
Raphael Jacquot's avatar
Raphael Jacquot committed
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
        # the reference is composed of a bibcode (19 non spaces) and, optionally, a space, and comments
        first_space = value.find(' ')
        if first_space == -1:
            first_space = len(value)
        if first_space != 19:
            self.append_error(field_name, "must start with a bibcode (19 non spaces)")
            return
        bibcode = value[0:first_space]
        comment = value[first_space:].strip()
        # we have something that may be a bibcode
        bc_valid = self.is_bibcode_valid(bibcode)
        if bc_valid is False:
            self.append_error(field_name, "bibcode portion is invalid")
        else:
            if bc_valid is None:
                self.append_warning(field_name, "bibcode portion is incompletely specified")
            self.set_field_value('bibcode', bibcode)
        if comment:
501
            self.append_warning('bibcode', 'bibcode extract ignore next part : %s' % comment)
Raphael Jacquot's avatar
Raphael Jacquot committed
502

Guillaume Mella's avatar
Guillaume Mella committed
503
504
505
506
    # force parsing before checking since starname check depends on this field
    def parse_sinpe(self, field_name, field_info, value):
        self.check_bool(field_name, field_info, value)

Raphael Jacquot's avatar
Raphael Jacquot committed
507
508
509
510
    #
    # object level checks
    #

511
512
513
    def check_star_ids(self):
        if hasattr(self, 'id1_id') and hasattr(self, 'id2_id'):
            if self.id1_id != self.id2_id:
514
515
                self.append_entry_error("id1 and id2 don't match, '%s' (oid %d) - '%s' (oid %d)" % (
                self.id1, self.id1_id, self.id2, self.id2_id))
516

517
518
519
520
521
522
523
    def set_id1_from_starname(self, starname):
        names_q = self.session.query(StarName).filter_by(star_id=self.starname_id).all()
        names = []
        for name in names_q:
            names.append(name.name)
        prefered_name = None
        for cat in self.PREFERED_CATALOGS:
524
            print('looking for cat %s - ' % (cat), end=' ')
525
526
527
528
529
530
531
532
533
534
535
536
            for name in names:
                print(name, end=' ')
                if name[:len(cat)] == cat:
                    print('found')
                    prefered_name = name
                    break
            if prefered_name:
                break
        if not prefered_name:
            # search for the main name
            pass
        if not prefered_name:
537
            self.append_error('id1', 'not set (star_id %d, \'%s\')' % (self.starname_id, prefered_name))
538
539
540
        else:
            self.id1 = prefered_name

Raphael Jacquot's avatar
Raphael Jacquot committed
541
542
    def set_star_ids(self):
        if hasattr(self, 'starname'):
543
            print("checking starname, identify id1 for star_id %d" % (self.starname_id))
544
545
546
            # attempt to guess ID1
            self.set_id1_from_starname(self.starname)
            self.id2 = self.starname
Raphael Jacquot's avatar
Raphael Jacquot committed
547

Raphael Jacquot's avatar
Raphael Jacquot committed
548
    def check_diameter_data(self):
549

Raphael Jacquot's avatar
Raphael Jacquot committed
550
551
        pass

Raphael Jacquot's avatar
Raphael Jacquot committed
552
    def set_band_code(self):
Raphael Jacquot's avatar
Raphael Jacquot committed
553
554
555
556
557
558
        bc = Band.parse_band_code(self.band)
        if not bc:
            self.append_error('band', 'unable to parse band')
        elif isinstance(bc, dict):
            warning = bc.get('warning', None)
            error = bc.get('error', None)
559
            bc = bc.get('band_code', None)
560
            if warning:
Raphael Jacquot's avatar
Raphael Jacquot committed
561
562
                self.append_warning('band', warning)
            if error:
Guillaume Mella's avatar
Guillaume Mella committed
563
564
565
566
567
                # Report error only if associated band_code is not valid
                if self.has_problem('band_code'):
                    self.append_error('band', error)
                else:
                    self.append_warning('band', error)
568
569
570
571
        if isinstance(bc, float) or isinstance(bc, int):
            # some lines have band_code -1
            if self.band_code is not None and self.band_code != -1:
                if self.band_code != bc:
572
573
                    self.append_error('band', 'calculated band_code %s differs from given band_code %s' % (
                    str(bc), str(self.band_code)))
574
            else:
575
576
577
                # there was no band_code given but bc seems ok.
                # GM: do not throw an error since next affection seems nice.
                # self.append_error('band', 'calculated band_code %s'%(repr(bc)))
578
                self.band_code = bc
Guillaume Mella's avatar
Guillaume Mella committed
579
        elif bc:
580
            self.append_error('band', 'calculated band_code invalid \'%s\'' % (repr(bc)))
581

582
    #
583
584
585
586
587
    # header parsing function
    #

    @staticmethod
    def parse_header(header):
588
        # identify mode :
589
590
        valid_modes = []
        for fields_list in StarInfo.FIELDS:
591
            mode_id = fields_list['mode_id']
592
            print("mode %d" % (mode_id), end=" ")
593
594
595
596
597
            fields = fields_list['fields']
            fields_present = []
            all_mandatory = True
            for f in fields.keys():
                field = fields[f]
598
599
#                print(f, end=' - ')
#                print(fields, end=' - ')
600
601
                if field.get('must', None):
                    if f not in header:
602
                        # print('%s not in header'%(f), end=' - ')
603
604
                        alternate = field.get('alternate', None)
                        if not alternate:
605
606
                            all_mandatory = False
                            # print('no alternate')
607
608
                            continue
                        if alternate not in header:
609
610
                            all_mandatory = False
                            # print('alternate not in header')
611
                            continue
612
613
                        # print("alternate found '%s'"%(alternate))
                        f = alternate
614
615
                    # else:
                    #     print('%s found'%(f))
616
                    fields_present.append(f)
617
618
619
620
621
                else:
                    if f not in header:
                        # print('not mandatory - skipping')
                        continue
                    # print('%s found'%(f))
622
623
                    if f not in fields_present:
                        fields_present.append(f)
624
            if not fields_present:
625
                all_mandatory = False
626
            if all_mandatory:
627
                valid_modes.append((fields_list.get('mode_id'), fields_present,))
628
629
630
631
            if all_mandatory:
                print("we have all mandatory fields         :", end=' ')
            else:
                print("we are missing some mandatory fields :", end=' ')
632
633
            print(fields_present)
        if len(valid_modes) == 0:
634
            print("FATAL: no parse mode found")
635
636
            return None
        if len(valid_modes) > 1:
637
            print("FATAL: too many parse modes valid")
638
639
            return None
            # return only one mode
640
        return valid_modes[0]
641

642
643
644
    #
    # record parsing functions
    #
Raphael Jacquot's avatar
Raphael Jacquot committed
645

646
647
648
649
650
651
652
653
654
655
656
    def parse(self, header, data):
        """
        parses the data according to the header, checking the header for mandatory fields
        """
        print(header, data)
        parse_mode = StarInfo.parse_header(header)
        print("parse mode", parse_mode)

        mode, field_list = parse_mode

        field_defs = self.FIELDS[mode]
657
        field_engine = field_defs['fields']
658
659
660
        for field_name in field_list:
            field_info = field_engine[field_name]
            index = header.index(field_name)
661
            value = data[index].strip()
662
            # print(field_info, index, "'%s'"%(value))
663
664
665
666
667
668
669

            # if we have a parse function
            parse_func_name = field_info.get('parse', None)
            if parse_func_name:
                parse_func = getattr(self, parse_func_name, None)
                if parse_func:
                    if callable(parse_func):
Raphael Jacquot's avatar
Raphael Jacquot committed
670
671
                        # print(parse_func)
                        parse_func(field_name, field_info, value)
672
                    else:
673
                        self.append_error(field_name, "FATAL: '%s' function is not callable" % (parse_func_name))
674
                else:
675
676
                    self.append_error(field_name, "FATAL: '%s' function is not present in '%s'" % (
                    parse_func_name, self.__class__.__name__))
677
678
679
680
681
682
683
684
685
                # the parse func is responsible for setting the field value
                continue

            # if we have a check function
            check_func_name = field_info.get('check', None)
            if check_func_name:
                check_func = getattr(self, check_func_name, None)
                if check_func:
                    if callable(check_func):
Raphael Jacquot's avatar
Raphael Jacquot committed
686
                        # print(check_func)
687
688
                        check_func(field_name, field_info, value)
                    else:
689
                        self.append_error(field_name, "FATAL: '%s' function is not callable" % (check_func_name))
690
                else:
691
692
                    self.append_error(field_name, "FATAL: '%s' function is not present in '%s'" % (
                    check_func_name, self.__class__.__name__))
693
694
695
696
                # the check func is responsible for setting the field value
                continue

            # nothing specified, just set the field value
Raphael Jacquot's avatar
Raphael Jacquot committed
697
698
699
            field_var = field_info.get('field', None)
            if field_var:
                self.set_field_value(field_var, value)
Raphael Jacquot's avatar
Raphael Jacquot committed
700
701
702
703

        # object level checks
        checks = self.FIELDS[mode]['checks']
        for check_func_name in checks:
704
705
706
707
            check_func = getattr(self, check_func_name, None)
            if check_func:
                if callable(check_func):
                    check_func()
708
709
710
711

        # and finally retest for uniqueness
        self.check_unique()

712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
    def csv_headers():
        return [ '"%s"' % field for field in StarInfo.MAINFIELDS ]

    def csv_values(self):
        ret_cols =[]
        fields = StarInfo.FIELDS_DUMP['fields']
        for field in StarInfo.MAINFIELDS:
            colname=fields[field]['field']
            col=self.__getattribute__(colname)
            if isinstance(col, str) :
#            if col.property.columns[0].type==String:
                ret_cols.append('"%s"'%col.strip())
            elif col == None:
                ret_cols.append("")
            else:
                ret_cols.append("%s" % col)
        return ret_cols