1#!/usr/bin/env python3
2
3"""Source-ID for S3 project.
4
5SCID based but using numeric SIDNUM for storage in system and ts tables."""
6
7from collections import defaultdict
8from fnmatch import fnmatch
9
10from chart.common.xml import SubElement
11from chart.project import settings
12from chart.common.xml import load_xml
13from chart.common.xml import parsechildbool
14from chart.common.xml import parsechildstr
15from chart.common.decorators import memoized
16from chart.common.xml import parsechilddatetime
17from chart.common.xml import parsechildint
18from chart.common.xml import parsechildtimedelta
19from chart.plots.sampling import Sampling
20from chart.sids.exceptions import BadSID
21
22
23ELEM_SCID = 'scid'
24ELEM_SATELLITE = 'satellite'
25ELEM_OPERATIONAL = 'operational'
26ELEM_SID_NUM = 'sid-num'
27SYS_NAME = 'SYS'
28SYS_LABEL = 'System'
29
30
31class _Satellite:
32 """Information about a satellite."""
33
34 def __init__(self, elem):
35 self.scid = parsechildstr(elem, 'scid')
36 self.name = parsechildstr(elem, 'name')
37 self.operational = parsechildbool(elem, 'operational', True)
38 self.visible = parsechildbool(elem, 'visible', True)
39 self.ground_test = parsechildbool(elem, 'ground-test', False)
40 self.has_ext_hktm = parsechildbool(elem, 'has-ext-hktm', False)
41 self.launch_date = parsechilddatetime(elem, 'launch-date', None)
42 # self.orbital = parsechildbool(sat_elem, 'orbital')
43 # 'instruments': par
44 self.orbit_duration = parsechildtimedelta(elem, 'orbit-duration', None)
45 self.time_offset = parsechildtimedelta(elem, 'time-offset', None)
46 # print('scid={}, time_offset={}'.format(self.scid, self.time_offset))
47 # self.processing_frequency = parsechildtimedelta(elem, 'processing_frequency', None)
48 self.acronym = parsechildstr(elem, 'acronym', None)
49 # basic widgets look for acronym
50 self.sid_num = parsechildint(elem, ELEM_SID_NUM)
51
52
53@memoized
54def all_satellites():
55 """Return a list of all <satellites>."""
56 result = []
57 for sat_elem in load_xml(settings.SATELLITES).findall('satellite'):
58 result.append(_Satellite(sat_elem))
59
60 return result
61
62
63# @memoized
64def Satellite(scid=None, name=None): # (invalid name) pylint: disable=C0103
65 """Cached constructor."""
66 for s in all_satellites():
67 if scid is not None and scid == s.scid:
68 return s
69
70 if name is not None and name == s.name:
71 return s
72
73 return None
74
75
76class SID_S3:
77 """EPS Source ID object.
78 Data is identified using SCID and sensing_time.
79 """
80
81 # based on this we include the correct xxx_selector.html file from sids/templates
82 selector = 's3'
83 default_sid = {'scid': 'S3A'} # to set when plot page in browser is first brought up
84
85 # sampling options used by this SID class
86 sampling_options = [Sampling.AUTO,
87 Sampling.ALL_POINTS,
88 Sampling.FIT,
89 Sampling.ORBITAL,
90 # Sampling.DAILY,
91 Sampling.DAILY_TOTALS,
92 ]
93
94 # subsampling options to be considered when auto-selecting best stats subsampling
95 # should be sorted from longest to shortest
96 def auto_sampling_options(self):
97 """Return the list of stats subsampling options available for this sid."""
98
99 if self.satellite is None:
100 return []
101
102 elif self.satellite.ground_test:
103 # return [Sampling.DAILY_TOTALS] - we might have it eventually (or HOURLY)
104 return []
105
106 else:
107 return [Sampling.ORBITAL]
108
109 def __init__(self, scid=None, sid_num=None, wildcard=False):
110 """If `wildcard` then allow specs which are not exact matches
111 but can be used for wildcards."""
112
113 # temporary heck to work around from_string and from_xml problems, when called from tables.py
114 # happens when plotting limits
115 if scid == '*':
116 # scid = 'S3A'
117 self.scid = '*'
118 return
119
120 self.scid = None
121
122 ok = False
123 if sid_num is not None:
124 for s in all_satellites():
125 if s.sid_num == sid_num:
126 # print(s.sid_num, sid_num)
127 ok = True
128 self.scid = s.scid
129 break
130
131 elif scid is not None:
132 if scid =='SYS':
133 self.scid = scid
134
135 else:
136 scid = scid.upper()
137 for s in all_satellites():
138 if s.scid == scid:
139 ok = True
140 self.scid = scid
141 break
142
143 # print ('ok={}'.format(ok))
144 # if not ok and not wildcard:
145 # raise BadSID(scid)
146
147 def __repr__(self):
148 return 'SID(scid={scid})'.format(scid=self.scid)
149
150 def __str__(self):
151 if self.scid is None:
152 return 'Non satellite specific'
153
154 else:
155 return self.scid
156
157 def __eq__(self, other):
158 if other is None:
159 # needed for 'if sid in sids...'
160 return False
161
162 else:
163 return self.scid == other.scid
164
165 def __ne__(self, other):
166 # this is required otherwise the SID non-uniqueness check in eventsfile.py
167 # gives a false failure
168 return self.scid != other.scid
169
170 def __hash__(self):
171 return hash(self.scid)
172
173 @property
174 def orbit(self):
175 """Retrieve orbital information. May (in future) return None for ground test
176 data SIDs. Should be cached."""
177 from chart.products.fdf.orbit_s3 import Orbit
178 return Orbit(self)
179
180 @property
181 def sid_num(self):
182 return self.satellite.sid_num if self.satellite is not None else None
183
184 #
185 ##
186 ### Web site support
187 ##
188 #
189
190 @staticmethod
191 def from_django_request(request):
192 """Extract a SID from `query` (URL parameter string)."""
193 scid = request.get('scid')
194 if scid is None:
195 return None
196
197 return SID_S3(scid=scid)
198
199 def as_url_query(self, base=None):
200 """Return a dictionary representation of ourselves suitable for passing into
201 the urllib.urlencode() function to create/extend a URL query fragment."""
202 if base is None:
203 return {'scid': self.scid}
204
205 else:
206 base['scid'] = self.scid
207 return base
208
209 #
210 ##
211 ### DDL creation support
212 ##
213 #
214
215 @staticmethod
216 def ddl_id_fields():
217 """SID fields implicitly added to the start of all TS tables."""
218 from chart.db.model.table import FieldInfo
219 return [
220 FieldInfo(name='SID_NUM',
221 description='Source ID Number',
222 datatype=int,
223 length=32),
224 ]
225
226 # change to function
227 @staticmethod
228 def ddl_ap_indexes():
229 """Configure the indexes for timeseries tables."""
230 return [{'suffix': '_C', 'fields': ('SID_NUM', 'SENSING_TIME', 'SPID'), 'compress': 1}]
231
232 # Indexes for stats tables
233 # change to function
234
235 @staticmethod
236 def ddl_stats_indexes():
237 """Configure indexes for timeseries stats tables."""
238 return [{'suffix': '_P', 'fields': ('SID_NUM', 'REGION', 'SENSING_TIME'), 'compress': 2},
239 # {'suffix': '_O', 'fields': ('SID_NUM', 'REGION', 'ORBIT'), 'compress': 2}]
240 ]
241
242 @staticmethod
243 def ddl_sys_fields(table_name=None): # (unused arg) pylint: disable=W0613
244 """Events, jobs, reports, subscriptions tables using the {{sid}} expansion."""
245 # don't import at module level because it probably imports back to us
246 from chart.db.model.table import FieldInfo
247 # if table_name == 'SUBSCRIPTIONS':
248 # return [
249 # FieldInfo(name='SCID', datatype=str, length=4, description='Spacecraft ID'),
250 # FieldInfo(name='DUMMYTEST', datatype=str, length=4, description='Spacecraft ID'),
251 # ]
252
253 # else:
254 return [
255 FieldInfo(name='SID_NUM',
256 datatype=int,
257 length=4,
258 allow_null=True,
259 description='Spacecraft ID'),
260 ]
261
262 # @staticmethod
263 # def ddl_sys_indexes(table_name=None):
264 # pass
265
266 #
267 ##
268 ### Generic binary ingester support
269 ##
270 #
271
272 @staticmethod
273 def rdr_kwargs(row):
274 """Extract selection args from a row."""
275 # rename to ts_sql_bindvars
276 return {'scid': row[0]}
277
278 rdr_insert = 'sid.scid'
279
280 @staticmethod
281 def from_sf(sfreader):
282 """Extract a SID from an SF reader object."""
283 # move to sf; not needed here
284 # sf files should be removed from msg also
285 return SID_S3(scid=sfreader.scid)
286
287 # When creating a timeseries ingest cursor, prepend it with these fields
288 insert_fields = ['SCID']
289
290 def bind_insert(self):
291 """When calling an insert cursor, pass these fields first."""
292 # rename to ts_insert_data
293 return [self.scid]
294
295 # This is now only used by sf00_ingester and can probably be removed completely.
296 # ddl_id_fields() replaces it
297 SCID_MAX_CHARS = 3
298
299
300 @staticmethod
301 def decode_cortex_scid(mission):
302 """Mission to SID."""
303
304 if mission == 'SN3A':
305 return SID_S3('S3A')
306
307 elif mission == 'SN3B':
308 return SID_S3('S3B')
309
310 else:
311 return None
312
313 #
314 ##
315 ### ts table support
316 ##
317 #
318
319 @staticmethod
320 def ts_indexes():
321 """Return list of TS AP table field names which should be indexed."""
322
323 return ['SID_NUM', ]
324
325
326 # Construct an SQL where clause which will filter for `sid`.
327 # Used by ingestion code (in MSG this means we have CFID but not OGSID)
328 # rename to ts_sql_where_bind
329 sql_where_bind = 'SID_NUM=:sid_num'
330
331 def sql_where(self, table_name=None, match_on_none=False, prefix=None):
332 """Return an SQL clause that will test for ts rows containing us.
333
334 If `table_name` is set it will be the name (not Tableinfo) of the table to be written
335 to, which may be a system table.
336
337 If `match_on_none` is set then we will match table rows contains null values in the SID
338 column.
339
340 `prefix` can be used to prefix column names for queries that join tables.
341
342 Avoids bind variables."""
343 if match_on_none:
344 return '(SID_NUM is null or SID_NUM={sidnum})'.format(sidnum=self.sid_num)
345
346 else:
347 return 'SID_NUM={sidnum}'.format(sidnum=self.sid_num)
348
349 #
350 ##
351 ### sys table support
352 ##
353 #
354
355 @staticmethod
356 def sql_sys_where(table_name, sid): # (unused arg) pylint: disable=W0613
357 """Return an SQL clause which will test for us in the JOBS or REPORTS or EVENTS table.
358 If `sid` is None then don't apply any filtering.
359 """
360 # should be removed and reported with all bind variables
361 if sid is None:
362 return '1=1'
363
364 elif sid.scid is None:
365 return 'SCID IS NULL'
366
367 else:
368 return "SID_NUM='{sid_num}'".format(sid_num=sid.sid_num)
369
370 @staticmethod
371 def sql_sys_where_bind(table_name): # (unused arg) pylint: disable=W0613
372 """SQL fragment to use in WHERE clauses filtering for us, using bind variables."""
373 return 'SID_NUM=:sid_num'
374
375 def bind_sys_where(self, table_name): # (unused arg) pylint: disable=W0613
376 """Convert ourselves into a bindvars dict for use with the sql emitted by
377 sql_sys_where_bind()."""
378 return {'sid_num': self.sid_num}
379
380 @staticmethod
381 def sql_sys_select(table_name): # (unused arg) pylint: disable=W0613
382 """SQL fragment to add to the (end of the) list of fields to retrieve
383 in order to construct a SID later with from_sys_select()."""
384 return ['SID_NUM']
385
386 @staticmethod
387 def from_sys_select(table_name, args): # (unused arg) pylint: disable=W0613
388 """Construct a SID from the fields requested by sql_sys_select()."""
389 if args[0] is None:
390 return None
391
392 else:
393 return SID_S3(sid_num=args[0])
394
395 @staticmethod
396 def sql_sys_insert(table_name):
397 """Return 2 SQL fragments for an insert statement:
398 - Fields
399 - Bind variables
400
401 E.g.:
402 sid_vars, sid_binds = SID.sql_sys_insert('JOBS')
403 insert_job.ins_cur = db_conn.prepared_cursor(
404 'INSERT INTO JOBS (CATEGORY, ACTIVITY, FILENAME, DIRNAME, {sidfield}ORBIT, '
405 'SENSING_START, SENSING_STOP, TABLENAME, EARLIEST_EXECUTION_TIME '
406 'VALUES '
407 '(:category, :activity, :filename, :dirname, {sidbind}:orbit, :sensing_start, '
408 ':sensing_stop, :tablename, earliest_execution_time'.format(
409 sidfield=sid_fields, sidbind=sid_binds))
410 .
411 """
412 # if table_name == 'PRODUCTS':
413 # return (',SID', ',:sid')
414 # else:
415 return (',SID_NUM', ',:sid_num')
416
417 @staticmethod
418 def sql_sys_update(table_name): # (unused arg) pylint: disable=W0613
419 """SQL fragment for updating our underlying table, using bind variables."""
420 return ',sid_num=:sid_num'
421
422 @staticmethod
423 def bind_sys_insert(table_name, sid):
424 """Convert ourselves into a dict to be inserted into an insert statement variable
425 list.
426 usage:
427 db_conn.query('INSERT INTO PRODUCTS (activity, filename, result{sidfields}, sensing_start) '
428 'VALUES (:activity, :filename, :result{sidbinds}, :sensing_start)'.format(
429 sidfields=sid_fields, sidbinds=sid_binds),
430 activity=activity,
431 filename=filename,
432 result=result,
433 sensing_start=sensing_start,
434 **SID.bind_sys_insert('PRODUCTS', sid))
435 .
436 """
437 if table_name == 'REPORTS' and sid is None:
438 return {'sid_num': SID_S3.SIDLESS.sid_num}
439
440 # elif table_name == 'PRODUCTS':
441 # return {'sid': sid.scid}
442
443 elif sid is None:
444 return {'sid_num': None}
445
446 else:
447 return {'sid_num': sid.sid_num}
448
449 #
450 ##
451 ### XML support
452 ##
453 #
454
455 def to_xml(self, elem):
456 """Annotate parent `elem` with ourselves, creating child nodes."""
457 # maybe rename to insert_in_xml_node ?
458 # or insert_to_xml?
459 if self.scid is not None:
460 scid_elem = elem.find(ELEM_SCID)
461 if scid_elem is None:
462 SubElement(elem, ELEM_SCID).text = self.scid
463
464 else:
465 scid_elem.text = self.scid
466
467 # if self.sid_num is not None:
468 # SubElement(elem, ELEM_SID_NUM).text = str(self.sid_num)
469
470 @staticmethod
471 def from_xml(elem, wildcard=False):
472 """Create a SID from childs of `elem`, or None if not found.."""
473 scid = parsechildstr(elem, ELEM_SCID, None)
474 if scid is not None:
475 return SID_S3(scid=scid, wildcard=wildcard)
476
477 sid_num = parsechildstr(elem, ELEM_SID_NUM, None)
478 if sid_num is not None:
479 return SID_S3(sid_num=sid_num, wildcard=wildcard)
480
481 return None
482
483 #
484 ##
485 ### Wildcard support
486 ##
487 #
488
489 @staticmethod
490 def from_string(instr, wildcard=False):
491 """Create a SID from a single string.
492 Will accept an actual satellite name, or 'SYS'.
493 Will also accept wildcard names only if `wildcard` is set.
494 This is used when:
495 - interpreting command line arguments (although we should probably allow --oper etc too)
496 - ?
497 .
498 """
499 if instr.upper() == SYS_NAME:
500 return SID_S3.SIDLESS
501
502 else:
503 return SID_S3(scid=instr, wildcard=wildcard)
504
505 def expand(self):
506 """If we were created as a wildcard M* return a series of separate SIDS.
507 Used for job expansion if the schedule file says MSG*.
508 Should also be used for activity matching but isn't.
509 """
510 result = []
511 for s in all_satellites():
512 if not s.operational:
513 continue
514
515 if fnmatch(s.scid, self.scid):
516 result.append(SID_S3(s.scid))
517
518 return result
519
520 def match(self, other):
521 """Test is we match `other`.
522 Handles wildcards."""
523 return fnmatch(other.scid, self.scid)
524
525 #
526 ##
527 ### Satellite database support
528 ##
529 #
530
531 # List of valid satellites. Cached XML load.
532 satellites_elem = None
533
534 @staticmethod
535 def all(operational=True):
536 """Yield all satellites matching parameters.
537 CFID and OGSID will not be set.
538 """
539
540 if SID_S3.satellites_elem is None:
541 SID_S3.satellites_elem = load_xml(settings.SATELLITES)
542
543 for sat_elem in SID_S3.satellites_elem.findall(ELEM_SATELLITE):
544 if operational is None or operational == parsechildbool(sat_elem, ELEM_OPERATIONAL):
545 yield SID_S3(scid=parsechildstr(sat_elem, ELEM_SCID))
546
547 @staticmethod
548 def django_all():
549 """Return SID information sent to the web client."""
550 return [sid.as_dict() for sid in SID_S3.all(operational=None)]
551
552 @property
553 def satellite(self):
554 """Retrieve the satellite associated with this SID or none."""
555 return Satellite(self.scid)
556
557 def as_dict(self):
558 """Convert ourselves into a dictionary of data for the web interface client."""
559 operational = self.satellite is not None and self.satellite.operational
560 return {'menu_value': {'scid': self.scid},
561 'initial_year': self.satellite.launch_date.year
562 if self.satellite.launch_date is not None else None,
563 'menu_name': '{name} ({scid})'.format(
564 scid=self.scid,
565 name=self.satellite.name),
566 'time_offset': self.satellite.time_offset.total_seconds()
567 if self.satellite is not None and self.satellite.time_offset is not None
568 else 0,
569 'title': str(self),
570 'operational': operational
571 }
572
573 #
574 ##
575 ### Reporting
576 ##
577 #
578
579 @staticmethod
580 def get_reportname_part(sid):
581 """Return the SID fragment of a report name e.g. the M02 in
582 SVM_DHSA_REPORT_M02_20130901.zip.
583 Also used as subdirectory name in the reports archive.
584 Also used as part of the statefiles name.
585 """
586 if sid is None or sid.scid is None:
587 return SID_S3.SIDLESS.scid
588
589 else:
590 return sid.scid
591
592 @staticmethod
593 def from_reportname_part(fragment):
594 """Decode a report name fragment."""
595 if fragment == SID_S3.SIDLESS.scid:
596 return SID_S3.SIDLESS
597
598 else:
599 return SID_S3(fragment)
600
601 @staticmethod
602 def ddl_cal_clause(sids):
603 """Return an SQL clauses suitable for a CASE statement which will recognise
604 any of `sids`."""
605 return '1=1'
606
607 @staticmethod
608 def all_report_sids():
609 """Scan REPORTs table and return a dictionary of report name against a set of
610 SIDs for which we have reports of that activity."""
611 # only used by reportviewer. can be removed
612 from chart.db.connection import db_connect
613 res = defaultdict(list)
614 db_conn = db_connect('REPORTS')
615 for activity, sid_num in db_conn.query(
616 'SELECT DISTINCT activity, sid_num FROM reports ORDER BY sid_num'):
617 res[activity].append(SID_S3(sid_num=sid_num) if sid_num is not None else SID_S3.SIDLESS)
618
619 return res
620
621 @staticmethod
622 def report_sids(activity):
623 """Only used by reportviewer. can be removed."""
624 from chart.db.connection import db_connect
625 db_conn = db_connect('REPORTS')
626 res = []
627 for activity, sid_num in db_conn.query(
628 'SELECT DISTINCT activity, sid_num FROM reports WHERE activity=:reportname',
629 reportname=activity.name):
630 res.append(SID_S3(sid_num=sid_num) if sid_num is not None else SID_S3.SIDLESS)
631
632 return res
633
634 def cal_dirname(self):
635 """Compute a subdirectory to write named calibration files to."""
636 return self.short_name
637
638 def cal_name(self):
639 """Compute a suitable plSQL function name suffix."""
640 return self.scid
641
642 @property
643 def name(self):
644 return self.scid
645
646 @property
647 def short_name(self):
648 return self.name
649
650
651
652# Used to represent a report which is has no source, usually a system
653# job like purge or a system digest report
654# uses wildcard to switch off the validity check
655SID_S3.SIDLESS = SID_S3(scid='SYS', wildcard=True)