1#!/usr/bin/env python3
2
3"""Source-ID for EPS project.
4
5Needs extending to include satellites database. We should be able to:
6
7- Collections:
8 - Iterate through all known satellites
9 - Iterate through active (NRT) satellites
10 - includine wildcard slicing
11 - Iterate through visible (in UI) satellites)
12- Metadata:
13 - Nominal orbit duration
14 - Name
15 - Acryonm (MA, MB, MC) for theme.mod_filename and report name generation
16 - SCID
17 - Launch date
18 - (instruments)
19 - has ext hktm(?)
20 - (decomissioning date)
21 - (ground test data)
22 - (has orb stats)
23
24There are several categories of SID:
25
26- Concrete SID for a spacecraft
27- Wildcard SID that matches many spacecraft
28- SIDless SID.
29
30"""
31
32from collections import defaultdict
33from fnmatch import fnmatch
34
35from chart.common.xml import SubElement
36
37from chart.project import settings
38from chart.common.xml import load_xml
39from chart.common.xml import parsechildbool
40from chart.common.xml import parsechildstr
41from chart.common.decorators import memoized
42from chart.common.xml import parsechilddatetime
43from chart.common.xml import parsechildtimedelta
44from chart.plots.sampling import Sampling
45from chart.sids.exceptions import BadSID
46from chart.sids import SIDBase
47
48ELEM_SCID = 'scid'
49ELEM_SATELLITE = 'satellite'
50ELEM_OPERATIONAL = 'operational'
51ELEM_DESCRIPTION = 'description'
52
53SYS_NAME = 'SYS'
54SYS_LABEL = 'System'
55
56
57class _Satellite:
58 """Information about a satellite."""
59
60 def __init__(self, elem):
61 self.scid = parsechildstr(elem, 'scid')
62 self.name = parsechildstr(elem, 'name')
63 self.description = parsechildstr(elem, ELEM_DESCRIPTION, None)
64 self.operational = parsechildbool(elem, 'operational', True)
65 self.visible = parsechildbool(elem, 'visible', True)
66 self.ground_test = parsechildbool(elem, 'ground-test', False)
67 self.has_ext_hktm = parsechildbool(elem, 'has-ext-hktm', False)
68 self.launch_date = parsechilddatetime(elem, 'launch-date', None)
69 # self.orbital = parsechildbool(sat_elem, 'orbital')
70 # 'instruments': par
71 self.orbit_duration = parsechildtimedelta(elem, 'orbit-duration', None)
72 self.time_offset = parsechildtimedelta(elem, 'time-offset', None)
73 self.acronym = parsechildstr(elem, 'acronym', None)
74 # basic widgets look for acronym
75 self.colour = parsechildstr(elem, 'colour', None)
76
77
78@memoized
79def all_satellites():
80 """Return a list of all <satellites>."""
81 result = []
82 for sat_elem in load_xml(settings.SATELLITES).findall('satellite'):
83 result.append(_Satellite(sat_elem))
84
85 return result
86
87
88# @memoized
89def Satellite(scid=None, name=None): # (invalid name) pylint: disable=C0103
90 """Cached constructor."""
91 for s in all_satellites():
92 if scid is not None and scid == s.scid:
93 return s
94
95 if name is not None and name == s.name:
96 return s
97
98 return None
99
100
101class SID_EPS(SIDBase):
102 """EPS Source ID object.
103 Data is identified using SCID and sensing_time."""
104
105 # based on this we include the correct xxx_selector.html file from sids/templates
106 selector = 'eps'
107 default_sid = {'name': 'M01', 'scid': 'M01'} # to set when plot page in browser is first brought up
108
109 plot_aux_events_label = 'ECL'
110
111 # sampling options used by this SID class
112 # This should be used only to populate the plot tool Sampling drop down
113 sampling_options = [Sampling.AUTO,
114 Sampling.ALL_POINTS,
115 Sampling.FIT,
116 Sampling.ORBITAL,
117 Sampling.DAILY_TOTALS,
118 Sampling.DAYTIME,
119 Sampling.NIGHT,
120 ]
121
122 # subsampling options to be considered when auto-selecting best stats subsampling
123 # should be sorted from longest to shortest
124 def auto_sampling_options(self):
125 """Return the list of stats subsampling options available for this sid."""
126 if self.satellite is None:
127 return []
128
129 elif self.satellite.ground_test:
130 # return [Sampling.DAILY_TOTALS] - we might have it eventually (or HOURLY)
131 return []
132
133 else:
134 return [Sampling.ORBITAL]
135
136
137 #
138 ##
139 ### Construction and testing
140 ##
141 #
142
143 def __init__(self, scid=None, wildcard=False):
144 """If `wildcard` then allow specs which are not exact matches
145 but can be used for wildcards."""
146
147 if scid is None or scid == SYS_NAME:
148 # sys, or sidless sid
149 self.scid = None
150 return
151
152 if scid is not None:
153 scid = scid.upper()
154
155 ok = False
156 for s in all_satellites():
157 if s.scid == scid:
158 ok = True
159
160 if not ok and not wildcard:
161 raise BadSID(scid)
162
163 self.scid = scid
164
165 # orbit now caches some information
166 if scid is not None:
167 # dangerous loop on startup; avoided here because settings.py constructs
168 # a SID()
169 from chart.products.fdf.orbit import Orbit
170 self.orbit = Orbit(self)
171
172 def __repr__(self):
173 return 'SID(scid={scid})'.format(scid=self.scid)
174
175 def __str__(self):
176 if self.scid is None:
177 return 'Non satellite specific'
178
179 else:
180 return self.scid
181
182 def __hash__(self):
183 return hash(self.scid)
184
185
186 #
187 ##
188 ### Web site support
189 ##
190 #
191
192 @staticmethod
193 def from_django_request(request):
194 """Extract SID(s) from `query`, either a Django request GET object
195 or a dictionary or extracted pairs from a plot URL datapoint (dict)."""
196 scid = request.get('sid')
197 if scid is None:
198 return None
199
200 return SID_EPS(scid)
201
202 @staticmethod
203 def from_django_params(params, remove=False):
204 """Extract a SID from `query` (URL parameter string)."""
205 if 'scid' in params:
206 res = SID_EPS(params['scid'])
207 if remove:
208 del params['scid']
209
210 return res
211
212 else:
213 return None
214
215 def as_url_query(self, base=None):
216 """Return a dictionary representation of ourselves suitable for passing into
217 the urllib.urlencode() function to create/extend a URL query fragment."""
218 if base is None:
219 if self.scid is None:
220 return {'sid': 'SYS'}
221
222 else:
223 return {'sid': self.scid}
224
225 else:
226 if self.scid is None:
227 base['sid'] = 'SYS'
228
229 else:
230 base['sid'] = self.scid
231
232 return base
233
234 #
235 ##
236 ### DDL creation support
237 ##
238 #
239
240 @staticmethod
241 def ddl_id_fields():
242 """SID fields implicitly added to the start of all TS tables."""
243 from chart.db.model.table import FieldInfo
244 return [
245 FieldInfo(name='SCID',
246 description='Spacecraft ID',
247 datatype=str,
248 length=4),
249 ]
250
251 # change to function
252 @staticmethod
253 def ddl_ap_indexes():
254 """Configure the indexes for timeseries tables."""
255 return [{'suffix': '_C', 'fields': ('SCID', 'SENSING_TIME'), 'compress': 1}]
256
257 @staticmethod
258 def ts_indexes():
259 """Return list of TS AP table field names which should be indexed."""
260 return ['SCID', ]
261
262 # Indexes for stats tables
263 # change to function
264 # @staticmethod
265 # def ddl_stats_indexes():
266 # """Configure indexes for timeseries stats tables."""
267 # return [{'suffix': '_P', 'fields': ('SCID', 'REGION', 'SENSING_TIME'), 'compress': 2},
268 # {'suffix': '_O', 'fields': ('SCID', 'REGION', 'ORBIT'), 'compress': 2}]
269
270 @staticmethod
271 def ddl_sys_fields(table_name=None): # (unused arg) pylint: disable=W0613
272 """Events, jobs, reports, subscriptions tables using the {{sid}} expansion."""
273 # don't import at module level because it probably imports back to us
274 from chart.db.model.table import FieldInfo
275 # if table_name == 'SUBSCRIPTIONS':
276 # return [
277 # FieldInfo(name='SCID', datatype=str, length=4, description='Spacecraft ID'),
278 # FieldInfo(name='DUMMYTEST', datatype=str, length=4, description='Spacecraft ID'),
279 # ]
280
281 # else:
282 return [
283 FieldInfo(name='SCID',
284 datatype=str,
285 length=4,
286 allow_null=True,
287 description='Spacecraft ID'),
288 ]
289
290 # @staticmethod
291 # def ddl_sys_indexes(table_name=None):
292 # pass
293
294 #
295 ##
296 ### Generic binary ingester support
297 ##
298 #
299
300 @staticmethod
301 def rdr_kwargs(row):
302 """Extract selection args from a row."""
303 # rename to ts_sql_bindvars
304 return {'scid': row[0]}
305
306 rdr_insert = 'sid.scid'
307
308 @staticmethod
309 def from_sf(sfreader):
310 """Extract a SID from an SF reader object."""
311 # move to sf; not needed here
312 # sf files should be removed from msg also
313 return SID_EPS(scid=sfreader.scid)
314
315 # When creating a timeseries ingest cursor, prepend it with these fields
316 insert_fields = ['SCID']
317
318 def bind_insert(self):
319 """When calling an insert cursor, pass these fields first."""
320 # rename to ts_insert_data
321 return [self.scid]
322
323 # This is now only used by sf00_ingester and can probably be removed completely.
324 # ddl_id_fields() replaces it
325 SCID_MAX_CHARS = 3
326
327 #
328 ##
329 ### ts table support
330 ##
331 #
332
333 # Construct an SQL where clause which will filter for `sid`.
334 # Used by ingestion code (in MSG this means we have CFID but not OGSID)
335 # rename to ts_sql_where_bind
336 sql_where_bind = 'SCID=:scid'
337
338 def sql_where(self, table_name=None, match_on_none=False, prefix=None):
339 """Return an SQL clause that will test for ts rows containing us.
340
341 If `table_name` is set it will be the name (not Tableinfo) of the table to be written
342 to, which may be a system table.
343
344 If `match_on_none` is set then we will match table rows contains null values in the SID
345 column.
346
347 `prefix` can be used to prefix column names for queries that join tables.
348
349 Avoids bind variables."""
350 if match_on_none:
351 return "(SCID is null or SCID='{scid}')".format(scid=self.scid)
352
353 else:
354 return "SCID='{scid}'".format(scid=self.scid)
355
356 #
357 ##
358 ### sys table support
359 ##
360 #
361
362 @staticmethod
363 def sql_sys_where(table_name, sid): # (unused arg) pylint: disable=W0613
364 """Return an SQL clause which will test for us in the JOBS or REPORTS or EVENTS table.
365 If `sid` is None then don't apply any filtering.
366 """
367 # should be removed and reported with all bind variables
368 if sid is None:
369 return '1=1'
370
371 elif sid.scid is None:
372 # return 'SCID IS NULL'
373 return 'SCID=\'SYS\''
374
375 else:
376 return "SCID='{scid}'".format(scid=sid.scid)
377
378 @staticmethod
379 def sql_sys_where_bind(table_name): # (unused arg) pylint: disable=W0613
380 """SQL fragment to use in WHERE clauses filtering for us, using bind variables."""
381 return 'SCID=:scid'
382
383 def bind_sys_where(self, table_name): # (unused arg) pylint: disable=W0613
384 """Convert ourselves into a bindvars dict for use with the sql emitted by
385 sql_sys_where_bind()."""
386 return {'scid': self.scid}
387
388 @staticmethod
389 def sql_sys_select(table_name): # (unused arg) pylint: disable=W0613
390 """Craft SQL fragment to retrieve the fields of a SID from a system table.
391
392 SID is constructed later from_sys_select()."""
393
394 return ['SCID']
395
396 @staticmethod
397 def from_sys_select(table_name, args): # (unused arg) pylint: disable=W0613
398 """Construct a SID from the fields requested by sql_sys_select()."""
399 if args[0] is None:
400 return None
401
402 else:
403 return SID_EPS(scid=args[0])
404
405 @staticmethod
406 def sql_sys_insert(table_name):
407 """Return 2 SQL fragments for an insert statement:
408 - Fields
409 - Bind variables
410
411 E.g.:
412 sid_vars, sid_binds = SID.sql_sys_insert('JOBS')
413 insert_job.ins_cur = db_conn.prepared_cursor(
414 'INSERT INTO JOBS (CATEGORY, ACTIVITY, FILENAME, DIRNAME, {sidfield}ORBIT, '
415 'SENSING_START, SENSING_STOP, TABLENAME, EARLIEST_EXECUTION_TIME '
416 'VALUES '
417 '(:category, :activity, :filename, :dirname, {sidbind}:orbit, :sensing_start, '
418 ':sensing_stop, :tablename, earliest_execution_time'.format(
419 sidfield=sid_fields, sidbind=sid_binds))
420 .
421 """
422 if table_name == 'PRODUCTS':
423 return (',SID', ',:sid')
424
425 else:
426 return (',SCID', ',:scid')
427
428 @staticmethod
429 def sql_sys_update(table_name): # (unused arg) pylint: disable=W0613
430 """SQL fragment for updating our underlying table, using bind variables."""
431 return ',scid=:scid'
432
433 @staticmethod
434 def bind_sys_insert(table_name, sid):
435 """Convert ourselves into a dict to be inserted into an insert statement variable
436 list.
437 usage:
438 db_conn.query('INSERT INTO PRODUCTS (activity, filename, result{sidfields}, sensing_start) '
439 'VALUES (:activity, :filename, :result{sidbinds}, :sensing_start)'.format(
440 sidfields=sid_fields, sidbinds=sid_binds),
441 activity=activity,
442 filename=filename,
443 result=result,
444 sensing_start=sensing_start,
445 **SID.bind_sys_insert('PRODUCTS', sid))
446 .
447 """
448 if table_name == 'REPORTS' and sid is None:
449 return {'scid': 'SYS'}
450
451 # elif table_name == 'PRODUCTS':
452 # return {'sid': sid.scid}
453
454 elif sid is None:
455 return {'scid': None}
456
457 else:
458 return {'scid': sid.scid}
459
460 #
461 ##
462 ### XML support
463 ##
464 #
465
466 def to_xml(self, elem):
467 """Annotate parent `elem` with ourselves, creating child nodes."""
468 # maybe rename to insert_in_xml_node ?
469 # or insert_to_xml?
470 if self.scid is not None:
471 SubElement(elem, ELEM_SCID).text = self.scid
472
473 @staticmethod
474 def from_xml(elem, wildcard=False):
475 """Create a SID from childs of `elem`, or None if not found.."""
476 scid = parsechildstr(elem, ELEM_SCID, None)
477 if scid is None:
478 return None
479
480 return SID_EPS.from_string(scid, wildcard)
481
482 #
483 ##
484 ### Wildcard support
485 ##
486 #
487
488 @staticmethod
489 def from_string(instr, wildcard=False):
490 """Create a SID from a single string.
491 Will accept an actual satellite name, or 'SYS'.
492 Will also accept wildcard names only if `wildcard` is set.
493 This is used when:
494 - interpreting command line arguments (although we should probably allow --oper etc too)
495 - ?
496 .
497 """
498 if instr.upper() == SYS_NAME:
499 return SID_EPS.SIDLESS
500
501 else:
502 return SID_EPS(scid=instr, wildcard=wildcard)
503
504 def expand(self):
505 """If we were created as a wildcard M* return a series of separate SIDS.
506 Used for job expansion if the schedule file says MSG*.
507 Should also be used for activity matching but isn't.
508 """
509 result = []
510 for s in all_satellites():
511 if not s.operational:
512 continue
513
514 if fnmatch(s.scid, self.scid):
515 result.append(SID_EPS(s.scid))
516
517 return result
518
519 def match(self, other):
520 """Test is we match `other`.
521 Handles wildcards."""
522 return fnmatch(other.scid, self.scid)
523
524 #
525 ##
526 ### Satellite database support
527 ##
528 #
529
530 # List of valid satellites. Cached XML load.
531 satellites_elem = None
532
533 @staticmethod
534 def all(operational=None):
535 """Yield all satellites matching parameters.
536 CFID and OGSID will not be set.
537 """
538
539 if SID_EPS.satellites_elem is None:
540 SID_EPS.satellites_elem = load_xml(settings.SATELLITES)
541
542 for sat_elem in SID_EPS.satellites_elem.findall(ELEM_SATELLITE):
543 if operational is None or operational == parsechildbool(sat_elem, ELEM_OPERATIONAL):
544 yield SID_EPS(parsechildstr(sat_elem, ELEM_SCID))
545
546 @staticmethod
547 def django_all():
548 """Return SID information sent to the web client."""
549 return [sid.as_dict() for sid in SID_EPS.all(operational=None)]
550
551 @property
552 def satellite(self):
553 """Retrieve the satellite associated with this SID or none."""
554 return Satellite(self.scid)
555
556 def as_dict(self):
557 """Convert ourselves into a dictionary of data for the web interface client."""
558 operational = self.satellite is not None and self.satellite.operational
559 return {'menu_value': {'sid': self.scid},
560 'initial_year': self.satellite.launch_date.year
561 if self.satellite.launch_date is not None else None,
562 'menu_name': '{name} ({scid})'.format(
563 scid=self.scid,
564 name=self.satellite.name),
565 'time_offset': self.satellite.time_offset.total_seconds()
566 if self.satellite is not None and self.satellite.time_offset is not None
567 else 0,
568 'title': str(self),
569 'operational': operational
570 }
571
572 #
573 ##
574 ### Reporting
575 ##
576 #
577
578 @staticmethod
579 def get_reportname_part(sid):
580 """Return the SID fragment of a report name e.g. the M02 in
581 SVM_DHSA_REPORT_M02_20130901.zip.
582 Also used as subdirectory name in the reports archive.
583 Also used as part of the statefiles name.
584 """
585 if sid is None or sid.scid is None:
586 return 'SYS'
587
588 else:
589 return sid.scid
590
591 @staticmethod
592 def from_reportname_part(fragment):
593 """Decode a report name fragment."""
594 if fragment == SID_EPS.SIDLESS.scid:
595 return SID_EPS.SIDLESS
596
597 else:
598 return SID_EPS(fragment)
599
600 @staticmethod
601 def ddl_cal_clause(sids):
602 """Return an SQL clause suitable for a CASE statement which will recognise
603 any of `sids`."""
604
605 if len(sids) == 1:
606 if sids[0].scid == '*':
607 return '1=1'
608
609 else:
610 return 'SCID=\'{scid}\''.format(scid=sids[0].scid)
611
612 else:
613 return 'SCID in ({members})'.format(
614 members=','.join('\'{scid}\''.format(scid=s.scid) for s in sids))
615
616 @staticmethod
617 def all_report_sids():
618 """Scan REPORTs table and return a dictionary of report name against a set of
619 SIDs for which we have reports of that activity."""
620 # only used by reportviewer. can be removed
621 from chart.db.connection import db_connect
622 res = defaultdict(list)
623 db_conn = db_connect('REPORTS')
624 for activity, scid in db_conn.query(
625 'SELECT DISTINCT activity, scid FROM reports ORDER BY scid'):
626 res[activity].append(SID_EPS(scid) if scid is not None else SID_EPS.SIDLESS)
627
628 return res
629
630 @staticmethod
631 def report_sids(activity):
632 """Only used by reportviewer. can be removed."""
633 from chart.db.connection import db_connect
634 db_conn = db_connect('REPORTS')
635 res = []
636 for activity, scid in db_conn.query(
637 'SELECT DISTINCT activity, scid FROM reports WHERE activity=:reportname',
638 reportname=activity.name):
639 res.append(SID_EPS(scid) if scid is not None else SID_EPS.SIDLESS)
640
641 return res
642
643 # representing jobs in a table
644 # (events, reports, subscriptions?)
645 # def columns(self, table):
646 # pass
647
648 @property
649 def acronym(self):
650 """Satellite acronym used for report and PDF naming. Does not need to be
651 implemented by non-EPS projects."""
652 return {None: 'Mx',
653 'M01': 'MB',
654 'M02': 'MA',
655 'M03': 'MC'}[self.sid]
656
657 @property
658 def name(self):
659 return self.scid
660
661 @property
662 def short_name(self):
663 return self.name
664
665
666# Used to represent a report which is has no source, usually a system
667# job like purge or a system digest report
668# uses wildcard to switch off the validity check
669SID_EPS.SIDLESS = SID_EPS()