1#!/usr/bin/env python3
2
3"""Source-ID for GSARMTG project."""
4
5import collections
6from fnmatch import fnmatch
7
8from chart.common.xml import SubElement
9from chart.common.xml import parsechildstr
10from chart.plots.sampling import Sampling
11
12
13ELEM_GSID = 'gsid'
14
15plot_aux_events_label = None
16
17
18class GroundStation:
19 """Metadata about a receiving station.
20 This should probably read from an XML file to make the list more visible."""
21
22 all = ('TTC1-MDA1', 'TTC2-MDA2')
23
24 def __init__(self, gsid=''):
25 self.gsid = gsid
26 if gsid.upper() == 'TTC1-MDA1':
27 # the ts tables use "sid_num" to identify source
28 self.sid_num = 1
29 # somehow we ended up with "gs_num" in some system tables
30 self.gs_num = 1
31 self.name = 'TTC1-MDA1'
32 self.description = 'GS Fucino, Italy'
33
34 elif gsid.upper() == 'TTC2-MDA2':
35 self.sid_num = 2
36 self.gs_num = 2
37 self.name = 'TTC2-MDA2'
38 self.description = 'GS Cheia, Romania'
39
40 else:
41 # hopefully this is the sidless / sys source
42 self.gsid = ''
43 self.gs_num = None
44 # self.gs_num = 0 # sys tables don't allow NULL for GS_NUM
45
46
47class SID_GSARMTG:
48 """GSAR Source ID object.
49 Data is identified using GSAR and sensing_time.
50 """
51 plot_aux_events_label = None
52 # based on this we include the correct xxx_selector.html file from sids/templates
53 selector = 'gsarmtg'
54 default_sid = {'gsid': 'TTC1-MDA1'} # to set when plot page in browser is first brought up
55 shifts = {} # default time shifts between SIDs.
56
57 # sampling options used by this SID class
58 sampling_options = [Sampling.AUTO,
59 Sampling.ALL_POINTS,
60 Sampling.FIT,
61 Sampling.HALF_HOURLY,
62 Sampling.DAILY,
63 ]
64
65 # subsampling options to be considered when auto-selecting best stats subsampling
66 # should be sorted from longest to shortest
67 def auto_sampling_options(self):
68 """Return the list of stats subsampling options available for this sid.
69 """
70 return [Sampling.HALF_HOURLY, Sampling.DAILY]
71
72 #
73 ##
74 ### Construction and testing
75 ##
76 #
77
78 def __init__(self, gsid=None, gs_num=None, wildcard=False):
79 """If `wildcard` then allow specs which are not exact matches
80 but can be used for wildcards."""
81 # print('SID cons gsid {gsid} gs_num {gs_num} wc {wc}'.format(
82 # gsid=gsid, gs_num=gs_num,wc=wildcard))
83 if gsid is None:
84 if gs_num == 1:
85 gsid = 'TTC1-MDA1'
86
87 elif gs_num == 2:
88 gsid = 'TTC2-MDA2'
89
90 else:
91 gsid = 'SYS'
92
93 self.gsid = gsid.upper()
94 if wildcard or gsid == 'SYS':
95 self.ground_station = None
96
97 else:
98 self.ground_station = GroundStation(gsid)
99
100 def __repr__(self):
101 return 'SID_GSARMTG(gsid={gsid})'.format(gsid=self.gsid)
102
103 def __str__(self):
104 if self.gsid is None:
105 return 'Non GS specific'
106
107 else:
108 return self.gsid
109
110 def __eq__(self, other):
111 if other is None:
112 # needed for 'if sid in sids...'
113 return False
114
115 else:
116 return self.gsid == other.gsid
117
118 def __ne__(self, other):
119 # this is required otherwise the SID non-uniqueness check in eventsfile.py
120 # gives a false failure
121 return self.gsid != other.gsid
122
123 def __hash__(self):
124 return hash(self.gsid)
125
126 @property
127 def sidnum(self):
128 """Numerical SID identifier for TS tables only."""
129 return self.ground_station.sidnum
130
131 @property
132 def orbit(self):
133 """GSARMTG doesn't use orbits."""
134 return None
135
136 #
137 ##
138 ### Web site support
139 ##
140 #
141
142 @staticmethod
143 def from_django_request(request):
144 """Extract SID(s) from `query`, either a Django request GET object
145 or a dictionary or extracted pairs from a plot URL datapoint (dict)."""
146 gsid = request.get('gsid')
147 if gsid is None:
148 gsid = request.get('domain')
149 if gsid is None:
150 gsid = request.get('sid')
151 if gsid is None:
152 return None
153
154 return SID_GSARMTG(gsid=gsid)
155 # return SID(request.GET['gsid'])
156
157 def as_url_query(self, base=None):
158 """Return a dictionary representation of ourselves suitable for passing into
159 the urllib.urlencode() function to create/extend a URL query fragment."""
160 if base is None:
161 return {'gsid': self.gsid}
162
163 else:
164 base['gsid'] = self.gsid
165 return base
166
167 #
168 ##
169 ### DDL creation support
170 ##
171 #
172
173 @staticmethod
174 def ddl_id_fields():
175 """SID fields implicitly added to the start of all TS tables.
176 Rename to 'ts_fields' and make it a function and return FieldInfo objects."""
177 from chart.db.model.table import FieldInfo
178 return [
179 #FieldInfo(name='GSID',
180 # datatype=str,
181 # length=4),
182 FieldInfo(name='SID_NUM',
183 datatype=int,
184 length=1),
185 ]
186
187 @staticmethod
188 def ddl_ap_indexes():
189 """TS AP index fields."""
190 return [{'suffix': '_C', 'fields': ('SID_NUM', 'KEY_NUM', 'SENSING_TIME'), 'compress': 2}]
191
192 # Indexes for stats tables
193 # change to function
194 @staticmethod
195 def ddl_stats_indexes():
196 """TS stats index fields."""
197 return [
198 {'suffix': '_P', 'fields': ('SID_NUM', 'REGION', 'KEY_NUM', 'SENSING_TIME'), 'compress': 2}
199 ]
200
201 @staticmethod
202 def ddl_sys_fields(table_name=None): # (unused arg) pylint:disable=W0613
203 """Events, jobs, reports, subscriptions tables using the {{sid}} expansion."""
204 # don't import at module level because it probably imports back to us
205 from chart.db.model.table import FieldInfo
206 # if table_name == 'SUBSCRIPTIONS':
207 # return [
208 # FieldInfo(name='SCID', datatype=str, length=4, description='Spacecraft ID'),
209 # FieldInfo(name='DUMMYTEST', datatype=str, length=4, description='Spacecraft ID'),
210 # ]
211
212 # else:
213 return [
214 FieldInfo(name='GS_NUM',
215 description='GSID SID identifier',
216 datatype=int,
217 allow_null=True,
218 length=1),
219 # FieldInfo(name='GSID',
220 # datatype=str,
221 # length=4,
222 # allow_null=True,
223 # description='Spacecraft ID'),
224 ]
225
226 @staticmethod
227 def ddl_sys_indexes(table_name=None):
228 """Return list if sys tables index fields."""
229 pass
230
231 #
232 ##
233 ### Generic binary ingester support
234 ##
235 #
236
237 # @staticmethod
238 # def rdr_kwargs(row):
239 # """Extract selection args from a row."""
240 # rename to ts_sql_bindvars
241 # return {'gsid': row[0]}
242
243 # rdr_insert = 'sid.gsid'
244
245 # @staticmethod
246 # def from_sf(sfreader):
247 # """Extract a SID from an SF reader object."""
248 # move to sf; not needed here
249 # sf files should be removed from msg also
250 # return SID_GSAR(gsid=sfreader.gsid)
251
252 # When creating a timeseries ingest cursor, prepend it with these fields
253 # insert_fields = ['SCID']
254
255 # def bind_insert(self):
256 # """When calling an insert cursor, pass these fields first."""
257 # rename to ts_insert_data
258 # return [self.scid]
259
260 # This is now only used by sf00_ingester and can probably be removed completely.
261 # ddl_id_fields() replaces it
262 # SCID_MAX_CHARS = 3
263
264 #
265 ##
266 ### ts table support
267 ##
268 #
269
270 # Construct an SQL where clause which will filter for `sid`.
271 # Used by ingestion code (in MSG this means we have CFID but not OGSID)
272 # rename to ts_sql_where_bind
273 sql_where_bind = 'GSID=:gsid'
274
275 def sql_where(self, table_name=None, match_on_none=False, prefix=None):
276 """Return an SQL clause that will test for ts rows containing us.
277
278 If `table_name` is set it will be the name (not Tableinfo) of the table to be written
279 to, which may be a system table.
280
281 If `match_on_none` is set then we will match table rows contains null values in the SID
282 column.
283
284 `prefix` can be used to prefix column names for queries that join tables.
285
286 Avoids bind variables."""
287 if match_on_none:
288 return '(SID_NUM is null or SID_NUM={sidnum})'.format(sidnum=self.ground_station.sid_num)
289
290 else:
291 return 'SID_NUM={sidnum}'.format(sidnum=self.ground_station.sid_num)
292
293 #
294 ##
295 ### sys table support
296 ##
297 #
298
299 @staticmethod
300 def sql_sys_where(table_name, sid): # (unused arg) pylint:disable=W0613
301 """Return an SQL clause which will test for us in the JOBS or REPORTS or EVENTS table."""
302 if sid is None:
303 return '1=1'
304
305 elif sid.ground_station is None:
306 return 'GS_NUM IS NULL'
307
308 else:
309 return "GS_NUM='{g}'".format(g=sid.gs_num)
310
311 @property
312 def gs_num(self):
313 """TS tables should use GS_NUM not SID_NUM."""
314 gs = self.ground_station
315 if gs is None:
316 return None
317
318 else:
319 return gs.gs_num
320
321 @staticmethod
322 def sql_sys_where_bind(table_name): # (unused arg) pylint:disable=W0613
323 """SQL fragment for a WHERE clause using bind variables."""
324 return 'GS_NUM=:gs_num'
325
326 def bind_sys_where(self, table_name): # (unused arg) pylint:disable=W0613
327 """Convert ourselves into a bindvars dict for use with the sql emitted by
328 sql_sys_where_bind()."""
329 return {'gs_num': self.gs_num}
330
331 @staticmethod
332 def sql_sys_select(table_name): # (unused arg) pylint:disable=W0613
333 """SQL fragment to add to the (end of the) list of fields to retrieve
334 in order to construct a SID later with from_sys_select()."""
335 return ['GS_NUM']
336
337 @staticmethod
338 def from_sys_select(table_name, args): # (unused arg) pylint:disable=W0613
339 """Construct a SID from the fields requested by sql_sys_select()."""
340 if args[0] is None:
341 return None
342
343 else:
344 return SID_GSARMTG(gs_num=args[0])
345
346 @staticmethod
347 def sql_sys_insert(table_name): # (unused arg) pylint:disable=W0613
348 """Return 2 SQL fragments for an insert statement:
349 - Fields
350 - Bind variables
351
352 E.g.:
353 sid_vars, sid_binds = SID.sql_sys_insert('JOBS')
354 insert_job.ins_cur = db_conn.prepared_cursor(
355 'INSERT INTO JOBS (CATEGORY, ACTIVITY, FILENAME, DIRNAME, {sidfield}ORBIT, '
356 'SENSING_START, SENSING_STOP, TABLENAME, EARLIEST_EXECUTION_TIME '
357 'VALUES '
358 '(:category, :activity, :filename, :dirname, {sidbind}:orbit, :sensing_start, '
359 ':sensing_stop, :tablename, earliest_execution_time'.format(
360 sidfield=sid_fields, sidbind=sid_binds)).
361 """
362 # if table_name == 'PRODUCTS':
363 return (',GS_NUM', ',:gs_num')
364
365 # else:
366 # return (',GSID', ',:gsid')
367
368 @staticmethod
369 def sql_sys_update(table_name): # (unused arg) pylint:disable=W0613
370 """SQL fragment, using bind variables, to update the SID part of a system table."""
371 return ',gs_num=:gs_num'
372
373 @staticmethod
374 def bind_sys_insert(table_name, sid): # (unused arg) pylint:disable=W0613
375 """Convert ourselves into a dict to be inserted into an insert statement variable
376 list.
377 usage:
378 db_conn.query('INSERT INTO PRODUCTS (activity, filename, result{sidfields}, sensing_start) '
379 'VALUES (:activity, :filename, :result{sidbinds}, :sensing_start)'.format(
380 sidfields=sid_fields, sidbinds=sid_binds),
381 activity=activity,
382 filename=filename,
383 result=result,
384 sensing_start=sensing_start,
385 **SID.bind_sys_insert('PRODUCTS', sid)).
386 """
387 # if table_name == 'REPORTS' and sid is None:
388 # return {'sidnum': SID_GSARMTG.SIDLESS.sid_num}
389
390 # elif table_name == 'PRODUCTS':
391 # return {'sid': sid.scid}
392
393 if sid is None:
394 return {'gs_num': None}
395
396 else:
397 return {'gs_num': sid.gs_num}
398
399 #
400 ##
401 ### XML support
402 ##
403 #
404
405 def to_xml(self, elem):
406 """Annotate parent `elem` with ourselves, creating child nodes."""
407 # maybe rename to insert_in_xml_node ?
408 # or insert_to_xml?
409 if self.gsid is not None:
410 SubElement(elem, ELEM_GSID).text = self.gsid
411
412 @staticmethod
413 def from_xml(elem, wildcard=False):
414 """Create a SID from childs of `elem`, or None if not found.."""
415 gsid = parsechildstr(elem, ELEM_GSID, None)
416 if gsid is None:
417 return None
418
419 return SID_GSARMTG.from_string(gsid, wildcard)
420
421 #
422 ##
423 ### Wildcard support
424 ##
425 #
426
427 @staticmethod
428 def from_string(instr, wildcard=False):
429 """Create a SID from a single string.
430 Will accept an actual satellite name, or 'SYS'.
431 Will also accept wildcard names only if `wildcard` is set.
432 This is used when:
433 - interpreting command line arguments (although we should probably allow --oper etc too)
434 - ?.
435 """
436 if instr == SID_GSARMTG.SIDLESS.gsid or instr.lower() == SID_GSARMTG.SIDLESS.gsid.lower():
437 return SID_GSARMTG.SIDLESS
438
439 else:
440 return SID_GSARMTG(gsid=instr, wildcard=wildcard)
441
442 def expand(self):
443 """If we were created as a wildcard return a series of separate SIDS.
444 Used for job expansion if the schedule file says MSG*.
445 Should also be used for activity matching but isn't.
446 """
447 result = []
448 for s in GroundStation.all:
449 # if not s.operational:
450 # continue
451
452 if fnmatch(s.upper(), self.gsid.upper()):
453 result.append(SID_GSARMTG(s))
454
455 return result
456
457 def match(self, other):
458 """Test is we match `other`.
459 Handles wildcards."""
460 return fnmatch(other.gsid, self.gsid)
461
462 #
463 ##
464 ### Satellite database support
465 ##
466 #
467
468 # List of valid satellites. Cached XML load.
469 satellites_elem = None
470
471 @staticmethod
472 def all(operational=True): # (unused arg) pylint:disable=W0613
473 """Yield all satellites matching parameters.
474 CFID and OGSID will not be set.
475 """
476 return [
477 SID_GSARMTG('TTC1-MDA1'),
478 SID_GSARMTG('TTC2-MDA2'),
479 ]
480
481 # if SID.satellites_elem is None:
482 # SID.satellites_elem = load_xml(settings.SATELLITES)
483
484 # for sat_elem in SID.satellites_elem.findall(ELEM_SATELLITE):
485 # if operational is None or operational == parsechildbool(sat_elem, ELEM_OPERATIONAL):
486 # yield SID_GSAR(parsechildstr(sat_elem, ELEM_GSID))
487 # raise NotImplementedError()
488
489 @staticmethod
490 def django_all():
491 """Return SID information sent to the web client."""
492 #test=SID_GSARMTG.all()
493 return [sid.as_dict() for sid in SID_GSARMTG.all()]
494 ## TBD this should come from xml.
495 #return [
496 #{'value': {'gsid': 'CDA1'}, 'menu_name': 'CDA 1', 'time_offset': 0, 'title': str(self)},
497 #{'value': {'gsid': 'CDA2'}, 'menu_name': 'CDA 2', 'time_offset': 0, 'title': 'CDA1'}
498 #]
499
500 @property
501 def satellite(self):
502 """We don't use satellites in this project."""
503 return None
504
505 # @property
506 # @memoized2
507 # def ground_station(self):
508 # return GroundStation(self.gsid)
509
510 # @property
511 # def entity(self):
512 # return self.ground_station
513
514 #
515 ##
516 ### Statistics / multiregion
517 ##
518 #
519
520 # not used yet. don't use, try common.regiontype instead
521 # regions = {
522 # 'ORB': {'name': 'Orbital',
523 # 'duration': timedelta(minutes=102),
524 # 'description': 'Stats per orbit'},
525 # 'DAY': {'name': 'Daily',
526 # 'duration': timedelta(hours=24, minutes=30),
527 # 'description': 'Stats for all orbits starting in a day'},
528 # }
529
530 #
531 ##
532 ### Reporting
533 ##
534 #
535
536 @staticmethod
537 def get_reportname_part(sid):
538 """Return the SID fragment of a report name e.g. the M02 in
539 SVM_DHSA_REPORT_M02_20130901.zip.
540 Also used as subdirectory name.
541 """
542 if sid is None or sid.gsid is None:
543 return SID_GSARMTG.SIDLESS.gsid
544
545 else:
546 return sid.gsid
547
548 @staticmethod
549 def from_reportname_part(fragment):
550 """Decode a report name fragment."""
551 if fragment == SID_GSARMTG.SIDLESS.gsid:
552 return SID_GSARMTG.SIDLESS
553
554 else:
555 return SID_GSARMTG(fragment)
556
557 def as_dict(self):
558 """Convert ourselves into a dictionary of data for the web interface client."""
559 return {'menu_value': {'gsid': self.gsid},
560 'initial_year': None,
561 'menu_name': '{gsid}'.format(gsid=self.gsid),
562 'time_offset': 0,
563 'title': str(self),
564 'operational': True
565 }
566
567 @staticmethod
568 def all_report_sids():
569 """Scan REPORTs table and return a dictionary of report name against a set of
570 SIDs for which we have reports of that activity."""
571 from chart.db.connection import db_connect
572 res = collections.defaultdict(set)
573 for activity, gs_num in db_connect('REPORTS').query(
574 'SELECT DISTINCT activity, gs_num FROM reports'):
575 res[activity].add(SID_GSARMTG(gs_num=gs_num) if gs_num is not None else SID_GSARMTG.SIDLESS)
576
577 return res
578
579 @staticmethod
580 def report_sids(activity):
581 """Only used by reportviewer. can be removed."""
582 from chart.db.connection import db_connect
583 db_conn = db_connect('REPORTS')
584 res = []
585 for activity, gs_num in db_conn.query(
586 'SELECT DISTINCT activity, gs_num FROM reports WHERE activity=:reportname',
587 reportname=activity.name):
588 res.append(SID_GSARMTG(gs_num=gs_num) if gs_num is not None else SID_GSARMTG.SIDLESS)
589
590 return res
591
592 @staticmethod
593 def report_sids_per_report():
594 """Scan REPORTs table and return a dictionary of report name against a set of
595 SIDs for which we have reports of that activity."""
596 from chart.db.connection import db_connect
597 res = collections.defaultdict(set)
598 for activity, gs_num in db_connect('REPORTS').query(
599 'SELECT DISTINCT activity, gs_num FROM reports'):
600 res[activity].add(SID_GSARMTG(gs_num=gs_num) if gs_num is not None else SID_GSARMTG.SIDLESS)
601
602 return res
603
604 @property
605 def name(self):
606 return self.gsid
607
608 @property
609 def short_name(self):
610 return self.gsid
611
612
613# Used to represent a report which is has no source, usually a system
614# job like purge or a system digest report
615# uses wildcard to switch off the validity check
616SID_GSARMTG.SIDLESS = SID_GSARMTG(gsid='SYS', wildcard=True)