1#!/usr/bin/env python3
2
3"""Source-ID for GSAR project."""
4
5import collections
6from fnmatch import fnmatch
7
8from chart.common.xml import SubElement
9from chart.common.xml import parsechildstr
10from chart.plots.sampling import Sampling
11
12ELEM_GSID = 'gsid'
13
14
15class GroundStation:
16 """Metadata about a receiving station.
17 This should probably read from an XML file to make the list more visible."""
18
19 all = ('CDA1', 'CDA2')
20
21 def __init__(self, gsid=''):
22 self.gsid = gsid
23 if gsid.upper() == 'CDA1':
24 # the ts tables use "sid_num" to identify source
25 self.sid_num = 1
26 # somehow we ended up with "gs_num" in some system tables
27 self.gs_num = 1
28 self.name = 'CDA 1'
29 self.description = 'Primary CDA'
30
31 elif gsid.upper() == 'CDA2':
32 self.sid_num = 2
33 self.gs_num = 2
34 self.name = 'CDA 2'
35 self.description = 'Secondary CDA'
36
37 else:
38 # hopefully this is the sidless / sys source
39 self.gsid = ''
40 self.gs_num = None
41 # self.gs_num = 0 # sys tables don't allow NULL for GS_NUM
42
43
44class SID_GSAR:
45 """GSAR Source ID object.
46 Data is identified using GSAR and sensing_time.
47 """
48
49 # based on this we include the correct xxx_selector.html file from sids/templates
50 selector = 'gsar'
51 default_sid = {'gsid': 'CDA1'} # to set when plot page in browser is first brought up
52 shifts = {} # default time shifts between SIDs.
53
54 # sampling options used by this SID class
55 sampling_options = [Sampling.AUTO,
56 Sampling.ALL_POINTS,
57 Sampling.FIT,
58 Sampling.HALF_HOURLY,
59 Sampling.DAILY,
60 ]
61
62 # subsampling options to be considered when auto-selecting best stats subsampling
63 # should be sorted from longest to shortest
64 def auto_sampling_options(self):
65 """Return the list of stats subsampling options available for this sid.
66 """
67 return [Sampling.DAILY, Sampling.HALF_HOURLY]
68
69 #
70 ##
71 ### Construction and testing
72 ##
73 #
74
75 def __init__(self, gsid=None, gs_num=None, wildcard=False):
76 """If `wildcard` then allow specs which are not exact matches
77 but can be used for wildcards."""
78 if gsid is None and gs_num is None:
79 # Just a dummy call some SID classes use to initialise config files
80 self.gsid = None
81 return
82 # print('SID cons gsid {gsid} gs_num {gs_num} wc {wc}'.format(
83 # gsid=gsid, gs_num=gs_num,wc=wildcard))
84 if gsid is None:
85 if gs_num == 1:
86 gsid = 'CDA1'
87
88 elif gs_num == 2:
89 gsid = 'CDA2'
90
91 else:
92 raise ValueError('Unknown GS_NUM {g}'.format(g=gs_num))
93
94 self.gsid = gsid.upper()
95
96 if wildcard or gsid == 'SYS':
97 self.ground_station = None
98
99 else:
100 self.ground_station = GroundStation(gsid)
101
102 def __repr__(self):
103 return 'SID_GSAR(gsid={gsid})'.format(gsid=self.gsid)
104
105 def __str__(self):
106 if self.gsid is None:
107 return 'Non GS specific'
108
109 else:
110 return self.gsid
111
112 def __eq__(self, other):
113 if other is None:
114 # needed for 'if sid in sids...'
115 return False
116
117 else:
118 return self.gsid == other.gsid
119
120 def __ne__(self, other):
121 # this is required otherwise the SID non-uniqueness check in eventsfile.py
122 # gives a false failure
123 return self.gsid != other.gsid
124
125 def __hash__(self):
126 return hash(self.gsid)
127
128 @property
129 def sidnum(self):
130 """Numerical SID identifier for TS tables only."""
131 return self.ground_station.sid_num
132
133 @property
134 def orbit(self):
135 """GSAR doesn't use orbits."""
136 return None
137
138 #
139 ##
140 ### Web site support
141 ##
142 #
143
144 @staticmethod
145 def from_django_request(request):
146 """Extract SID(s) from `query`, either a Django request GET object
147 or a dictionary or extracted pairs from a plot URL datapoint (dict)."""
148 gsid = request.get('gsid')
149 if gsid is None:
150 return None
151
152 return SID_GSAR(gsid=gsid)
153 # return SID(request.GET['gsid'])
154
155 def as_url_query(self, base=None):
156 """Return a dictionary representation of ourselves suitable for passing into
157 the urllib.urlencode() function to create/extend a URL query fragment."""
158 if base is None:
159 return {'gsid': self.gsid}
160
161 else:
162 base['gsid'] = self.gsid
163 return base
164
165 #
166 ##
167 ### DDL creation support
168 ##
169 #
170
171 @staticmethod
172 def ddl_id_fields():
173 """SID fields implicitly added to the start of all TS tables.
174 Rename to 'ts_fields' and make it a function and return FieldInfo objects."""
175 from chart.db.model.table import FieldInfo
176 return [
177 FieldInfo(name='SID_NUM',
178 datatype=int,
179 length=15),
180 ]
181
182 @staticmethod
183 def ddl_ap_indexes():
184 """TS AP index fields."""
185 return [{'suffix': '_C', 'fields': ('SID_NUM', 'KEY_NUM', 'SENSING_TIME'), 'compress': 2}]
186
187 # Indexes for stats tables
188 # change to function
189 @staticmethod
190 def ddl_stats_indexes():
191 """TS stats index fields."""
192 return [
193 {'suffix': '_P', 'fields': ('SID_NUM', 'REGION', 'KEY_NUM', 'SENSING_TIME'), 'compress': 2},
194 ]
195
196 @staticmethod
197 def ddl_sys_fields(table_name=None): # (unused arg) pylint:disable=W0613
198 """Events, jobs, reports, subscriptions tables using the {{sid}} expansion."""
199 # don't import at module level because it probably imports back to us
200 from chart.db.model.table import FieldInfo
201 # if table_name == 'SUBSCRIPTIONS':
202 # return [
203 # FieldInfo(name='SCID', datatype=str, length=4, description='Spacecraft ID'),
204 # FieldInfo(name='DUMMYTEST', datatype=str, length=4, description='Spacecraft ID'),
205 # ]
206
207 # else:
208 return [
209 FieldInfo(name='GS_NUM',
210 description='GSID SID identifier',
211 datatype=int,
212 allow_null=True,
213 length=1),
214 # FieldInfo(name='GSID',
215 # datatype=str,
216 # length=4,
217 # allow_null=True,
218 # description='Spacecraft ID'),
219 ]
220
221 @staticmethod
222 def ddl_sys_indexes(table_name=None):
223 """Return list if sys tables index fields."""
224 pass
225
226 #
227 ##
228 ### Generic binary ingester support
229 ##
230 #
231
232 # @staticmethod
233 # def rdr_kwargs(row):
234 # """Extract selection args from a row."""
235 # rename to ts_sql_bindvars
236 # return {'gsid': row[0]}
237
238 # rdr_insert = 'sid.gsid'
239
240 # @staticmethod
241 # def from_sf(sfreader):
242 # """Extract a SID from an SF reader object."""
243 # move to sf; not needed here
244 # sf files should be removed from msg also
245 # return SID_GSAR(gsid=sfreader.gsid)
246
247 # When creating a timeseries ingest cursor, prepend it with these fields
248 # insert_fields = ['SCID']
249
250 # def bind_insert(self):
251 # """When calling an insert cursor, pass these fields first."""
252 # rename to ts_insert_data
253 # return [self.scid]
254
255 # This is now only used by sf00_ingester and can probably be removed completely.
256 # ddl_id_fields() replaces it
257 # SCID_MAX_CHARS = 3
258
259 #
260 ##
261 ### ts table support
262 ##
263 #
264
265 # Construct an SQL where clause which will filter for `sid`.
266 # Used by ingestion code (in MSG this means we have CFID but not OGSID)
267 # rename to ts_sql_where_bind
268 sql_where_bind = 'GSID=:gsid'
269
270 def sql_where(self, table_name=None, match_on_none=False, prefix=None):
271 """Return an SQL clause that will test for ts rows containing us.
272
273 If `table_name` is set it will be the name (not Tableinfo) of the table to be written
274 to, which may be a system table.
275
276 If `match_on_none` is set then we will match table rows contains null values in the SID
277 column.
278
279 `prefix` can be used to prefix column names for queries that join tables.
280
281 Avoids bind variables."""
282 if match_on_none:
283 return '(SID_NUM is null or SID_NUM={sidnum})'.format(sidnum=self.sid_num)
284
285 else:
286 return 'SID_NUM={sidnum}'.format(sidnum=self.sidnum)
287
288 #
289 ##
290 ### sys table support
291 ##
292 #
293
294 @staticmethod
295 def sql_sys_where(table_name, sid): # (unused arg) pylint:disable=W0613
296 """Return an SQL clause which will test for us in the JOBS or REPORTS or EVENTS table."""
297 if sid is None:
298 return '1=1'
299
300 elif sid.ground_station is None:
301 return 'GS_NUM IS NULL'
302
303 else:
304 return "GS_NUM='{g}'".format(g=sid.gs_num)
305
306 @property
307 def gs_num(self):
308 """TS tables should use GS_NUM not SID_NUM."""
309 gs = self.ground_station
310 if gs is None:
311 return None
312
313 else:
314 return gs.gs_num
315
316 @staticmethod
317 def sql_sys_where_bind(table_name): # (unused arg) pylint:disable=W0613
318 """SQL fragment for a WHERE clause using bind variables."""
319 return 'GS_NUM=:gs_num'
320
321 def bind_sys_where(self, table_name): # (unused arg) pylint:disable=W0613
322 """Convert ourselves into a bindvars dict for use with the sql emitted by
323 sql_sys_where_bind()."""
324 return {'gs_num': self.gs_num}
325
326 @staticmethod
327 def sql_sys_select(table_name): # (unused arg) pylint:disable=W0613
328 """SQL fragment to add to the (end of the) list of fields to retrieve
329 in order to construct a SID later with from_sys_select()."""
330 return ['GS_NUM']
331
332 @staticmethod
333 def from_sys_select(table_name, args): # (unused arg) pylint:disable=W0613
334 """Construct a SID from the fields requested by sql_sys_select()."""
335 if args[0] is None:
336 return None
337
338 else:
339 return SID_GSAR(gs_num=args[0])
340
341 @staticmethod
342 def sql_sys_insert(table_name): # (unused arg) pylint:disable=W0613
343 """Return 2 SQL fragments for an insert statement:
344 - Fields
345 - Bind variables
346
347 E.g.:
348 sid_vars, sid_binds = SID.sql_sys_insert('JOBS')
349 insert_job.ins_cur = db_conn.prepared_cursor(
350 'INSERT INTO JOBS (CATEGORY, ACTIVITY, FILENAME, DIRNAME, {sidfield}ORBIT, '
351 'SENSING_START, SENSING_STOP, TABLENAME, EARLIEST_EXECUTION_TIME '
352 'VALUES '
353 '(:category, :activity, :filename, :dirname, {sidbind}:orbit, :sensing_start, '
354 ':sensing_stop, :tablename, earliest_execution_time'.format(
355 sidfield=sid_fields, sidbind=sid_binds)).
356 """
357 # if table_name == 'PRODUCTS':
358 return (',GS_NUM', ',:gs_num')
359
360 # else:
361 # return (',GSID', ',:gsid')
362
363 @staticmethod
364 def sql_sys_update(table_name): # (unused arg) pylint:disable=W0613
365 """SQL fragment, using bind variables, to update the SID part of a system table."""
366 return ',gs_num=:gs_num'
367
368 @staticmethod
369 def bind_sys_insert(table_name, sid): # (unused arg) pylint:disable=W0613
370 """Convert ourselves into a dict to be inserted into an insert statement variable
371 list.
372 usage:
373 db_conn.query('INSERT INTO PRODUCTS (activity, filename, result{sidfields}, sensing_start) '
374 'VALUES (:activity, :filename, :result{sidbinds}, :sensing_start)'.format(
375 sidfields=sid_fields, sidbinds=sid_binds),
376 activity=activity,
377 filename=filename,
378 result=result,
379 sensing_start=sensing_start,
380 **SID.bind_sys_insert('PRODUCTS', sid)).
381 """
382 # if table_name == 'REPORTS' and sid is None:
383 # return {'sidnum': SID_GSAR.SIDLESS.sid_num}
384
385 # elif table_name == 'PRODUCTS':
386 # return {'sid': sid.scid}
387
388 if sid is None:
389 return {'gs_num': None}
390
391 else:
392 return {'gs_num': sid.gs_num}
393
394 #
395 ##
396 ### XML support
397 ##
398 #
399
400 def to_xml(self, elem):
401 """Annotate parent `elem` with ourselves, creating child nodes."""
402 # maybe rename to insert_in_xml_node ?
403 # or insert_to_xml?
404 if self.gsid is not None:
405 SubElement(elem, ELEM_GSID).text = self.gsid
406
407 @staticmethod
408 def from_xml(elem, wildcard=False):
409 """Create a SID from childs of `elem`, or None if not found.."""
410 gsid = parsechildstr(elem, ELEM_GSID, None)
411 if gsid is None:
412 return None
413
414 return SID_GSAR.from_string(gsid, wildcard)
415
416 #
417 ##
418 ### Wildcard support
419 ##
420 #
421
422 @staticmethod
423 def from_string(instr, wildcard=False):
424 """Create a SID from a single string.
425 Will accept an actual satellite name, or 'SYS'.
426 Will also accept wildcard names only if `wildcard` is set.
427 This is used when:
428 - interpreting command line arguments (although we should probably allow --oper etc too)
429 - ?.
430 """
431 if instr == SID_GSAR.SIDLESS.gsid or instr.lower() == SID_GSAR.SIDLESS.gsid.lower():
432 return SID_GSAR.SIDLESS
433
434 else:
435 return SID_GSAR(gsid=instr, wildcard=wildcard)
436
437 def expand(self):
438 """If we were created as a wildcard return a series of separate SIDS.
439 Used for job expansion if the schedule file says MSG*.
440 Should also be used for activity matching but isn't.
441 """
442 result = []
443 for s in GroundStation.all:
444 # if not s.operational:
445 # continue
446
447 if fnmatch(s.upper(), self.gsid.upper()):
448 result.append(SID_GSAR(s))
449
450 return result
451
452 def match(self, other):
453 """Test is we match `other`.
454 Handles wildcards."""
455 return fnmatch(other.gsid, self.gsid)
456
457 #
458 ##
459 ### Satellite database support
460 ##
461 #
462
463 # List of valid satellites. Cached XML load.
464 satellites_elem = None
465
466 @staticmethod
467 def all(operational=True): # (unused arg) pylint:disable=W0613
468 """Yield all satellites matching parameters.
469 CFID and OGSID will not be set.
470 """
471 return [
472 SID_GSAR('CDA1'),
473 SID_GSAR('CDA2'),
474 ]
475
476 # if SID.satellites_elem is None:
477 # SID.satellites_elem = load_xml(settings.SATELLITES)
478
479 # for sat_elem in SID.satellites_elem.findall(ELEM_SATELLITE):
480 # if operational is None or operational == parsechildbool(sat_elem, ELEM_OPERATIONAL):
481 # yield SID_GSAR(parsechildstr(sat_elem, ELEM_GSID))
482 # raise NotImplementedError()
483
484 @staticmethod
485 def django_all():
486 """Return SID information sent to the web client."""
487 return [sid.as_dict() for sid in SID_GSAR.all()]
488 ## TBD this should come from xml.
489 #return [
490 #{'value': {'gsid': 'CDA1'}, 'menu_name': 'CDA 1', 'time_offset': 0, 'title': str(self)},
491 #{'value': {'gsid': 'CDA2'}, 'menu_name': 'CDA 2', 'time_offset': 0, 'title': 'CDA1'}
492 #]
493
494 @staticmethod
495 def ts_indexes():
496 """Return list of TS AP table field names which should be indexed."""
497 return ['SID_NUM', ]
498
499 @property
500 def satellite(self):
501 """We don't use satellites in this project."""
502 return None
503
504 # @property
505 # @memoized2
506 # def ground_station(self):
507 # return GroundStation(self.gsid)
508
509 # @property
510 # def entity(self):
511 # return self.ground_station
512
513 #
514 ##
515 ### Statistics / multiregion
516 ##
517 #
518
519 # not used yet. don't use, try common.regiontype instead
520 # regions = {
521 # 'ORB': {'name': 'Orbital',
522 # 'duration': timedelta(minutes=102),
523 # 'description': 'Stats per orbit'},
524 # 'DAY': {'name': 'Daily',
525 # 'duration': timedelta(hours=24, minutes=30),
526 # 'description': 'Stats for all orbits starting in a day'},
527 # }
528
529 #
530 ##
531 ### Reporting
532 ##
533 #
534
535 @staticmethod
536 def get_reportname_part(sid):
537 """Return the SID fragment of a report name e.g. the M02 in
538 SVM_DHSA_REPORT_M02_20130901.zip.
539 Also used as subdirectory name.
540 """
541 if sid is None or sid.gsid is None:
542 return SID_GSAR.SIDLESS.gsid
543
544 else:
545 return sid.gsid
546
547 @staticmethod
548 def from_reportname_part(fragment):
549 """Decode a report name fragment."""
550 if fragment == SID_GSAR.SIDLESS.gsid:
551 return SID_GSAR.SIDLESS
552
553 else:
554 return SID_GSAR(fragment)
555
556 def as_dict(self):
557 """Convert ourselves into a dictionary of data for the web interface client."""
558 return {'menu_value': {'gsid': self.gsid},
559 'initial_year': None,
560 'menu_name': '{gsid}'.format(gsid=self.gsid),
561 'time_offset': 0,
562 'title': str(self),
563 'operational': True
564 }
565
566 @staticmethod
567 def all_report_sids():
568 """Scan REPORTs table and return a dictionary of report name against a set of
569 SIDs for which we have reports of that activity."""
570 from chart.db.connection import db_connect
571 res = collections.defaultdict(set)
572 for activity, gs_num in db_connect('REPORTS').query(
573 'SELECT DISTINCT activity, gs_num FROM reports'):
574 res[activity].add(SID_GSAR(gs_num=gs_num) if gs_num is not None else SID_GSAR.SIDLESS)
575
576 return res
577
578 @staticmethod
579 def report_sids(activity):
580 """Only used by reportviewer. can be removed."""
581 from chart.db.connection import db_connect
582 db_conn = db_connect('REPORTS')
583 res = []
584 for activity, gs_num in db_conn.query(
585 'SELECT DISTINCT activity, gs_num FROM reports WHERE activity=:reportname',
586 reportname=activity.name):
587 res.append(SID_GSAR(gs_num=gs_num) if gs_num is not None else SID_GSAR.SIDLESS)
588
589 return res
590
591 @staticmethod
592 def report_sids_per_report():
593 """Scan REPORTs table and return a dictionary of report name against a set of
594 SIDs for which we have reports of that activity."""
595 from chart.db.connection import db_connect
596 res = collections.defaultdict(set)
597 for activity, gs_num in db_connect('REPORTS').query(
598 'SELECT DISTINCT activity, gs_num FROM reports'):
599 res[activity].add(SID_GSAR(gs_num=gs_num) if gs_num is not None else SID_GSAR.SIDLESS)
600
601 return res
602
603 @property
604 def name(self):
605 return self.gsid
606
607 @property
608 def short_name(self):
609 return self.gsid
610
611
612# Used to represent a report which is has no source, usually a system
613# job like purge or a system digest report
614# uses wildcard to switch off the validity check
615SID_GSAR.SIDLESS = SID_GSAR(gsid='SYS', wildcard=True)