1#!/usr/bin/env python3
2
3"""MICMICS-CHART Source IDentifier."""
4
5from enum import Enum
6from fnmatch import fnmatch
7from collections import defaultdict
8
9from chart.common.xml import SubElement
10from chart.common.xml import load_xml
11from chart.common.xml import parsechildbool
12from chart.common.xml import parsechildstr
13from chart.common.xml import parsechilddatetime
14from chart.db.func import ANY
15from chart.plots.sampling import Sampling
16from chart.sids.exceptions import BadSID
17
18# FIXME(mschwarz) on 07/03/22: various sonarqube blockers - several classes and 'constants' undefined.
19# Unknown how to resolve. Should this file be deleted?
20class SID_MICMICS:
21 """Mission Source ID object.
22 Data is identified using:
23 SCID : spacecraft ID,
24 INSTID: instrument ID
25 CFID : monitoring chain ID (VAL or OPE)
26 and sensing_time.
27 """
28
29 # based on this we include the correct xxx_selector.html file from sids/templates
30 selector = 'micmics'
31
32 # to set when plot page in browser is first brought up
33 default_sid = {'scid': 'M01', 'rawcount': 1000}
34 shifts = {}
35
36 # sampling options used by this SID class
37 sampling_options = [Sampling.AUTO,
38 Sampling.ALL_POINTS,
39 Sampling.FIT,
40 Sampling.DAILY,
41 #Sampling.HOURLY,
42 # Sampling.HALF_HOURLY,
43 ]
44
45 # subsampling options to be considered when auto-selecting best stats subsampling
46 # should be sorted from longest to shortest
47 def auto_sampling_options(self):
48 """Return the list of stats subsampling options available for this sid.
49 """
50 return []
51
52 def __init__(self,
53 mission=None,
54 instrument=None,
55 channel=None,
56 operational=True,
57 launch_date=None,
58 stop_date=None,
59 description=None,
60 colour=None,
61 visible=True):
62 self.mission = mission
63 self.instrument = instrument
64 self.channel = channel
65 self.operational = operational
66 self.launch_date = launch_date
67 self.colour = colour
68 self.visible = visible
69
70 def __str__(self):
71 res = 'micmics name'
72 return res
73
74 def __eq__(self, other):
75 if other is None:
76 # needed for 'if sid in sids...'
77 return False
78
79 else:
80 return self.scid == other.scid and self.cfid == other.cfid
81
82 def __ne__(self, other):
83 # this is required otherwise the SID non-uniqueness check in eventsfile.py
84 # gives a false failure
85 return self.scid != other.scid or self.cfid != other.cfid #or self.ogsid != other.ogsid
86
87 def __hash__(self):
88 things = ['scid={scid}'.format(scid=self.scid), 'instid={instid}'.format(instid=self.instid)]
89
90 if self.cfid is not None:
91 # things.append('cfid={cfid}'.format(cfid=self.cfid))
92 things.append('cfid={cfid}'.format(cfid=self.cfid.name))
93
94# if self.ogsid is not None:
95# # things.append('ogsid={ogsid}'.format(ogsid=self.ogsid))
96# things.append('ogsid={ogsid}'.format(ogsid=self.ogsid.name))
97
98 return hash('SID_MICMICS({things})'.format(things=', '.join(things)))
99
100 def replace(self, scid=ANY, instid=ANY, cfid=ANY):
101 """Return a new SID with any of our fields replaced."""
102 return SID_MICMICS(scid=self.scid if scid is ANY else scid,
103 instid=self.instid if instid is ANY else instid,
104 cfid=self.cfid if cfid is ANY else cfid)
105
106 @staticmethod
107 def from_django_request(get):
108 """Extract a SID from `query` (URL parameter string).
109 Allowed inputs:
110
111 scid:MSG1
112 scid:MSG1 cfid:PRIM # shouldn't happen
113 scid:MSG1 cfid:OPER
114 scid:MSG1 cfid:VALI.
115 """
116 sid = {}
117
118 if 'scid' in get:
119 sid['scid'] = get['scid']
120
121 if 'instid' in get:
122 sid['instid'] = get['instid']
123
124 if 'cfid' in get:
125 sid['cfid'] = get['cfid']
126
127 if len(sid) <= 1:
128 return None
129
130 # currently the gs parameter is always called CFID in the URL even if its an OGSID
131
132 elif len(sid) == 2:
133 # scid only
134 return SID_MICMICS(scid=sid['scid'], instid=sid['instid'])
135
136 elif len(sid) == 3:
137 if sid['cfid'] == CFID.OPER.name:
138 return SID_MICMICS(scid=sid['scid'], instid=sid['instid'], cfid=CFID.OPER.name)
139
140 elif sid['cfid'] == CFID.VALI.name:
141 return SID_MICMICS(scid=sid['scid'], instid=sid['instid'], cfid=CFID.VALI)
142
143 else:
144 raise BadSID('Invalid source ID')
145
146 def as_url_query(self, base=None):
147 """Return a dictionary representation of ourselves suitable for passing into
148 the urllib.urlencode() function to create/extend a URL query fragment."""
149 # we should insert either ogsid or cfid depending on which was explicitly selected on
150 # construction
151 if base is None:
152 base = {'scid': self.scid, 'instid': self.instid}
153
154 else:
155 base['scid'] = self.scid
156 base['instid'] = self.instid
157
158 if self.cfid is not None:
159 base['cfid'] = self.cfid
160
161 return base
162
163 # Construct an SQL where clause which will filter for `sid`.
164 # Used by ingestion code to test/delete duplicates (this means we have CFID but not OGSID)
165 sql_where_bind = "SCID=:scid AND CFID=:cfid"
166
167 def sql_where(self):
168 """Construct an SQL WHERE clause (without bind variables) that will search for
169 us in TS tables. Note the MSG TS tables have an index on OGSID but not CFID.
170 Not used for JOBS, EVENTS or REPORTS tables.
171 """
172 return "SCID='{scid}'".format(
173 scid=self.scid,
174 cfid=" AND cfid='{cfid}'".format(cfid=self.cfid.name) if self.cfid is not None else '')
175 # ogsid="OGSID IN ('PRIM','BACK')" if self.ogsid is None else "OGSID='{ogsid}'".format(
176 # ogsid=self.ogsid.name),
177 # cfid="" if self.cfid is None else " AND CFID='{cfid}'".format(cfid=self.cfid.name))
178
179 def sql_where(self, table_name=None, match_on_none=False, prefix=None):
180 """Return an SQL clause that will test for ts rows containing us.
181
182 If `table_name` is set it will be the name (not Tableinfo) of the table to be written
183 to, which may be a system table.
184
185 If `match_on_none` is set then we will match table rows contains null values in the SID
186 column.
187
188 `prefix` can be used to prefix column names for queries that join tables.
189
190 Avoids bind variables."""
191 if table_name == 'EVENT_SUBSCRIPTIONS':
192 # In event subscriptions table we just store the SCID and don't allow subscribing
193 # to non-primary events
194 if match_on_none:
195 return "(SCID is null or SCID='{scid}')".format(scid=self.scid)
196
197 else:
198 return "SCID='{scid}'".format(scid=self.scid)
199
200 else:
201 return "SCID='{scid}'".format(
202 scid=self.scid,
203 cfid=" AND cfid='{cfid}'".format(cfid=self.cfid.name) if self.cfid is not None else '')
204 # ogsid="OGSID IN ('PRIM','BACK')" if self.ogsid is None else "OGSID='{ogsid}'".format(
205 # ogsid=self.ogsid.name),
206 # cfid="" if self.cfid is None else " AND CFID='{cfid}'".format(cfid=self.cfid.name))
207
208 # When creating an insert cursor, prepend it with these fields
209 insert_fields = ['SCID', 'CFID']
210
211 def bind_insert(self):
212 """When executing an insert cursor with bind variables as an array,
213 pass these fields first."""
214 return [self.scid, self.cfid.name]
215
216 @staticmethod
217 def sql_sys_where(table_name, sid):
218 """Return an SQL clause which will test for us in the JOBS or REPORTS table."""
219 if table_name == 'REPORTS' or table_name == 'EVENTS' or table_name == 'JOBS':
220 if sid is None:
221 return '1=1'
222
223 elif sid.scid is None:
224 return 'SCID IS NULL'
225
226 else:
227 return "SCID='{scid}'".format(scid=sid.scid)
228
229 else:
230 return '{scid}{cfid}'.format(
231 scid='' if sid.scid is None else "SCID='{scid}'".format(scid=sid.scid),
232 cfid='' if sid.cfid is None else " AND CFID='{cfid}'".format(cfid=sid.cfid.name))
233
234 @staticmethod
235 def sql_sys_where_bind(table_name): # (unused arg) pylint:disable=W0613
236 """Where clause using bind variables."""
237 if table_name != 'PRODUCTS':
238 return 'SCID=:scid AND CFID=:cfid'
239
240 else:
241 return 'SCID=:scid AND CFID=:cfid'
242
243
244 def bind_sys_where(self, table_name):
245 """Bind variable values fo us using SQL from sql_sys_where_bind."""
246 if table_name != 'PRODUCTS':
247 return {'scid': self.scid,
248 'cfid': self.cfid.name}
249
250 else:
251 return {'scid': self.scid,
252 'cfid': self.cfid.name}
253
254 @staticmethod
255 def sql_sys_select(table_name):
256 """SQL fragment to add to the (end of the) list of fields to retrieve
257 in order to construct a SID later with from_sys_select()."""
258 if table_name == 'EVENTS' or table_name == 'REPORTS' or table_name == 'JOBS':
259 # no reason for events to be special but I haven't added CFID and OGSID to the tablers
260 # return ',SCID'
261 return ['SCID']
262
263 else:
264 # subscriptions, products
265 # return ',SCID, CFID, OGSID'
266 return ['SCID','CFID']
267
268 @staticmethod
269 def from_sys_select(table_name, args):
270 """Construct a SID from the fields requested by sql_sys_select()."""
271 if table_name == 'EVENTS' or table_name == 'REPORTS' or table_name == 'JOBS':
272 if args[0] is None and args[1] is None:
273 return None
274
275 else:
276 return SID_MICMICS(scid=args[0], instid=args[1])
277
278 else:
279 return SID_MICMICS(scid=args[0], instid=args[1], cfid=args[1])
280
281 @staticmethod
282 def sql_sys_insert(table_name):
283 """SQL fragment to insert a new row.
284 Return value is a tuple of field names, values.
285 """
286 if table_name == 'EVENTS' or table_name == 'REPORTS' or table_name == 'JOBS':
287 return (',SCID', ',:scid')
288
289 elif table_name == 'PRODUCTS':
290 return (',SCID, CFID', ',:scid, :cfid')
291
292 else:
293 return (',SCID, CFID', ',:scid, :cfid')
294
295 @staticmethod
296 def sql_sys_update(table_name):
297 """Modify the SID in a system table."""
298 if table_name == 'EVENTS':
299 return ',scid=:scid'
300
301 else:
302 raise NotImplementedError()
303
304 @staticmethod
305 def bind_sys_insert(table_name, sid):
306 """Bind variables to go with sys_sys_update()."""
307 # convert ourselves into a list to be inserted into an insert statement variable list
308 if table_name == 'EVENTS' or table_name == 'JOBS':
309 if sid is None:
310 return {'scid': None}
311
312 else:
313 return {'scid': sid.scid}
314
315 elif table_name == 'PRODUCTS':
316 return {'scid': sid.scid, 'cfid': sid.cfid.name}
317
318 elif table_name == 'REPORTS':
319 # in the reports table only we use 'SYS' for sidless reports
320 # print('we are reports scid is ' + str(self.scid) + ' type ' + str(type(self.scid)))
321 # print(len(self.scid))
322 if sid is None:
323 # print(' is sidless')
324 return {'scid': SID_MICMICS.SIDLESS.scid}
325
326 else:
327 # print(' is not sidless')
328 return {'scid': sid.scid}
329
330 else:
331 return {'scid': sid.scid,
332 'cfid': None if sid.cfid is None else sid.cfid.name}
333
334 def to_xml(self, elem):
335 """Annotate parent `elem` with ourselves, creating child nodes."""
336 SubElement(elem, ELEM_SCID).text = self.scid
337 if self.cfid is not None:
338 SubElement(elem, ELEM_CFID).text = self.cfid.name
339
340 @staticmethod
341 def from_xml_sat(elem, wildcard=False):
342 """Create a SID from childs of `elem`."""
343 scid = parsechildstr(elem, ELEM_SCID, None)
344 if scid is None or scid == '':
345 return None
346
347 elif scid == SID_MICMICS.SIDLESS.scid:
348 return SID_MICMICS.SIDLESS
349
350 return SID_MICMICS(scid=scid,instid='SEVIRI',
351 cfid=parsechildstr(elem, ELEM_CFID, None),
352 wildcard=wildcard)
353
354 @staticmethod
355 def from_xml_inst(elem, wildcard=False):
356 """Create a SID from childs of `elem`."""
357 instid = parsechildstr(elem, ELEM_INSTID, None)
358 if instid is None or instid == '':
359 return None
360
361 elif instid == SID_MICMICS.SIDLESS.instid:
362 return SID_MICMICS.SIDLESS
363
364 return SID_MICMICS(scid=scid,instid=instid,
365 cfid=parsechildstr(elem, ELEM_CFID, None),
366 wildcard=wildcard)
367
368 @staticmethod
369 def from_string(instr, wildcard=False):
370 """Create a SID from a single string
371 (wildcards in activity and schedule files)."""
372 if instr == SID_MICMICS.SIDLESS.scid or instr.lower() == SID_MICMICS.SIDLESS.scid.lower():
373 return SID_MICMICS.SIDLESS
374
375 else:
376 cfid = None
377 if ':' in instr:
378 bits = instr.split(':')
379 scid = bits[0]
380 for bit in bits[1:]:
381 if bit.upper() in ('OPER', 'VALI'):
382 cfid = bit.upper()
383
384 else:
385 scid = instr
386
387 return SID_MICMICS(scid=scid, instid=instid, cfid=cfid, wildcard=wildcard)
388
389 def expand(self):
390 """If we were created as a wildcard M* return a series of separate SIDS.
391 Used for job expansion if the schedule file says MSG*.
392 # yield instead?.
393 """
394 # raise NotImplementedError()
395 result = []
396 for s in SID_MICMICS.all():
397 # if not s.operational:
398 # continue
399
400 if fnmatch(s.scid, self.scid):
401 result.append(SID_MICMICS(s.scid))
402
403 return result
404
405 def match(self, other):
406 """Test is we match `other`.
407 Handles wildcards."""
408 return fnmatch(other.scid, self.scid)
409
410 @staticmethod
411 def from_cmdline(arg):
412 """Convert a command line parameter to a SID.
413 We assume PRIM data unless otherwise specified.
414 -s MSG2
415 -s MSG2:ogsid=BACK
416 -s MSG2:bare
417 -s MSG2:ogsid=none
418 -s MSG2:cfid=oper
419 -s MSG%:cfid=oper
420 -s MSG*:cfid=oper
421 -s MSG[1,2]:cfid=oper
422
423 ok symbols: # @ : .
424 """
425 return SID_MICMICS(scid=arg, instid='SEVIRI')
426
427 # List of valid satellites. Cached XML load.
428 satellites_elem = None
429
430 # List of valid instruments. Cached XML load.
431 instruments_elem = None
432
433 @staticmethod
434 def all_sat(operational=True):
435 """Yield all satellites matching parameters.
436 CFID and OGSID will not be set.
437 """
438 from chart.project import settings
439 if SID_MICMICS.satellites_elem is None:
440 SID_MICMICS.satellites_elem = load_xml(settings.SATELLITES)
441
442 for sat_elem in SID_MICMICS.satellites_elem.findall(ELEM_SATELLITE):
443 if operational is None or operational == parsechildbool(sat_elem, ELEM_OPERATIONAL):
444 yield SID_MICMICS(parsechildstr(sat_elem, ELEM_SCID))
445
446 @staticmethod
447 def all_inst(operational=True):
448 """Yield all satellites matching parameters.
449 CFID and OGSID will not be set.
450 """
451 from chart.project import settings
452 if SID_MICMICS.instruments_elem is None:
453 SID_MICMICS.instruments_elem = load_xml(settings.INSTRUMENTS)
454
455 for inst_elem in SID_MICMICS.instruments_elem.findall(ELEM_INSTRUMENT):
456 yield SID_MICMICS(parsechildstr(inst_elem, ELEM_INSTID))
457
458 @staticmethod
459 def django_all():
460 """Return SID information sent to the web client."""
461 # return [sid.as_dict() for sid in SID_MICMICS.all(operational=None)]
462 return {
463 'missions': [1,2,3],
464 'instruments': [4,5,6],
465 }
466
467 def as_dict(self):
468 """Convert ourselves into a dictionary of data for the web interface client."""
469 operational = self.satellite is not None and self.satellite.operational
470 return {'menu_value': {'scid': self.scid, 'cfid': self.cfid},
471 'initial_year': None,
472 'menu_name': '{name} ({scid})'.format(scid=self.scid, name=self.satellite.name),
473 'time_offset': 0,
474 'title': str(self),
475 'operational': operational
476 }
477 @staticmethod
478 def all_report_sids():
479 """Scan REPORTs table and return a dictionary of report name against a set of
480 SIDs for which we have reports of that activity."""
481 # only used by reportviewer. can be removed
482 from chart.db.connection import db_connect
483 res = defaultdict(list)
484 db_conn = db_connect('REPORTS')
485 for scid, activity in db_conn.query(
486 'SELECT DISTINCT scid, activity FROM reports ORDER BY scid'):
487 res[activity].append(SID_MICMICS(scid) if scid is not None else SID_MICMICS.SIDLESS)
488
489 return res
490
491 @staticmethod
492 def report_sids(activity):
493 """Only used by reportviewer. can be removed."""
494 from chart.db.connection import db_connect
495 db_conn = db_connect('REPORTS')
496 res = []
497 for scid, activity in db_conn.query(
498 'SELECT DISTINCT scid, activity FROM reports '
499 'WHERE activity=:reportname ORDER BY scid',
500 reportname=activity.name):
501 res.append(SID_MICMICS(scid) if scid is not None else SID_MICMICS.SIDLESS)
502
503 return res
504
505 @staticmethod
506 def get_reportname_part(sid):
507 """Return a string to be the source ID part of a report filename
508 e.g. SVM_DHSA_REPORT_M02_20130901.zip."""
509 if sid is None or sid.scid is None:
510 return SID_MICMICS.SIDLESS.scid
511
512 else:
513 return sid.scid
514
515 @staticmethod
516 def from_reportname_part(fragment):
517 """Decode a report name fragment."""
518 if fragment == SID_MICMICS.SIDLESS.scid:
519 return SID_MICMICS.SIDLESS
520
521 else:
522 return SID_MICMICS(fragment)
523
524 @staticmethod
525 def ddl_id_fields():
526 """SID fields implicitly added to the start of all TS tables."""
527 from chart.db.model.table import FieldInfo
528 return [
529 FieldInfo(name='SID_NUM',
530 description='Source ID number',
531 datatype=int,
532 length=8),
533 ]
534
535 @staticmethod
536 def ts_indexes():
537 """Return list of TS AP table field names which should be indexed."""
538 return []
539
540
541 @staticmethod
542 def ddl_cal_clause(sids):
543 """Return an SQL clause suitable for a CASE statement which will recognise
544 any of `sids`.
545 We don't distinguish ground segments here because calibration is only
546 satellite dependant.
547 """
548
549 if len(sids) == 1:
550 if sids[0].scid == '*':
551 return '1=1'
552
553 else:
554 return 'SCID=\'{scid}\''.format(scid=sids[0].scid)
555
556 else:
557 return 'SCID in ({members})'.format(
558 members=','.join('\'{s}\''.format(s=s.scid) for s in sids))
559
560 @property
561 def acronym(self):
562 return self.scid
563
564 @property
565 def name(self):
566 return self.scid
567
568 @property
569 def short_name(self):
570 return self.name