1#!/usr/bin/env python3
2
3"""Web interface to time-series tables definition with parameters."""
4
5from collections import defaultdict
6from collections import namedtuple
7import itertools
8from io import StringIO
9from operator import attrgetter
10
11from django.http import HttpResponseServerError
12import numpy as np
13# always import matplotlib_agg before matplotlib
14from chart.common import matplotlib_agg # (unused import) pylint: disable=W0611
15
16from matplotlib import figure
17from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
18from django.shortcuts import render
19from django.http import HttpResponse
20from django.urls import reverse
21from django.views.decorators.cache import cache_page
22
23from chart.common.prettyprint import float_to_str
24from chart.db.model.exceptions import NoDataRange
25from chart.db.model.table import TableInfo
26from chart.common.xml import parsechildstr
27from chart.common.xml import parsechildstrs
28from chart.db.settings import SF_LINEAR_CAL
29from chart.db.settings import SF_LINEAR_RAW
30from chart.db.settings import SF_POLY
31from chart.common.prettyprint import Table
32from chart.db.model.table import TableType
33from chart.db.model.table import TableSourceType
34from chart.products.pus.packetdef import PacketDef
35from chart.db.model.exceptions import NoSuchTable
36from chart.common.traits import is_listlike
37from chart.common.traits import name_of_thing
38from chart.project import SID
39from chart.project import settings
40from chart.products.pus.packetdef import PacketDef
41from chart.products.pus.packetdef import PacketDomain
42
43@cache_page(43200) # 12h
44def index(request):
45 """Show list of available tables."""
46 tables = []
47 sid_tables = []
48 for info in TableInfo.all_ts():
49 source_elem = info.elem.find('source-sf00')
50 if source_elem is None:
51 source = ''
52
53 else:
54 # source = 'SF00 Assy ' + parsechildstr(source_elem, 'assembly-id') +\
55 # ' (' + parsechildstr(source_elem, 'name') + ')'
56 source = parsechildstr(source_elem, 'name')
57
58 table_source = info.source
59 if table_source['type'] is TableSourceType.SCOS:
60 source = ' '.join(p.name for p in table_source['packets'])
61 # add sid infromation for sid specific table_info objects.
62
63 if (settings.SID_SPECIFIC_DIR in info.filename.parents):
64 sid_tables.append({
65 'sid' : info.sid,
66 'name': info.name,
67 'type': info.table_type,
68 'source': source,
69 'description': info.description,
70 'period': info.period,
71 'field_count': len(info.fields),
72 })
73 else:
74 tables.append({
75 'name': info.name,
76 'type': info.table_type,
77 'source': source,
78 'description': info.description,
79 'period': info.period,
80 'field_count': len(info.fields),
81 })
82
83 sys_tables = []
84 for table_info in TableInfo.all_sys():
85 sys_tables.append({'name': table_info.name,
86 'description': table_info.description})
87
88 return render(request,
89 'db/index.html',
90 dict(tables=tables,
91 sys_tables=sys_tables,
92 sid_tables=sid_tables))
93
94
95def view_table(request, table_name):
96 """Show web page describing a single non-system table."""
97
98 try:
99 sid_name = request.GET.get('sid')
100 if sid_name:
101 table_info = TableInfo(table_name,sid=SID(sid_name))
102 else:
103 table_info = TableInfo(table_name)
104 except NoSuchTable:
105 return HttpResponseServerError('Unknown table {t}'.format(t=table_name))
106
107 parameters = []
108
109 if table_info.table_type is TableType.RAW:
110 default_length = 8
111
112 else:
113 default_length = ''
114
115 include_validity = False
116
117 for field_info in table_info.fields.values():
118 if len(field_info.cal) == 0 or list(field_info.cal.cals.keys())[0] == None:
119 calibration = 'none'
120
121 else:
122 calibration = field_info.cal[
123 list(field_info.cal.cals.keys())[0]].calibration_type.capitalize()
124
125 if field_info.invalid_value is not None:
126 calibration = '{calibration} (invalid {invalid})'.format(
127 calibration=calibration,
128 invalid=field_info.invalid_value)
129
130 if field_info.mask is None:
131 mask = ''
132
133 else:
134 try:
135 mask = '0x{0:x}'.format(int(field_info.mask))
136
137 except ValueError:
138 mask = field_info.mask
139
140 if field_info.validity_group is not None:
141 include_validity = True
142
143 parameters.append({
144 'name': field_info.name,
145 'description': field_info.description,
146 'datatype': field_info.datatype,
147 'position': field_info.position,
148 'length': default_length if field_info.length is None else field_info.length,
149 'allow_null': field_info.allow_null,
150 'mask': mask,
151 'shift': field_info.shift,
152 'calibration': calibration,
153 'unit': field_info.unit,
154 'lang': '', # !?
155 'param_readers': field_info.param_readers,
156 'validity': field_info.validity_group
157
158 # optionally include a '(FR)' marker on fields with French language descriptions
159 # in the table page
160 # 'lang': '<span style="color:red">(FR)</span>' if
161 # guess_lang(field_info.description) == 'fr' else '',
162 })
163
164 # create URLs pointing to any calibration files
165 # cals = []
166 # for cal in table_info.cals_elems:
167 # filename = xml_filename(cal)
168 # cals.append({'text': os.path.splitext(os.path.basename(filename))[0],
169 # 'url': '{base}{name}'.format(base=settings.BROWSE_CALIBRATION_DIR,
170 # name=os.path.basename(filename))})
171 cals = table_info.browse_cals_elems()
172
173 return render(request,
174 'db/table.html',
175 dict(include_validity=include_validity,
176 table=table_info,
177 cals=cals,
178 parameters=parameters))
179
180
181def error(message):
182 """Return 'message' as a Django response object."""
183 return HttpResponse(message, content_type='text/html')
184
185
186def view_field(request, table_name, field_name):
187 """Extract information on a single field."""
188 sid = request.GET.get('SID')
189 table_info = TableInfo(name=table_name,sid=SID(sid))
190 if table_info.table_type is TableType.RAW:
191 default_length = 8
192
193 else:
194 default_length = ''
195
196 field_info = table_info.fields.get(field_name)
197 if field_info is None:
198 return error('No such field {field} in table {table}'.format(field=field_name,
199 table=table_name))
200
201 field_length = default_length if field_info.length is None else field_info.length
202 field_datatype = 'uint' if field_info.datatype is None else field_info.datatype
203
204 try:
205 raw_data_range = field_info.raw_data_range
206 except NoDataRange:
207 raw_data_range = None
208
209 # list of dictionaries containing information about each per-SID calibration
210 # for this field
211 cals = []
212
213 # in rare cases (MHS PRT) we suppress the visual calibration curve
214 suppress_curve = False
215
216 if table_info.table_type is TableType.RAW:
217 keys = list(field_info.cal.cals.keys())
218 keys = sorted(keys, key=name_of_thing)
219
220 for sids in keys:
221 # A key can be None meaning a generic calibration applied to all,
222 # or a list of the SIDs it applies to.
223 # Unfortunately it can also be a single value which makes this more
224 # complex and is probably not necessary
225 if not is_listlike(sids):
226 sids = (sids,)
227
228 # for sids, cal in field_info.cal.iteritems():
229 cal = field_info.cal[sids]
230
231 if cal.calibration_type == 'linear':
232 definition = ('<p>Linear calibration</p><table><thead>'
233 '<tr><th>Raw</th><th>Calibrated</th></tr></thead><tbody>')
234 for raw, eng in cal.pairs:
235 definition += '<tr><td>{raw}</td><td>{eng}</td></tr>'.format(
236 raw=raw if isinstance(raw, str) else float_to_str(raw, SF_LINEAR_RAW),
237 eng=eng if isinstance(eng, str) else float_to_str(eng, SF_LINEAR_CAL))
238
239 definition += '</tbody></table>'
240
241 elif cal.calibration_type == 'poly':
242 definition = '<p>Polynomial calibration:</p><pre>cal = '
243 terms = []
244 for i, coeff in enumerate(cal.coeffs):
245 if i == 0:
246 terms.append(float_to_str(coeff, SF_POLY))
247
248 elif i == 1:
249 terms.append('{c} * raw'.format(c=float_to_str(coeff, SF_POLY)))
250
251 else:
252 terms.append('{c} * raw^{i}'.format(
253 c=coeff,
254 # c=float_to_str(coeff, SF_POLY),
255 i=i))
256
257 definition += ' + '.join(terms) + '</pre>'
258
259 elif cal.calibration_type == 'plsql':
260 definition = cal.plsql_function + '(raw)'
261
262 elif cal.calibration_type == 'sql':
263 definition = '<p>Raw SQL:</p><pre>{sql}</pre>'.format(sql=cal.sql)
264 # suppress_curve = True
265
266 else:
267 definition = 'Unknown calibration type'
268
269 cal_range = None
270
271 # Build sid_str as a string for the web page describing the sources this calibration
272 # applies to
273 # first_sid is used to find the calibrated limits of this parameter as some extra info
274 # for the user
275 if sids is None:
276 sid_str = 'All'
277 first_sid = None
278
279 else:
280 sid_str = ', '.join((sid.name if sid else 'fallback' for sid in sids))
281 first_sid = sids[0]
282
283
284 limits = [
285 {'sid': sid_str,
286 'limit': field_info.limits(first_sid) if field_info.limits is not None else None}]
287
288 cals.append({'sids': [sid_str],
289 'definition': definition,
290 'url': None, # cal.browse_source_url,
291 'range': cal_range,
292 'limits': limits})
293 # 'limits': field_info.limits(sids[0])})
294
295 # try to guess the language the description is written in. If French,
296 # the template output includes a link to Google Translate to convert the text.
297 # lang = guess_lang(field_info.description)
298 lang = None
299
300 else:
301 # For System tables the description should always be in English
302 lang = None
303
304 def getkey(x):
305 """Sort calibrations by SID."""
306 if len(x['sids']) > 0:
307 return str(x['sids'][0])
308
309 return ''
310
311 cals.sort(key=getkey)
312
313 validity_group_members = []
314 # if field_info.validity_group_prime is not None:
315 # for f in field_info.table.fields.values():
316 # if f.validity_group == field_info.name:
317 # validity_group_members.append(f)
318
319 return render(request,
320 'db/field.html',
321 dict(table=table_info,
322 field=field_info,
323 field_length=field_length,
324 field_datatype=field_datatype,
325 desc_lang=lang,
326 raw_data_range=raw_data_range,
327 cals=cals,
328 suppress_curve=suppress_curve,
329 validity_group_members=validity_group_members))
330
331
332def systable(request, table_name): # (unused arg) pylint: disable=W0613
333 """Show web page describing a system table."""
334
335 table_info = TableInfo(table_name)
336
337 uniques = []
338 for unique_elem in table_info.elem.findall('unique'):
339 uniques.append(', '.join(parsechildstrs(unique_elem, 'name')))
340
341 return render(request,
342 'db/systable.html',
343 {'table': table_info,
344 'uniques': uniques})
345
346
347def search(request):
348 """Database parameters search function."""
349 search_string = request.GET['q'].strip()
350
351 # list of results to be presented to the user.
352 # A list of tuples of (TableInfo, matching field element)
353 results = []
354
355 def search_error(message):
356 """Return an error code to the browse to be displayed in place of search results."""
357 return HttpResponse('<p style=\'color:red\'>' + message + '</p>', content_type='text/html')
358
359 if ':' in search_string:
360 field, value = search_string.split(':')
361 for table_info in itertools.chain(TableInfo.all_ts(), TableInfo.all_sys()):
362 if field == 'field':
363 matches = table_info.elem.xpath('//field/name[contains(.,"' + value.upper() + '")]')
364
365 elif field == 'type':
366 matches = table_info.elem.xpath(
367 '//field/type[contains(translate(.,"abcdefghijklmnopqrstuvwxyz",'
368 '"ABCDEFGHIJKLMNOPQRSTUVWXYZ"),"' + value.upper() + '")]')
369
370 elif field == 'unit':
371 matches = table_info.elem.xpath(
372 '//field/unit[contains(translate(.,"abcdefghijklmnopqrstuvwxyz",'
373 '"ABCDEFGHIJKLMNOPQRSTUVWXYZ"),"' + value.upper() + '")]')
374
375 elif field == 'desc':
376 matches = table_info.elem.xpath(
377 '//field/description[contains(translate(.,"abcdefghijklmnopqrstuvwxyz",'
378 '"ABCDEFGHIJKLMNOPQRSTUVWXYZ"),"' + value.upper() + '")]')
379
380 else:
381 return search_error('Unknown prefix: ' + field)
382
383 for m in matches:
384 results.append((table_info, m.getparent()))
385
386# <dt>desc:
387# <dd>Search inside field descriptions
388# <dt>unit:
389# <dd>Search for fields using a given unit
390# <dt>cal:
391# <dd>Search for a given calibration name
392
393 else:
394 # look for name matches first
395 for table_info in itertools.chain(TableInfo.all_ts(), TableInfo.all_sys()):
396 matches = table_info.elem.xpath(
397 '//field/name[contains(.,"' + search_string.upper() + '")]')
398 for m in matches:
399 results.append((table_info, m.getparent()))
400
401 def result_sorter(a):
402 """Sort by shortest name first, then alphabetically.
403 Matches on name appear before matches on description if no prefix is used."""
404 # a_name = parsechildstr(a[1], 'name')
405 # b_name = parsechildstr(b[1], 'name')
406 # if len(a_name) != len(b_name):
407 # return cmp(len(a_name), len(b_name))
408
409 # else:
410 # return cmp(a_name, b_name)
411 return parsechildstr(a[1], 'name')
412
413 results.sort(key=result_sorter)
414
415 if ':' not in search_string:
416 # now append description matches
417 for table_info in itertools.chain(TableInfo.all_ts(), TableInfo.all_sys()):
418 matches = table_info.elem.xpath('//field/description[contains('
419 'translate(.,"abcdefghijklmnopqrstuvwxyz",'
420 '"ABCDEFGHIJKLMNOPQRSTUVWXYZ"),"' +
421 search_string.upper() + '")]')
422 for m in matches:
423 f = m.getparent()
424 if f not in results:
425 # don't add the same parameter twice even if the name and description both match
426 results.append((table_info, f))
427
428 # limit results to 200 entries
429 max_results = 200
430 trunc = len(results) > max_results
431 results = results[0:max_results]
432
433 html = []
434
435 if len(results) > 0:
436 html.append('<h2>Fields which match search string "{search}"</h2>'
437 '<table class="table"><thead><tr><th>Table</th>'
438 '<th>Parameter</th>'
439 '<th>Unit</th>'
440 '<th>Calibration</th>'
441 '<th>Datatype</th>'
442 '<th>Description</th></tr></thead><tbody>'.format(search=search_string))
443
444 for r in results:
445 table_name = r[0].name
446 table_url = r[0].browse_url
447 param_name = parsechildstr(r[1], 'name')
448 param_url = reverse('db:field', kwargs={'table_name': table_name,
449 'field_name': param_name})
450
451 html.append('<tr><td><a href="{table_url}">{table}</a></td>'
452 '<td><a href="{param_url}">{param}</a></td>'
453 '<td>{unit}</td>'
454 '<td>{calibration}</td>'
455 '<td>{type}</td>'
456 '<td>{desc}</td></tr>'.format(
457 table=table_name,
458 table_url=table_url,
459 param=param_name,
460 param_url=param_url,
461 unit=parsechildstr(r[1], 'unit', ''),
462 calibration=parsechildstr(r[1], 'calibration', ''),
463 type=parsechildstr(r[1], 'type', 'uint'),
464 desc=parsechildstr(r[1], 'description', '')))
465
466 html.append('</tbody></table>')
467
468 if trunc:
469 html.append('<p>Some additional results were truncated</p>')
470
471 # now look for table matches (presented first)
472 table_matches = []
473 for t in TableInfo.all_ts():
474 if search_string.lower() in t.name.lower():
475 table_matches.append(t)
476
477 if len(table_matches) > 0:
478 table = Table(headings=('Table', 'Description'))
479 for match in table_matches:
480 table.append(({'text': match.name,
481 'url': match.browse_url},
482 match.description))
483
484 dummy = StringIO()
485 table.write_html(dummy)
486 html.insert(0, dummy.getvalue())
487 html.insert(0, '<h2>Tables which match string "{search}"</h2>'.format(search=search_string))
488
489 if len(html) == 0:
490 html.append('<h2>No results found for "{search}"</h2>'.format(search=search_string))
491
492 return HttpResponse('\n'.join(html), content_type='text/html')
493
494
495def calibration_curve(request, # (unused arg 'request') pylint: disable=W0613
496 table_name,
497 field_name,
498 size):
499 """Plot all calibrations curve for a field."""
500
501 size = int(size)
502
503 fig = figure.Figure(figsize=(size / 100, size / 100), dpi=100)
504 # canvas =
505 FigureCanvas(fig)
506 # ax1 = fig.add_axes((0,0,1,1)) #(0.06, 0.07, 0.90, 0.80))
507 ax1 = fig.add_subplot(111)
508
509 table_info = TableInfo(table_name)
510 field_info = table_info.fields[field_name]
511
512 fontsize = 8
513
514 if size > 500:
515 ax1.set_title('{field} calibration curve'.format(field=field_name))
516 fontsize = 12
517
518 if size > 300:
519 ax1.set_xlabel('Raw', fontsize=fontsize)
520 ax1.set_ylabel('Calibrated ({unit})'.format(unit=field_info.unit),
521 fontsize=fontsize)
522
523 data_range = field_info.raw_data_range
524 # subsample = max(1, (data_range[1] - data_range[0]) // size)
525
526 x = np.linspace(data_range[0], data_range[1], size)
527 ax1.set_xlim(*data_range)
528
529 cal = None
530
531 row = None # pylint
532 for row in field_info.cal.cals.items():
533 if row[0] is None:
534 # if row[0] == (None, ):
535 cal = row[1]
536
537 if cal is None:
538 cal = row[1]
539
540 y = np.fromiter((cal.calibrate_value(raw) for raw in x), dtype=float)
541 ax1.plot(x, y)
542 # ax1.plot(x, y, label='Default' if sids[0] is None else ', '.join(sfids))
543
544 for tick in ax1.xaxis.get_major_ticks():
545 tick.label1.set_fontsize(fontsize)
546
547 for tick in ax1.yaxis.get_major_ticks():
548 tick.label1.set_fontsize(fontsize)
549
550 # ax1.legend()
551 response = HttpResponse(content_type='image/png')
552 fig.savefig(response, format='png')
553 return response
554
555
556def packets(request):
557 """Show list of available packets."""
558 sid = request.GET.get('SID')
559 pkt_type = request.GET.get('PKT_TYPE')
560 packetdefs = {p.name: p for p in PacketDef.all(domain=PacketDomain(pkt_type),sid=SID(sid))}
561 return render(request,
562 'db/packets.html',
563 {'packets': packetdefs.items(),
564 'sid':sid,
565 'pkt_type':pkt_type.upper()}
566 )
567
568def sid_packets(request):
569 """Show list of available packets for sid's."""
570 sids_packets = {}
571 for each_sid in SID.all():
572 if each_sid.name:
573 packets= []
574 xml_gen = settings.SID_SPECIFIC_DIR.joinpath(each_sid.name,settings.SID_PACKETS_DIR).glob('*')
575 for filename in sorted(xml_gen):
576 packets.append(filename.name)
577 sids_packets[each_sid.name] = packets
578 return render(request,
579 'db/sid_packets.html',
580 {'sids_packets': sids_packets.items()})
581
582def packet(request, packet_name):
583 """Show details of a Sid packets ingestion."""
584 sid = request.GET.get('SID')
585 pkt_type = request.GET.get('PKT_TYPE')
586 spid = request.GET.get('SPID')
587 reader = PacketDef.find_one(name=packet_name,sid=SID(sid),domain=PacketDomain(pkt_type),spid=spid)
588 params = defaultdict(list)
589 for param in reader.paramlists:
590 #To get packet parameters
591 for eachparam in param.params:
592 params[(eachparam.byte, eachparam.bit)].append(eachparam)
593 Position = namedtuple('Position', 'byte bit fields')
594 positions = []
595 for pos in sorted(params.keys()):
596 positions.append(Position(pos[0], pos[1], params[pos]))
597 return render(request,
598 'db/packet.html',
599 {'packet': reader,
600 'positions': positions})