1#!/usr/bin/env python3
2
3# Copyright (c) 2020 EUMETSAT
4# License: Proprietary
5
6"""Implementation of Limits widget."""
7
8import logging
9import re
10from collections import OrderedDict
11
12import numpy as np
13
14# always import matplotlib_agg before matplotlib
15from chart.common import matplotlib_agg # (unused import) pylint: disable=W0611
16from matplotlib import ticker
17from matplotlib import figure
18from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
19
20from chart.db import ts
21from chart.db.model.exceptions import NoDataRange
22from chart.common.util import nvl
23from chart.project import SID
24from chart.project import settings
25from chart.plots.sampling import region_duration
26from chart.plots.sampling import Sampling
27from chart.common.path import Path
28from chart.plots.plot import Plot
29from chart.reports.widget import Widget
30from chart.common.prettyprint import float_to_str
31from chart.common.exceptions import ConfigError
32from chart.db.func import Min
33from chart.db.func import Max
34from chart.db.func import Sum
35from chart.db.func import Avg
36from chart.db.model.calibration import NamedCalibrationCluster
37from chart.db.model.field import RowcountFieldInfo
38from chart.common.prettyprint import Table
39from chart.common.util import nvl_min
40from chart.common.util import nvl_max
41from chart.plotviewer.make_url import make_plot_url
42from chart.plots.retrieve import round_time
43from chart.db.model.limits_model import load_limits_from_xml_files, Limit
44from chart.reports.widget import WidgetOptionChoice
45from chart.products.fdf.orbit import NoSuchOrbit
46from chart.widgets.utils.limits import retrieve_limits, get_applicable_onboard_limits
47
48logger = logging.getLogger()
49
50DEFAULT_STATS_SF = 4
51
52# limits for the number of significant figures selected for display by the auto algorithm
53# for the legend
54MIN_SF = 4
55MAX_SF = 7
56
57# if the user has not specified a filename, use this template
58DEFAULT_FILENAME = 'LIMIT{number}{ext}'
59
60
61def pad(in_str, sf):
62 """Pad `in_str` to have at least `sf` digits after the decimal place.
63
64 Right padding with zeroes if not.
65 """
66
67 if '.' not in in_str:
68 return in_str
69
70 places = len(in_str) - in_str.rfind('.') - 1
71
72 if places < sf:
73 return in_str + '0' * (sf - places)
74
75 else:
76 return in_str
77
78
79class LimitsGraph:
80 """Render a bar graph showing min/max values for a range of parameters, all of the same unit.
81 The background to each row shows the potential data range deduced from the XML config file
82 showing the total data range, and regions where values are classed red, yellow of green using
83 coloured backgrounds. Data values should have been previously read using the retrieve_limits
84 function.
85 """
86
87 # color to render the extent of actual data found (black)
88 DATA_COL = (0, 0, 0)
89 LIMIT_BAR_HEIGHT = 1.0
90 # height of actual data bar
91 DATA_BAR_HEIGHT = 0.8
92 # colour to render Red limit region
93 RED = (1, 0, 0)
94 # colour to render Yellow limit region
95 YELLOW = (1, 1, 0)
96 # colour to render Green limit region
97 GREEN = (0, 1, 0)
98 # colour to render onboard_limits bars (RGBA)
99 OBLIMITS_COLOR = (1, 1, 1, 0.75)
100 # colour to render Green limit region
101 BORDER_COLOUR = (0, 0, 0)
102 # width of the outline around limits bars
103 BORDER_WIDTH = 0.5
104 # file extension for saved images
105 EXT = '.png'
106
107 def __init__(self, width, title=None, filename=None):
108 self.filename = filename
109 self.title = title
110 # size is set at the end when we know how many values there were
111 self.fig = figure.Figure()
112 FigureCanvas(self.fig) # needed internally by matplotlib
113 self.ax1 = self.fig.add_subplot(111)
114
115 self.width = width
116 self.height = None
117
118 self.names = []
119 self.unit = None
120 self.limits = [None, None]
121
122 def finalise(self, title, filename, ylabel, label_fontsize):
123 """Write barchart image to disk, returning a dictionary which can be passed to the
124 report.add_image function."""
125
126 # allow graph title to be set here if not already done
127 if title is not None:
128 self.title = title
129
130 if self.title is not None:
131 self.ax1.set_title(self.title)
132
133 # set to y axis limits
134 self.ax1.set_ylim((0, len(self.names)))
135 self.ax1.xaxis.set_ticks_position('both')
136
137 self.height = 66 + len(self.names) * 26
138
139 # label the x axis
140 self.ax1.yaxis.set_major_locator(
141 ticker.FixedLocator(
142 np.array(list(range(len(self.names))))
143 + LimitsGraph.LIMIT_BAR_HEIGHT / 2
144 )
145 )
146 self.ax1.yaxis.set_major_formatter(ticker.FixedFormatter(self.names))
147
148 # hide vertical tick marks
149 for i in self.ax1.yaxis.get_majorticklines():
150 i.set_visible(False)
151
152 self.ax1.yaxis.get_label().set_fontsize(label_fontsize)
153 for label in self.ax1.get_yticklabels():
154 label.set_fontsize(label_fontsize)
155
156 self.ax1.xaxis.get_label().set_fontsize(label_fontsize)
157 for label in self.ax1.get_xticklabels():
158 label.set_fontsize(label_fontsize)
159
160 # set x-axis limits to the maximum data range of all input values
161 self.ax1.set_xlim(self.limits)
162
163 # fixed y-axis label
164 if ylabel is not None:
165 self.ax1.set_ylabel(ylabel)
166
167 self.fig.set_size_inches((self.width / Plot.DPI, self.height / Plot.DPI))
168 # allow filename to be set in finalise() if not done in constructor
169 if filename is not None:
170 self.filename = filename
171
172 if self.filename is None and self.title is not None:
173 self.filename = self.title.replace(' ', '_').replace('/', '_')
174
175 if self.filename is None:
176 raise ConfigError(
177 'Filename must be specified either in constructor or finalise '
178 'function'
179 )
180
181 # the units becomes the x-axis label
182 if self.unit is not None:
183 self.ax1.set_xlabel(self.unit)
184
185 # add a .png filename extension is not already present
186 if self.filename.suffix.lower() != LimitsGraph.EXT.lower():
187 self.filename = Path(str(self.filename) + LimitsGraph.EXT)
188
189 self.fig.tight_layout()
190 # this is probably caused by a stray inf value that should be stripped out instead
191 self.fig.savefig(str(self.filename), dpi=Plot.DPI)
192
193 # return a data structure suitable for passing to report.add_image()
194 return {'filename': self.filename, 'title': self.title}
195
196 def add_param(
197 self,
198 datapoint,
199 stat,
200 relevant_ob_limit,
201 min_x=None,
202 max_x=None,
203 default_min_x=None,
204 default_max_x=None,
205 ):
206 """Add extra single parameter to graph."""
207
208 name = datapoint.name # stat['param']
209
210 unit = datapoint.unit # stat['unit']
211 if self.unit is None:
212 self.unit = unit
213 else:
214 if self.unit != unit:
215 logger.info(
216 'Limits plot with mixed units (found {new} in {table} already '
217 'had {exist})'.format(new=unit, table=name, exist=self.unit)
218 )
219
220 # find the total x axis range needed for this field.
221 # We use the theoretical limits if present,
222 # otherwise (i.e. for a floating point value)
223 # use the actual data range.
224 # If red/yellow limits are defined the bar will be extended to allow for them.
225
226 left = default_min_x
227 right = default_max_x
228
229 if stat['data_range'] is not None:
230 if stat['data_range'][0] is not None:
231 left = nvl_min(left, stat['data_range'][0])
232
233 if stat['data_range'][1] is not None:
234 right = nvl_max(right, stat['data_range'][1])
235
236 else:
237 if stat['min'] is not None:
238 left = nvl_min(left, stat['min'])
239
240 if stat['max'] is not None:
241 right = nvl_max(right, stat['max'])
242
243 if min_x is not None:
244 left = min_x
245
246 if max_x is not None:
247 right = max_x
248
249 bar_left = left
250 bar_right = right
251
252 # Expand the limits for the entire plot if needed
253 self.limits[0] = nvl_min(self.limits[0], left)
254 self.limits[1] = nvl_max(self.limits[1], right)
255
256 # y-position of the bar we are about to draw
257 ypos = len(self.names) + LimitsGraph.LIMIT_BAR_HEIGHT / 2
258
259 if left is None:
260 left = 0
261 logger.warning(
262 'Using {min} as min value for {field} as no other value could be '
263 'found'.format(min=left, field=name)
264 )
265
266 if right is None:
267 right = 1
268 logger.warning(
269 'Using {max} as max value for {field} as no other value could be '
270 'found'.format(max=right, field=name)
271 )
272
273 limits = stat['limits']
274 if limits is not None:
275 # limits can be:
276 # upper and lower red and yellow given and different
277 # upper and lower red and yellow given but red and yellow same
278 # only red given
279 # only yellow given
280
281 red = limits.red
282 yellow = limits.yellow
283
284 # if red limits are given, plot the entire data range in red
285 if red.low is not None or red.high is not None:
286 self.ax1.barh(
287 ypos,
288 right - left,
289 left=left,
290 color=LimitsGraph.RED,
291 edgecolor=LimitsGraph.BORDER_COLOUR,
292 height=LimitsGraph.LIMIT_BAR_HEIGHT,
293 linewidth=LimitsGraph.BORDER_WIDTH,
294 )
295
296 if red.low is not None:
297 left = max(red.low, left)
298
299 if red.high is not None:
300 right = min(red.high, right)
301
302 # if yellow limits are given, fill all/remaining space with yellow
303 if yellow.low is not None or yellow.high is not None:
304 self.ax1.barh(
305 ypos, # bottom
306 right - left, # width
307 left=left,
308 color=LimitsGraph.YELLOW,
309 edgecolor=LimitsGraph.BORDER_COLOUR,
310 height=LimitsGraph.LIMIT_BAR_HEIGHT,
311 linewidth=LimitsGraph.BORDER_WIDTH,
312 )
313
314 if yellow.low is not None:
315 left = max(yellow.low, left)
316
317 if yellow.high is not None:
318 right = min(yellow.high, right)
319
320 # fill remaining space with green
321 self.ax1.barh(
322 ypos,
323 right - left,
324 left=left,
325 color=LimitsGraph.GREEN,
326 edgecolor=LimitsGraph.BORDER_COLOUR,
327 height=LimitsGraph.LIMIT_BAR_HEIGHT,
328 linewidth=LimitsGraph.BORDER_WIDTH,
329 )
330
331 if relevant_ob_limit:
332 ob_min, ob_max, ob_validity_start = relevant_ob_limit
333 self.ax1.barh(
334 ypos,
335 ob_max - ob_min,
336 left=ob_min,
337 color=LimitsGraph.OBLIMITS_COLOR,
338 edgecolor=LimitsGraph.BORDER_COLOUR,
339 height=LimitsGraph.LIMIT_BAR_HEIGHT / 4,
340 linewidth=LimitsGraph.BORDER_WIDTH,
341 )
342
343 # plot main data point, if data was found
344 # the nominal bar height is 1 but if DATA_BAR_HEIGHT is different we make sure the data
345 # bar is entered
346
347 # note numpy 1.13.1 on CentOS has a bug, the np.isinf function always
348 # returns False (yes, even "print(numpy.isinf(numpy.inf))" gives "False")
349 # This does not occur in 1.13.1 on openSUSE 13
350 # So we do this ugly thing
351 MAX_BAR = 10**12
352
353 if stat['min'] is not None and stat['max'] < MAX_BAR:
354 self.ax1.barh(
355 ypos,
356 stat['max'] - stat['min'],
357 left=stat['min'],
358 color=LimitsGraph.DATA_COL,
359 height=LimitsGraph.DATA_BAR_HEIGHT,
360 edgecolor=LimitsGraph.BORDER_COLOUR,
361 linewidth=LimitsGraph.BORDER_WIDTH,
362 )
363
364 self.names.append(name)
365
366 def trim_labels(self, unprefix):
367 """Remove `unprefix` from the start of each label for the legend.
368 Unfortunately matplotlib will draw them offscreen otherwise if they are too long.
369 """
370
371 self.names = [n[len(unprefix) :] for n in self.names]
372
373
374class Limits(Widget):
375 """Display horizontal bar chart showing the min/max values for a list of parameters
376 in the given time range, and also show the MCS limits on the same axis.
377 """
378
379 name = 'limits'
380
381 thumbnail = 'widgets/limits.png'
382
383 options = OrderedDict(
384 [
385 ('title', {'type': 'string', 'default': 'Parameter limits'}),
386 ('filename', {'type': 'string', 'optional': True}),
387 ('datapoint', {'type': 'field', 'multiple': True}),
388 (
389 'width',
390 {
391 'type': 'uint',
392 'default': 680,
393 'unit': 'pixels',
394 'description': (
395 'Width of output image (height is not configurable '
396 'and depends on number of values shown)'
397 ),
398 },
399 ),
400 (
401 'thumbnail-width',
402 {
403 'type': 'uint',
404 'default': 680,
405 'description': 'Width of initially displayed thumbnail image',
406 'unit': 'pixels',
407 },
408 ),
409 (
410 'sampling',
411 {
412 'type': str,
413 'default': 'all',
414 'choices': [
415 WidgetOptionChoice(
416 name='all',
417 description='Use orbital stats and all points tables',
418 ),
419 WidgetOptionChoice(
420 name='stats-only',
421 description=(
422 'Use stats averages only '
423 '(from the shortest stats region available)'
424 ),
425 ),
426 WidgetOptionChoice(
427 name='orbital-avg',
428 description='Use per-orbit averages only',
429 ),
430 ],
431 'description': 'Quantisation of input data',
432 },
433 ),
434 (
435 'calibrated',
436 {
437 'type': 'boolean',
438 'default': True,
439 'description': 'Use calibrated data',
440 },
441 ),
442 (
443 'label-unprefix',
444 {
445 'type': 'string',
446 'optional': True,
447 'description': 'Prefix string to remove from all y-axis labels',
448 },
449 ),
450 ('y-label', {'type': 'string', 'default': 'Mnemonic'}),
451 (
452 'label-fontsize',
453 {
454 'type': 'uint',
455 'unit': 'pt',
456 'default': 8,
457 'description': 'Font size for axis labels',
458 },
459 ),
460 (
461 'title-fontsize',
462 {
463 'type': 'uint',
464 'unit': 'pt',
465 'default': 0,
466 'description': (
467 'Font size for the title. 0 disabled embedded '
468 'title. Only 0 and 10 can be used.'
469 ),
470 },
471 ),
472 (
473 'x-min',
474 {'type': 'float', 'optional': True, 'description': 'Left axis limit'},
475 ),
476 (
477 'x-max',
478 {'type': 'float', 'optional': True, 'description': 'Right axis limit'},
479 ),
480 (
481 'default-x-min',
482 {'type': 'float', 'optional': True, 'description': 'Left axis limit'},
483 ),
484 (
485 'default-x-max',
486 {'type': 'float', 'optional': True, 'description': 'Right axis limit'},
487 ),
488 (
489 'table',
490 {
491 'type': 'string',
492 'default': 'legend+stats',
493 'choices': [
494 WidgetOptionChoice(name='none', description='No summary table'),
495 WidgetOptionChoice(
496 name='legend',
497 description='Legend showing field descriptions only',
498 ),
499 WidgetOptionChoice(
500 name='legend+stats',
501 description='Legend including per-field statistics',
502 ),
503 ],
504 },
505 ),
506 (
507 'stats-sf',
508 {
509 'type': int,
510 'optional': True,
511 # 'default': 4,
512 'description': (
513 'If given force the significant figures for rendering the '
514 'stats table'
515 ),
516 },
517 ),
518 (
519 'dynamic-stats-sf',
520 {
521 'type': bool,
522 'default': False,
523 'description': (
524 'Use an algorithm to guess the correct number of '
525 'significant figured in legend table'
526 ),
527 },
528 ),
529 ]
530 )
531
532 document_options = OrderedDict(
533 [
534 ('sid', {'type': 'sid'}),
535 ('sensing_start', {'type': 'datetime'}),
536 ('sensing_stop', {'type': 'datetime'}),
537 ]
538 )
539
540 def __str__(self):
541 """String representation of this widget, appears in template log file."""
542
543 # trap self.config here in case an exception was thrown early in construction but
544 # someone is still trying to print us.
545 if hasattr(self, 'config') and self.config is not None:
546 return 'Limits({title})'.format(title=self.config.get('title', ''))
547
548 else:
549 return 'Limits'
550
551 def pre_html(self, document):
552 """Insert ourselves into the List of Figures widget."""
553
554 c = self.config
555 document.figures.append(c['title'])
556
557 def html(self, document):
558 """Render ourselves."""
559
560 c = self.config
561 html = document.html
562 dc = document.config
563
564 if 'filename' not in c:
565 # `anon_counts` gives the number of anonymous images already created
566 # for this report
567 if 'limits' not in document.anon_counts:
568 document.anon_counts['limits'] = 1
569 else:
570 document.anon_counts['limits'] += 1
571
572 base_filename = DEFAULT_FILENAME.format(
573 number=document.anon_counts['limits'], ext=Plot.EXT
574 )
575
576 else:
577 base_filename = c['filename']
578
579 filename = document.theme.mod_filename(document, base_filename)
580
581 bars = LimitsGraph(
582 title=c['title'] if c['title-fontsize'] > 0 else None,
583 filename=filename,
584 width=c['width'],
585 )
586
587 if c['table'] == 'legend+stats':
588 desc_table = Table(
589 title=c['title'],
590 headings=('Mnemonic', 'Description', 'Min', 'Max', 'Avg', 'Unit'),
591 )
592
593 else:
594 desc_table = Table(title=c['title'], headings=('Mnemonic', 'Description'))
595
596 stats_only = c['sampling'] == 'orbital-avg' or c['sampling'] == 'stats-only'
597
598 # to accumulate stats info for the second datapoint loop below
599 full_stats = {}
600
601 for dp in reversed(c['datapoint']):
602 stats = retrieve_limits(
603 dc['sid'],
604 dc['sensing_start'],
605 dc['sensing_stop'],
606 c['calibrated'],
607 stats_only,
608 dp,
609 )
610
611 full_stats[dp] = stats
612 onboard_limits = get_applicable_onboard_limits(
613 dp.name, dc['sid'].name, dc['sensing_start'], dc['sensing_stop']
614 )
615 relevant_ob_limit = onboard_limits[-1] if onboard_limits else []
616
617 bars.add_param(
618 dp,
619 stats,
620 relevant_ob_limit,
621 c.get('x-min'),
622 c.get('x-max'),
623 c.get('default-x-min'),
624 c.get('default-x-max'),
625 )
626
627 desc = dp.description
628 if '_;_' in desc:
629 # fields like NTPLM43 have really long descriptions with no spaces
630 # so we add one to allow the browser to break up the table cell
631 # holding the description
632 desc = desc.replace('_;_', ' ')
633
634 if c['table'] == 'legend+stats':
635 # Work how how many decimal places to use for these stats
636 if stats['min'] is not None and stats['max'] is not None:
637 if c.get('stats-sf') is None and c['dynamic-stats-sf'] is True:
638 dynrange = stats['max'] - stats['min']
639 if dynrange > 0:
640 # this is probably wrong and we should strip out the inf values instead
641 try:
642 sf = min(
643 MAX_SF,
644 max(MIN_SF, abs(int(np.log10(dynrange))) + 1),
645 )
646 except OverflowError:
647 sf = 1
648
649 elif stats['max'] > 0:
650 # take care of cases like min=max=5.123
651 # min() added to handle limits plot of
652 # FNV2014 M02 2016-05-01 to 2016-05-02
653 sf = min(MAX_SF, max(2, int(np.log10(stats['max']))))
654
655 else:
656 sf = 1
657
658 # this is verbose and also may crash the daemon due to non-ascii chars
659 # not handled properly in log.py
660 # logger.debug('dp {name} desc {desc} unit {unit} sf {sf} min {mn} max {mx} '
661 # 'dr {dr}'.format(
662 # name=dp.name, desc=desc, unit=dp.unit, sf=sf,
663 # mn=stats['min'], mx=stats['max'], dr=dynrange))
664
665 else:
666 sf = c.get('stats-sf', DEFAULT_STATS_SF)
667
668 desc_table.append(
669 (
670 dp.name,
671 desc,
672 pad(float_to_str(stats['min'], sf), sf),
673 pad(float_to_str(stats['max'], sf), sf),
674 pad(float_to_str(stats['avg'], sf), sf),
675 dp.unit,
676 )
677 )
678
679 else:
680 # occurs if we have no data in specified range, and the field
681 # is floating point (theoretical max is NaN)
682 desc_table.append((dp.name, desc, 'None', 'None', 'None', dp.unit))
683
684 else:
685 desc_table.append((dp.name, dp.description))
686
687 # if specified, remove a common prefix from the y-axis labels
688 if 'label-unprefix' in c:
689 bars.trim_labels(c['label-unprefix'])
690 for row in desc_table.rows:
691 row[0] = '({common}) {item}'.format(
692 common=c['label-unprefix'], item=row[0][len(c['label-unprefix']) :]
693 )
694
695 bars.finalise(
696 title=None,
697 filename=None,
698 ylabel=c['y-label'],
699 label_fontsize=c['label-fontsize'],
700 )
701
702 # prepare per-datapoint statistics for the info page
703 # TBD: make this look nicer with 2 rows of headers and merged cells
704 datapoint_info = Table(
705 headings=(
706 'Field',
707 'Table',
708 'Low (theo)',
709 'High (theo)',
710 'Red low',
711 'Red high',
712 'Yellow low',
713 'Yellow high',
714 'Low (act)',
715 'High (act)',
716 'Avg',
717 )
718 )
719 for dp in c['datapoint']:
720 stat = full_stats[dp]
721 stat['field'] = dp
722
723 if full_stats[dp]['raw_data_range'] is not None:
724 dynrange = max(full_stats[dp]['raw_data_range']) - min(
725 full_stats[dp]['raw_data_range']
726 )
727 sf = max(4, int(np.log10(dynrange)) + 1)
728 data_range = [
729 pad(float_to_str(stat['data_range'][0], sf), sf),
730 pad(float_to_str(stat['data_range'][1], sf), sf),
731 ]
732
733 else:
734 sf = 4
735 data_range = (None, None)
736
737 if stat['limits'] is not None:
738 red = stat['limits'].red
739 yellow = stat['limits'].yellow
740 else:
741 red = Limit(None, None)
742 yellow = Limit(None, None)
743
744 datapoint_info.append(
745 (
746 dp.name,
747 dp.table.name,
748 data_range[0],
749 data_range[1],
750 red.low,
751 red.high,
752 yellow.low,
753 yellow.high,
754 pad(float_to_str(stat['min'], sf), sf),
755 pad(float_to_str(stat['max'], sf), sf),
756 pad(float_to_str(stat['avg'], sf), sf),
757 )
758 )
759
760 url = make_plot_url(
761 plot_type='LIMITS',
762 datapoints=[{'field': x} for x in c['datapoint']],
763 sid=dc['sid'],
764 sensing_start=dc['sensing_start'],
765 sensing_stop=dc['sensing_stop'],
766 calibrated=c['calibrated'],
767 )
768
769 document.append_figure(
770 title=c['title'],
771 filename=filename,
772 width=c['thumbnail-width'],
773 zoom_filename=filename,
774 zoom_width=bars.width,
775 zoom_height=bars.height,
776 datapoint_info=datapoint_info,
777 live_url=url,
778 widget=self,
779 )
780
781 if c['table'] != 'none':
782 desc_table.reverse()
783 desc_table.write_html(html)