1#!/usr/bin/env python3
2
3"""Basic widgets to render graphs of single or multiple data points."""
4
5import math
6import logging
7import warnings
8import itertools
9from collections import OrderedDict
10from collections import defaultdict
11from datetime import datetime
12from datetime import timedelta
13
14from django.template import Template
15from django.template import Context
16import numpy as np
17
18# always import matplotlib_agg before matplotlib
19from chart.common import matplotlib_agg # (unused import) pylint: disable=W0611
20
21from chart.common.path import Path
22from chart.project import settings
23from chart.common.traits import to_htmlstr
24from chart.common.traits import name_of_thing
25from chart.common.util import nvl_min
26from chart.common.util import nvl_max
27from chart.common.util import fp_eq
28from chart.common.prettyprint import Table
29from chart.common.prettyprint import show_timedelta
30from chart.common.exceptions import ConfigError
31from chart.common.texttime import texttime_to_datetime
32from chart.plots.retrieve import retrieve_series
33from chart.reports.widget import Widget
34from chart.plotviewer.plot_utils import flot_palette
35from chart.common.traits import str_to_field
36from chart.common.traits import to_str
37from chart.plotviewer.make_url import make_plot_url
38from chart.plots.sampling import sampling_from_name
39from chart.project import SID
40from chart.plots.render import Axis
41from chart.plots.render import Series
42from chart.plots.render import Stats
43from chart.plots.render import Annotation
44from chart.plots.render import Presentation
45from chart.reports.widget import WidgetOption
46from chart.reports.widget import WidgetOptionChoice
47from chart.plots.render_matplotlib import matplotlib_render
48from chart.common.timerange import TimeRange
49from chart.plots.sampling import Sampling
50from chart.widgets.utils.limits import retrieve_limits, get_applicable_onboard_limits
51from chart.plots.sampling import region_duration
52
53
54# switch off matplotlib warnings
55np.seterr(all='raise') # all math errors cause exceptions
56warnings.simplefilter('ignore', np.RankWarning)
57warnings.simplefilter('ignore', UserWarning)
58warnings.simplefilter('ignore', RuntimeWarning)
59
60MIN_BINS = 3
61
62# redirect certain numpy warnings from stderr to standard logging
63# module
64class NumpyLog:
65 """Handle Numpy log messages."""
66
67 def write(self, message):
68 """Log `message` to info stream."""
69 logger.info(message.strip())
70
71
72np_log = NumpyLog()
73np.seterrcall(np_log)
74np.seterr(over='log') # except floating point overflow, as this is triggered by
75# set_major_locator()
76
77# if the user has not forced a filename, and we are plotting multiple points,
78# use this template
79DEFAULT_FILENAME = 'FIG{number}{ext}'
80
81DEFAULT_AUTO_AXIS_MARGIN = Axis.DEFAULT_AUTO_MARGIN
82
83# used to select width/height if only one given
84DEFAULT_ASPECT_RATIO = 1.5
85
86# remove and replace with multiplier
87DEFAULT_ZOOM_FACTOR = 1.5
88
89# Maximum number of regions to show in the log file
90MAX_SHOW_REGIONS = 10
91
92# if oversampling is used (stats plot, no trim, not disabled by widget config)
93# we extend sensing start/stop by a proportion of the normal amount, subject to a minimum
94# expansion. This is to avoid seeing fake gaps at the edges of the graph from 'missing' stats
95MIN_OVERSAMPLE = timedelta(hours=2)
96
97logger = logging.getLogger()
98
99
100def nan_pad(inputs, pad):
101 """Return a copy of `input` right-padded with `pad` NaN values.
102
103 In numpy-1.11 get could use the numpy.pad function to do this quickly, if not
104 neatly. But that doesn't compile on AIX and numpy-1.9.0, the last version which
105 works there, has a broken pad() implementation. So we have this gruesome hack.
106 """
107 # the relatively nice (or at least fast) way
108
109 # np.pad(
110 # o,
111 # (0, pad),
112 # 'constant',
113 # constant_values=np.nan,).reshape(target_buckets, modulus), axis=1)
114
115 # you may wish to look away now...
116
117 # return np.concatenate((inputs, [np.nan] * pad))
118
119 # update: for unknown reasons, with Python 3.6.2 and numpy-1.3.1,
120 # the algorithm above causes matplotlib to crash *only* on CentOS 7 (SLES 13 is fine).
121 # So, for this terrible reason we use the following completely wrong algorithm
122 # to get something to render
123 return np.concatenate((inputs, [inputs[-1]] * pad))
124
125
126# threshold where we retrieve stats not ap
127STATS_CUTOFF = timedelta(days=1)
128
129# only show the Orbits top axis if more than this number of orbits
130SHOW_ORBITS_THRESHOLD = 300
131
132# not sure if used, but mark y2 series in the legend
133Y2_LEGEND_LABEL = ' (y2)'
134
135
136class GraphWidget(Widget):
137 """A line or scatter plot of either timeseries data or event properties.
138 Either all points data or orbital statistics can be shown with optional trend lines.
139 """
140
141 name = 'graph'
142
143 image = 'widgets/graph.png'
144 thumbnail = 'widgets/graph_sm.png'
145
146 url = 'http://tctrac/projects/chart/wiki/GraphWidget'
147
148 options = [
149 WidgetOption(
150 name='title',
151 description='Force title for this graph',
152 datatype=str,
153 optional=True,
154 ),
155 WidgetOption(
156 name='title-fontsize',
157 description=(
158 'Font size for title. 0 to disable. Only 0 or 10 (not used) are '
159 'accepted'
160 ),
161 default=settings.GRAPH_TITLE_FONTSIZE,
162 unit='pt',
163 ),
164 WidgetOption(
165 name='filename',
166 description='Force name of output file',
167 datatype=str,
168 optional=True,
169 ),
170 WidgetOption(name='bar-width',
171 description='barwidth floating value',
172 datatype=float,
173 default=Series.DEFAULT_BAR_WIDTH),
174
175 WidgetOption(
176 name='datapoint',
177 description='Data points to be plotted',
178 multiple=True,
179 datatype=[
180 WidgetOption(
181 name='field',
182 description=(
183 'Timeseries data point to plot. Enter just fieldname if '
184 'unique, otherwise table.fieldname'
185 ),
186 datatype=str, # we parse fieldnames as strings and expand them
187 # later so that computed fields "<field>LND1000 + 1</field>" work
188 optional=True,
189 ),
190 WidgetOption(
191 name='table',
192 description=('Rowcount plot (stats only)'),
193 datatype='table',
194 optional=True,
195 ),
196 WidgetOption(
197 name='event',
198 description=(
199 'Plot a numeric or duration property from an ' 'event class'
200 ),
201 datatype='eventproperty',
202 optional=True,
203 ),
204 WidgetOption(
205 name='axis', description='Obsolete', datatype=int, optional=True
206 ),
207 WidgetOption(
208 name='y-axis',
209 description='Give name of y-axis to attach to',
210 default='left',
211 datatype=str,
212 restrict_to_choices=False,
213 choices=[
214 WidgetOptionChoice(name='left'),
215 WidgetOptionChoice(name='right'),
216 ],
217 ),
218 WidgetOption(
219 name='label',
220 description='Legend label',
221 datatype=str,
222 optional=True,
223 ),
224 WidgetOption(
225 name='accumulate',
226 description=(
227 'Keep a count of number of events found. Only for use with event '
228 'plots with no property selected'
229 ),
230 default=False,
231 optional=True,
232 ),
233 WidgetOption(
234 name='constraint',
235 description='Per-field filtering',
236 datatype=[
237 WidgetOption(
238 name='table',
239 description='Table containing the conditional clauses',
240 datatype='table',
241 optional=True,
242 ),
243 WidgetOption(
244 name='clause',
245 description=(
246 'Filter this data series by a condition '
247 '(e.g. MHS_MODE=3)'
248 ),
249 multiple=True,
250 datatype=str,
251 ),
252 ],
253 multiple=True,
254 optional=True,
255 ),
256 WidgetOption(
257 name='colour',
258 description='Line and shade colour and default marker colour',
259 datatype='colour',
260 optional=True,
261 ),
262 WidgetOption(name='edge-colour',
263 description='Edge colour if different to line edge colour',
264 datatype='colour',
265 optional=True),
266 WidgetOption(
267 name='marker',
268 description='Marker type for scatter plots',
269 datatype=str,
270 default='',
271 ),
272
273 WidgetOption(
274 name='marker-colour',
275 description='Marker colour if different to line colour',
276 datatype='colour',
277 optional=True,
278 ),
279 WidgetOption(name='marker-size',
280 description='Marker size for scatter plots',
281 datatype=float,
282 default=Series.DEFAULT_MARKER_SIZE),
283 WidgetOption(name='bar-width',
284 description='barwidth floating value',
285 datatype=float,
286 default=Series.DEFAULT_BAR_WIDTH),
287
288 WidgetOption(name='trendline',
289 description='Show a trendline for this series',
290 default=False),
291 WidgetOption(
292 name='linestyle',
293 description='Line type for scatter plots',
294 datatype=str,
295 default='solid',
296 ),
297 WidgetOption(
298 name='linewidth',
299 description='Line width for scatter plots',
300 datatype=float,
301 default=1,
302 ),
303 ]),
304 WidgetOption(name='absolute-start-time',
305 description='Set start time for plot to literal time or `launch`',
306 datatype=datetime,
307 optional=True),
308
309 WidgetOption(name='absolute-stop-time',
310 description='Set stop time for plot to literal time or `now`',
311 datatype=str,
312 optional=True),
313
314 WidgetOption(name='relative-start-time',
315 description='Obsolete',
316 datatype=timedelta,
317 optional=True),
318
319 WidgetOption(name='relative-stop-time',
320 description='Obsolete',
321 datatype=timedelta,
322 optional=True),
323
324 WidgetOption(name='auto-axis-margin',
325 description='Obsolete',
326 default=DEFAULT_AUTO_AXIS_MARGIN,
327 optional=True),
328
329 WidgetOption(name='scid',
330 description=('For SCID-less reports, individual graphs can be plotted using '
331 'data from selected satellites'),
332 datatype=str,
333 optional=True),
334
335 WidgetOption(name='region',
336 description='Stats region if stats table used (ORB, 30MIN, DAY, HOUR etc.)',
337 datatype=str,
338 optional=True),
339
340 WidgetOption(
341 name='limits',
342 description='to display red/yellow/onboard limits if configured',
343 default='none',
344 choices=[
345 WidgetOptionChoice(name='none'),
346 WidgetOptionChoice(name='ground'),
347 WidgetOptionChoice(name='onboard'),
348 WidgetOptionChoice(name='all'),
349 ],
350 ),
351 WidgetOption(
352 name='sampling',
353 description='Source table class for samples',
354 default='auto',
355 choices=[
356 WidgetOptionChoice(
357 name='auto',
358 description=(
359 'Select all-points if <{cutoff} data, '
360 'otherwise stats'.format(cutoff=show_timedelta(STATS_CUTOFF))
361 ),
362 ),
363 WidgetOptionChoice(name='all-points'),
364 WidgetOptionChoice(name='stats'),
365 WidgetOptionChoice(name='orbital-stats'),
366 WidgetOptionChoice(name='daily-totals'),
367 ],
368 ),
369 WidgetOption(name='subsampling',
370 description='Subsampling mode',
371 default='auto',
372 choices=[
373 WidgetOptionChoice(
374 name='auto',
375 description=('Subsample if more than 3 times as many points as pixels '
376 '(not implemented) or if this is a coverage (ROWCOUNT) '
377 'plot')),
378 WidgetOptionChoice(
379 name='none',
380 description='No subsampling'),
381 WidgetOptionChoice(name='subsampled',
382 description='Subsample points to 1 row per pixel')]),
383
384 WidgetOption(name='modulus',
385 description='Subsampling factor',
386 optional=True,
387 datatype=int),
388
389 WidgetOption(name='width',
390 description='Width of the normal non-zoomed image',
391 default=680,
392 unit='pixels'),
393
394 WidgetOption(name='height',
395 description=('Height of the normal image. If height is omited but width is '
396 'specified, an aspect ratio of 3/2 is used'),
397 datatype=int,
398 optional=True,
399 unit='pixels'),
400
401 WidgetOption(name='thumbnail-width',
402 description= ('Obsolete option. If <thumbnail-width> is specified, <width> '
403 'becomes zoomed width'),
404 datatype=int,
405 optional=True,
406 unit='pixels1'),
407
408 WidgetOption(name='thumbnail-height',
409 description='Obsolete',
410 datatype='uint',
411 optional=True,
412 unit='pixels'),
413
414 WidgetOption(name='y-max',
415 description='Obsolete',
416 datatype=float,
417 optional=True),
418
419 WidgetOption(name='y-min',
420 description='Obsolete',
421 datatype=float,
422 optional=True),
423
424 WidgetOption(name='default-y-max',
425 description='Obsolete',
426 datatype=float,
427 optional=True),
428
429 WidgetOption(name='default-y-min',
430 description='Obsolete',
431 datatype=float,
432 optional=True),
433
434 WidgetOption(name='trendline',
435 description='Obsolete',
436 default=False),
437
438 # WidgetOption(name='trendline-degree',
439 # description='Obsolete',
440 # default=1),
441 WidgetOption(
442 name='appearance',
443 description='Appearance of graph',
444 default='auto',
445 choices=[
446 WidgetOptionChoice(
447 name='auto',
448 description=(
449 'Select "min-max-avg" normally for a single '
450 'data point, otherwise "line". Coverage plot '
451 'uses "bar"'
452 ),
453 ),
454 WidgetOptionChoice(name='line', description='Line graph'),
455 WidgetOptionChoice(name='scatter', description='Scatter plot'),
456 WidgetOptionChoice(
457 name='line-dots', description='Scatter plot with connecting line'
458 ),
459 WidgetOptionChoice(name='bar-over', description='Overlayed Bar graph'),
460 WidgetOptionChoice(name='bar-istar', description='ISTAR-MSG styled Bar graph'),
461 WidgetOptionChoice(name='bar-istar-gapless',
462 description='ISTAR-MSG styled Bar graph without gaps'),
463 WidgetOptionChoice(
464 name='bar-side', description='Side-by_side Bar graph'
465 ),
466 WidgetOptionChoice(name='bar', description='Bar graph'),
467 WidgetOptionChoice(
468 name='min-max-avg',
469 description=(
470 'For single data point plot only, using '
471 'statistics or subsampling only. Graph used '
472 'separate lines to show the min, avg and max '
473 'values.'
474 ),
475 ),
476 WidgetOptionChoice(
477 name='min-max',
478 description=(
479 'For single-point subsampled or stats plots '
480 'only, draw separate min and max lines'
481 ),
482 ),
483 WidgetOptionChoice(
484 name='dynrange',
485 description=(
486 'For subsampled or stats plots only, '
487 'show min/max ranges as filled polygons with '
488 'an averages line'
489 ),
490 ),
491 WidgetOptionChoice(name='pie', description='Pie graph'),
492 ],
493 ),
494 WidgetOption(
495 name='calibrated', description='Plot calibrated values', default=True
496 ),
497 WidgetOption(
498 name='trim',
499 description=(
500 'Clip all x-axes to only include the time range containing '
501 'non-null data'
502 ),
503 default=False,
504 ),
505 WidgetOption(
506 name='show-orbits',
507 description='Show orbit numbers along top axis',
508 default='auto',
509 choices=[
510 WidgetOptionChoice(
511 name='auto', description='Show orbits if height > 300 pixels'
512 ),
513 WidgetOptionChoice(name='true', description='Always show orbits'),
514 WidgetOptionChoice(name='false', description='Never show orbits'),
515 ],
516 ),
517 WidgetOption(
518 name='legend',
519 default='auto',
520 choices=[
521 WidgetOptionChoice(
522 name='auto',
523 description=(
524 'None for min-max-avg plot, otherwise ' 'outside-right'
525 ),
526 ),
527 WidgetOptionChoice(
528 name='none', description='No legend'),
529 WidgetOptionChoice(
530 name='embedded', description='Embedded, auto position'
531 ),
532 WidgetOptionChoice(
533 name='embedded-top-left', description='Embedded, top left corner'
534 ),
535 WidgetOptionChoice(
536 name='embedded-top-right', description='Embedded, top right corner'
537 ),
538 WidgetOptionChoice(
539 name='embedded-bottom-left',
540 description='Embedded, bottom left corner',
541 ),
542 WidgetOptionChoice(
543 name='embedded-bottom-right',
544 description='Embedded, bottom right corner',
545 ),
546 WidgetOptionChoice(
547 name='outside-right', description='To the right of the graph'
548 ),
549 WidgetOptionChoice(
550 name='below-desc',
551 description=(
552 'HTML table underneath the graph, include ' 'field descriptions'
553 ),
554 ),
555 WidgetOptionChoice(
556 name='below-desc-stats',
557 description=(
558 'HTML table underneath the graph, include '
559 'field descriptions and statistics'
560 ),
561 ),
562 ],
563 ),
564 # WidgetOption(name='label-fontsize',
565 # description='Font size for axis labels',
566 # default=8,
567 # unit='pt'),
568 # WidgetOption(name='thumbnail-label-fontsize',
569 # description='Font size for axis labels in thumbnails',
570 # default=8,
571 # unit='pt'),
572 WidgetOption(
573 name='y-label', description='Obsolete', datatype=str, optional=True
574 ),
575 # WidgetOption(name='y2-label',
576 # description='Specify label for right hand y-axis',
577 # datatype=str,
578 # optional=True),
579 WidgetOption(
580 name='annotation',
581 description='Annotate the plot with additional text',
582 multiple=True,
583 datatype=[
584 WidgetOption(name='text', datatype=str),
585 WidgetOption(name='colour', datatype='colour', default='black'),
586 WidgetOption(name='size', default=10),
587 WidgetOption(name='x', default=0.02),
588 WidgetOption(name='y', default=0.92),
589 ],
590 ),
591 WidgetOption(
592 name='radius', description='Obsolete', datatype=int, optional=True
593 ),
594 WidgetOption(
595 name='marker-size',
596 description='Default marker size for datapoints in a scatter plot',
597 datatype=float,
598 default=Series.DEFAULT_MARKER_SIZE,
599 ),
600 WidgetOption(
601 name='normalise',
602 description='Shift lines vertically so they all begin at the same level',
603 default=False,
604 ),
605 WidgetOption(
606 name='rowcount-threshold',
607 description=(
608 'For stats plots only, discard all points where the rowcount '
609 'is less than THRESH (as a proportion of the nominal rowcount'
610 ),
611 default=0.7,
612 metavar='THRESH',
613 unit='proportion',
614 ),
615 WidgetOption(
616 name='anomaly-response',
617 description='Control how to handle anomalous data points',
618 default='none',
619 choices=[
620 WidgetOptionChoice(name='none', description='No anomaly processing'),
621 WidgetOptionChoice(
622 name='axis',
623 description=(
624 'Exclude anomalies from automatic y-axis ' 'adjustment'
625 ),
626 ),
627 WidgetOptionChoice(name='remove', description='Remove anomalies'),
628 WidgetOptionChoice(name='remove_histogram',
629 description='Remove anomalies using a histogram-based approach'),
630 ],
631 ),
632 WidgetOption(
633 name='anomaly-threshold',
634 datatype=float,
635 description=(
636 'Adjustment for anomaly detection. Valid for stats or subsampled '
637 'data only. Number of standard deviations from the mean of the '
638 'standard deviations which is considered an anomaly'
639 ),
640 default=2.5,
641 ),
642 WidgetOption(name='anomaly-bin-ratio',
643 description=('Adjustment for anomaly detection for response option "remove". '
644 'Mean number of data points per bin. The lower the number, '
645 'the more precise the anomaly detection at the expense of '
646 'computation time.'),
647 default=50),
648
649 WidgetOption(name='anomaly-cutoff-percentage',
650 description=('Adjustment for anomaly detection for response option "remove". '
651 'Maximum percentage of data points removed from each end of the value range. '
652 'The higher the value, the more data points are removed.'),
653 default=2),
654 WidgetOption(
655 name='dynrange-alpha',
656 description=(
657 'With dynrange appearance, vary the transparency of the vertical '
658 'min-max line'
659 ),
660 default=0.5,
661 ),
662 WidgetOption(
663 name='oversampling',
664 description=(
665 'If <trim> is not in use, read a bit more data either side of '
666 'nominal report duration to avoid appearance of data gaps'
667 ),
668 default=0.01,
669 ),
670 # WidgetOption(name='date-locator',
671 # description=('Determine whether to use the matplotlib autolocator for date axis '
672 # 'or the CHART locator'),
673 # default='auto',
674 # restrict_to_choices=False,
675 # choices=[
676 # WidgetOptionChoice(name='auto',
677 # description='Use matplotlib for 1-month graphs only'),
678 # WidgetOptionChoice(name='matplotlib',
679 # description='Use matplotlib locator'),
680 # WidgetOptionChoice(name='chart',
681 # description='Use CHART locator')]),
682 # WidgetOption(name='date-format',
683 # description=('Select text formatter for dates in x-axis labels. Pass either '
684 # '"auto" or a `strftime`-style format string (see '
685 # 'https://docs.python.org/2/library/time.html#time.strftime for '
686 # 'options'),
687 # default='auto',
688 # restrict_to_choices=False,
689 # choices=[
690 # WidgetOptionChoice(name='auto',
691 # description='Use automatic formatting')]),
692 WidgetOption(
693 name='x-axis',
694 description='Configure an x-axis',
695 multiple=True,
696 datatype=[
697 # one anonymous axis can be given as a default (time) axes
698 # second if present must be called 'orbit'
699 # WidgetOption(name='id'),
700 WidgetOption(
701 name='position',
702 description=(
703 'Location of axis. Can only have one bottom (time) and '
704 'one top (orbit) x-axis'
705 ),
706 default='bottom',
707 choices=[
708 WidgetOptionChoice(name='top'),
709 WidgetOptionChoice(name='bottom'),
710 ],
711 ),
712 WidgetOption(
713 name='locator',
714 description=(
715 'Method used to work how where to place major and '
716 'minor tick-marks. Pass an integer for fixed number'
717 ),
718 default='chart',
719 restrict_to_choices=False,
720 choices=[
721 WidgetOptionChoice(
722 name='chart', description='Custom CHART method'
723 ),
724 WidgetOptionChoice(
725 name='matplotlib', description='Matplotlib default method'
726 ),
727 WidgetOptionChoice(
728 name='hour', description='Ticks based on count of hours'
729 ),
730 WidgetOptionChoice(
731 name='day', description='Ticks based on count of hours'
732 ),
733 ],
734 ),
735 WidgetOption(
736 name='locator-modulus',
737 description=(
738 'For fixed interval locators (hour etc) specify number '
739 'of units between major ticks'
740 ),
741 default=1,
742 datatype=int,
743 ),
744 # WidgetOption(name='values',
745 # description=('Purpose of axis. By default bottom is sensing time, '
746 # 'top is orbit'),
747 # datatype=str,
748 # choices=[
749 # WidgetOptionChoice(name='sensing-time'),
750 # WidgetOptionChoice(name='orbit'),
751 # ],),
752 WidgetOption(name='format', datatype=str, optional=True),
753 WidgetOption(
754 name='absolute-start-time',
755 description='Adjust graph start time if different to report start time',
756 datatype=datetime,
757 optional=True,
758 ),
759 WidgetOption(
760 name='absolute-stop-time',
761 description='Adjust graph stop time if different to report stop time',
762 datatype=str,
763 optional=True,
764 ),
765 WidgetOption(
766 name='relative-start-time',
767 description='Adjust graph start time if different to report start time',
768 datatype=timedelta,
769 optional=True,
770 ),
771 WidgetOption(
772 name='relative-stop-time',
773 description=(
774 'Set the stop time of this graph to an offset from the '
775 'report start time'
776 ),
777 datatype=timedelta,
778 optional=True,
779 ),
780 WidgetOption(
781 name='round-start-time',
782 description='Round down the start time to a nice value',
783 choices=[
784 WidgetOptionChoice(name='day'),
785 WidgetOptionChoice(name='week'),
786 WidgetOptionChoice(name='month'),
787 WidgetOptionChoice(name='year'),
788 ],
789 datatype=str,
790 optional=True,
791 ),
792 WidgetOption(
793 name='fontsize',
794 description='Font size for tick marks and label',
795 default=8,
796 ),
797 WidgetOption(
798 name='colour',
799 description='Use a named (matplotlib) colour, or #RRGGBB (hex values)',
800 datatype=str,
801 default='black',
802 optional=True,
803 ),
804 WidgetOption(
805 name='label',
806 description='Override the default axis label',
807 datatype=str,
808 optional=True,
809 ),
810 WidgetOption(
811 name='label-colour',
812 description='Specify label colour, if different to tick mark colour',
813 datatype='colour',
814 optional=True,
815 ),
816 WidgetOption(
817 name='label-fontsize',
818 description='Font size for label if different to tick mark font size',
819 default=8,
820 ),
821 ],
822 ),
823 WidgetOption(
824 name='y-axis',
825 description='Configure default or additional y-axes',
826 multiple=True,
827 datatype=[
828 # one anonymous axis can be given which becomes the default
829 # additional axes must have names
830 # currently default must be left, additional must be right
831 # WidgetOption(name='name',
832 # description='Name the axis if not the default',
833 # datatype=str,
834 # optional=True),
835 # WidgetOption(name='locator'),
836 # WidgetOption(name='format'),
837 WidgetOption(
838 name='min',
839 description='Force axis start (min) point',
840 datatype=float,
841 optional=True,
842 ),
843 WidgetOption(
844 name='default-min',
845 description='Suggested axis start, shifted if data needs it',
846 datatype=float,
847 optional=True,
848 ),
849 WidgetOption(
850 name='max',
851 description='Force axis stop (max) point',
852 datatype=float,
853 optional=True,
854 ),
855 WidgetOption(
856 name='default-max',
857 description='Suggested axis stop, shifted if data needs it',
858 datatype=float,
859 optional=True,
860 ),
861 WidgetOption(
862 name='margin',
863 description=(
864 'With automatic y-axis limits selection a small margin is added to '
865 'the computed axis limits'
866 ),
867 default=DEFAULT_AUTO_AXIS_MARGIN,
868 ),
869 WidgetOption(
870 name='fontsize',
871 description='Font size for tick marks and label',
872 default=8,
873 ),
874 WidgetOption(
875 name='colour',
876 description='Colour for tick marks and default label colour',
877 datatype=str,
878 optional=True,
879 ),
880 WidgetOption(
881 name='label',
882 description='Force axis label',
883 datatype=str,
884 optional=True,
885 ),
886 WidgetOption(
887 name='label-colour',
888 description='Specify label colour, if different to tick mark colour',
889 datatype='colour',
890 optional=True,
891 ),
892 WidgetOption(
893 name='label-fontsize',
894 description='Font size for label if different to tick mark font size',
895 default=8,
896 ),
897 WidgetOption(name='anomaly-threshold', datatype=float, optional=True),
898 WidgetOption(
899 name='position',
900 default='left',
901 choices=[
902 WidgetOptionChoice(name='left'),
903 WidgetOptionChoice(name='right'),
904 ],
905 ),
906 WidgetOption(
907 name='minor-ticks',
908 optional=True,
909 description='Distance between minor tick marks',
910 datatype=float,
911 ),
912 ],
913 ),
914 WidgetOption(
915 name='zoom',
916 description=(
917 'Specify appearance of enlarged image, when zoomed in the web '
918 'interface of for PDF rendering'
919 ),
920 optional=True,
921 datatype=[
922 WidgetOption(
923 name='width',
924 description='Width of zoomed image',
925 optional=True,
926 datatype='uint',
927 # default=1000,
928 unit='pixels',
929 ),
930 WidgetOption(
931 name='height',
932 optional=True,
933 datatype='uint',
934 description='If omited, non-zoomed aspect ratio is used',
935 unit='pixels',
936 ),
937 WidgetOption(
938 name='x-axis',
939 optional=True,
940 datatype=[
941 WidgetOption(name='format', datatype=str, optional=True),
942 WidgetOption(name='locator', datatype=str, optional=True),
943 WidgetOption(
944 name='label-fontsize',
945 description='Font size for label if different to tick mark font size',
946 default=8,
947 ),
948 ],
949 ),
950 ],
951 ),
952 ]
953
954 document_options = OrderedDict(
955 [
956 ('sid', {'type': 'sid'}),
957 ('sensing_start', {'type': 'datetime'}),
958 ('sensing_stop', {'type': 'datetime'}),
959 ]
960 )
961
962 def __init__(self):
963 super().__init__()
964 self.title = None
965
966 def __str__(self):
967 # build a list of displayeable datapoint names
968 prints = []
969 for dp in self.config['datapoint']:
970 if 'field' in dp:
971 prints.append(dp['field'])
972
973 elif 'event' in dp:
974 prints.append(dp['event']['eventclass'].name)
975
976 else:
977 prints.append('unknown')
978
979 return 'GraphWidget({data})'.format(data=', '.join(to_str(p) for p in prints))
980 # return 'GraphWidget({data})'.format(
981 # data=', '.join(c.get('field', c.get('event')) for c in self.config['datapoint']))
982
983 def pre_html(self, document):
984 """Compute our final title for table of contents."""
985 config = self.config
986 doc_config = document.config
987
988 # get rid of an <title></title> if exists
989 if 'title' in config and config.get('title', None) is None:
990 del config['title']
991
992 # has to be done here as it affects auto generated titles
993 self.decode_datapoints(config['datapoint'])
994
995 relative_start_time = config.get('relative-start-time')
996 relative_stop_time = config.get('relative-stop-time')
997 absolute_start_time = config.get('absolute-start-time')
998 absolute_stop_time = config.get('absolute-stop-time')
999 x_axis = config.get('x-axis')
1000 if x_axis is not None:
1001 if len(x_axis) != 1:
1002 logging.warning('Multiple X-axes may not be interpreted correctly')
1003
1004 if 'relative-start-time' in x_axis[0]:
1005 relative_start_time = x_axis[0]['relative-start-time']
1006
1007 if 'relative-stop-time' in x_axis[0]:
1008 relative_stop_time = x_axis[0]['relative-stop-time']
1009
1010 if 'absolute-start-time' in x_axis[0]:
1011 absolute_start_time = x_axis[0]['absolute-start-time']
1012
1013 if 'absolute-stop-time' in x_axis[0]:
1014 absolute_stop_time = x_axis[0]['absolute-stop-time']
1015
1016 self.sensing_start, self.sensing_stop = self.get_time_range(
1017 sid=doc_config['sid'],
1018 sensing_start=doc_config['sensing_start'],
1019 sensing_stop=doc_config['sensing_stop'],
1020 absolute_start_time=absolute_start_time,
1021 absolute_stop_time=absolute_stop_time,
1022 relative_start_time=relative_start_time,
1023 relative_stop_time=relative_stop_time,
1024 )
1025
1026 # expand <title> macros
1027 if 'title' in config:
1028 expansions = Context(
1029 {
1030 'scid': document.config.get('sid'),
1031 'gs': document.config.get('gs'),
1032 'start': self.sensing_start,
1033 'stop': self.sensing_stop,
1034 }
1035 )
1036 self.title = Template(config['title']).render(expansions)
1037
1038 else:
1039 # generate a title if none is specified.
1040 if len(config['datapoint']) == 1:
1041 # for plots of a single parameter this is "field: description"
1042 if 'field' in config['datapoint'][0]:
1043 if config['datapoint'][0]['field'] is not None:
1044 self.title = '{field}: {desc}'.format(
1045 field=config['datapoint'][0]['field'].name,
1046 desc=to_htmlstr(
1047 config['datapoint'][0]['field'].description
1048 ),
1049 )
1050
1051 else:
1052 self.title = '{table} coverage'.format(
1053 table=config['datapoint'][0]['table'].name
1054 )
1055
1056 else:
1057 self.title = '{event} {prop}'.format(
1058 event=config['datapoint'][0]['event']['eventclass'],
1059 prop=config['datapoint'][0]['event']['property'],
1060 )
1061
1062 else:
1063 # otherwise we don't bother making up one for multiple datapoints
1064 self.title = 'No title set'
1065
1066 document.figures.append(self.title)
1067
1068 def get_time_range(
1069 self,
1070 sid,
1071 sensing_start,
1072 sensing_stop,
1073 absolute_start_time,
1074 absolute_stop_time,
1075 relative_start_time,
1076 relative_stop_time,
1077 ):
1078 """Choose the time range for this graph based on the report start and stop settings
1079 and the config settings absolute-start-time, relative-start-time and relative-stop-time.
1080 """
1081
1082 if relative_stop_time is not None:
1083 sensing_stop = sensing_start + relative_stop_time
1084
1085 if relative_start_time is not None:
1086 sensing_start += relative_start_time
1087
1088 if absolute_start_time is not None:
1089 if isinstance(absolute_start_time, datetime):
1090 sensing_start = absolute_start_time
1091
1092 else:
1093 sensing_start = texttime_to_datetime(absolute_start_time, sid=sid)
1094
1095 if absolute_stop_time is not None:
1096 if isinstance(absolute_stop_time, datetime):
1097 sensing_stop = absolute_stop_time
1098
1099 else:
1100 sensing_stop = texttime_to_datetime(absolute_stop_time, sid=sid)
1101
1102 if sensing_stop < sensing_start:
1103 sensing_stop = sensing_start + timedelta(seconds=1)
1104
1105 return sensing_start, sensing_stop
1106
1107 def get_legend_type(self, c):
1108 """Choose the legend type for this graph."""
1109 if c['legend'] == 'auto':
1110 if len(c['datapoint']) > 1:
1111 # if we have multiple data points place the legend outside the graph, to the right
1112 legend = 'outside-right'
1113 else:
1114 # if only one point is plotted no legend is needed
1115 legend = 'none'
1116
1117 else:
1118 # allow the user to specify legend position
1119 legend = c['legend']
1120
1121 return legend
1122
1123 def get_coverage(self, datapoints):
1124 """See if this is a data coverage plot.
1125 Coverage plots are selected by choosing "tablename.ROWCOUNT" as a field to plot
1126 i.e. <field>MHS_NEDT.ROWCOUNT</field>.
1127 Might change this to just <field>MHS_NEDT</field>
1128 to match point plots."""
1129
1130 # Check if this is a rowcount (coverage) plot
1131 coverage = False
1132 non_coverage = False
1133 for d in datapoints:
1134 # try:
1135 if (
1136 'field' in d
1137 and d['field'] is not None
1138 and 'ROWCOUNT' in d['field'].name.upper()
1139 ):
1140 coverage = True
1141
1142 else:
1143 non_coverage = True
1144
1145 if coverage and non_coverage:
1146 raise ConfigError(
1147 'Cannot plot both ROWCOUNT and non-ROWCOUNT data in the same plot'
1148 )
1149
1150 return coverage
1151
1152 def gen_all_limits_series(self, sid, start, stop, limits_config, datapoints):
1153 ret_series = []
1154
1155 if limits_config == 'none':
1156 return None
1157
1158 all_limits = self.get_all_limits(datapoints, sid, start, stop)
1159 try:
1160 self.assert_same_limits(all_limits)
1161 except DifferentLimitsError as e:
1162 logger.error(
1163 "Omitting Limits Plots, as parameters "
1164 f"{[dp['field'].name for dp in datapoints]} have "
1165 f"different limits: {e.limits1} vs {e.limits2}"
1166 )
1167 return
1168
1169 applicable_limits = all_limits[0]
1170
1171 if limits_config in ('ground', 'all'):
1172 logger.debug("Limits ground")
1173 if not applicable_limits["yellow"]:
1174 logger.info(
1175 "Plotting of Ground Limits requested but no ground yellow limits "
1176 "available for "
1177 f"{[dp['field'].name for dp in datapoints]}"
1178 )
1179 if not applicable_limits["red"]:
1180 logger.info(
1181 "Plotting of Ground Limits requested but no ground red limits "
1182 "available for "
1183 f"{[dp['field'].name for dp in datapoints]}"
1184 )
1185 ret_series.extend(
1186 self.gen_limit_series(applicable_limits["yellow"], "#FFD300")
1187 )
1188 ret_series.extend(
1189 self.gen_limit_series(applicable_limits["red"], "#FF0000")
1190 )
1191
1192 if limits_config in ('onboard', 'all'):
1193 logger.debug("Limits onboard")
1194 ob_limits = applicable_limits['onboard']
1195 if not ob_limits:
1196 logger.info(
1197 "Plotting of Onboard Limits requested but no onboard limits"
1198 "available for "
1199 f"{[dp['field'].name for dp in datapoints]}"
1200 )
1201
1202 plot_limits = self.gen_plottable_ob_limits(ob_limits, start, stop)
1203 ret_series.extend(
1204 self.gen_limit_series(plot_limits, "#000000", linestyle="dashdot")
1205 )
1206 return ret_series
1207
1208 def gen_limit_series(self, limits, color, linestyle="solid"):
1209 ret = []
1210 for limval, start, stop in limits:
1211 ret.append(
1212 Series(
1213 times=[start, stop],
1214 values=[limval, limval],
1215 line_colour=color,
1216 marker_colour=color,
1217 regions=[[0, 2]],
1218 line_type=linestyle,
1219 label="_nolegend_",
1220 )
1221 )
1222 return ret
1223
1224 def get_all_limits(self, datapoints, sid, start, stop):
1225 all_limits = []
1226 for dp in datapoints:
1227 result = retrieve_limits(
1228 sid,
1229 start,
1230 stop,
1231 True, # assuming its calibrated
1232 False, # stats are marked as false
1233 dp["field"],
1234 )
1235 current_limits = self.extract_limits(result, start, stop)
1236 current_limits['onboard'] = get_applicable_onboard_limits(
1237 dp["field"].name, sid.name, start, stop
1238 )
1239 all_limits.append(current_limits)
1240 return all_limits
1241
1242 def assert_same_limits(self, all_limits):
1243 limits_to_check_against = all_limits[0]
1244 for current_limits in all_limits[1:]:
1245 if current_limits != limits_to_check_against:
1246 raise DifferentLimitsError(current_limits, limits_to_check_against)
1247
1248 def extract_limits(self, result, start, stop):
1249 if(result['limits'] is None):
1250 return {}
1251
1252 red_limits = (result['limits'].red.low, result['limits'].red.high)
1253 yellow_limits = (result['limits'].yellow.low, result['limits'].yellow.high)
1254 return {
1255 'red': [(x, start, stop) for x in red_limits if x is not None],
1256 'yellow': [(x, start, stop) for x in yellow_limits if x is not None],
1257 }
1258
1259 def gen_plottable_ob_limits(self, ob_limits, start, stop):
1260 if not ob_limits:
1261 return []
1262 plot_limits = []
1263 for (low1, high1, t1), (low2, high2, t2) in zip(ob_limits, ob_limits[1:]):
1264 actual_start = t1 if t1 is not None else start
1265 plot_limits.append((low1, actual_start, t2))
1266 plot_limits.append((high1, actual_start, t2))
1267
1268 low, high, t = ob_limits[-1]
1269
1270 actual_start = t if t is not None else start
1271 plot_limits.append((low, actual_start, stop))
1272 plot_limits.append((high, actual_start, stop))
1273 return plot_limits
1274
1275 def choose_appearance(self, appearance, datapoints): # , sampling, subsampling):
1276 """Find the look and feel of this graph."""
1277 if appearance == 'auto':
1278 if self.get_coverage(datapoints):
1279 # data coverage plots are always bar charts
1280 result = 'bar'
1281
1282 elif len(datapoints) == 1:
1283 result = settings.DEFAULT_SINGLE_DATAPOINT_APPEARANCE
1284
1285 else:
1286 result = settings.DEFAULT_MULTIPLE_DATAPOINT_APPEARANCE
1287
1288 else:
1289 result = appearance
1290
1291 if result in ('min-max-avg', 'min-max'):
1292 if len(datapoints) != 1:
1293 raise ConfigError(
1294 'Cannot choose {app} plot with multiple data points (found '
1295 '{act}'.format(app=appearance, act=len(datapoints))
1296 )
1297
1298 logger.debug('Selected appearance of {a}'.format(a=result))
1299 return result
1300
1301 def markup_series(self, series, datapoint, marker_size, trendline, counter):
1302 """Set the non-data metadata values for seach series"""
1303
1304 if 'field' in datapoint:
1305 series.field = datapoint['field']
1306
1307 if 'table' in datapoint:
1308 series.table = datapoint['table']
1309
1310 if 'event' in datapoint:
1311 series.event = datapoint['event']
1312
1313 series.y_axis = datapoint.get('axis', datapoint.get('y-axis', None))
1314
1315 if 'colour' in datapoint:
1316 series.line_colour = datapoint['colour']
1317 series.marker_colour = datapoint['colour']
1318
1319 if 'label' in datapoint:
1320 series.label = datapoint['label']
1321
1322 if 'marker-colour' in datapoint:
1323 series.marker_colour = datapoint['marker-colour']
1324
1325 if 'edge-colour' in datapoint:
1326 series.edge_color = datapoint['edge-colour']
1327
1328 if series.line_colour is None:
1329 series.line_colour = flot_palette[counter % len(flot_palette)]
1330
1331 if series.marker_colour is None:
1332 series.marker_colour = flot_palette[counter % len(flot_palette)]
1333
1334 if 'marker-size' in datapoint:
1335 series.marker_size = datapoint['marker-size']
1336
1337 else:
1338 series.marker_size = marker_size
1339
1340 if 'marker' in datapoint:
1341 series.marker_type = datapoint['marker']
1342 else:
1343 series.marker_type = ''
1344
1345 if 'linestyle' in datapoint:
1346 series.line_type = datapoint['linestyle']
1347 series.linestyle = datapoint['linestyle']
1348
1349 if 'linewidth' in datapoint:
1350 series.line_size = datapoint['linewidth']
1351
1352 if datapoint['trendline'] or trendline:
1353 series.trendline = True
1354
1355 if 'bar-width' in datapoint and datapoint['bar-width'] != series.DEFAULT_BAR_WIDTH:
1356 series.bar_width = datapoint['bar-width']
1357
1358 def subsample(self, series, modulus): # width, force_modulus=None):
1359 """Subsample the data in `dp` (a single series) to fit `width`.
1360 We assume width is a count of pixels for the target output, and we want to make sure
1361 each subsampling bucket is less than 1 pixel wide.
1362
1363 Args:
1364 `series`: Series object ot process
1365 `width`: Output image width in pixels
1366
1367 Sample of tricky subsampling: L7025L, MSG1, 2002-08-01 to 2005-01-01
1368 """
1369
1370 # input number of samples
1371 # orig_len = len(series.times)
1372
1373 logging.debug(
1374 'Subsampling {orig} points by {mod}'.format(
1375 orig=len(series.times), mod=modulus
1376 )
1377 )
1378
1379 if modulus == 1:
1380 return None
1381
1382 # handle some empty table cases
1383 if series.values is not None and len(series.values) == 0:
1384 return
1385
1386 # You probably don't want to read this algorithm and I hope it never goes wrong
1387 # If it does go wrong, an alternative to fixing this is to implement subsampling
1388 # entirely at database level usign the slower but more accurate time bucket
1389 # method instead of using modulus, which is not really what we want as all anyway
1390
1391 # all-points data set, subsampled using the new per-region algorithm
1392 if series.values is not None:
1393 # we rebuild the regions struture giving new offsets after subsampling
1394 new_regions = []
1395 orig_times = series.times
1396 series.times = []
1397 # accumulator for current length of output arrays
1398 acc = 0
1399 for region in series.regions:
1400 # we trim the input samples down to an exact multiple of `modulus` first
1401 # (TBD this is bad because it's clipping data before gaps and at end of plot)
1402 region_len = region[1] - region[0]
1403
1404 full_buckets = region_len // modulus
1405 target_buckets = full_buckets + 1 # !
1406 # new_region_len = target_buckets * modulus
1407 # number of rows to add to input before subsampling it
1408 pad = (target_buckets * modulus) - region_len
1409
1410 # logging.debug('Region {region_len} target buckets {target_buckets} '
1411 # 'pad {pad}'.format(
1412 # region_len=region_len,
1413 # target_buckets=target_buckets,
1414 # pad=pad))
1415
1416 for dst, fn in (
1417 ('mins', np.nanmin),
1418 ('maxs', np.nanmax),
1419 ('avgs', np.nanmean),
1420 ):
1421 # d = dp['values'][region[0]:region[1]][:trim_len]
1422 # logging.debug('reshaping {d} dims {dd} to {ddd}'.format(d=d, dd=d.shape,
1423 # ddd=(new_len,localmod)))
1424 # nv = fn(dp['values'][region[0]:region[1]][:trim_len].reshape(
1425 # (new_len, localmod)), axis=1)
1426
1427 # nv = fn(
1428 # np.pad(
1429 # dp['values'][region[0]:region[1]],
1430 # (0, remainder),
1431 # 'constant',
1432 # constant_values=np.nan,).reshape(target_buckets, modulus), axis=1)
1433
1434 o = series.values[region[0] : region[1]]
1435 # dummy = nan_pad(o, pad).reshape(target_buckets, modulus)
1436 # nv = fn(dummy, axis=1)
1437
1438 nv = fn(nan_pad(o, pad).reshape(target_buckets, modulus), axis=1)
1439
1440 if series.has(dst):
1441 # a = series.get(dst)
1442 # b = np.concatenate(a, nv)
1443 # series.set(dst, b)
1444 series.set(dst, np.concatenate((series.get(dst), nv)))
1445
1446 else:
1447 series.set(dst, nv)
1448
1449 # there is presumably a neater way to extend a numpy array of datetimes
1450 # but I couldn't find it so...
1451 region_times = orig_times[region[0] : region[1]]
1452 pad_date = np.array((orig_times[region[1] - 1],))
1453 # y = np.pad(x, (0, pad), 'constant', constant_values=np.array((pad_date,)))
1454 new_times = region_times[::modulus]
1455 while len(new_times) < len(nv):
1456 new_times = np.append(new_times, pad_date)
1457
1458 series.times.extend(new_times)
1459
1460 # continue building the new `regions` array
1461 new_acc = acc + target_buckets
1462 new_regions.append([acc, new_acc])
1463 # logging.debug('added new values {nv} ({tv}) new times {nt} ({tt}) '
1464 # 'new region {nr}'.format(
1465 # nv=len(nv), nt=len(new_times), nr=(acc, new_acc), tv=len(dp[dst]),
1466 # tt=len(dp['times'])))
1467 acc = new_acc
1468
1469 # drop the original AP non-subsampled data
1470 series.values = None
1471 series.regions = new_regions
1472
1473 # stats data set
1474 else:
1475 orig_regions = series.regions
1476 orig_mins = series.mins
1477 orig_maxs = series.maxs
1478 orig_avgs = series.avgs
1479 orig_times = series.times
1480 series.mins = np.array([])
1481 series.maxs = np.array([])
1482 series.avgs = np.array([])
1483 series.times = []
1484 series.regions = []
1485 acc = 0
1486 for region in orig_regions:
1487 region_len = region[1] - region[0]
1488 full_buckets = region_len // modulus
1489 target_buckets = full_buckets + 1 # !
1490 pad = (target_buckets * modulus) - region_len
1491
1492 # logging.debug('Region {region_len} target buckets {target_buckets} '
1493 # 'pad {pad}'.format(
1494 # region_len=region_len,
1495 # target_buckets=target_buckets,
1496 # pad=pad))
1497
1498 for src, dst, fn in (
1499 (orig_mins, 'mins', np.nanmin),
1500 (orig_maxs, 'maxs', np.nanmax),
1501 (orig_avgs, 'avgs', np.nanmean),
1502 ):
1503 orig_data = src[region[0] : region[1]]
1504 new_data = fn(
1505 nan_pad(orig_data, pad).reshape(target_buckets, modulus), axis=1
1506 )
1507 series.set(dst, np.concatenate((series.get(dst), new_data)))
1508
1509 region_times = orig_times[region[0] : region[1]]
1510 pad_date = np.array((orig_times[region[1] - 1],))
1511 new_times = region_times[::modulus]
1512 while len(new_times) < len(new_data):
1513 new_times = np.append(new_times, pad_date)
1514
1515 series.times.extend(new_times)
1516
1517 # build the new `regions` array
1518 new_acc = acc + target_buckets
1519 series.regions.append([acc, new_acc])
1520 # logging.debug('added new values {nv} ({tv}) new times {nt} ({tt}) '
1521 # 'new region {nr}'.format(
1522 # nv=len(nv), nt=len(new_times), nr=(acc, new_acc), tv=len(dp[dst]),
1523 # tt=len(dp['times'])))
1524 acc = new_acc
1525
1526 # we trim the input samples down to an exact multiple of `modulus` first
1527 # trim_len = orig_len - orig_len % modulus
1528 # (TBD this is bad because it's clipping data before gaps and at end of plot)
1529
1530 # output length
1531 # new_len = trim_len // modulus
1532
1533 # logger.debug('Subsampling {orig} rows to target of {width} by selecting '
1534 # 'modulus of {mod}'.format(orig=orig_len, width=width, mod=modulus))
1535
1536 # series.avgs = series.avgs[:trim_len].reshape((new_len, modulus)).mean(axis=1)
1537
1538 # if series.mins is not None:
1539 # series.mins = series.mins[:trim_len].reshape((new_len, modulus)).min(axis=1)
1540 # series.maxs = series.maxs[:trim_len].reshape((new_len, modulus)).max(axis=1)
1541
1542 # reduce the times (suspect this breaks with <1week plots?)
1543 # series.times = series.times[::modulus][0:new_len]
1544
1545 # this block needs rerwriting to handle regions properly. Here we recompute
1546 # the intra-gap regions according to the new modulus but it's basically
1547 # a wing and a prayer
1548 # series.regions = [[r[0] // modulus, r[1] // modulus] for r in series.regions]
1549
1550 # the gap threshold increases
1551 # if series.gap_threshold is not None:
1552 # series.gap_threshold *= modulus
1553
1554 # logger.debug('subsample final len {cc}'.format(cc=len(dp['times'])))
1555 # logger.info('Resample modulus {mod} reduced {orig} to {new} rows in {cc} regions'.format(
1556 # mod=modulus, orig=orig_len, new=len(series.times), cc=len(series.regions)))
1557
1558 return modulus
1559
1560 def get_filename(self, filename, datapoint, document):
1561 """Return a filename for the primary output of this plot.
1562 If the user supplied one, use that.
1563 Otherwise for a single parameter plot use the parameter name,
1564 with duplicate avoidance.
1565
1566 Args:
1567 `filename` (Path): Base file name.
1568 `datapoint` (dict): Information about the data to be plotted.
1569 `document` (Document): document object.
1570 """
1571
1572 if filename is not None:
1573 return filename
1574
1575 if len(datapoint) == 1:
1576 # this is a plot of a single datapoint
1577 if 'field' in datapoint[0]:
1578 if datapoint[0]['field'] is None:
1579 result = '{table}{ext}'.format(
1580 table=datapoint[0]['table'].name, ext=Presentation.DEFAULT_EXT
1581 )
1582
1583 else:
1584 # single datapoint is time series
1585 result = '{field}{ext}'.format(
1586 field=datapoint[0]['field'].name, ext=Presentation.DEFAULT_EXT
1587 )
1588
1589 else:
1590 # single datapoint is an event property
1591 if datapoint[0]['event']['property'] is not None:
1592 result = '{prop}{ext}'.format(
1593 prop=datapoint[0]['event']['property']['name'],
1594 ext=Presentation.DEFAULT_EXT,
1595 )
1596
1597 else:
1598 result = '{evt}{ext}'.format(
1599 evt=datapoint[0]['event']['eventclass'].name,
1600 ext=Presentation.DEFAULT_EXT,
1601 )
1602
1603 else:
1604 result = DEFAULT_FILENAME.format(
1605 number=document.figure_count + 1, ext=Presentation.DEFAULT_EXT
1606 )
1607
1608 # convert it to a report specific filename
1609 filename = document.theme.mod_filename(document, result)
1610
1611 # check for duplicates
1612 # tidy this up - str<>Path has really messed up this alg
1613 # because "a in b" failed if a is a str and b a list of Path
1614 a = [str(a) for a in document.aux_files]
1615 if str(filename) in a:
1616 suffix = 1
1617 filename_without_ext = (
1618 filename.stem
1619 ) # remove extension to the complete filename
1620
1621 while (
1622 str(
1623 '{filename}_{suffix}{ext}'.format(
1624 filename=filename_without_ext,
1625 suffix=suffix,
1626 ext=Presentation.DEFAULT_EXT,
1627 )
1628 )
1629 in a
1630 ):
1631 suffix += 1
1632
1633 filename = Path(
1634 '{filename}_{suffix}{ext}'.format(
1635 filename=filename_without_ext,
1636 suffix=suffix,
1637 ext=Presentation.DEFAULT_EXT,
1638 )
1639 )
1640
1641 return filename
1642
1643 def choose_sampling(
1644 self,
1645 appearance,
1646 sid,
1647 sensing_start,
1648 sensing_stop,
1649 datapoint,
1650 requested_sampling,
1651 requested_subsampling,
1652 requested_region,
1653 # requested_modulus,
1654 width,
1655 ):
1656 """Decide whether to do an `all-points` or `stats` plot.
1657 If requested sampling is `auto`, return `all-points` if plot is short enough,
1658 otherwise `stats`.
1659
1660 Args:
1661 sid (SID): Source ID
1662 sensing_start (datetime):
1663 sensing_stop (datetime):
1664 datapoint (dict): configuration from widget XML.
1665 Pre-decoded so 'table' is a TableInfo if TS read, 'field' is a fieldinfo
1666 if non-rowcount plot, 'event' is EventInfo if events. 'mod' set to a string
1667 if modification requested.
1668 requested_sampling (str): User preference
1669 requested_subsampling (str): User preference
1670 requested_region (str): Allow user to forse stats region
1671 request_modulus (None or int):
1672 width (int): Width of zoomed graph
1673
1674 Returns:
1675 sampling (Sampling)
1676 modulus (int)
1677 region (str or None)
1678 """
1679 if 'event' in datapoint:
1680 return Sampling.FIT
1681
1682 Statistics = object()
1683
1684 # Basic user preference
1685 if requested_sampling == 'auto':
1686 # we could do a ts.count() here instead
1687 if (sensing_stop - sensing_start) < STATS_CUTOFF:
1688 sampling = Sampling.FIT
1689
1690 else:
1691 sampling = Statistics
1692
1693 elif requested_sampling == 'orbital-stats' or requested_sampling == 'stats':
1694 sampling = Statistics
1695
1696 elif requested_sampling == 'all-points' or requested_sampling == 'daily-totals':
1697 if requested_sampling == 'auto' or requested_subsampling == 'subsampled':
1698 sampling = Sampling.FIT
1699
1700 else:
1701 sampling = Sampling.ALL_POINTS
1702
1703 else:
1704 raise ValueError('unknown sampling')
1705
1706 # switch off stats for sources that don't have them
1707 if sampling is Statistics:
1708 if 'event' in datapoint:
1709 sampling = Sampling.FIT
1710
1711 elif not datapoint['table'].has_stats:
1712 sampling = Sampling.FIT
1713
1714 # switch off AP subsampling if the user doesn't want it
1715 if sampling is Sampling.FIT and requested_subsampling == 'none':
1716 sampling = Sampling.ALL_POINTS
1717
1718 # no subsampling for scatter plots unless the user really wants it
1719 if appearance == 'scatter' and requested_subsampling == 'auto':
1720 sampling = Sampling.ALL_POINTS
1721
1722 # Now see about subsamping modulus factor
1723 # if sampling is Sampling.FIT or Statistics:
1724 # or not
1725 # modulus = True
1726 # else:
1727 # modulus = None
1728
1729 # Pick the region if we're using stats
1730 if sampling is Statistics:
1731 if requested_region is not None:
1732 sampling = sampling_from_name(requested_region)
1733
1734 else:
1735 # plot duration
1736 plot_seconds = (sensing_stop - sensing_start).total_seconds()
1737
1738 # determine the best sampling rate for graph plot
1739 for sampling in reversed(sid.auto_sampling_options()):
1740 region_seconds = region_duration(sampling, sid=sid).total_seconds()
1741 regions_per_series = plot_seconds // region_seconds
1742 if regions_per_series > width:
1743 break
1744
1745 logger.debug(
1746 'Sampling of {sampling} (stats {stats})'.format(
1747 sampling=sampling.name, stats=sampling is Statistics
1748 )
1749 )
1750
1751 return sampling
1752
1753 def remove_nonfinite(self, series):
1754 """Locate non finite (NaN or Inf) values in 'values', 'mins' or 'maxs' and remove those
1755 entries from all arrays."""
1756
1757 if series.values is not None:
1758 # this looks like a bug in numpy but np.isfinite doesn't catch all the NaNs
1759 # in one dataset (EPS field UZNBRDEB for 2017-08) so we do it with another method
1760 # that is slower but for some reason more reliable
1761 # keep = np.isfinite(series.values)
1762 keep = [math.isfinite(v) for v in series.values]
1763
1764 else:
1765 # keep = np.logical_or(np.isfinite(series.mins), np.isfinite(series.maxs))
1766 # the following algorithm is appallingly inefficient and messy but appears to be needed
1767 # on the current system (python-3.6.2, numpy-1.13.1, centos-7) because every other
1768 # method of detecting NaN values fails and never detects them:
1769 keep = []
1770 inf_nan = ('inf', 'nan')
1771 for i in range(len(series.times)):
1772 keep.append(
1773 str(series.maxs[i]) not in inf_nan
1774 and str(series.mins[i]) not in inf_nan
1775 )
1776
1777 for s in ('mins', 'avgs', 'maxs', 'orbits', 'times', 'rowcounts', 'values'):
1778 if getattr(series, s, None) is not None:
1779 setattr(series, s, np.compress(keep, getattr(series, s)))
1780
1781 if len(keep) != len(series.times):
1782 logging.info(
1783 'Finite filter reduced {old} points to {new}'.format(
1784 old=len(keep), new=len(series.times)
1785 )
1786 )
1787
1788 def remove_anomalies(self, series, anomaly_threshold):
1789 """Strip out values that look like anomalies.
1790
1791 Detection of based on values more than `anomaly_threshold` standard deviations away
1792 from the mean.
1793 Only works for stats or subsampled data."""
1794
1795 logging.debug(
1796 'Removing anomalies with threshold {thresh}'.format(
1797 thresh=anomaly_threshold
1798 )
1799 )
1800
1801 if series.mins is not None:
1802 if len(series.mins) == 0:
1803 return
1804
1805 # this means mins/avgs/maxs are all present. Could be AP subs or stats data
1806 # find the min/std of mins
1807 # min_std = np.nanstd(dp['mins'])
1808 # min_avg = np.nanmean(dp['mins'])
1809 min_std = np.std(series.mins)
1810 min_avg = np.mean(series.mins)
1811
1812 # find the max/std of maxs
1813 # max_std = np.nanstd(dp['maxs'])
1814 # max_avg = np.nanmean(dp['maxs'])
1815 max_std = np.std(series.maxs)
1816 max_avg = np.mean(series.maxs)
1817
1818 # build an array of bools where a True means keep that index, False means strip it
1819 cond = (
1820 # (np.isfinite(dp['maxs'])) &
1821 # (np.isfinite(dp['mins'])) &
1822 (series.maxs < (max_avg + anomaly_threshold * max_std))
1823 & (series.maxs > (max_avg - anomaly_threshold * max_std))
1824 & (series.mins < (min_avg + anomaly_threshold * min_std))
1825 & (series.mins > (min_avg - anomaly_threshold * min_std))
1826 )
1827
1828 # apply `cond` to all arrays in the datapoint
1829 precount = len(series.mins)
1830 for s in ('mins', 'avgs', 'maxs', 'orbits', 'times', 'rowcounts', 'values'):
1831 if series.has(s):
1832 series.set(s, np.compress(cond, series.get(s)))
1833
1834 logging.info(
1835 'Anomaly removal reduced {pre} points to {post}'.format(
1836 pre=precount, post=len(series.mins)
1837 )
1838 )
1839
1840 def reduce_regions(index):
1841 """Each time we remove an anomalous data point, shift all the regions.
1842
1843 Moving values down by one."""
1844 for r in series.regions:
1845 if r[0] > index:
1846 r[0] -= 1
1847
1848 if r[1] > index:
1849 r[1] -= 1
1850
1851 try:
1852 if self.config['anomaly-response'] == 'remove_histogram':
1853 cond = self.find_spikes_histogram(series)
1854 else:
1855 cond = self.find_spikes(series)
1856
1857 # apply `cond` to all arrays in the datapoint
1858 precount = len(series.mins)
1859 for s in ('mins', 'avgs', 'maxs', 'orbits', 'times', 'rowcounts', 'values'):
1860 if series.has(s):
1861 series.set(s, np.compress(cond, series.get(s)))
1862
1863 logging.info('Anomaly removal reduced {pre} points to {post}'.format(
1864 pre=precount, post=len(series.mins)))
1865
1866 # reset the regions
1867 i = 0
1868 for cc, val in enumerate(cond):
1869 if not val:
1870 reduce_regions(cc - i)
1871 i += 1
1872
1873 except ValueError as err:
1874 logging.warning(err)
1875
1876 elif series.avgs is not None:
1877 # actually I'm not sure this happens
1878 logging.info('Remove anomalies not implemented for avg only plots')
1879 return
1880
1881 elif series.values is not None:
1882 logging.info('Remove anomalies not implemented for AP non-subsampled data')
1883 return
1884
1885 def find_spikes(self, series):
1886 """ Finds spikes in series using a avg/std-based approach. """
1887 anomaly_threshold = self.config['anomaly-threshold']
1888
1889 # this means mins/avgs/maxs are all present. Could be AP subs or stats data
1890 # find the min/std of mins
1891 min_std = np.std(series.mins)
1892 min_avg = np.mean(series.mins)
1893
1894 # find the max/std of maxs
1895 max_std = np.std(series.maxs)
1896 max_avg = np.mean(series.maxs)
1897
1898 # build an array of bools where a True means keep that index, False means strip it
1899 return (
1900 (series.maxs < (max_avg + anomaly_threshold * max_std)) &
1901 (series.maxs > (max_avg - anomaly_threshold * max_std)) &
1902 (series.mins < (min_avg + anomaly_threshold * min_std)) &
1903 (series.mins > (min_avg - anomaly_threshold * min_std)))
1904
1905 def find_spikes_histogram(self, series):
1906 """ Finds spikes in series using a histogram-based approach. """
1907 bin_ratio = self.config['anomaly-bin-ratio']
1908 cutoff_percentage = self.config['anomaly-cutoff-percentage']
1909
1910 logger.info(f'Finding spikes using histogram-based approach '
1911 f'with bin-ratio = {bin_ratio} and cutoff-percentage = {cutoff_percentage}')
1912
1913 max_bin_ratio = len(series.times) // MIN_BINS
1914 if not (1 < bin_ratio < max_bin_ratio):
1915 raise ValueError('The bin ratio is not within the valid value range '
1916 f'between 1 and {max_bin_ratio}')
1917
1918 if not (0 < cutoff_percentage < 50):
1919 raise ValueError('The cutoff percentage is not within the valid value range '
1920 'between 0 and 50 percent')
1921
1922 bins = len(series.times) // bin_ratio
1923 max_dp_remove = int(len(series.times) * cutoff_percentage / 100)
1924
1925 if bins < MIN_BINS or max_dp_remove < 1:
1926 raise ValueError('Cannot remove anomalies with current settings '
1927 f'({bins} bins, max points to remove: {2 * max_dp_remove})')
1928
1929 logging.info(f'Removing up to {2 * max_dp_remove} anomalous points using {bins} histogram bins')
1930
1931 limit_low, limit_high = self.calculate_limits(series, bins, max_dp_remove)
1932 logging.info(f'Limits used for spike removal: low = {limit_low}, high = {limit_high}')
1933
1934 return ((series.mins >= limit_low) &
1935 (series.maxs <= limit_high))
1936
1937 def calculate_limits(self, series, bins, max_dp_remove):
1938 """ Creates a histogram from mins / maxs and finds the lower / upper limit
1939 that separates data from potential outliers. """
1940 histogram_min = np.histogram(series.mins, bins=bins)
1941 histogram_max = np.histogram(series.maxs, bins=bins)
1942 limit_low = self.find_lower_limit(histogram_min, max_dp_remove)
1943 limit_high = self.find_upper_limit(histogram_max, max_dp_remove)
1944 return limit_low, limit_high
1945
1946 def find_lower_limit(self, histogram, max_dp_remove):
1947 """ Finds the value that separates data from potential outliers in the lower value range. """
1948 bins, thresholds = histogram
1949 dp_removed = 0
1950 for low_id, bin_content in enumerate(bins):
1951 dp_removed += bin_content
1952 if dp_removed > max_dp_remove:
1953 break
1954 return thresholds[low_id]
1955
1956 def find_upper_limit(self, histogram, max_dp_remove):
1957 """ Finds the value that separates data from potential outliers in the upper value range. """
1958 bins, thresholds = histogram
1959 dp_removed = 0
1960 for high_id in reversed(range(len(bins))):
1961 dp_removed += bins[high_id]
1962 if dp_removed > max_dp_remove:
1963 high_id += 1
1964 break
1965 return thresholds[high_id]
1966
1967 def find_series_range(self, series, anomaly_threshold):
1968 """Find the min and max values in a series."""
1969
1970 # subsampled or stats data
1971 if series.mins is not None:
1972 # anomaly clip algorithm
1973 if anomaly_threshold is not None:
1974 # logging.debug('find y axis mins anom')
1975 # we disregard any points more than anomaly_threshold
1976 # standard deviations from mean
1977 min_std = np.std(series.mins)
1978 min_avg = np.mean(series.mins)
1979 max_std = np.std(series.maxs)
1980 max_avg = np.mean(series.maxs)
1981
1982 if not np.isnan(min_std): # no need to test all 4
1983 # bool array identifying buckets inside and outside the thresholds
1984 # True means keep that index, False means strip it
1985 cond = (
1986 (series.maxs < (max_avg + anomaly_threshold * max_std))
1987 & (series.maxs > (max_avg - anomaly_threshold * max_std))
1988 & (series.mins < (min_avg + anomaly_threshold * min_std))
1989 & (series.mins > (min_avg - anomaly_threshold * min_std))
1990 )
1991
1992 clip_mins = np.compress(cond, series.mins)
1993 if len(clip_mins) > 0:
1994 start = np.nanmin(clip_mins)
1995
1996 else:
1997 # all points failed the anomaly test. This can happen if they're
1998 # all the same
1999 start = None
2000
2001 clip_maxs = np.compress(cond, series.maxs)
2002 if len(clip_maxs) > 0:
2003 stop = np.nanmax(clip_maxs)
2004
2005 else:
2006 stop = None
2007
2008 else:
2009 start = None
2010 stop = None
2011
2012 # just some info
2013 unclipped_limits = (np.nanmin(series.mins), np.nanmax(series.maxs))
2014 logging.info(
2015 'Anomaly limit processing gives limits of {c1}-{c2} instead of '
2016 '{unclipped}'.format(c1=start, c2=stop, unclipped=unclipped_limits)
2017 )
2018
2019 else:
2020 # simple clip algorithm
2021 start = np.nanmin(series.mins)
2022 stop = np.nanmax(series.maxs)
2023 logging.debug(
2024 'Range of {f} is {mn} to {mx}'.format(
2025 f=name_of_thing(series.field), mn=start, mx=stop
2026 )
2027 )
2028
2029 elif series.avgs is not None:
2030 # this can happen in subsampled line plots
2031
2032 # Otherwise it's a stats plot so use that
2033 # Note, we should actually just use 'avgs' here if only
2034 # averages are to be plotted.
2035
2036 start = np.nanmin(series.avgs)
2037 stop = np.nanmax(series.avgs)
2038
2039 else:
2040 if anomaly_threshold is not None:
2041 avg = np.nanmean(series.values)
2042 std = np.nanstd(series.values)
2043
2044 # np.isfinite also works
2045 v = series.values[np.logical_not(np.isnan(series.values))]
2046
2047 mn = avg - anomaly_threshold * std
2048 mx = avg + anomaly_threshold * std
2049 logging.info(
2050 'Avg {a} std {s} min {mn} max {mx}'.format(
2051 a=avg, s=std, mn=mn, mx=mx
2052 )
2053 )
2054
2055 cond = (v < mx) & (v > mn)
2056
2057 start = np.nanmin(np.compress(cond, v))
2058 stop = np.nanmax(np.compress(cond, v))
2059
2060 # just for info
2061 unclipped_limits = (np.nanmin(series.values), np.nanmax(series.values))
2062
2063 logging.info(
2064 'Anomaly limit processing gives limits of {clipped} instead of '
2065 '{unclipped}'.format(
2066 clipped=(start, stop), unclipped=unclipped_limits
2067 )
2068 )
2069
2070 else:
2071 start = np.nanmin(series.values)
2072 stop = np.nanmax(series.values)
2073
2074 return start, stop
2075
2076 def find_y_axis_limits(self, y_axis, all_series, config):
2077 """Compute the actual min and max for `y-axis`, writing to the `start` and `stop` members.
2078
2079 `config` is the dict for this axis if explicit.
2080
2081 """
2082 # logging.debug('Setting y axis limits for {y}'.format(y=y_axis))
2083
2084 # default (extendable) min start value
2085 start = config.get('default-min')
2086 # default (extendable) max stop value
2087 stop = config.get('default-max')
2088 # we may be configured to ignore anomalous values when computing axis limits
2089 anomaly_threshold = config.get('anomaly-threshold')
2090 # add a bit on so the data doesn't touch the stop or bottom of plot
2091 auto_axis_margin = config.get('margin', DEFAULT_AUTO_AXIS_MARGIN)
2092
2093 logging.debug(
2094 'Limits for {axis} default {strt} stop {stop} anomaly {thresh} '
2095 'margin {margin}'.format(
2096 axis=y_axis.position.name,
2097 strt=start,
2098 stop=stop,
2099 thresh=anomaly_threshold,
2100 margin=auto_axis_margin,
2101 )
2102 )
2103
2104 # first pass: make sure we have sufficient entries in the y_limits array
2105 # (currently the rendering code will only handle a max of 2 entries)
2106 for series in all_series:
2107 if series.y_axis is not y_axis:
2108 continue
2109
2110 if len(series.times) == 0:
2111 continue
2112
2113 series_range = self.find_series_range(series, anomaly_threshold)
2114 start = nvl_min(start, series_range[0])
2115 stop = nvl_max(stop, series_range[1])
2116
2117 # we now have start, stop based on the data
2118
2119 # allow user to force (series.start/stop already set earlier)
2120 if y_axis.start is not None:
2121 start = y_axis.start
2122
2123 if y_axis.stop is not None:
2124 stop = y_axis.stop
2125
2126 logging.debug('found start {strt} stop {stop}'.format(strt=start, stop=stop))
2127
2128 # handle empty plots
2129 if start is None or stop is None:
2130 start = 0
2131 stop = 0
2132
2133 # handle plots where all data has the same value
2134 elif start == stop:
2135 start -= 0.5
2136 stop += 0.5
2137
2138 # just move the axis limits out by 5%
2139 elif stop != float('inf'):
2140 # !!! np.isinfinite doesn't work here!!!
2141 # charteps report -t test_scaomac1 -s m01 --start 2015-05-01 --stop 2015-06-01
2142 # --db argus-val-enduser -o ~/tmp/report/aocs
2143 # fails if we use np.isinfinite because it is a special type of infinity that
2144 # numpy doesn't recognise (?!)
2145 av = (start + stop) / 2
2146 dx = stop - av
2147 mdx = dx * (1 + auto_axis_margin)
2148 start = av - mdx
2149 stop = av + mdx
2150
2151 y_axis.start = start
2152 y_axis.stop = stop
2153 logging.debug(
2154 'axis margin {marg} gives start {strt} stop {stop}'.format(
2155 marg=auto_axis_margin, strt=y_axis.start, stop=y_axis.stop
2156 )
2157 )
2158
2159 def normalise(self, all_series):
2160 """Align all datapoints so their first value is zero by adding or subtracting
2161 a fixed value to every point in each series.
2162 """
2163
2164 for series in all_series:
2165 if series.values is not None:
2166 attrs = ('values',)
2167
2168 else:
2169 # Make sure we fix the mins and maxs (even if only avgs might be plotted)
2170 # so the axis limits get computed properly.
2171 attrs = ('avgs', 'mins', 'maxs')
2172
2173 if len(series.get(attrs[0])) == 0:
2174 continue
2175
2176 for attr in attrs:
2177 # logger.debug('Normalising by {factor}'.format(factor=dp[key][0]))
2178 values = series.get(attr)
2179 values -= series.get(attr)[0]
2180
2181 def assign_y_axis_labels(self, y_axis, all_series):
2182 """Work out the y-axis labels.
2183 We start by using the field units as labels, if possible.
2184 If it has no unit use the field name.
2185 After processing all datapoints, if we find one axis would need 2 different
2186 labels (mixed units for example) we deactivate the label as it would be confusing.
2187 If the datapoint has but <axis> and <label> that is used instead.
2188 Otherwise look for widget ylabels.
2189
2190 This should be replaced by a better system using a proper axis
2191 configuration in the report template.
2192
2193 Must be called after referencing y-axis objects.
2194 """
2195 if y_axis.label is not None:
2196 return
2197
2198 # find all units used to plot this axis
2199 units = set()
2200 for series in all_series:
2201 if series.y_axis != y_axis:
2202 continue
2203
2204 if series.field is not None:
2205 if not series.calibrated:
2206 units.add(series.field.name)
2207 continue
2208
2209 # if series['field'] is None:
2210 # rowcounts
2211 # units.add('Count')
2212
2213 if series.field.unit is not None:
2214 units.add(series.field.unit)
2215
2216 else:
2217 units.add(series.field.name)
2218
2219 elif series.event is not None:
2220 if series.event['property'] is not None:
2221 if 'unit' in series.event['property']:
2222 units.add(series.event['property']['unit'])
2223
2224 elif series.event['property']['type'] == 'duration':
2225 units.add('s')
2226
2227 else:
2228 units.add('Event')
2229
2230 y_axis.label = ', '.join(units)
2231
2232 def choose_show_orbits(self, show_orbits_request, sid, height):
2233 """Decide if we will show orbit numbers along the top of the output."""
2234
2235 if not hasattr(sid, 'orbit') or sid.orbit is None:
2236 return False
2237
2238 # Choose whether to show orbits along top axis if the user did not force choice
2239 if show_orbits_request == 'auto':
2240 # if the user left <show-orbits> at the default setting of 'auto' then
2241 # we decide whether to show orbits based on the duration of this report
2242 if height > SHOW_ORBITS_THRESHOLD:
2243 # It is a long report so show orbits
2244 show_orbits = True
2245
2246 else:
2247 # It is a short term report so don't show orbits
2248 show_orbits = False
2249
2250 else:
2251 show_orbits = show_orbits_request == 'true'
2252
2253 return show_orbits
2254
2255 def decode_datapoints(self, datapoints):
2256 """Preprocessing of datapoint field and event config items
2257
2258 Examine all field and event options.
2259 Convert the strings to FieldInfo and EventClass objects.
2260 Fill `modifier`s.
2261 We cannot set the datatype of our field options to Field since users can enter
2262 formulae e.g. "<fieldname>NEDT_H2 * 3</fieldname>"."""
2263
2264 for dp in datapoints:
2265 # if dp['field'] contains only a field name, we replace the original string
2266 # with a FieldInfo object.
2267 # If it is a computed field we set:
2268 # dp['mod'] :: The text as entered by the user
2269 # dp['field'] :: The FieldInfo
2270 # dp['fieldname'] :: The field name as a string including tablename
2271
2272 if 'field' in dp: # or 'event' in dp
2273 fieldname = ''
2274 # We extract the field name from the complete calculation
2275 reading = False
2276 for c in dp['field']:
2277 if not reading and (
2278 (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z')
2279 ):
2280 # Look for the start of the field name text
2281 reading = True
2282
2283 if reading:
2284 if c in (' ', '+', '-', '*', '/', '^'):
2285 # Stop scanning if we hit something that cannot be a field name
2286 break
2287
2288 else:
2289 fieldname += c
2290
2291 if fieldname != dp['field']:
2292 # we have a computed series
2293 dp['fieldname'] = fieldname
2294 dp['mod'] = dp['field']
2295 logger.debug(
2296 'Detected field name {fieldname} mod {mod}'.format(
2297 fieldname=fieldname, mod=dp['field']
2298 )
2299 )
2300
2301 # try:
2302 dp['field'] = str_to_field(fieldname)
2303 dp['table'] = dp['field'].table
2304
2305 def find_regions(self, times, threshold, trim=0):
2306 """Examine list of `times` and look for gaps greater than `threshold`.
2307
2308 Yield a list of non-gap ranges.
2309 """
2310 # logger.debug('gaps threshold {t}'.format(t=threshold))
2311 # if x-axis is not time, do not look for gaps
2312 if len(times) == 0 or not isinstance(times[0], datetime):
2313 return
2314
2315 if threshold is None:
2316 # disable gap check
2317 yield [0, len(times) - trim]
2318 return
2319
2320 logger.info(
2321 'Computing regions with gap threshold {thresh} trim {trim}'.format(
2322 thresh=threshold, trim=trim
2323 )
2324 )
2325 prev_t = None
2326 start = 0
2327 region_count = 0
2328 for cc, t in enumerate(times):
2329 # logging.debug('{cc} {time} {delta}'.format(cc=cc, time=t, delta=not prev_t or (t-prev_t)))
2330 if prev_t is not None and (t - prev_t) > threshold:
2331 region_start = start
2332 region_stop = cc - 1
2333 record_stop = cc
2334 if region_count < MAX_SHOW_REGIONS:
2335 logger.info(
2336 'Region ({cc1}-{cc2}) {strt} to {stop} record {record} jump {jump}'.format(
2337 cc1=region_start,
2338 cc2=region_stop,
2339 strt=times[region_start],
2340 stop=times[region_stop],
2341 record=(region_start, record_stop),
2342 jump=t,
2343 )
2344 )
2345
2346 elif region_count == MAX_SHOW_REGIONS:
2347 logger.info('Suppressing additional regions')
2348
2349 yield [region_start, record_stop]
2350 region_count += 1
2351 start = cc
2352
2353 prev_t = t
2354
2355 if prev_t is not None:
2356 logger.debug(
2357 'Final region ({cc1}-{cc2}) {strt} to {stop}'.format(
2358 cc1=start, cc2=cc + 1, strt=times[start], stop=times[cc]
2359 )
2360 )
2361 yield [start, cc + 1 - trim]
2362 region_count += 1
2363
2364 logger.info(
2365 'Gap scan identified {rc} regions across {cc} times'.format(
2366 rc=region_count, cc=cc
2367 )
2368 )
2369
2370 def apply_mods(self, series, datapoint):
2371 """If any fields use computed data series, calculate the final values now."""
2372
2373 if datapoint.get('mod') is not None:
2374 # Note, this replacement may not be needed
2375 command = datapoint['mod'].replace(datapoint['field'].name, 'x')
2376
2377 # If the user specifies a modification of a fully qualified field name, strip out the
2378 # table part now
2379 if datapoint['table'].name in datapoint['mod']:
2380 command = command.replace('{}.'.format(datapoint['table'].name), '')
2381
2382 if series.values is not None:
2383 # use the sandboxed version of `eval` to block user code from access
2384 # to anything except timeseries data points
2385 series.values = eval(
2386 command, {'__builtins__': None}, {'x': series.values}
2387 )
2388 else:
2389 series.avgs = eval(command, {'__builtins__': None}, {'x': series.avgs})
2390 # ? why can we have avgs but not mins or maxs?
2391 if series.mins is not None:
2392 series.mins = eval(
2393 command, {'__builtins__': None}, {'x': series.mins}
2394 )
2395
2396 if series.maxs is not None:
2397 series.maxs = eval(
2398 command, {'__builtins__': None}, {'x': series.maxs}
2399 )
2400
2401 def fix_axes_references(self, all_series, x_axes, y_axes):
2402 """Convert each series `x_axis` and `y_axis` attributes to pointers to Axis
2403 objects."""
2404
2405 for series in all_series:
2406 if (
2407 series.y_axis is None
2408 or series.y_axis == 'left'
2409 or series.y_axis == 1
2410 or series.y_axis is Axis.Position.LEFT
2411 ):
2412 series.y_axis = y_axes[0]
2413
2414 else:
2415 series.y_axis = y_axes[1]
2416
2417 series.x_axis = x_axes[0]
2418
2419 def post_retrieve(self, all_series):
2420 """Derived clases like HIRSNedn may do something."""
2421 pass
2422
2423 def oversample(self, sensing_start, sensing_stop, oversampling):
2424 if oversampling is not None and not fp_eq(oversampling):
2425 # timedelta doesn't allow multiplications
2426 delta = (sensing_stop - sensing_start).total_seconds()
2427 extra_time = max(MIN_OVERSAMPLE, timedelta(seconds=delta * oversampling))
2428 # logger.debug('oversampling {o} delta {d} extra_time {e}'.format(
2429 # o=oversampling, d=delta, e=extra_time))
2430 sensing_start -= extra_time
2431 sensing_stop += extra_time
2432 # logging.info('Oversampling by {extra} to {strt} - {stop}'.format(
2433 # extra=extra_time, strt=sensing_start, stop=sensing_stop))
2434
2435 return sensing_start, sensing_stop
2436
2437 def html(self, document):
2438 """Render a graph and output document HTML."""
2439 config = self.config
2440 dc = document.config
2441
2442 # we will retrieve data from the report sid, unless this widget
2443 # has a <scid> (scidfree) parameter which overrides the report data source
2444 if 'scid' in config:
2445 sid = SID(config['scid']) # scidfree
2446
2447 else:
2448 sid = dc['sid']
2449
2450 sensing_start = self.sensing_start
2451 sensing_stop = self.sensing_stop
2452
2453 if sensing_start > sensing_stop:
2454 sensing_start = sensing_stop
2455 logging.info(
2456 'Adjusting sensing start time to {t} to avoid reversal'.format(
2457 t=sensing_start
2458 )
2459 )
2460
2461 # Choose a legend type
2462 legend = self.get_legend_type(config)
2463
2464 # Determine the look and feel of this graph
2465 appearance = self.choose_appearance(config['appearance'], config['datapoint'])
2466
2467 # determine normal dimensions
2468 width = config['width']
2469 if 'height' in config:
2470 height = config['height']
2471
2472 else:
2473 height = int(width // DEFAULT_ASPECT_RATIO)
2474
2475 # Determine full sized (zoomed / pdf) dimensions
2476 zoom_width = int(width * DEFAULT_ZOOM_FACTOR)
2477 zoom_height = None
2478 if 'zoom' in config:
2479 if 'width' in config['zoom']:
2480 zoom_width = config['zoom']['width']
2481
2482 if 'height' in config['zoom']:
2483 zoom_height = config['zoom']['height']
2484
2485 if zoom_height is None:
2486 # the int seems to be important...
2487 unzoomed_aspect = width / height
2488 logging.debug('unzoomed aspect ' + str(unzoomed_aspect))
2489 zoom_height = int(zoom_width // unzoomed_aspect)
2490
2491 # handle legacy sizing
2492 if 'thumbnail-width' in config:
2493 zoom_width = width
2494 zoom_height = height
2495 width = config['thumbnail-width']
2496 if 'thumbnail-height' in config:
2497 height = config['thumbnail-height']
2498
2499 else:
2500 height = int(width // DEFAULT_ASPECT_RATIO)
2501
2502 logging.debug(
2503 'unzoomed size {w1} x {h1} zoomed size {w2} x {h2}'.format(
2504 w1=width, h1=height, w2=zoom_width, h2=zoom_height
2505 )
2506 )
2507
2508 # Build x-axis objects
2509 # first x axis is time
2510 x_axes = []
2511
2512 purge = []
2513
2514 # look for specific configuration for bottom axis
2515 if 'x-axis' in config:
2516 for x_axis in config['x-axis']:
2517 if x_axis.get('position', 'bottom') == 'bottom':
2518 logging.debug(
2519 'Building explicit x bottom axis from {strt} to {stop}'.format(
2520 strt=sensing_start, stop=sensing_stop
2521 )
2522 )
2523 bottom_axis = Axis(
2524 start=sensing_start,
2525 stop=sensing_stop,
2526 locator=x_axis.get('locator'),
2527 locator_modulus=x_axis['locator-modulus'],
2528 formatter=x_axis.get('format'),
2529 fontsize=x_axis['fontsize'],
2530 label_fontsize=x_axis['label-fontsize'],
2531 position=Axis.Position.BOTTOM,
2532 colour=x_axis.get('colour'),
2533 label_colour=x_axis.get('label-colour'),
2534 )
2535 x_axes.append(bottom_axis)
2536 purge.append(x_axis)
2537
2538 # add an automatic one otherwise
2539 if len(x_axes) == 0:
2540 x_axes.append(
2541 Axis(
2542 position=Axis.Position.BOTTOM,
2543 start=sensing_start,
2544 stop=sensing_stop,
2545 )
2546 )
2547
2548 # look for specific top axis configuration
2549 if 'x-axis' in config:
2550 for x_axis in config['x-axis']:
2551 if x_axis.get('position') == 'top':
2552 top_axis = Axis(position=Axis.Position.TOP)
2553 x_axes.append(top_axis)
2554 purge.append(x_axis)
2555 show_orbits = True
2556
2557 for x in purge:
2558 config['x-axis'].remove(x)
2559
2560 if len(config['x-axis']) > 0:
2561 raise ConfigError('Unrecognised x-axis')
2562
2563 # now check if we need to automatically create a top axis
2564 if len(x_axes) == 1:
2565 # second x axis is orbit
2566 show_orbits = self.choose_show_orbits(
2567 show_orbits_request=config['show-orbits'], sid=sid, height=height
2568 )
2569 if show_orbits:
2570 orbit_min = None
2571 orbit_max = None
2572 x_axes.append(
2573 Axis(start=orbit_min, stop=orbit_max, position=Axis.Position.TOP)
2574 )
2575
2576 # oversampling reads a bit of extra data to the left and right of the graph
2577 # to avoid false gaps at the edges
2578 retrieve_start, retrieve_stop = self.oversample(
2579 sensing_start,
2580 sensing_stop,
2581 config['oversampling'] if not config['trim'] else None,
2582 )
2583
2584 # basic data retrieval
2585 time_limits = TimeRange()
2586 all_series = []
2587
2588 # Determine the type of limits
2589 limits_series = self.gen_all_limits_series(
2590 sid, sensing_start, sensing_stop, config.get('limits'), config['datapoint']
2591 )
2592
2593 if limits_series is not None:
2594 all_series.extend(limits_series)
2595
2596 for counter, dp in enumerate(config['datapoint']):
2597 # sampling -> Sampling (AUTOx, ALL_POINTS, FIT, stats types)
2598 sampling = self.choose_sampling(
2599 appearance=appearance,
2600 sid=sid,
2601 sensing_start=retrieve_start,
2602 sensing_stop=retrieve_stop,
2603 datapoint=dp, # includes modulus
2604 requested_sampling=config['sampling'], # ap, stats
2605 requested_subsampling=config['subsampling'], # yes/no
2606 # requested_modulus=config.get('modulus'), # force exact subsampling
2607 requested_region=config.get('region'), # force stats region
2608 width=zoom_width,
2609 )
2610 series = retrieve_series(
2611 sid=sid,
2612 sensing_start=retrieve_start,
2613 sensing_stop=retrieve_stop,
2614 datapoint=dp,
2615 calibrated=config['calibrated'],
2616 sampling=sampling,
2617 rowcount_threshold=config['rowcount-threshold'],
2618 time_limits=time_limits,
2619 )
2620
2621 # check if the report uses calculated series and if so, apply the calculations
2622 self.apply_mods(series, dp)
2623
2624 # get rid if +/- inf values
2625 self.remove_nonfinite(series)
2626
2627 # choose modulus
2628 if sampling is Sampling.FIT or sampling.stats is True:
2629 if config['subsampling'] == 'none':
2630 modulus = None
2631
2632 elif 'modulus' in config:
2633 modulus = config['modulus']
2634
2635 else:
2636 # subsampling factor
2637 modulus = max(1, len(series.times) // zoom_width)
2638 logging.debug(
2639 'times {t} width {w} gives modulus {m}'.format(
2640 t=len(series.times), w=zoom_width, m=modulus
2641 )
2642 )
2643
2644 else:
2645 modulus = None
2646
2647 # determine the bar-width
2648 if 'bar-width' in config:
2649 series.bar_width = config['bar-width']
2650
2651 series.modulus = modulus
2652
2653 # Enhance the Series objects with user choices
2654 self.markup_series(
2655 series,
2656 dp,
2657 marker_size=config.get('radius', config.get('marker-size')),
2658 trendline=config['trendline'],
2659 counter=counter,
2660 )
2661
2662 # add a regions structure to data containing tuples of (start,stop)
2663 # contiguous data regions
2664 if appearance != 'scatter' and series.gap_threshold is not None:
2665 series.regions = list(
2666 self.find_regions(
2667 series.times,
2668 series.gap_threshold
2669 if modulus is None
2670 else series.gap_threshold * modulus,
2671 )
2672 )
2673
2674 else:
2675 series.regions = ((0, len(series.times)),)
2676
2677 # apply subsampling to either AP or stats data
2678 if modulus is not None:
2679 # we always allow stats plots to be subsampled
2680 self.subsample(series, modulus)
2681 # series.regions = list(self.find_regions(series.times,
2682 # series.gap_threshold * modulus,
2683 # 0)) # 1
2684
2685 # if enabled, this hides gap in plot of
2686 # <datapoint><field>PLM_ASCAT_2.BNT0007</field></datapoint>
2687 # it was needed for something else though
2688 # else:
2689 # series.regions = ((0, len(series.times)),)
2690
2691 else:
2692 logging.debug('No subsampling with modulus {m}'.format(m=modulus))
2693
2694 all_series.append(series)
2695
2696 if config['trim']:
2697 logger.debug('Trimming time range')
2698 for series in all_series:
2699 if len(series.times) > 0:
2700 logger.debug(
2701 'Trim {series} to {strt} to {stop}'.format(
2702 series=series, strt=series.times[0], stop=series.times[-1]
2703 )
2704 )
2705 sensing_start = max(sensing_start, series.times[0])
2706 sensing_stop = min(sensing_stop, series.times[-1])
2707 for x_axis in x_axes:
2708 if x_axis.start is not None:
2709 x_axis.start = sensing_start
2710
2711 if x_axis.stop is not None:
2712 x_axis.stop = sensing_stop
2713
2714 # do it after subsampling because the alg doesn't handle non-subsampled data
2715 if config['anomaly-response'].startswith('remove'):
2716 for series in all_series:
2717 self.remove_anomalies(series, config['anomaly-threshold'])
2718
2719 else:
2720 logging.debug('Not removing anomalies')
2721
2722 self.post_retrieve(all_series)
2723
2724 # apply left side normalisation if selected
2725 # this is done before finding limits as we may have to rescale the axis afterwards
2726 if config.get('normalise'):
2727 self.normalise(all_series)
2728
2729 # Decide what to call the output file
2730 filename = self.get_filename(
2731 config.get('filename'), config['datapoint'], document
2732 )
2733 filename = Path(filename)
2734
2735 if config['trim']:
2736 # user has requested that the x-axis be trimmed to match the period where
2737 # data is actually available.
2738 # If no data was found we don't touch the limits
2739 if time_limits.start is not None:
2740 sensing_start = time_limits.start
2741
2742 if time_limits.stop is not None:
2743 sensing_stop = time_limits.stop
2744
2745 # We need a better way to set up plots. For now, just use one of the objects from
2746 # rpeorts/render.py. Currently it's only used for the call to render()
2747 # but can be expanded out in both directions
2748 # if config['date-format'] == 'auto':
2749 # date_formatter = Axis.AUTO_FORMATTER
2750
2751 # else:
2752 # date_formatter = config['date-format']
2753
2754 # if config['date-locator'].isdigit():
2755 # date_locator = int(config['date-locator'])
2756
2757 # else:
2758 # date_locator = Axis.Locator(config['date-locator'])
2759
2760 # build y-axes structure as list of Axis objects.
2761 # The first is the default axis, with no ID code, for left hand side
2762 # Optionally a second right hand axis can be specified
2763
2764 def y_axis_from_config(axis_config):
2765 """Convert an item from the config['y-axis'] array into an Axis object."""
2766 return Axis(
2767 name=axis_config.get('name'),
2768 start=axis_config.get('min'),
2769 stop=axis_config.get('max'),
2770 formatter=axis_config.get('format'),
2771 locator=axis_config.get('locator'),
2772 position=Axis.Position(axis_config.get('position')),
2773 fontsize=axis_config.get('fontsize'), # default?
2774 colour=axis_config.get('colour'), # default
2775 label=axis_config.get('label'), # auto
2776 label_fontsize=axis_config.get('label-fontsize'),
2777 label_colour=axis_config.get('label-colour'),
2778 minor_ticks=axis_config.get('minor-ticks'),
2779 )
2780
2781 # create y axes
2782 y_axes = []
2783 # we remember the configration block for explicit axes in order to do more
2784 # setup later, using values that aren't stored in the Axis object
2785 y_axes_configs = {}
2786 # used to
2787 purge = []
2788
2789 if 'y-axis' in config:
2790 # look for explicit left axis config
2791 for y_config in config['y-axis']:
2792 if y_config['position'] == 'left':
2793 logging.debug('Configuring explicit left axis')
2794 left_y_axis = y_axis_from_config(y_config)
2795 y_axes.append(left_y_axis)
2796 y_axes_configs[left_y_axis] = y_config
2797 purge.append(y_config)
2798 break
2799
2800 # make a left axis if not specifically requested
2801 if len(y_axes) == 0:
2802 logging.debug('Configuring implicit left axis')
2803 default_y_axis = Axis(
2804 start=config.get('y-min'),
2805 stop=config.get('y-max'),
2806 position=Axis.Position.LEFT,
2807 label=config.get('y-label'),
2808 )
2809 y_axes.append(default_y_axis)
2810 # this is probably dumb, we can allow .margin in Axis objects
2811 y_axes_configs[default_y_axis] = {
2812 'margin': config['auto-axis-margin'],
2813 }
2814
2815 if 'y-axis' in config:
2816 # look for explicit right axis config
2817 for y_config in config['y-axis']:
2818 # if 'id' in y:
2819 if y_config['position'] == 'right':
2820 logging.debug('Configuring explicit right axis')
2821 right_y_axis = y_axis_from_config(y_config)
2822 y_axes.append(right_y_axis)
2823 y_axes_configs[right_y_axis] = y_config
2824 purge.append(y_config)
2825 break
2826
2827 for y in purge:
2828 config['y-axis'].remove(y)
2829
2830 if len(config['y-axis']) > 0:
2831 raise ConfigError('Unrecognised y-axis')
2832
2833 # make a default right axis if needed
2834 if len(y_axes) < 2:
2835 for d in config['datapoint']:
2836 if d.get('axis') == 2 or d.get('y-axis') == 'right':
2837 logging.debug('Configuring legacy or implicit right axis')
2838 right_y_axis = Axis(
2839 start=config.get('y-min'),
2840 stop=config.get('y-max'),
2841 position=Axis.Position.RIGHT,
2842 margin=config['auto-axis-margin'],
2843 )
2844 y_axes.append(right_y_axis)
2845 y_axes_configs[right_y_axis] = {}
2846 break
2847
2848 # If the widget anomaly-response is 'axis', we are going by default to ignore
2849 # anomalous results when computing y-axis limits
2850 if config['anomaly-response'] == 'axis':
2851 for y_axis in y_axes:
2852 if 'anomaly-threshold' not in y_axes_configs[y_axis]:
2853 y_axes_configs[y_axis]['anomaly-threshold'] = config[
2854 'anomaly-threshold'
2855 ]
2856
2857 # now convert any default / named x_axis to references to Axis objects
2858 self.fix_axes_references(all_series, x_axes, y_axes)
2859
2860 if 'default-y-min' in config and 'default-min' not in y_axes_configs[y_axes[0]]:
2861 y_axes_configs[y_axes[0]]['default-min'] = config['default-y-min']
2862
2863 if 'default-y-max' in config and 'default-max' not in y_axes_configs[y_axes[0]]:
2864 y_axes_configs[y_axes[0]]['default-max'] = config['default-y-max']
2865
2866 # determine actual min/max ranges
2867 for y_axis in y_axes:
2868 self.find_y_axis_limits(
2869 y_axis=y_axis, all_series=all_series, config=y_axes_configs[y_axis]
2870 )
2871
2872 # set y axis labels
2873 for y_axis in y_axes:
2874 self.assign_y_axis_labels(y_axis, all_series)
2875
2876 # set tick labels if the plot uses choices
2877 self.assign_y_axis_ticklabels(y_axes, all_series)
2878
2879 # Create a list of Axis structures for the x-axes to pass to the render()
2880 # function later
2881
2882 annotations = []
2883 if 'annotation' in config: # but it's multiple?
2884 for a in config['annotation']:
2885 annotations.append(
2886 Annotation(
2887 text=a['text'],
2888 colour=a['colour'],
2889 size=a['size'],
2890 x=a['x'],
2891 y=a['y'],
2892 )
2893 )
2894
2895 presentation = Presentation(
2896 title=self.title,
2897 title_fontsize=config['title-fontsize'],
2898 )
2899
2900 logging.debug('Render zoom graph {w}x{h}'.format(w=zoom_width, h=zoom_height))
2901
2902 # y_axes[0].fontsize=settings.GRAPH_FONT_SIZE_ZOOM
2903 # y_axes[0].label_fontsize=settings.GRAPH_FONT_SIZE_ZOOM
2904 # x_axes[0].fontsize=settings.GRAPH_FONT_SIZE_ZOOM
2905 matplotlib_render(
2906 presentation=presentation,
2907 sid=sid,
2908 filename=filename,
2909 width=zoom_width,
2910 height=zoom_height,
2911 legend=legend,
2912 all_series=all_series,
2913 annotations=annotations,
2914 sampling=config['sampling'],
2915 appearance=appearance,
2916 dynrange_alpha=config['dynrange-alpha'],
2917 x_axes=x_axes,
2918 y_axes=y_axes,
2919 )
2920
2921 thumbnail_filename = Path(
2922 '{base}{thumb}{ext}'.format(
2923 base=filename.stem,
2924 ext=Presentation.DEFAULT_EXT,
2925 thumb=document.theme.thumbnail_suffix,
2926 )
2927 )
2928
2929 logging.debug('Render normal graph {w}x{h}'.format(w=width, h=height))
2930 # y_axes instead of: ylabels, ylimits
2931 y_axes[0].fontsize = settings.GRAPH_FONT_SIZE
2932 y_axes[0].label_fontsize = settings.GRAPH_FONT_SIZE
2933 x_axes[0].fontsize = settings.GRAPH_FONT_SIZE
2934 matplotlib_render(
2935 presentation=presentation,
2936 sid=sid,
2937 filename=thumbnail_filename,
2938 width=width,
2939 height=height,
2940 legend=legend,
2941 all_series=all_series,
2942 annotations=annotations,
2943 sampling=config['sampling'],
2944 appearance=appearance,
2945 dynrange_alpha=config['dynrange-alpha'],
2946 x_axes=x_axes,
2947 y_axes=y_axes,
2948 )
2949
2950 # TBD: support live link to event based plot here
2951 event_plot = False
2952 for dp in config['datapoint']:
2953 if 'event' in dp:
2954 event_plot = True
2955
2956 # Embed a URL in the plot info page which will jump to the same plot
2957 # in the plot tool
2958 if event_plot:
2959 url = ''
2960
2961 else:
2962 # per-series trendlines are not passed though
2963 url = make_plot_url(
2964 datapoints=config['datapoint'],
2965 sid=sid,
2966 sensing_start=sensing_start,
2967 sensing_stop=sensing_stop,
2968 sampling=sampling,
2969 # subsampling=subsampling, # first sampling?
2970 calibrated=config['calibrated'],
2971 appearance=appearance,
2972 y_min=config.get('y-min'),
2973 y_max=config.get('y-max'),
2974 )
2975
2976 stats_table, below_graph_table = compute_stats(all_series, legend)
2977
2978 document.append_figure(
2979 title=self.title,
2980 filename=thumbnail_filename,
2981 width=width,
2982 height=height,
2983 zoom_filename=filename,
2984 zoom_width=zoom_width,
2985 zoom_height=zoom_height,
2986 live_url=url,
2987 summary_info=None,
2988 datapoint_info=stats_table,
2989 widget=self,
2990 )
2991
2992 if below_graph_table is not None:
2993 below_graph_table.write_html(document.html)
2994
2995 def assign_y_axis_ticklabels(self, y_axes, all_series):
2996 """Set tick labels from choices if possible.
2997
2998 We only do it if there's a y-axis with only a single dataseries plotted against
2999 it and that series has <choices> defined."""
3000 consider = defaultdict(list) # Axis : list(FieldInfo)
3001 # Make a list of Series used by eac haxis
3002 for series in all_series:
3003 consider[series.y_axis].append(series)
3004
3005 for y_axis, series_list in consider.items():
3006 # Only look for choices if we have an axis with a single series using it
3007 if len(series_list) == 1:
3008 series = series_list[0]
3009 field_info = series.field
3010 if field_info is not None:
3011 choices = field_info.choices
3012 if choices is not None:
3013 # Insert tick labels for all defined choices
3014 # The plotting code clips the axis to only the range of values
3015 # for which we have data
3016 y_axis.tick_label_pairs = []
3017 for min_value, item in choices.items.items():
3018 y_axis.tick_label_pairs.append([min_value, item.name])
3019
3020
3021def compute_stats(all_series, legend):
3022 stats_table = Table(
3023 headings=(
3024 'Table',
3025 'Field',
3026 'Description',
3027 'Min',
3028 'Avg',
3029 'Max',
3030 'Slope',
3031 'Plotted points',
3032 )
3033 )
3034
3035 if legend == 'below-desc':
3036 below_graph_table = Table(
3037 title='Legend',
3038 cssclass='table nobreak nowrap',
3039 headings=('Colour', 'Field', 'Description'),
3040 )
3041
3042 elif legend == 'below-desc-stats':
3043 below_graph_table = Table(
3044 title='Legend',
3045 cssclass='table nobreak nowrap',
3046 headings=(
3047 'Colour',
3048 'Field',
3049 'Description',
3050 'Min',
3051 'Max',
3052 'Avg',
3053 'Stddev',
3054 'Slope',
3055 ),
3056 )
3057
3058 else:
3059 below_graph_table = None
3060
3061 for series in all_series:
3062 stats = Stats(series)
3063
3064 stats_table.append(
3065 (
3066 series.table.name if series.table is not None else '',
3067 series.field.name if series.field is not None else '',
3068 series.field.description if series.field is not None else '',
3069 stats.min,
3070 stats.avg,
3071 stats.max,
3072 stats.slope,
3073 stats.count,
3074 )
3075 )
3076
3077 if legend == 'below-desc':
3078 # if 'label' in dp:
3079 # name = dp['label']
3080
3081 # else:
3082 # name = dp['field'].name
3083
3084 if series.label is not None:
3085 name = series.label
3086
3087 elif series.field is not None:
3088 name = series.field.name
3089
3090 elif series.table is not None:
3091 name = series.table.name
3092
3093 elif series.event is not None:
3094 name = series.event.name
3095
3096 below_graph_table.append(
3097 (
3098 '<div style="background-color:{col}"> </div>'.format(
3099 col=series.line_colour or series.marker_colour
3100 ),
3101 name,
3102 stats.html_description(),
3103 )
3104 )
3105
3106 elif legend == 'below-desc-stats':
3107 if series.field is None:
3108 continue
3109
3110 below_graph_table.append(
3111 (
3112 '<div style="background-color:{col}"> </div>'.format(
3113 col=series.line_colour or series.marker_colour
3114 ),
3115 series.field.name,
3116 stats.html_description(),
3117 stats.min,
3118 stats.max,
3119 stats.avg,
3120 stats.std,
3121 stats.slope,
3122 )
3123 )
3124
3125 return stats_table, below_graph_table
3126
3127
3128class DifferentLimitsError(Exception):
3129 """This error indicates when two sets of limits (onboard, yellow, red)
3130 are different."""
3131
3132 def __init__(self, limits1, limits2):
3133 self.limits1 = limits1
3134 self.limits2 = limits2