1#!/usr/bin/env python3
2
3"""Server side code for CHART event viewer."""
4
5import csv
6import sys
7import json
8import re
9import operator
10import traceback
11from io import StringIO
12from datetime import datetime, timedelta
13from urllib.parse import unquote
14import logging
15
16from lxml.etree import Element, ElementTree
17from django.http import HttpResponse
18from django.shortcuts import render
19from django.shortcuts import redirect
20from django.views.decorators.cache import never_cache
21from django.http import StreamingHttpResponse
22
23from chart.project import settings
24from chart.db.func import Reversed
25from chart.events.eventclass import EventClass
26from chart.events.eventclass import get_event_xml
27from chart.events.db import find_events
28from chart.events.db import count_events
29from chart.events.db import find_single_event
30from chart.events.event import json_id_to_event
31from chart.common.xml import XMLSyntaxError
32from chart.common.texttime import texttime_to_timerange
33from chart.common.prettyprint import Table
34from chart.events.ingest import ingest_events
35from chart.backend.eventsfile import EventsFileReader
36from chart.plots.geoloc import Geoloc
37from chart.plots.geoloc import CannotGeolocate
38from chart.common.util import nvl
39from chart.web.proxy import proxy_if_configured
40from chart.web.responses import json_response
41from chart.project import SID
42from chart.events.exceptions import InvalidEvent
43from chart.events.event import show_event_time
44from chart.events.event import interpret_filter_clause
45from chart.events.event import event_to_json_id
46from chart.events.eventclass import EventTimeFilter
47from chart.common.xml import XMLElement
48from chart.web.user import User
49
50logger = logging.getLogger()
51
52# max number of events for property filtering
53MAX_EVENTS_FOR_FILTERING = 50000
54
55# maximum number of events for geoplot
56MAX_GEOLOCATED_EVENTS = 5000
57
58# maximum number of events for table
59MAX_TABLE_EVENTS = 500000
60
61# maximum number of events for timeline
62MAX_TIMELINE_EVENTS = 1000
63
64# name of uploaded XML file object stored in server session
65# TBD: these should be cleaned up
66XML_UPLOAD_NAME = 'upfile_xml'
67
68# get the database entry, nearest in time before the start_time,
69# used to get list of OOL events at a given time
70KEYWORD_SNAP = 'SNAP'
71
72# Time format for CSV export
73CSV_TIME_FORMAT_EXCEL = '%Y/%m/%d %H:%M:%S'
74
75# private global variable to map key code to class names
76_REVERSE_CODE_CLASS_MAPPING = {}
77
78# TBD this must be derived from request
79MAX_COLUMNS = 100
80
81# Geolocator objects have to be created with a timerange, for performance with large
82# numbers of retrievals, but if we're only creating a few like in the events tabulated
83# view we just build them with a nominal timerange
84GEOLOC_DUMMY_RANGE = timedelta(minutes=1)
85
86
87class EventViewerException(Exception):
88 """The user has requested a plot (2d geolocated or 3d globe) with too many events."""
89 def __init__(self, message):
90 super(EventViewerException, self).__init__()
91 self.message = message
92
93 def __str__(self):
94 return self.message
95
96
97def ingest_uploaded(request):
98 """Ingest events from the uploaded file stored in session."""
99 user = request.user.username
100 if user is None:
101 return send_alert('Not logged in')
102
103 if request.GET.get('flag') == 'cancel':
104 if XML_UPLOAD_NAME in request.session:
105 del request.session[XML_UPLOAD_NAME]
106 return HttpResponse('', content_type='text/html')
107
108 # we remove the file object from the session when it is used
109 file_xml = request.session.pop(XML_UPLOAD_NAME, None)
110
111 if file_xml is None or len(file_xml) == 0:
112 return send_alert('Nothing to ingest')
113
114 input_stream = StringIO(file_xml)
115
116 # request.session.modified = True
117
118 try:
119 input_counter, stored_counter, deleted_counter, updated_counter, unchanged_counter = \
120 ingest_events(input_stream, gen_method='upload-' + user)
121
122 # except IOError:
123 # return send_alert('Could not open events file')
124 # except ValueError:
125 # return send_alert('Events file parsing failed')
126 # except InvalidEvent:
127 # return send_alert('Invalid event (tag clash)')
128 except Exception: # pylint: disable=W0703
129 exc_type, exc_value, exc_traceback = sys.exc_info()
130 return send_alert('\n'.join(traceback.format_exception(exc_type, exc_value, exc_traceback)))
131
132 return send_alert('Total events: {input}<br/>New: {stored}<br/>'
133 'Deleted: {deleted}<br/>Updated: {updated}<br/>'
134 'Unchanged: {unchanged}<br/>Username: {user}'.format(
135 input=input_counter,
136 stored=stored_counter,
137 deleted=deleted_counter,
138 updated=updated_counter,
139 unchanged=unchanged_counter,
140 user=user
141 ))
142
143
144def directupload(request):
145 """Store the uploaded file in session."""
146 if request.method == 'POST':
147 form_file = request.FILES.get('file')
148 if form_file is None:
149 return render(request, 'eventviewer2/fileupload.html',
150 {'event_list': '', 'disable_ingest': 'true'})
151
152 request.session[XML_UPLOAD_NAME] = form_file.file.getvalue().decode('utf-8')
153 request.session.set_expiry(0) # session will expire when browser is closed
154
155 try:
156 form_file.file.seek(0)
157 file_reader = EventsFileReader(form_file.file)
158
159 t = Table(headings=('Event class', 'SCID', 'Start', 'Properties', 'Description'),
160 cssclass='eventtable table table-striped')
161
162 # Initial pass - we parse each event in order to build
163 # an HTML table.
164 for e in file_reader.gen_events():
165 t.append(
166 (e.event_classname,
167 e.sid, # EPS only
168 show_event_time(e.start_time),
169 '<br/>'.join([x[0] + '=' + x[1] for x in e.property_pairs()]),
170 e.description())
171 )
172
173 except (ValueError, InvalidEvent, XMLSyntaxError) as e:
174 result = 'Error parsing xml file: {}'.format(e)
175 return render(
176 request,
177 'eventviewer2/fileupload.html',
178 {'event_list': '<div style="font-weight:bold; margin:20px">{message}</div>'.format(
179 message=result), 'disable_ingest': 'true'})
180
181 output = StringIO()
182 t.write_html(target=output)
183 result = output.getvalue()
184
185 return render(request, 'eventviewer2/fileupload.html',
186 {'event_list': result, 'disable_ingest': 'false'})
187
188 return render(request, 'eventviewer2/fileupload.html', {
189 'event_list': '', 'disable_ingest': 'true'})
190
191
192def send_alert(message):
193 """Return error to be displayed as alert on the web page."""
194 return HttpResponse(json.dumps({'alert': message}),
195 content_type='application/json')
196
197
198def get_sensing_time(request):
199 """Get times from the request and try to convert to datetime."""
200 start = request.GET.get('sstart')
201 stop = request.GET.get('sstop')
202 sid = SID.from_django_request(request.GET)
203
204 # converting request.GET from QueryDict to standard dict (we need this when creating events)
205 # turns all values into lists (since QueryDict allows multiple values per key)
206 # thus we must be prepared to get lists instead of scalars here.
207 if isinstance(start, list):
208 start = start[0]
209
210 if start is None or start == '':
211 raise ValueError('Start time must be specified')
212
213 if isinstance(stop, list):
214 stop = stop[0]
215
216 if stop == '':
217 stop = None
218
219 # disable this for now as its annoying to get this error when you've changed the start
220 # time and are just about to change the stop time too
221
222 # if stop is not None and start > stop:
223 # raise ValueError("Start time cannot be greater than stop time")
224
225 return texttime_to_timerange(start, stop, sid)
226
227
228@proxy_if_configured
229def interpret_times(request):
230 """Interpret sensing start/stop times received from browser (things like `now`, `launch`, etc.)
231 and return a json object with times in milliseconds.
232 """
233 epoch = datetime(1970, 1, 1)
234
235 try:
236 start, stop = get_sensing_time(request)
237
238 except ValueError as e:
239 return HttpResponse(json.dumps({'alert': str(e)}), content_type='application/json')
240
241 start = int((start - epoch).total_seconds() * 1000)
242 stop = int((stop - epoch).total_seconds() * 1000)
243 return HttpResponse(json.dumps([start, stop]))
244
245
246def parse_events_request(request):
247 """Extract common parameters from Django request object, since multiple functions
248 require this processing.
249
250 Args:
251 `request` (Request): Browser request object
252
253 Returns:
254 Tuple of: (
255 SID (various): Source ID
256 start (datetime): Start of search range
257 stop (datetime): Stop of search range
258 class_list (list of str): List of classes to search for
259 search_props (list of tuple): Property search criteria. Tuple is:
260 (name, value, operator)
261
262 Raises:
263 ValueError
264 """
265 class_list = []
266
267 if parse_events_request.pattern is None:
268 parse_events_request.pattern = re.compile(
269 r"""(?P<key>\w+)((\[(?P<index>\d+)\]\[(?P<name>\w+)\])|(.*))""")
270
271 for k, v in request.GET.items():
272 v = unquote(v)
273 match_obj = parse_events_request.pattern.search(k)
274 key = match_obj.group('key')
275 # index = match_obj.group('index')
276 name = match_obj.group('name')
277 if key == 'classes': # event classes
278 class_code_list = v.split('|')
279 for code in class_code_list:
280 class_title = None
281 try:
282 class_title = _REVERSE_CODE_CLASS_MAPPING[code]
283 except KeyError:
284 # This error might occur in two scenarios:
285 # 1. A user refresh the page AFTER a server restart.
286 # Just rebuild the class tree and try again.
287 # 2. Or the querystring has been tampered with invalid values.
288 # Try again and pass.
289 if len(_REVERSE_CODE_CLASS_MAPPING) == 0:
290 build_class_tree(get_event_xml().elem.xpath('/class')[0])
291 try:
292 class_title = _REVERSE_CODE_CLASS_MAPPING[code]
293 except KeyError:
294 pass
295 if class_title is not None:
296 class_list.append(class_title)
297
298 # we must convert the unicode representation to string, otherwise the entire query
299 # will become unicode and be rejected
300 # print (filters)
301 # for v in filters.itervalues():
302 # filter_items.append((v['pname'], v['pval'], getattr(operator, v['pop'])))
303
304 try:
305 start, stop = get_sensing_time(request)
306
307 except ValueError as e:
308 exc_msg = str(e).replace('"', '\"').replace("'", "\'").replace("\n", " ")
309 raise ValueError("Time error: {0}".format(exc_msg))
310
311 sid = SID.from_django_request(request.GET)
312
313 # collect property search clauses if any
314 search_props = []
315 for i in range(MAX_COLUMNS):
316 name = request.GET.get('columns[{}][name]'.format(i))
317 if name is None:
318 break
319
320 search_clause = request.GET.get('columns[{}][search][value]'.format(i)).strip()
321 if search_clause is None or search_clause == '':
322 continue
323
324 value, op = interpret_filter_clause(search_clause)
325 search_props.append((name, value, op))
326
327 if len(search_props) == 0:
328 search_props = None
329
330 return sid, start, stop, class_list, search_props
331
332parse_events_request.pattern = None
333
334
335def geolocate_events_request(sid,
336 start,
337 stop,
338 class_list,
339 search_props,
340 width,
341 height):
342 """Yield a series of events, annotated with lat/lon positions.
343 This is much faster than geolocating each event individually as we can cache
344 geolocator objects.
345 This is used for the 2d and 3d renderers.
346 """
347
348 total_events = count_events(
349 sid=sid,
350 start_time=start,
351 stop_time=stop,
352 event_class=class_list,
353 properties=search_props)
354
355 if total_events > MAX_GEOLOCATED_EVENTS:
356 raise EventViewerException('Too many events ({cc}), can geolocate up to {max}'.format(
357 cc=total_events, max=MAX_GEOLOCATED_EVENTS))
358
359 # Performance issues:
360 # - There is some overhead to constructing a geolocator object
361 # as the constructor created a cursor
362 # - This delay increases with the duration of the geolocator and can be substantial
363 # (>10s) is the duration is in years
364 # - It takes a substantial time (>1s) to retrieve coords from a geolocator
365 # if there is a gap of many days between one retrieval and the next
366 # - Therefore we throw away the geolocator and create a new one if there is more that
367 # new_geo_thresh (3 hours) between events
368 # - Each geolocator only has a validity of geo_duration (24 hours) on construction,
369 # after this time we throw it away and make a new one
370
371 events = find_events(
372 sid=sid,
373 start_time=start,
374 stop_time=stop,
375 event_class=class_list,
376 properties=search_props)
377
378 geo_duration = timedelta(hours=24)
379 geo_timeout = start + geo_duration
380 geolocator = Geoloc(sid, start, geo_timeout, width, height)
381 prev_time = None
382 new_geo_thresh = timedelta(hours=3)
383
384 for event in events:
385 if event.start_time > geo_timeout or \
386 (prev_time is not None and (event.start_time - prev_time) > new_geo_thresh):
387
388 geo_timeout = event.start_time + geo_duration
389 try:
390 geolocator = Geoloc(sid, event.start_time, geo_timeout, width, height)
391
392 except CannotGeolocate:
393 continue
394
395 yield event, geolocator
396 # x, y = geolocator.x_y(event.start_time)
397 prev_time = event.start_time
398
399
400@proxy_if_configured
401def location(request):
402 """Build geolocated events' image in cylindrical projection."""
403
404 try:
405 sid, start, stop, class_list, search_props = parse_events_request(request)
406
407 except ValueError as e:
408 return HttpResponse(str(e), content_type='text/html')
409
410 response = []
411 try:
412 for event, geolocator in geolocate_events_request(
413 sid,
414 start,
415 stop,
416 class_list,
417 search_props,
418 float(request.GET.get('gridx', 0)),
419 float(request.GET.get('gridy', 0))):
420
421 # subtract 2 pixels to center the 4x4 event image
422 x, y = geolocator.x_y(event.start_time)
423 response.append((event.event_classname,
424 show_event_time(event.start_time),
425 x - 2,
426 y - 2))
427
428 except EventViewerException as e:
429 return send_alert(str(e))
430
431 except CannotGeolocate as e:
432 return send_alert(str(e))
433
434 return HttpResponse(json.dumps(response, separators=(',', ':')), content_type='application/json')
435
436
437@proxy_if_configured
438def xml_export(request):
439 """Respond to a browser request for all displayed events as an XML file."""
440 try:
441 sid, start, stop, class_list, search_props = parse_events_request(request)
442
443 except ValueError as e:
444 return HttpResponse(json.dumps({'alert': str(e)}), content_type='application/json')
445
446 events = find_events(
447 sid=sid,
448 start_time=start,
449 stop_time=stop,
450 event_class=class_list,
451 properties=search_props
452 )
453
454 elem = Element('events')
455 for event in events:
456 event.to_xml(elem)
457
458 response = HttpResponse(content_type='text/xml')
459 ElementTree(elem).write(response, pretty_print=True, encoding='utf-8')
460 response['Content-Disposition'] = 'attachment; filename={filename}'.format(
461 filename='Result.xml')
462
463 return response
464
465
466@proxy_if_configured
467def csv_export(request):
468 """Respond to a browser request for all displayed events as an XML file."""
469 try:
470 sid, start, stop, class_list, search_props = parse_events_request(request)
471
472 except ValueError as e:
473 return HttpResponse(json.dumps({'alert': str(e)}), content_type='application/json')
474
475 # Check for special filtering options on the sensing_time / execution_time field
476 filtering = EventTimeFilter.SENSING_TIME
477 if settings.EVENTVIEWER_TIME_FILTER_OPTIONS:
478 filtering_str = request.GET.get('timefilter')
479 if filtering_str == EventTimeFilter.EXECUTION_TIME.value:
480 filtering = EventTimeFilter.EXECUTION_TIME
481
482 # extract user, if set, for filtering
483 user = User.from_django_request(request) if settings.TABLE_SECURITY_RESTRICTIONS else None
484
485 def data():
486 """Django callback to request additional csv rows."""
487 for row in retrieve_csv_data(sid, start, stop, class_list, search_props, filtering, user):
488 # This algorithm has gone through several evolutions.
489 # Initial there was no data() function, and csv_export() just returned the result
490 # as a big string.
491 # That caused the web server to crash when retrieving big (>1GB) files, so was replaced
492 # with a streaming algorithm that sent one line at a time to the user.
493 # That had a bug injecting Null values into the output (and was also slow).
494 # Unfortunately the AR wasn't fixed very well, instead the code was replaced with the
495 # pointless algorithm below that uses an internal function - as though for streaming -
496 # but doesn't actually stream the output therefore we have the original big files
497 # bug again.
498 # The correct solution is to assemble the final output into batches of a few kb each
499 # and send them to the user
500 csvfile = StringIO()
501 csvwriter = csv.writer(csvfile)
502 csvwriter.writerow(row)
503 yield csvfile.getvalue()
504
505 response = StreamingHttpResponse(data(), content_type='text/csv')
506 response['Content-Disposition'] = 'attachment; filename={filename}'.format(
507 filename='events.csv')
508 response.streaming = True
509 return response
510
511
512def retrieve_csv_data(sid, start, stop, class_list, search_props, filtering, user):
513 """Callback function to stream CSV file to server."""
514
515 if isinstance(stop, str) and stop.upper() == KEYWORD_SNAP:
516 # get all the entries, nearest in time before the start time.
517 # SNAP gets the time stamp value of the first record found before start_time
518 # then sets start and stop range to that time stamp.
519 # Used to get list of OOL events at a given time.
520 event = find_single_event(sid=sid,
521 stop_time=start,
522 event_class=class_list,
523 ordering=Reversed('start_time'))
524
525 # update start and stop, with this located entry,
526 # cover case when no event before start time
527 if event is None:
528 # no events before start time
529 stop = start
530
531 else:
532 start = event[0].start_time
533 stop = None
534
535 order_by = 'start_time'
536
537 # get events within time range, limit numbers to 'from_to'. Note find_events handles
538 # where to read events from, either EVENT table or a ts table
539 events = list(find_events(sid=sid,
540 start_time=start,
541 stop_time=stop,
542 event_class=class_list,
543 ordering=order_by,
544 properties=search_props,
545 filtering=filtering,
546 user=user))
547
548 if len(events) == 0:
549 return
550
551 # get all instance properties from all events
552 properties = set()
553 for event in events:
554 properties |= set(event.instance_properties.keys())
555
556 # by default find_events sorts by time
557 # this should use the fast geoloc algorithm in geolocate_events_request
558 # but for now we just disable location for long retrievals
559 # For EUM/JasonCS/NCR/2499 user requests that all lat/lon values are included in
560 # CSV export regardless of number of events.
561 # While there is a good reason to limit number of geolocated events for the Geolocate
562 # visualisation - we can easily overload the web browser - it's less important to
563 # limit them in CSV export.
564 get_geoloc = True # len(events) <= MAX_GEOLOCATED_EVENTS
565
566 # Output first line of CSV file with column headings
567 # properties = list(properties)
568 properties = [str(p) for p in properties]
569 properties.sort(key=str.lower)
570 yield (['event_class',
571 'sid',
572 'start_time',
573 'start_time_us',
574 'stop_time',
575 'stop_time_us',
576 'latitude',
577 'longitude'] +
578 properties)
579
580 if get_geoloc:
581 geo_duration = timedelta(hours=24)
582 geo_timeout = start + geo_duration
583 geolocator = Geoloc(sid, start, geo_timeout)
584 prev_time = None
585 new_geo_thresh = timedelta(hours=3)
586
587 for event in events:
588 pos = ['', '']
589 if get_geoloc:
590 if event.start_time > geo_timeout or \
591 (prev_time is not None and (event.start_time - prev_time) > new_geo_thresh):
592
593 geo_timeout = event.start_time + geo_duration
594 try:
595 geolocator = Geoloc(sid, event.start_time, geo_timeout)
596
597 except CannotGeolocate:
598 pass
599
600 if geolocator is not None:
601 try:
602 pos = geolocator.lat_lon(event.start_time)
603 except CannotGeolocate:
604 pass
605
606 row = [event.event_classname,
607 event.sid.name,
608 event.start_time.strftime(CSV_TIME_FORMAT_EXCEL),
609 event.start_time.microsecond,
610 None if event.stop_time is None else event.stop_time.strftime(CSV_TIME_FORMAT_EXCEL),
611 None if event.stop_time is None else event.stop_time.microsecond,
612 pos[0],
613 pos[1]]
614
615 for p in properties:
616 row.append(event.instance_properties.get(p, ''))
617
618 yield row
619
620
621def timeline_events(request):
622 """Provide events for the timeline as json object."""
623 try:
624 sid, start, stop, class_list, search_props = parse_events_request(request)
625
626 except ValueError as e:
627 return HttpResponse(str(e), content_type='text/html')
628
629 total_events = count_events(
630 sid=sid,
631 start_time=start,
632 stop_time=stop,
633 event_class=class_list,
634 properties=search_props)
635
636 if total_events > MAX_TIMELINE_EVENTS:
637 return HttpResponse(json.dumps({'alert': 'Too many events ({}), can display up to {}'.
638 format(total_events, MAX_TIMELINE_EVENTS)}, separators=(',', ':')),
639 content_type='application/json')
640
641 events = find_events(sid=sid,
642 start_time=start,
643 stop_time=stop,
644 event_class=class_list,
645 properties=search_props)
646
647 result = []
648
649 for event in events:
650 event_start = event.start_time.strftime('%Y%m%dT%H%M%SZ')
651 event_stop = event.stop_time
652 if event_stop is None:
653 event_stop = ''
654
655 else:
656 event_stop = event_stop.strftime('%Y%m%dT%H%M%SZ')
657
658 result.append({'start': event_start,
659 'end': event_stop,
660 'title': event.event_classname,
661 # 'isDuration': False,
662 'ix': class_list.index(event.event_classname),
663 'description': event_to_json_id(event)})
664
665 return HttpResponse(json.dumps({'dateTimeFormat': 'iso8601',
666 'events': result},
667 separators=(',', ':')),
668 content_type='application/json')
669
670
671def process_event(event):
672 """Create Eventviewer row from event.
673 Apply colour coding and choices to the parameters in the row where appropriate.
674 """
675 pairs = event.property_pairs()
676 try:
677 property_string = '\n'.join([x[0] + '=' + x[1] for x in pairs])
678
679 except UnicodeDecodeError:
680 # APT-ANOMALY on 2013-01-28
681 lines = []
682 for pair in pairs:
683 key = str(pair[0])
684 value = str(pair[1], 'latin')
685 lines.append('{k}={v}'.format(k=key, v=value))
686
687 property_string = '\n'.join(lines)
688
689 # If we don't do this Firefox reports Malformed URI when displaying MHS NEDT TREND
690 # events because they have '%' chars.
691 property_string = property_string.replace('%', ' percent') # Not sure why.
692 property_string = property_string.replace('percent20', '') # Not sure why.
693 property_string += '\n\nDouble-click for more...'
694
695 row = dict(pairs)
696
697 # Build a map of column name against colour
698 colours = {}
699 associated_param = None
700 for k, v in row.items():
701 prop_def = event.event_class.instance_properties[k]
702 if 'choices' in prop_def:
703 for choice in prop_def['choices']:
704 # The cast to str() here is ugly and is because choice['value'] uses native type (i.e. int for colours of spid
705 # property in CHART-MTG but v has already been converted to a displayable value (str in all cases)
706 # This is another reason to get rid of the ugly property_pairs() function, and iterate through instance_properties
707 # here, and convert native values to displayable ones below are they get put in the rows structure
708 #
709 # In the choice defintion it is possible to apply the colour applied to a parameter(associated parameter) to
710 # another parameter also. In JCS and MTG this is used to apply the colour associate with the STATUS parameter to
711 # the RGTOASS112299CC parameter
712 if 'use-colour' in choice.keys() and choice['use-colour'] is not None:
713 # get parameter name from associated parameter
714 associated_param = choice['use-colour']
715 colour_prop = k
716
717 else:
718 if 'value' in choice.keys() and str(choice['value']) == v or \
719 'name' in choice.keys() and str(choice['name']) == v:
720 row[k] = choice['name']
721 if 'colour' in choice:
722 colours[k] = choice['colour']
723 if k == associated_param:
724 # parameter is the associated param so colour the other parameter also
725 # defined by colour_prop
726 colours[colour_prop] = colours[k]
727
728 # special patch for computed latitude and longitude virtual properties
729 if 'latitude' in event.event_class.instance_properties:
730 geolocator = Geoloc(event.sid, event.start_time, event.start_time + GEOLOC_DUMMY_RANGE)
731 lat, lon = geolocator.lat_lon(event.start_time)
732 row['latitude'] = lat
733 # assume we have both
734 row['longitude'] = lon
735
736 row['__colours'] = colours
737 row['id'] = event_to_json_id(event)
738 # logger.debug('sending id {id}'.format(id=row['id']))
739 row['event_class'] = event.event_classname
740 row['scid'] = event.sid.name
741 row['start_time'] = show_event_time(event.start_time) if event.start_time is not None else '-'
742 row['stop_time'] = show_event_time(event.stop_time) if event.stop_time is not None else '-'
743 row['tooltip'] = property_string
744
745 return row
746
747
748@proxy_if_configured
749def retrieve_events(request):
750 """Reply to request from jquery Datatables to populate main events table."""
751 try:
752 sid, start, stop, class_list, search_props = parse_events_request(request)
753
754 except ValueError as e:
755 return json_response({'alert': str(e)})
756
757 except EventViewerException as e:
758 return send_alert(str(e))
759
760 iDisplayLength = int(request.GET['length'])
761 iDisplayStart = int(request.GET['start'])
762
763 # Check for special filtering options on the sensing_time / execution_time field
764 filtering = EventTimeFilter.SENSING_TIME
765 if settings.EVENTVIEWER_TIME_FILTER_OPTIONS:
766 filtering_str = request.GET.get('timefilter')
767 if filtering_str == EventTimeFilter.EXECUTION_TIME.value:
768 filtering = EventTimeFilter.EXECUTION_TIME
769
770 # find all currently used instance properties' names
771 used_instance_properties = set([])
772
773 if search_props is not None:
774 new_search_props = []
775 for ev_class in class_list:
776 iprops = EventClass(ev_class).instance_properties
777 for name in iprops.keys():
778 used_instance_properties.add(name)
779
780 # replace choice value for raw value for db search
781 for search_prop in search_props:
782 if name == search_prop[0]:
783 prop_def = iprops[name]
784
785 # allow wildcards in search
786 if 'choices' in prop_def:
787 for choice in prop_def['choices']:
788 if str(choice['name']) == search_prop[1]:
789 # replace choice name with 'raw' value for db search
790 new_search_props.append((search_prop[0], str(choice['value']), search_prop[2]))
791
792 else:
793 new_search_props.append(search_prop)
794
795 search_props = new_search_props
796
797 sort_columns = []
798 # compute sortable columns
799 # refer to https://datatables.net/manual/server-side
800 # XXX: It's important to note that column names MUST match actual database
801 # columns
802 regex = re.compile(r'columns\[(?P<number>[\d])*\]\[orderable\]')
803 sort_col_count = 0
804 for param in request.GET:
805 match = regex.match(param)
806 if match is not None:
807 sort_col_count += 1
808 for i in range(sort_col_count):
809 sort_col_number = request.GET.get('order[{}][column]'.format(i))
810 if sort_col_number is None:
811 continue
812 sort_col = request.GET.get('columns[{}][name]'.format(sort_col_number))
813 sort_dir = request.GET.get('order[{}][dir]'.format(i), '')
814 sort_columns.append((sort_col, sort_dir))
815
816 if isinstance(stop, str) and stop.upper() == KEYWORD_SNAP:
817 # get all the entries, nearest in time before the start time.
818 # SNAP gets the time stamp value of the first record found before start_time
819 # then sets start and stop range to that time stamp.
820 # Used to get list of OOL events at a given time.
821 event = find_single_event(sid=sid,
822 stop_time=start,
823 event_class=class_list,
824 ordering=Reversed('start_time'))
825
826 # update start and stop, with this located entry,
827 # cover case when no event before start time
828 if event is None:
829 # no events before start time
830 stop = start
831
832 else:
833 start = event.start_time
834 # to ensure get all entries with this start_time only
835 stop = event.start_time + timedelta(microseconds=1)
836
837 order_by = 'start_time'
838 from_to = (iDisplayStart, (iDisplayStart + iDisplayLength))
839
840 # if only one class and default sort 'time asc' can we take advantage of not having to
841 # sorting across classes. Worth while as often default selection case and does save
842 # time and memory, see below
843 if len(class_list) != 1 or sort_columns[0][0]+sort_columns[0][1] != 'start_timeasc':
844 from_to = None
845
846 result = {'data': []}
847
848 # check number of events in request to prevent display hanging, there can be many TM-Packets in
849 # small time range
850 total_events = count_events(
851 sid=sid,
852 start_time=start,
853 stop_time=stop,
854 event_class=class_list)
855
856 if total_events > settings.EVENTVIEWER_MAX_TABLE_EVENTS:
857 # TBD would like to raise alert using the following, but this is causing errors and multiple
858 # pop-up alerts, so I have implemented the following until this is resolved...
859 # return HttpResponse(json.dumps({'alert': 'Too many events ({}), can display up to {}\n{}'.
860 # format(total_events, settings.EVENTVIEWER_MAX_TABLE_EVENTS, message)}, separators=(',', ':')),
861 # content_type='application/json')
862 # Temporary fix for above issue.
863 # Create a dummy event to display in table
864 row = {}
865 row['id'] = 1
866 row['event_class'] = 'WARNING - Too many events ({m}), can process up to {t}'.format(m=total_events, t=settings.EVENTVIEWER_MAX_TABLE_EVENTS)
867 row['scid'] = sid.name
868 row['start_time'] = '-'
869 row['tooltip'] = '-'
870
871 result['data'].append(row)
872 result['recordsTotal'] = 1
873 result['recordsFiltered'] = 1
874
875 return json_response(result)
876
877 try:
878 # get events within time range, limit numbers to 'from_to'. Note find_events handles
879 # where to read events from, either EVENT table or a ts table
880 events = find_events(
881 sid=sid,
882 start_time=start,
883 stop_time=stop,
884 event_class=class_list,
885 ordering=order_by,
886 properties=search_props,
887 from_to=from_to,
888 filtering=filtering,
889 user=User.from_django_request(
890 request) if settings.TABLE_SECURITY_RESTRICTIONS else None)
891
892 except ValueError:
893 return send_alert('Request error. Bad filter?')
894
895 # datatables API recommends to cast sEcho as int here (as a precaution)?
896
897 for event in events:
898 # create Eventviewer row from event
899 if isinstance(event, int):
900 total_events = event
901
902 else:
903 row = process_event(event)
904 result['data'].append(row)
905
906 if from_to is not None:
907 # see above, a useful short cut because a common default selection
908 result['recordsTotal'] = total_events
909 result['recordsFiltered'] = total_events
910
911 else:
912 # either multiple classes or non-default sort selection
913 # TBD need to have some clever code to handle multi-class selections. Code as is,
914 # we retrieve all events from all requested classes within the time period and then sort.
915 # There could be a lot of events, in case of TM and MOF events
916 res_include_col = []
917 res_exclude_col = []
918 # now sort on required sort order column
919 if len(sort_columns) > 0:
920 if sort_columns[0][1] == 'asc':
921 reverse = False
922
923 else:
924 reverse = True
925
926 # sort on column, if included in the results. First remove entries which do not include
927 # sort column (case when mixed class lists), perform sort then add to end of list
928 for row in result['data']:
929 # put row in included or excluded lists accordingly
930 if sort_columns[0][0] in row:
931 # first, convert all ints to ints from string, for correct sorting numerical or alphabetic
932 row = {k: int(v) if isinstance(v, str) and v.isdigit() else v for k, v in row.items()}
933 res_include_col.append(row)
934
935 else:
936 res_exclude_col.append(row)
937
938 # now sort
939 res_include_col.sort(key=operator.itemgetter(sort_columns[0][0]), reverse=reverse)
940 res_include_col += res_exclude_col
941
942
943 total_events = len(res_include_col)
944 # limit output to a page
945 result['data'] = res_include_col[iDisplayStart : (iDisplayStart + iDisplayLength - 1)]
946
947 result['recordsTotal'] = total_events
948 result['recordsFiltered'] = total_events
949
950 return json_response(result)
951
952
953@proxy_if_configured
954def json_event(request):
955 """Return a single event as HTML table fragment."""
956 # If event read from EVENTS table then id is an event_id and an integer
957 # else the event must be built via multitable support
958 ev_id = request.GET.get('id')
959
960 if ev_id is not None:
961 event = json_id_to_event(
962 ev_id,
963 User.from_django_request(request) if settings.TABLE_SECURITY_RESTRICTIONS else None)
964
965 else:
966 return send_alert('Missing event identifier in URL')
967
968 if event is None:
969 return send_alert('Event {id} not found'.format(id=ev_id))
970
971 return json_response({'event_classname': event.event_classname,
972 'html': event.as_table().to_html_str()})
973
974
975def event_views(request):
976 """Check browser, then either report an error or render the template."""
977 # the initial request response cannot be cached due to browser check
978 # return event_views_imp(request)
979 if 'MSIE' in request.META.get('HTTP_USER_AGENT', '') and \
980 request.COOKIES.get('allow_ie', '') != 'yes':
981 return render(request,
982 'ie_error.html', {})
983
984 return event_views_imp(request)
985
986
987def get_available_instance_properties(request):
988 """For a list of selected classes return the dictionary of property types keyed by all
989 properties present in these classes."""
990
991 # TBD: make this a set because the type of a property
992 # should not affect the search options
993
994 class_list = request.GET.getlist('event[]', [])
995
996 result = {}
997 for ev_class in class_list:
998 iprops = EventClass(EventClass.all()[int(ev_class)].name).instance_properties
999 for name, props in iprops.items():
1000 result[name] = props['type']
1001
1002 res = [{'name': k, 'type': result[k]} for k in result.keys()]
1003
1004 return json_response(res)
1005
1006
1007# @cache_page(86400*365)
1008@never_cache
1009def event_views_imp(request):
1010 """Return the main event viewer HTML page, rendered via the Django template engine."""
1011 return render(request, 'eventviewer2/event_viewer.html',
1012 {'project': settings.APPNAME,
1013 'WIKI_URL': settings.PROJECT_HOMEPAGE_URL,
1014 'class_tree': json.dumps(build_class_tree(get_event_xml().elem.xpath('/class')[0])),
1015 'all_classes': json.dumps(dict([(c.name, c.instance_properties) for c in EventClass.all()])),
1016 'geoloc': 'false' if settings.GEOLOC_CONSTRUCTOR is None else 'true',
1017 # 'all_classes': json.dumps([c.name for c in EventClass.all()]),
1018 'SID': SID(), # the Django template has trouble reading class statics
1019 # so we pass a dummy instance in
1020 'all_sids': json.dumps(SID.django_all()),
1021 'request': request})
1022
1023# @never_cache
1024# def get_events_js(request): # (unused argument) pylint: disable=W0613
1025 # """Run javascript through the template engine."""
1026 # return render(request, 'eventviewer2/event_viewer.js',
1027 # {'class_tree': json.dumps(build_class_tree(get_event_xml().elem.xpath('/class')[0])),
1028 # 'all_classes': json.dumps([c.name for c in EventClass.all()]),
1029 # 'request': request,
1030 # })
1031
1032
1033def build_class_tree(root):
1034 """Build json structure of event classes for the jquery dynatree plugin."""
1035 # TBD: Use EventClass as the parameter to this function
1036
1037 title = root.xpath('./name/text()')[0]
1038 key_code = root.xpath('./key-code/text()')[0]
1039 _REVERSE_CODE_CLASS_MAPPING[key_code] = title
1040 return {
1041 'title': title,
1042 'key': key_code,
1043 'tooltip': root.xpath('./name/text()')[0] + ': ' +
1044 nvl(EventClass(XMLElement(root)).description, '') +
1045 ' (double-click for more)',
1046 'icon': False,
1047 'children': sorted([build_class_tree(cls) for cls in root.xpath('child::class')], key=operator.itemgetter('title'))
1048 }
1049
1050
1051def redirect_to_index(request): # (unused param) pylint:disable=W0613
1052 """Redirect an old-style page request ("http://chart/event_viewer/event_views") to
1053 "http://chart/event_viewer".
1054 """
1055 return redirect('chart.eventviewer2.views.event_views')
1056
1057
1058@proxy_if_configured
1059def globe_content(request):
1060 """Populate the 3d globe render by sending a list of objects to plot on the scene."""
1061 try:
1062 sid, start, stop, class_list, search_props = parse_events_request(request)
1063
1064 except ValueError as e:
1065 return HttpResponse(str(e), content_type='text/html')
1066
1067 # each class gets is given a colour
1068 palette = (0xff0000, 0x00ff00, 0x0000ff, 0xffff00, 0x00ffff, 0xff00ff, 0x000000)
1069
1070 # keep a record of which classes have already been assigned
1071 # classes = []
1072
1073 # build map of event classnames against lists of events
1074 scatter_artifacts = {}
1075
1076 # loop through all events matching the specifications from the browser
1077 try:
1078 for event, geolocator in geolocate_events_request(
1079 sid,
1080 start,
1081 stop,
1082 class_list,
1083 search_props,
1084 float(request.GET.get('gridx', 0)), # dummy values could have been 0,0
1085 float(request.GET.get('gridy', 0))):
1086
1087 # geolocate the event
1088 lat, lon = geolocator.lat_lon(event.start_time)
1089
1090 # assign a colour to each new class in a round-robin fashion, taking the colours from
1091 # `palette`
1092 cls = event.event_class
1093 # if cls in classes:
1094 # pos = classes.index(cls)
1095
1096 # else:
1097 # classes.append(cls)
1098 # pos = len(classes) - 1
1099
1100 # artifacts.append({ # 'id': event.event_id,
1101 # 'color': palette[pos % len(palette)],
1102 # 'radius': 1,
1103 # 'lat': lat,
1104 # 'lon': lon})
1105
1106 if cls in scatter_artifacts:
1107 scatter_artifacts[cls].append((lat, lon))
1108
1109 else:
1110 scatter_artifacts[cls] = [(lat, lon)]
1111
1112 except CannotGeolocate as e:
1113 return send_alert(str(e))
1114
1115 except EventViewerException as e:
1116 return send_alert(str(e))
1117
1118 # json objects to be sent to the browser
1119 artifacts = []
1120 for cc, (k, v) in enumerate(scatter_artifacts.items()):
1121 artifacts.append({'id': k.name,
1122 'color': palette[cc % len(palette)],
1123 'radius': 1,
1124 'lat': [i[0] for i in v],
1125 'lon': [i[1] for i in v]})
1126
1127 return json_response(artifacts)