1#!/usr/bin/env python3
2
3"""This file handle requests for any algorithm related pages."""
4
5import zipfile
6import urllib.request, urllib.parse, urllib.error
7import json
8import mimetypes
9import logging
10import subprocess
11from io import StringIO
12from datetime import datetime, time
13from datetime import timedelta
14from collections import namedtuple
15import os
16import random
17import codecs
18
19from chart.common.path import Path
20from django.urls import reverse
21from django.shortcuts import render
22from django.shortcuts import redirect
23from django.http import HttpResponse
24from django.http import Http404
25from django.template.loader import get_template
26from django.template import RequestContext
27import lxml.html
28
29from chart.db.connection import db_connect
30from chart.backend.activity import Activity
31from chart.common.decorators import memoized
32from chart.reports.archive import get_report_name
33from chart.reports.archive import decode_report_name
34from chart.reports.archive import retrieve_file
35from chart.reports.archive import update_file
36from chart.reports.archive import get_report_abspath
37from chart.reports.archive import find_reports
38from chart.reports.manifest import Manifest
39from chart.common.scid import get_satellites
40from chart.common.xml import xml_to_str
41from chart.common.xml import mixed_decoder
42from chart.project import settings
43import chart.alg.settings
44from chart.backend.worker import add_job
45from chart.backend.job import Job
46from chart.web.user import User
47from chart.web.proxy import proxy_if_configured
48from chart.project import SID
49from chart.sids.exceptions import BadSID
50from chart.reports.archive import MissingReport
51from chart.reports.report_group import get_groups
52from chart.backend.activity import UnknownActivity
53from chart.reports.publish import publish_report
54from chart.reports.publish import unpublish_report
55from chart.db.settings import DatabaseEngine
56from chart.common.traits import str_to_datetime
57
58logger = logging.getLogger()
59
60# configure mixed-mode (latin-1 and utf-8) decoding
61codecs.register_error('mixed', mixed_decoder)
62
63db_conn = db_connect('REPORTS') # also used for report_revisions
64
65# Job category to use when creating PUBLISH jobs
66CATEGORY = 'SCHEDULER'
67
68# Number of entries to retrieve from database for latest reports page
69LATEST_REPORTS_LIST_SIZE = 20
70
71# Template for report info tooltip in calendar screen
72TOOLTIP_TMPL = """Start time: {start}
73Stop time: {stop}
74Generation time: {gen}{ed}{pub}"""
75
76# MIME type for returning PDF documents
77MIME_PDF = 'application/pdf'
78
79# Temporary PDF file name
80PDFNAME = Path('report.pdf')
81
82# 1 day
83DAY = timedelta(days=1)
84
85# Where to temporarily unpack report to create PDF
86PDF_TEMP_DIR_TMPL = '/tmp/chart-pdf-{code}'
87
88ReportInfo = namedtuple('ReportInfo', 'sensing_start sensing_stop gen_time last_publish_time '\
89 'revision_count last_revision_time')
90
91def error_response(mess):
92 """Package `mess` into a Django error response."""
93 return HttpResponse(mess)
94
95
96def reverse_with_query( # (dangerous default value) pylint:disable=W0102
97 function,
98 groups={},
99 query={}):
100 """Improved version of Django reverse() which can set URL query parameters too."""
101 return reverse(function, **groups) + '?' + urllib.parse.urlencode(query)
102
103
104def json_response(**kwargs):
105 """Helper function to return a json structure to the browser."""
106 return HttpResponse(json.dumps(kwargs), content_type='application/json')
107
108
109@memoized
110def min_max_cur():
111 """Return sensing time range for activity `activity` of SCID `scid`."""
112 return db_conn.prepared_cursor('SELECT min(sensing_start),max(sensing_start) '
113 'FROM reports WHERE activity=:activity')
114
115
116@proxy_if_configured
117def index(request):
118 """Draw the list of report types.
119 Template parameters are:
120
121 - `reports` (list of dict)
122 - `name` (str): Name of group
123 - `reports` (list): Reports within group
124 - `name` (str)
125 - `description` (str)
126 - `activity` (Activity)
127 - `template` (str)
128 - `scids` (list of dict)
129 - `name` (str)
130 - `url` (str)
131
132 - `num_scids` (int)
133
134 The first group has no name and consists of all activities without a <group> element.
135 """
136
137 # Collect an unsorted list of which reports exist for each satellite
138 activity_sids = SID.all_report_sids()
139
140 # sort reports using information retrieved from the report groups file
141 group_handler = get_groups()
142 groups = {}
143
144 # if a reports group file exists, use it to sort the reports index page
145 if group_handler is not None:
146 for group_name, group_info in group_handler.groups.items():
147 groups[group_name] = group_info
148 group_activities = []
149 for activity in group_info['activities']:
150 sids = []
151 for sid in activity_sids[activity.name]:
152 # URL query parameters for calendar view
153 query = {'report': activity.name}
154 query.update(sid.as_url_query())
155 sids.append({'name': sid,
156 'url': reverse_with_query('reportviewer:calendar', query=query)})
157
158 group_activities.append(dict(activity=activity, sids=sids))
159
160 group_info['activities_sids'] = group_activities
161
162 # Otherwise, just iterate through the available 'report' activities to generate the index
163 else:
164 for activity in Activity.all():
165 if activity.classname == 'report':
166 if activity.group not in groups:
167 groups[activity.group] = {'name': activity.group,
168 'description': None,
169 'duration': None,
170 'navigation': False,
171 'activities': [],
172 'activities_sids': []}
173
174 group_info = groups[activity.group]
175 group_info['activities'].append(activity)
176
177 sids = []
178 for sid in activity_sids[activity.name]:
179 # URL query parameters for calendar view
180 query = {'report': activity.name}
181 query.update(sid.as_url_query())
182 sids.append({'name': sid,
183 'url': reverse_with_query('reportviewer:calendar', query=query)})
184
185 group_info['activities_sids'].append(dict(activity=activity, sids=sids))
186
187 # shift groupless reports to the end in a 'Misc' group
188 if None in groups:
189 groups['Miscellaneous'] = groups[None]
190 del groups[None]
191
192 return render(request,
193 'reportviewer/index.html',
194 dict(groups=groups,
195 request=request))
196
197
198def latest_reports(request):
199 """Draw the list of latest reports."""
200 reports = []
201
202 # access DB to retrive latest reports
203 # rows = db_conn.query('SELECT * FROM ('
204 # 'SELECT archive_filename, gen_time, sensing_stop '
205 # 'FROM reports '
206 # 'ORDER BY GEN_TIME DESC '
207 # ') where rownum <= :num_rows',
208 # num_rows=settings.LATEST_REPORTS_LIST_SIZE)
209
210 now = datetime.now()
211 for row in find_reports(
212 fields=('ARCHIVE_FILENAME', 'SENSING_START', 'SENSING_STOP', 'ACTIVITY', 'GEN_TIME') +
213 tuple(SID.sql_sys_select('REPORTS')),
214 ordering='sensing_start DESC',
215 limit=LATEST_REPORTS_LIST_SIZE):
216 # print('row ' + str(row))
217 archive_filename, start_time, stop_time, activity_name, gen_time = row[:5]
218 sid = SID.from_sys_select('REPORTS', row[5:])
219 name = Path(archive_filename).stem
220 gen_time_delta = now - gen_time
221 report_url = reverse(
222 'reportviewer:report/file',
223 kwargs={'report_name': name,
224 'filename': chart.alg.settings.REPORT_FILENAME})
225
226 # URL query parameters for calendar view
227 calendar_url = reverse_with_query(
228 'reportviewer:calendar',
229 query=sid.as_url_query(
230 {'report': activity_name}))
231
232 activity = Activity(activity_name)
233 if activity.use_report_log is True:
234 entry = {'activity': activity_name,
235 'sensing_start': start_time,
236 'report_url': report_url,
237 'calendar_url': calendar_url,
238 'sid': sid,
239 'gen_time_delta': gen_time_delta,
240 'sensing_stop': stop_time,
241 'gen_time': gen_time}
242 reports.append(entry)
243
244 return render(request,
245 'reportviewer/latest_reports.html',
246 dict(request=request,
247 latest_reports=reports))
248
249
250def get_calendar_url(activity,
251 sid,
252 year=None):
253 """Compute the URL for the calendar page."""
254 query = {'report': activity.name}
255 if year is not None:
256 query['year'] = year
257
258 query.update(sid.as_url_query())
259 return reverse_with_query('reportviewer:calendar', query=query)
260
261
262@proxy_if_configured
263def calendar(request):
264 """Generate a calendar view for specified `sid` in `year` for `reportname`."""
265 activity_name = request.GET.get('report')
266 if activity_name is None:
267 return error_response('No activity in URL')
268
269 try:
270 activity = Activity(activity_name)
271 except (IOError, UnknownActivity):
272 raise Http404('Bad activity name')
273
274 year = request.GET.get('year')
275 try:
276 sid = SID.from_django_request(request.GET)
277 except BadSID:
278 raise Http404('Bad source ID')
279
280 # list of sources for which this report has been generated
281 sids = []
282
283 for s in SID.report_sids(activity):
284 sids.append({'sid': s,
285 'url': get_calendar_url(activity, s, year)})
286
287 # find list of years for which this report has been generated with `scid`
288 mn, mx = min_max_cur().execute(None, activity=activity.name).fetchone()
289 if db_conn.engine is DatabaseEngine.SQLITE:
290 if mn is not None:
291 mn = str_to_datetime(mn)
292
293 if mx is not None:
294 mx = str_to_datetime(mx)
295
296 years = []
297 now = datetime.utcnow()
298
299 if year is None:
300 year = datetime.utcnow().year
301
302 else:
303 year = int(year)
304
305 for i in range(mn.year, max(mx.year, now.year) + 1):
306 years.append({'url': get_calendar_url(activity, sid, i),
307 'name': i,
308 'highlight': i == year})
309
310 # first pass - retrieve all relevant data for the year
311 # and store as start date against info
312 reports = {}
313 for sensing_start, sensing_stop, gen_time, last_publish_time, revision_count,\
314 last_revision_time in find_reports(
315 fields=('SENSING_START', 'SENSING_STOP', 'GEN_TIME', 'LAST_PUBLISH_TIME',
316 'REVISION_COUNT', 'LAST_REVISION_TIME'),
317 start_time_ge=datetime(year, 1, 1),
318 start_time_lt=datetime(year + 1, 1, 1),
319 sid=sid,
320 activity=activity):
321 # only one report/day allowed so collect based on date only, not datetime
322 reports[sensing_start.date()] = ReportInfo(
323 sensing_start=sensing_start.date(),
324 sensing_stop=sensing_stop,
325 gen_time=gen_time,
326 last_publish_time=last_publish_time,
327 revision_count=revision_count or 0,
328 last_revision_time=last_revision_time)
329
330 # build the calendar
331 table = StringIO()
332 table.write('<table class="cal"><tr>\n')
333
334 MONTHS_PER_ROW = 4
335
336 # For reports that cover multiple days keep track of the one we are currently rendering
337 current_report = None
338
339 # include full year for historical years, if showing current year
340 # just show months up to current month
341 for month in range(1, 13): # if year < now.year else now.month + 1):
342 table.write('<td class="month"><div class="monthname">{month}</div><br>'
343 '<table class="month"><tr>'
344 '<th> </th><th>Mon</th><th>Tue</th><th>Wed</th><th>Thu</th><th>Fri</th>'
345 '<th>Sat</th><th>Sun</th></tr><tr>'.format(
346 month=datetime(year, month, 1).strftime('%B')))
347
348 first_week = True # left pad cells in the first week
349 write_week_number = True # flag to write week number at the first column
350
351 for day in range(1, 32):
352 try:
353 today = datetime(year, month, day)
354
355 except ValueError:
356 # if the current month has less than 31 days don't show them
357 break
358
359 if write_week_number:
360 table.write('<td class="week">{week}</td>'.format(
361 week=today.isocalendar()[1]))
362 write_week_number = False
363
364 if first_week:
365 for _ in range((datetime(year, month, 1).weekday()) % 7):
366 # if month doesn't begin on Sunday leave some spaces
367 table.write('<td class="day"></td>')
368 first_week = False
369
370 # If a report starts on the day we are processing then update current_report
371 new_report = reports.get(today.date())
372 if new_report is not None:
373 current_report = new_report
374
375 if current_report is not None:
376 # access reports table to retrieve info
377 report_name = get_report_name(activity, sid, current_report.sensing_start)
378
379 # show generation time and number of times edited in the tooltip
380 if current_report.gen_time is None:
381 tooltip_gen = 'Unknown'
382
383 else:
384 tooltip_gen = current_report.gen_time
385
386 if current_report.revision_count > 0:
387 tooltip_ed = '\nEdited {n} times.'.format(n=current_report.revision_count)
388
389 else:
390 tooltip_ed = ''
391
392 # we colour the background depending on the publish status of the report
393 if current_report.last_publish_time is None:
394 publish_marker = 'unpublished'
395
396 elif current_report.last_revision_time is None or \
397 current_report.last_publish_time > current_report.last_revision_time:
398 publish_marker = 'up_to_date'
399
400 elif current_report.last_publish_time < current_report.last_revision_time:
401 publish_marker = 'outdated'
402
403 # add publish status to tooltip (only for project with publishable reports)
404 if settings.REPORT_PUBLISH_HOST:
405 tooltip_pub = '\nPublish status: {st}'.format(st=publish_marker)
406
407 else:
408 tooltip_pub = ''
409
410 # show start and stop time in the tooltip
411 tooltip = TOOLTIP_TMPL.format(
412 start=current_report.sensing_start,
413 stop=current_report.sensing_stop,
414 gen=tooltip_gen,
415 ed=tooltip_ed,
416 pub=tooltip_pub)
417
418 # set stop time to time 00:00 as in some cases reports do not end on day bounadary
419 is_last_day = today == datetime.combine(current_report.sensing_stop, time.min) - DAY
420
421 # if so create a link to it
422 table.write('<td class="day {marker}{start}{stop}">'
423 '<a class="report" href="{url}" '
424 'title="{tooltip}"><div style="height:100%;width:100%">{day}{edited} </div></a></td>'.format(
425 marker=publish_marker,
426 start=' start' if today.date() == current_report.sensing_start else '',
427 stop=' stop' if is_last_day else '',
428 day=day,
429 edited='<sup>*</sup>' if current_report.revision_count > 0 else '',
430 url=reverse('reportviewer:report/index',
431 kwargs={'report_name': report_name}),
432 tooltip=tooltip))
433
434 if is_last_day:
435 current_report = None
436
437 else:
438 table.write('<td class="day">{day}</td>'.format(day=day))
439
440 # advance to next week
441 if today.weekday() == 6: # sunday
442 table.write('</tr><tr>')
443 write_week_number = True
444
445 # end of month
446 table.write('</tr></table></td>\n')
447
448 # advance to a new row every 3 mnths
449 if month % MONTHS_PER_ROW == 0:
450 table.write('</tr><tr>\n')
451
452 # end of year
453 table.write('</tr></table>\n')
454
455 if settings.LOCKDOWN_RESTRICTED_ACCESS:
456 info_url = None
457
458 else:
459 info_url = reverse('backend:activities/single',
460 kwargs={'activityname': activity.name})
461
462 return render(request,
463 'reportviewer/calendar.html',
464 dict(back=reverse('reportviewer:index'),
465 activity=activity,
466 info=info_url,
467 sid=sid,
468 sids=sids,
469 year=year,
470 description=activity.description,
471 years=years,
472 table=table.getvalue(),
473 request=request))
474
475
476def report_index(request, report_name):
477 """Just redirect to report.html.
478 This is needed because the initial, natural URL for a report is:
479
480 http://chart/dev/eps/reports/report/ADCS_TRENDING_REPORT_M02_20150202
481
482 but the report contains relative links like 'a.png' which must be addressed as:
483
484 http://chart/dev/eps/reports/report/ADCS_TRENDING_REPORT_M02_20150202/a.png
485
486 not:
487
488 http://chart/dev/eps/reports/report/a.png
489
490 Therefore we must redirect the initial request from http://{base}/{report}
491 to http://{base}/{report}/report.html (we use report.html because then the URL matches
492 the filename, it could be anything).
493 """
494 return redirect(reverse('reportviewer:report/file',
495 kwargs={'report_name': report_name,
496 'filename': chart.alg.settings.REPORT_FILENAME}))
497
498
499@proxy_if_configured
500def report_file(request, report_name, filename):
501 """Retrieve one file from a report.
502 If `filename` is not specified we return report.html, the index page, after
503 modifying it to include a report viewer header.
504 If `filename` is report.html we return the literal raw file.
505 """
506 if filename == str(chart.alg.settings.REPORT_FILENAME):
507 # manipulate report.html to add our header
508 try:
509 buff = retrieve_file(report_name,
510 chart.alg.settings.REPORT_FILENAME)
511
512 except zipfile.BadZipfile as bzf:
513 return HttpResponse(
514 'Filesystem error opening zipfile: {e}<br>'
515 'It may be possible to fix this error by restarting the web application server '
516 'process'.format(e=bzf))
517
518 except MissingReport as e:
519 return HttpResponse(str(e))
520
521 # a little piece of python3 xml craziness here. the PLM reports used to include
522 # "± " to encode a +/- sign. This ended up as a literal \xb1 code in report.html.
523 # python2 lxml had no problem converting this between buffer and HTML so no problems.
524 # but python3 lxml, for some reason, throws a very unhelpful exception on the same code
525 # (SerialisationError / IO_ENCODER)
526 # for the v1.9 release the bad sign was removed from the PLM report but because there
527 # are many instances of this symbol in the EPS reports archive, we sanitise every
528 # report.html file just in case
529 # \xb0 is the degree symbol that appears in EPS_SC_STATUS report with some events
530 # (M01 2017w37 hits it)
531 buff = buff.decode('utf-8', 'mixed') # replace(b'\xb1', b' ').replace(b'\xb0', b'°')
532
533 # logging.debug('building tree from ' + str(type(buff)) + ' len ' + str(len(buff)))
534 html_elem = lxml.html.fromstring(buff)
535 # bulk = lxml.html.tostring(html_elem)
536 # logging.debug('html elem ' + str(html_elem))
537
538 head_elem = html_elem.find('head')
539 if head_elem is None:
540 html_elem.insert(0, lxml.html.fromstring('<head></head>'))
541 head_elem = html_elem.find('head')
542
543 # We can't directly insert the output of the Django templates here
544 # since lxml doesn't allow blobs of text inserted into a parsed element tree.
545 # So we create a second tree based on the Django template and insert it
546 # element-by-element into the main page.
547 head_template = get_template('reportviewer/fragment_head.html')
548 for pos, head_fragment in enumerate(lxml.html.fragments_fromstring(
549 head_template.render({'settings': settings}))):
550 # head_template.render(RequestContext(request, {'settings': settings})))):
551
552 head_elem.append(head_fragment)
553
554 body_elem = html_elem.find('body')
555
556 # In case report.html is simply a redirect with no body
557 if body_elem is None:
558 # i.e. report.html is a redirect page with no <body>
559 return HttpResponse(lxml.html.tostring(html_elem))
560
561 # put together the `related` structure giving other similar reports the user
562 # can jump to
563 navigation = []
564 parts = decode_report_name(report_name)
565 activity = parts['activity']
566 sid = parts['sid']
567 sensing_start = parts['sensing_start']
568 related = activity.related
569 if related is not None:
570 navigation.append({'name': 'Related',
571 'related': [Activity(r) for r in related]})
572
573 # if a report group file exists, use it to group drop down activities
574 group_handler = get_groups()
575 if group_handler is not None:
576 for group_name, group_info in group_handler.groups.items():
577 if activity in group_info['activities']:
578 navigation.append({'name': group_name,
579 'related': [act for act in group_info['activities']]})
580
581 # get current user to later check whether he or she can publish
582 if request.user.id is not None:
583 my_user = User(user_id=request.user.id)
584 can_publish = my_user.can_publish_reports
585
586 else:
587 can_publish = False
588
589 # open manifest from archived report to retrieve report info
590 try:
591 manifest = Manifest(buff=retrieve_file(report_name, chart.alg.settings.MANIFEST_FILENAME))
592 sensing_stop = manifest.sensing_stop
593 publish_time = manifest.get_latest_publish_time()
594 revision_count = manifest.revision_count()
595 revisions = manifest.get_revisions()
596 history = manifest.get_history()
597
598 except IOError: # Report does not contain a manifest
599 sensing_stop = None
600 publish_time = None
601 revision_count = None
602 revisions = None
603 history = None
604
605 # get additional info from records table to show in info page
606 row = find_reports(fields=('GEN_TIME',
607 'LAST_PUBLISH_TIME',
608 'REVISION_COUNT',
609 'LAST_REVISION_TIME'),
610 start_time_ge=sensing_start,
611 start_time_lt=sensing_start+timedelta(days=1),
612 sid=sid,
613 activity=activity).fetchone()
614 if row is None:
615 return HttpResponse('Report not found')
616
617 gen_time, last_publish_time, revision_count, last_revision_time = row
618
619 for pos, body_fragment in enumerate(
620 lxml.html.fragments_fromstring(
621 get_template('reportviewer/fragment_body.html').render(
622 # RequestContext(
623 # request,
624 dict(settings=settings,
625 sid=sid,
626 back=reverse_with_query(
627 'reportviewer:calendar',
628 query=sid.as_url_query(dict(report=activity.name,
629 year=sensing_start.year))),
630 raw=reverse(
631 'reportviewer:report/raw',
632 kwargs={'report_name': report_name,
633 'filename': str(chart.alg.settings.REPORT_FILENAME)}),
634 sats=SID.all(operational=True),
635 navigation=navigation,
636 activity=activity,
637 edit=None if settings.LOCKDOWN_RESTRICTED_ACCESS else reverse(
638 'reportviewer:report/edit',
639 kwargs={'report_name': report_name,
640 'filename': str(chart.alg.settings.REPORT_FILENAME)}),
641 download=reverse(
642 'reportviewer:report/zip',
643 kwargs={'report_name': report_name}),
644 publish_url=None if settings.LOCKDOWN_RESTRICTED_ACCESS else reverse(
645 'reportviewer:report/publish',
646 kwargs={'report_name': report_name}),
647 unpublish_url=None if settings.LOCKDOWN_RESTRICTED_ACCESS else reverse(
648 'reportviewer:report/unpublish',
649 kwargs={'report_name': report_name}),
650 can_publish=can_publish,
651 is_published=last_publish_time is not None,
652 pdf=reverse('reportviewer:report/pdf',
653 kwargs={'report_name': report_name}),
654 name=report_name,
655 sensing_start=sensing_start,
656 sensing_stop=sensing_stop,
657 publish_time=last_publish_time,
658 revision_count=revision_count,
659 gen_time=gen_time,
660 last_revision_time=last_revision_time,
661 revisions=revisions,
662 history=history,
663 gap_warning=request.GET.get('gap_warning'),
664 # just a little piece of Django madness, after the 1.9 upgrade
665 # the middleware that automatically passes the User object to page
666 # templates stopped working, so we pass it manually
667 user=request.user,
668 )))):
669 body_elem.insert(pos, body_fragment)
670
671 bulk = lxml.html.tostring(html_elem, pretty_print=True).decode('utf-8') # encoding='utf-8').
672 # bulk = lxml.html.tostring(html_elem)
673 # bulk = bulk.decode('utf-8')
674 # don't use xml_to_str as it makes invalid html links
675 # bulk = xml_to_str(html_elem, pretty_print=True)
676 return HttpResponse('<!doctype html>\n' + bulk)
677
678 else:
679 try:
680 return HttpResponse(retrieve_file(report_name, filename),
681 mimetypes.guess_type(str(filename))[0])
682 except KeyError as e:
683 # print('hello')
684 # logger.debug(e)
685 raise Http404('Cannot find item {path}'.format(path=filename))
686 # return HttpResponse(
687
688
689@proxy_if_configured
690def raw(_, report_name, filename):
691 """Display just a report without header."""
692
693 try:
694 return HttpResponse(retrieve_file(report_name, filename),
695 mimetypes.guess_type(filename)[0])
696
697 except KeyError:
698 # return HttpResponse('No {filename} found in report archive'.format(
699 # filename=filename))
700 raise Http404('Cannot find {file} in {report}'.format(
701 file=filename, report=report_name))
702
703
704def edit(request, report_name, filename):
705 """Set up an editable report page.
706 If `filename` is `report.html` return the complete editor page.
707 Otherwise return raw files from the report.
708 """
709 parts = decode_report_name(report_name)
710 activity = parts['activity']
711 sid = parts['sid']
712 sensing_start = parts['sensing_start']
713
714 html = lxml.html.fromstring(retrieve_file(report_name, chart.alg.settings.REPORT_FILENAME))
715 titles = html.xpath('head/title/text()')
716 if len(titles) > 0:
717 title = titles[0]
718
719 else:
720 title = ''
721
722 body = html.find('body')
723 body.tag = 'div'
724
725 # extract only the report content, removing the <head> section and the <body> and </body> tags
726 # body_content = lxml.html.tostring(html_elem.find('body'))[6:-7]
727
728 if filename == str(chart.alg.settings.REPORT_FILENAME):
729 return render(request,
730 'reportviewer/edit.html',
731 dict(back=get_calendar_url(activity=activity,
732 sid=sid,
733 year=sensing_start.year),
734 report_name=report_name,
735 # reportname=activity,
736 cancel_url=reverse('reportviewer:report/file',
737 kwargs={'report_name': report_name,
738 'filename': filename}),
739 request=request,
740 report=lxml.html.tostring(body)
741 # Convert from bytestring to regular string
742 .decode('UTF-8')
743 # Remove non-HTML whitespace characters
744 .replace('\n', '')
745 .replace('\t', ''),
746 title=title))
747
748 else:
749 try:
750 return HttpResponse(retrieve_file(report_name, filename),
751 mimetypes.guess_type(filename)[0])
752 except IOError:
753 raise Http404('Cannot find item {p}'.format(p=filename))
754
755
756@proxy_if_configured
757def zip_download(_, report_name):
758 """Return a complete .zip archive to browser."""
759
760 response = HttpResponse(content_type='application/zip')
761 filename = get_report_abspath(report_name)
762 response['Content-Disposition'] = 'attachment; filename={name}'.format(
763 name=filename.name)
764 response.write(filename.open('rb').read())
765 return response
766
767
768def handle_publish_report(request, report_name):
769 """Publish a report using the implementation function."""
770 # get current user to later check whether he or she can publish
771 if request.user.id is not None:
772 my_user = User(user_id=request.user.id)
773 can_publish = my_user.can_publish_reports
774
775 else:
776 can_publish = False
777
778 if not can_publish:
779 return json_response(error='You do not have permission to publish reports.')
780
781 publish_report(report_name, my_user)
782 return json_response(message='The report has been published.')
783
784
785def handle_unpublish_report(request, report_name):
786 """Unpublish a report using the implementation function."""
787 # get current user to later check whether he or she can publish
788 if request.user.id is not None:
789 my_user = User(user_id=request.user.id)
790 can_publish = my_user.can_publish_reports
791
792 else:
793 can_publish = False
794
795 if not can_publish:
796 return json_response(error='You do not have permission to unpublish reports.')
797
798 unpublish_report(report_name)
799 return json_response(message='The report has been removed from the publish area.')
800
801
802def zip_to_pdf(zip_filename):
803 """Convert `zip_filename` to a PDF, returning a file handle, using
804 settings.PDF_CONVERTER_COMMAND.
805 If settings.PDF_CONVERTER_HOST is non-null the archive will be piped o that machine
806 over SSH, uncompressed to a temp dir /tmp/chart-pisa-$$, converted, piped back
807 then cleaned up.
808 A better solution would be to create PDFs at initial generation or ingestion time
809 generate them on the fly if missing.
810 We could maybe go for some kind of PDF file cache instead but I'm not sure that's worth it.
811 """
812 if settings.PDF_CONVERTER_HOST is None:
813 logging.debug('Local PDF conversion')
814 # I think tempfile caused problems on one of the MPSTAR systems
815 # with tempfile.TemporaryDirectory(prefix='chartpdf-') as tempdir:
816 tempdir = Path(PDF_TEMP_DIR_TMPL.format(code=random.randint(0,1000000)))
817 # tempdir = Path(PDF_TEMP_DIR_TMPL.format(code=os.getpid()))
818 # logging.debug('Temp dir {t}'.format(t=tempdir))
819 tempdir.mkdir(parents=True)
820 # os.chdir(str(tempdir))
821 with tempdir.chdir():
822 # zipfile might be faster
823 subprocess.call(('unzip', '{z}'.format(z=zip_filename)))
824 command = \
825 (Path(settings.PDF_CONVERTER_COMMAND[0]).expand(),) +\
826 settings.PDF_CONVERTER_COMMAND[1:] +\
827 (chart.alg.settings.REPORT_FILENAME, PDFNAME)
828 logging.debug('Spawning {c}'.format(c=' '.join(str(s) for s in command)))
829 child = subprocess.Popen((str(s) for s in command), stdout=subprocess.PIPE)
830 out, _ = child.communicate()
831 res = PDFNAME.open('rb').read()
832
833 tempdir.rmtree()
834 return res
835
836 else:
837 logging.debug('Remote PDF conversion on {host}'.format(host=settings.PDF_CONVERTER_HOST))
838 command = ('ssh',
839 settings.PDF_CONVERTER_HOST, (
840 'a=$$; '
841 'mkdir -p /tmp/chart/pisa/$a; '
842 'cd /tmp/chart/pisa/$a; '
843 'cat > a.zip; '
844 'sync; '
845 'unzip -q a.zip; '
846 '{tool} {index} a.pdf; '
847 'cat a.pdf; '
848 # 'rm -rf /tmp/chart/pisa/$a'
849 ).format(tool=' '.join(settings.PDF_CONVERTER_COMMAND),
850 index=chart.alg.settings.REPORT_FILENAME),
851 )
852 # print('COMMAND ', command)
853 child = subprocess.Popen(command,
854 stdin=zip_filename.open('rb'),
855 stdout=subprocess.PIPE)
856 out, _ = child.communicate()
857 return out
858
859
860@proxy_if_configured
861def pdf_convert(_, report_name):
862 """Return a complete PDF file to browser."""
863 h = zip_to_pdf(get_report_abspath(report_name=report_name))
864 response = HttpResponse(content_type=MIME_PDF)
865 response.write(h)
866 return response
867
868
869def save(request):
870 """Receive edited report."""
871 report_name = request.POST['name']
872
873 # we get the plain body from the browser. If will include a wrapper <div> added above.
874 # it is <div> only
875 body = request.POST['html']
876
877 # pull the original <head> from storage
878 html_elem = lxml.html.fromstring(retrieve_file(report_name, chart.alg.settings.REPORT_FILENAME))
879 head_elem = html_elem.find('head')
880 head = lxml.html.tostring(head_elem).decode()
881
882 # Make a new complete HTML string
883 content = """<!doctype html>
884<html>
885{head}
886<body>
887{body}
888</body>
889</html>""".format(head=head, body=body)
890
891 # push back to the archive
892 update_file(report_name,
893 chart.alg.settings.REPORT_FILENAME,
894 username=request.user.username,
895 content=content)
896
897 from chart.reports.archive import retrieve_archive
898 retrieve_archive.last_args = None
899 retrieve_archive.last_result = None
900
901 return HttpResponse(reverse('reportviewer:report/file',
902 kwargs=dict(report_name=report_name,
903 filename=chart.alg.settings.REPORT_FILENAME)),
904 'text/plain')
905
906
907# def revision(request, report_id, filename, revision_number):
908# (unused argument) pylint: disable=W0613
909 # """Show a diff of a single file."""
910#
911 # this_revision = int(revision_number)
912#
913 # a = retrieve_file(report_id, filename, this_revision)
914 # b = retrieve_file(report_id, filename, this_revision - 1)
915#
916 # return render(request,
917 # 'reportviewer/revision.html',
918 # dict(diff=difflib.HtmlDiff(tabsize=4, wrapcolumn=80).make_file(a.splitlines(1),
919 # b.splitlines(1))))
920
921@proxy_if_configured
922def nav(request):
923 """From report viewer page allow navigation to related reports
924 (change of activity name or sid).
925
926 URL parameters are:
927
928 `name` (str): Existing report name
929 `op` (str): Set to '<' or '>' to navigate back/forwards in time
930 `sid` (str): !!! switch SID !!!
931 `activity` (str): New activity.
932 """
933
934 # get GET parameters from HTTP request
935 name = request.GET.get('name')
936 op = request.GET.get('op')
937 new_activity = request.GET.get('new_activity')
938 if 'sid' in request.GET:
939 new_sid = SID(request.GET['sid'])
940
941 else:
942 new_sid = None
943
944 # new_sid = SID.from_django_request(request.GET) # .get('new_sid'))
945
946 # decode report name to obtain activity, sid, sensing_start
947 parts = decode_report_name(name)
948 activity = parts['activity']
949 sid = parts['sid']
950 sensing_start = parts['sensing_start']
951
952 gap_detected = False
953
954 new_name = None
955
956 # `op` may be '<' or '>' Note: Report sensing_start maynot be at time 00:00.
957 # Currently can only have maximum 1 report per day
958 if op is not None:
959 if op == '>':
960 sensing_start += timedelta(days=1)
961
962 new_time, = db_conn.query('SELECT {agg}(sensing_start) '
963 'FROM reports '
964 'WHERE activity=:activity '
965 'AND {sid_clause} '
966 'AND sensing_start{op}:sensing_start'.format(
967 agg='max' if op == '<' else 'min', op=op, sid_clause=SID.sql_sys_where('', sid)),
968 activity=activity.name,
969 sensing_start=sensing_start).fetchone()
970
971 if new_time is None:
972 return json_response(error='No report found')
973
974 # check the report group file (if available), to check for gaps between reports
975 group_handler = get_groups()
976 if group_handler is not None:
977 duration = group_handler.activities[activity]['duration']
978 if duration is not None and abs(new_time - sensing_start) > duration:
979 gap_detected = True
980
981 new_name = get_report_name(activity, sid, new_time)
982
983 elif new_activity is not None:
984 # find the report with max sensing_time, less than or equal to `sensing_start`
985 report_start = db_conn.query('SELECT max(sensing_start) '
986 'FROM reports '
987 'WHERE activity=:new_activity '
988 'AND {sid_clause} '
989 'AND sensing_start<=:sensing_start'.format(
990 sid_clause=SID.sql_sys_where('', sid)),
991 new_activity=new_activity,
992 sensing_start=sensing_start).fetchone()[0]
993
994 if report_start is not None:
995 new_name = get_report_name(new_activity, sid, sensing_start)
996
997 elif new_sid is not None:
998 cc = db_conn.query('SELECT count(*) '
999 'FROM reports '
1000 'WHERE activity=:activity '
1001 'AND {sid_clause} '
1002 'AND sensing_start>=:sensing_start'.format(
1003 sid_clause=SID.sql_sys_where('', new_sid)),
1004 activity=activity.name,
1005 sensing_start=sensing_start).fetchone()[0]
1006
1007 if cc > 0:
1008 new_name = get_report_name(activity, new_sid, sensing_start)
1009
1010 else:
1011 return json_response(error='Internal error')
1012
1013 if new_name is None:
1014 return json_response(error='No report found')
1015
1016 url = reverse('reportviewer:report/file', kwargs={'report_name': new_name,
1017 'filename': chart.alg.settings.REPORT_FILENAME})
1018
1019 # manually add warning param if we detect a gap
1020 if gap_detected:
1021 warning_param = '?gap_warning={val}'.format(val=str(gap_detected))
1022 url += warning_param
1023
1024 return json_response(url=url)