1#!/usr/bin/env python3
2
3"""Allow retrievals of timeseries data and configuration information from simple HTTP calls.
4
5One day when we bump the prerequisites for CHART we can include django-rest or some other
6REST framework which will handle the documentation, errors and parameters a bit more nicely,
7but for now we just do everything manually. A proper library can also generate an OpenAPI / Swagger
8description, or even autogenerate a proper web UI."""
9
10import sys
11import socket
12import json
13import csv
14from datetime import datetime
15from datetime import timedelta
16from enum import Enum
17from operator import itemgetter
18from typing import Optional
19from typing import List
20from collections import namedtuple
21import mimetypes
22
23from django.http import StreamingHttpResponse
24from django.shortcuts import render
25from django.http import HttpResponse
26from django.urls import reverse
27
28import chart
29from chart.project import SID
30from chart.project import settings
31from chart.common.traits import str_to_datetime
32from chart.common.traits import str_to_boolean
33from chart.common.traits import Choices
34from chart.common.xml import datetime_to_xml
35from chart.common.xml import timedelta_to_xml
36from chart.db.model.table import TableInfo as StorageInfo
37from chart.db.model.table import NoSuchParameter
38from chart.db.model.table import find_param_by_name
39from chart.db.model.field import FieldInfo
40from chart.db import ts
41from chart.plots.sampling import sampling_from_name
42from chart.plots.sampling import NoSuchSampling
43from chart.db.model.table import NoSuchTable as NoSuchStore
44from chart.db.func import SensingTime
45from chart.products.pus.packetdef import PacketDef
46from chart.products.pus.packetdef import PacketDomain
47from chart.plots.geoloc import Geoloc
48from chart.plots.geoloc import CannotGeolocate
49from chart.events.eventclass import EventClass
50from chart.events.exceptions import NoSuchEventClass
51from chart.events.db import find_events
52from chart.db.ts import merge_ts_or
53from chart.products.scos2000.srdb_version_info import get_srdb_version
54from chart.products.scos2000.srdb_version_info import get_srdb_date_implemented
55from chart.common.traits import name_of_thing
56from chart.alg.pus_state import build_state
57from chart.db.model.calibration import PolyCalibration
58from chart.backend.jobs import find_jobs
59from chart.backend.activity import Activity
60from chart.backend.activity import UnknownActivity
61from chart.backend.job import JobStatus
62from chart.backend.processes import find_processes
63from chart.common.path import Path
64from chart.products.products import find_products
65
66class TimeFormat(Enum):
67 """Date format to return, for timeseries function sensing time"""
68 ISO8601 = 'iso8601' # 2023-06-02T07:43:00.123456
69 PYTHON = 'python' # 2023-06-02 07:43:00.123456
70 EXCEL = 'excel' # 2023/06/02 07:43:00
71 EXCEL_FINE = 'excel_fine' # 2023/06/02 07:43:00,123456
72
73TimeFormat.EXCEL.format = '%Y/%m/%d %H:%M:%S'
74TimeFormat.EXCEL_FINE.format = '%Y/%m/%d %H:%M:%S'
75
76# Refuse to return a jobs list of it contains too many entries
77MAX_JOBS_LIST = 100000
78
79DEFAULT_TIMEFORMAT = TimeFormat.PYTHON
80
81# Supply a version number for the API itself in the /meta endpoint
82# Breaking changes must increment the major version
83API_VERSION = 2
84
85# Non breaking changes and additions should increment the minor version
86API_MINOR_VERSION = 5
87
88# Line delimiter
89NEWLINE = '\n'
90
91# HTTP return code for good result
92HTTP_OK = 200
93
94# HTTP return code for bad result
95HTTP_ERROR = 500
96
97# Note: Don't pass any exception text from internal functions into any API response as it
98# could be a security risk by exposing internal information
99
100# No /ts all-points retrievals more than this unless it's from a table with no stats
101MAX_TS_AP_DURATION = timedelta(days=14)
102
103# No /ts retrievals using stats if the estimated number of rows is more than this
104MAX_TS_STATS_ROWS = 100000
105
106# No /packets retrieval over this unless SPID filtering is on
107MAX_PACKETS_UNFILTERED_DURATION = timedelta(days=3)
108
109# No /packets SPID filtered retrieval over this
110MAX_PACKETS_FILTERED_DURATION = timedelta(days=90)
111
112# Max events retrieval time
113MAX_EVENTS_DURATION = timedelta(days=7)
114
115
116
117class APIException(Exception):
118 """Base class for API specific errors."""
119 def as_json(self):
120 return {}
121
122class RequiredParameterMissing(APIException):
123 """User has forgotten to include a required endpoint parameter."""
124 def __init__(self, parameter):
125 self.parameter = parameter
126
127 def __str__(self):
128 return 'Required parameter {p} missing from request'.format(p=self.parameter)
129
130 def as_json(self):
131 return {'parameter': self.parameter}
132
133class ParameterError(APIException):
134 """User has passed an invalid parameter value to an API endpoint."""
135 def __init__(self, parameter, message):
136 self.parameter = parameter
137 self.message = message
138
139 def __str__(self):
140 return 'Problem decoding parameter {p}: {m}'.format(p=self.parameter, m=self.message)
141
142 def as_json(self):
143 return {'parameter': self.parameter, 'message': self.message}
144
145class NotImplementedError(APIException):
146 """User requested an unimplemented feature."""
147 def __str__(self):
148 return 'Requested feature is not implemented'
149
150class DataError(APIException):
151 """Problem in the data prevents returning it."""
152 def __init__(self, message):
153 self.message = message
154
155 def __str__(self):
156 return self.message
157
158 def as_json(self):
159 return {'message': self.message}
160
161class TooLongError(APIException):
162 """Client has requested data over a long duration."""
163 def __init__(self, start, stop, maximum, suggestion):
164 self.start = start
165 self.stop = stop
166 self.maximum = maximum
167 self.suggestion = suggestion
168
169 def __str__(self):
170 return ('Requested time duration of {act} is greater than maximum allowed of {maximum}. '
171 '{suggestion}'.format(act=self.stop - self.start,
172 maximum=self.maximum,
173 suggestion=self.suggestion))
174
175 def as_json(self):
176 return {'start': self.start,
177 'stop': self.stop,
178 'maximum': self.maximum,
179 'suggestion': self.suggestion}
180
181class TooManyError(APIException):
182 """Client has requested too many data items."""
183 def __init__(self,
184 maximum,
185 # start=None,
186 # stop=None,
187 estimated=None,
188 actual=None,
189 suggestion=''):
190 # self.start = start
191 # self.stop = stop
192 self.maximum = maximum
193 self.estimated = estimated
194 self.suggestion = suggestion
195 self.actual = actual
196
197 def __str__(self):
198 # return ('Requested retrieval from {start} to {stop} is likely to contain {est} items, more '
199 # 'than the maximum {maximum}. {suggestion}').format(
200 # start=self.start,
201 # stop=self.stop,
202 # maximum=self.maximum,
203 # est='{est},'.format(est=self.estimated),
204 # suggestion=self.suggestion)
205 return ('Requested retrieval {got}, more '
206 'than the maximum {maximum}. {suggestion}').format(
207 got='is likely to contain {est} items'.format(est=self.estimated) if self.estimated is not None\
208 else 'contains {act} items'.format(act=self.actual),
209 maximum=self.maximum,
210 suggestion=self.suggestion)
211
212 def as_json(self):
213 return {#'start': self.start,
214 # 'stop': self.stop,
215 'maximum': self.maximum,
216 'estimated': self.estimated,
217 'actual': self.actual,
218 'suggestion': self.suggestion}
219
220
221def digits_to_datetime(s):
222 """Convert a timestamp expressed as numbers to a datetime object.
223
224 If 4 digits, assume it's the start of a year
225 If 6 digits, assume it's a year + month
226 If 8 digits, assume it's a year + month + day
227 If 10 digits, assume it's a year + month + day + hour
228 If 12 digits, assume it's a year + month + day + hour + minute
229 If 14 digits, assume it's a year + month + day + hour + minute + second
230 If 17 digits, assume it's a year + month + day + hour + minute + second + ms
231 If 20 digits, assume it's a year + month + day + hour + minute + second + us"""
232
233 if len(s) == 4:
234 return datetime.strptime(s, '%Y')
235
236 if len(s) == 6:
237 return datetime.strptime(s, '%Y%m')
238
239 if len(s) == 8:
240 return datetime.strptime(s, '%Y%m%d')
241
242 if len(s) == 10:
243 return datetime.strptime(s, '%Y%m%d%H')
244
245 if len(s) == 12:
246 return datetime.strptime(s, '%Y%m%d%H%M')
247
248 if len(s) == 14:
249 return datetime.strptime(s, '%Y%m%d%H%M%S')
250
251 if len(s) == 17:
252 return datetime.strptime(s, '%Y%m%d%H%M%S%f') # yes this works for milliseconds
253 # even though it's not documented
254
255 if len(s) == 20:
256 return datetime.strptime(s, '%Y%m%d%H%M%S%f')
257
258 else:
259 raise ValueError('Cannot decode numerical time of {s}'.format(s=s))
260
261
262def json_response(obj, status=HTTP_OK):
263 """Package an arbitrary Python object as a successful JSON return value."""
264 return HttpResponse(json.dumps(obj, default=json_default_encoder) + NEWLINE,
265 content_type='application/json;charset=utf-8',
266 status=status)
267
268
269def json_exception(exc):
270 """Package exception as a json response."""
271 if settings.DEBUG_API:
272 raise
273
274 else:
275 response = {'exception': exc.__class__.__name__,
276 'message': str(exc)}
277 response.update(exc.as_json())
278 return json_response(response, status=HTTP_ERROR)
279
280
281def json_default_encoder(obj:object):
282 """Tell the json module how to encode objects it can't handle by default."""
283 if isinstance(obj, timedelta):
284 return timedelta_to_xml(obj)
285
286 else:
287 return str(obj)
288
289
290class QueryParamDef:
291 def __init__(self,
292 name:str,
293 datatype:object=str,
294 default:object=None,
295 description:str=None,
296 optional:bool=False,
297 multiple:bool=False,
298 choices:List[str]=None,
299 decoder:str=None,
300 minor_version_added:int=None):
301 self.name = name
302 self.datatype = datatype
303 self.default = default
304 self.description = description
305 self.optional = optional
306 self.multiple = multiple
307 self.choices = choices
308 self.decoder = decoder
309 self.minor_version_added = minor_version_added
310
311 def show_datatype(self):
312 """Show in the API2 help page query parameters table datatype column."""
313 if isinstance(self.datatype, str):
314 result = self.datatype
315
316 elif self.datatype is str:
317 result = 'string'
318
319 else:
320 result = self.datatype.__name__
321
322 if self.optional:
323 result += ', optional'
324
325 if self.multiple:
326 result += ', multiple'
327
328 if self.default:
329 result += ', default {d}'.format(d=self.default)
330
331 if self.choices:
332 result += ', [{choices}]'.format(
333 choices='|'.join('"{val}"'.format(val=v) for v in self.choices))
334
335 return result
336
337class Endpoint:
338 def __init__(self,
339 url_name:str,
340 description:str=None,
341 query_param_defs:List[QueryParamDef]=None,
342 return_def:str=None,
343 minor_version_added:int=None):
344 assert url_name is not None
345 self.description = description
346 self.query_param_defs = query_param_defs
347 self.return_def = return_def
348 self.url_name = url_name
349 self.minor_version_added = minor_version_added
350
351# List of endpoint information for building help page
352endpoints = []
353
354def api_call(description:str=None,
355 query_param_defs:List[QueryParamDef]=None,
356 return_def:str=None,
357 url_name:str=None,
358 minor_version_added:int=None):
359 """Wrapper to declare an API2 endpoint."""
360 def wrapper(fn):
361 endpoints.append(Endpoint(description=description,
362 query_param_defs=query_param_defs,
363 return_def=return_def,
364 url_name=url_name,
365 minor_version_added=minor_version_added))
366 def imp(request):
367 try:
368 return fn(request)
369
370 except APIException as exc:
371 if settings.DEBUG_API:
372 raise
373
374 response = {'exception': exc.__class__.__name__,
375 'message': str(exc)}
376 response.update(exc.as_json())
377 return json_response(response, status=HTTP_ERROR)
378
379 except Exception as exc:
380 if settings.DEBUG_API:
381 raise
382
383 return json_response({
384 'exception': 'InternalError',
385 'message': str(exc)}, status=HTTP_ERROR)
386
387 return imp
388
389 return wrapper
390
391
392@api_call(
393 description='Generate this help page',
394 return_def='This HTML document',
395 url_name='api2:index',
396)
397def view_index(request):
398 """Index help page."""
399 return render(request, 'api2/index.html', dict(
400 endpoints=endpoints,
401 API_VERSION=API_VERSION,
402 API_MINOR_VERSION=API_MINOR_VERSION))
403
404
405class Echo:
406 """An object that implements the write method of the file-like interface.
407
408 Handy for Django HTTP streaming responses."""
409 def write(self, row):
410 """Write the value by returning it, instead of storing in a buffer."""
411 # result += ','.join(str(r) for r in row) + '\n'
412 # return value
413 # return (str(i) for i in row)
414 return row
415
416
417def param_str(request, param, allow_none=False):
418 """Generic extract str parameter from `request`."""
419 str_value = request.GET.get(param)
420 if str_value is None:
421 if allow_none:
422 return None
423
424 raise RequiredParameterMissing(param)
425
426 return str_value
427
428
429def param_strs(request, params, allow_none=False):
430 """Generic extract a list of strings from request.
431
432 If `params` is a list then first is the normal parameter name, subsequent values
433 are alts'"""
434 if isinstance(params, str):
435 params = [params]
436
437 for param in params:
438 result = request.GET.getlist(param)
439 if len(result) > 0:
440 return result
441
442 if allow_none:
443 return []
444
445 raise RequiredParameterMissing(params[0])
446
447
448def param_bool(request, param, default):
449 """Extract generic boolean parameter value from `request`."""
450 str_value = request.GET.get(param)
451 if str_value is not None:
452 try:
453 result = str_to_boolean(str_value)
454 except ValueError as e:
455 raise ParameterError(param, e)
456
457 else:
458 result = default
459
460 return result
461
462
463def param_int(request, param, allow_none=False):
464 """Extract int parameter from `request`."""
465 str_value = request.GET.get(param)
466 if str_value is None:
467 if allow_none:
468 return None
469
470 raise RequiredParameterMissing(param)
471
472 return int(str_value)
473
474
475def param_sid(request, allow_none=False) -> Optional[SID]:
476 """Extract a source-id from `request` parameter `sid`.
477
478 Raises exceptions if the parameter is missing or does not match a known source.
479 Case insensitive.
480
481 We also accept "source" as a synonym."""
482 SID() # unfortunately class needs manual initialisation
483 if 'source' in request.GET:
484 try:
485 return SID.from_string(request.GET.get('source'))
486 except ValueError as e:
487 raise ParameterError('source', e)
488
489 elif 'sid' in request.GET:
490 try:
491 return SID.from_string(request.GET.get('sid'))
492 except ValueError as e:
493 raise ParameterError('sid', e)
494
495 elif allow_none:
496 return None
497
498 raise RequiredParameterMissing('source')
499
500param_sid.help = QueryParamDef(
501 name='source',
502 description=(
503 'Source ID, also known as SID (usually a spacecraft or ground station identifier). '
504 'See the ``/meta/project`` endpoint for possible values for the project.'))
505
506def param_packetdomain(request):
507 """Extract a PacketDomain from the `domain` parameter of `request`.
508
509 Raise exceptions if missing or not recognised"""
510 domain_s = request.GET.get('domain')
511 if domain_s is None:
512 raise RequiredParameterMissing('domain')
513
514 try:
515 domain = PacketDomain(domain_s)
516 except ValueError as e:
517 raise ParameterError('domain', e)
518
519 return domain
520
521param_packetdomain.help = QueryParamDef(
522 name='domain',
523 choices=['tm', 'tc', 'ev'],
524 description='Domain of packets to retrieve from')
525
526
527
528def param_numeric_value(request):
529 """Extract the `value` parameter of `request`.
530
531 Raise exceptions if missing or not a number."""
532 value_s = request.GET.get('value')
533 if value_s is None:
534 raise RequiredParameterMissing('value')
535
536 if '.' in value_s:
537 try:
538 value = float(value_s)
539 except ValueError as e:
540 raise ParameterError('value', e)
541
542 else:
543 try:
544 value = int(value_s)
545 except ValueError as e:
546 raise ParameterError('value', e)
547
548 return value
549
550
551def param_store(sid, request):
552 """Extract a StorageInfo from `request` parameter `store`.
553
554 Also accepts `table` because that's how they are also refereed to.
555 Raises exceptions if parameter missing or unknown.
556 `sid` is request for future per-source storage."""
557 if 'store' not in request.GET:
558 if 'table' not in request.GET:
559 raise RequiredParameterMissing('store')
560
561 store_str = request.GET.get('table')
562
563 else:
564 store_str = request.GET.get('store')
565
566 try:
567 store = StorageInfo(store_str)
568 except NoSuchStore as e:
569 raise ParameterError('store', e)
570
571 return store
572
573
574def param_param(store, request):
575 """Return a parameter within a storage from `request`."""
576 if 'param' not in request.GET:
577 raise RequiredParameterMissing('param')
578
579 param_str = request.GET['param']
580 param = store.fields.get(param_str)
581 if param is None:
582 raise ParameterError('param', 'Not found')
583
584 else:
585 if param.choice_name is not None:
586 # load choices, if not done already
587 choices = Choices(choice_id=param.choice_name)
588 param.choices = choices.load_xml(sid=sid)
589
590 return param
591
592
593def param_category(request):
594 """Return category parameter from `request`."""
595 category = request.GET.get('category')
596 if category is None:
597 raise RequiredParameterMissing('category')
598
599 return category.upper()
600
601
602def param_activity(request, allow_none:bool=False) -> Activity:
603 """Return activity parameter from `request`."""
604 activity_str = request.GET.get('activity')
605 if activity_str is None:
606 if allow_none:
607 return None
608
609 raise RequiredParameterMissing('activity')
610
611 try:
612 return Activity(activity_str)
613
614 except UnknownActivity as e:
615 raise ParameterError('activity', e)
616
617
618def param_jobstatus(request, allow_none:bool=False) -> JobStatus:
619 """Return status parameter from `request` as a JobStatus."""
620 status_str = request.GET.get('status')
621 if status_str is None:
622 if allow_none:
623 return None
624
625 raise RequiredParameterMissing('status')
626
627 try:
628 return JobStatus[status_str.upper()]
629
630 except ValueError as e:
631 raise ParameterError('status', e)
632
633
634def param_id(request) -> int:
635 """Return an ID parameter `request`."""
636 id_str = request.GET.get('id')
637 if id_str is None:
638 raise RequiredParameterMissing('id')
639
640 return int(id_str)
641
642
643def param_start_stop(request,
644 allow_none_start=False,
645 allow_none_stop=False,
646 param_start:str='start',
647 param_stop:str='stop'):
648 """Return tuple of start, stop times as datetimes from `request`."""
649 start_str = request.GET.get(param_start, None)
650 if start_str is None:
651 if not allow_none_start:
652 raise RequiredParameterMissing(param_start)
653
654 start = None
655
656 else:
657 try:
658 # We try to decode the start time as a pure numerical format (20010506)
659 # and drop back to the same decoder as the plot tool otherwise
660 if start_str.isdigit():
661 start = digits_to_datetime(start_str)
662
663 else:
664 start = str_to_datetime(request.GET[param_start])
665
666 except ValueError as e:
667 raise ParameterError(param_start, e)
668
669 stop_str = request.GET.get(param_stop, None)
670 if stop_str is None:
671 if not allow_none_stop:
672 raise RequiredParameterMissing(param_stop)
673
674 stop = None
675
676 else:
677 try:
678 if stop_str.isdigit():
679 stop = digits_to_datetime(stop_str)
680
681 else:
682 stop = str_to_datetime(stop_str)
683
684 except ValueError as e:
685 raise ParameterError(param_stop, e)
686
687 return start, stop
688
689# Don't write a standardised help text for param_start_stop because most functions need
690# different wording for time range description
691# param_start_stop.help = [
692 # QueryParamDef(
693
694
695@api_call(
696 description="""Retrieve timeseries data. The tables and fields available are match those in
697the web plot tool or DB info page.
698
699For PUS missions only additional packet header information is available via the "/packets"
700endpoint.
701
702All-points retrievals are limited to a maximum time range of {maxap} and stats retirevals
703are limited to {maxstat} rows.
704
705The synonym "/ts" is also accepted for this endpoint as an alternate name.""".format(
706 maxap=MAX_TS_AP_DURATION, maxstat=MAX_TS_STATS_ROWS),
707 query_param_defs=[
708 QueryParamDef(
709 name='sid',
710 description=('Source ID (usually a spacecraft or ground station identifier). See '
711 '/meta/project endpoint for available values.')),
712 QueryParamDef(
713 name='start',
714 description=(
715 'Retrieve data starting with specified sensing time. Format is YYYYMMDDhhmmss. '
716 'Any missing digits are assumed to be zeroes dropped from the right hand side e.g. '
717 '20230401 is midnight, April 1 and 2023040101 is one hour later.'),
718 datatype='timestamp'),
719 QueryParamDef(
720 name='stop',
721 description=('Retrieve data up to but not including specified sensing time. Format as '
722 'above.'),
723 datatype='timestamp'),
724 QueryParamDef(
725 name='store',
726 description=(
727 'Name of data store to retrieve from. This value is required and if missing the '
728 'code will not attempt to search stores for the parameters. "table" is accepted '
729 'as a synonym.')),
730 QueryParamDef(
731 name='param',
732 multiple=True,
733 description=(
734 'Name(s) of parameter(s) retrieve. Pass multiple times to read several params '
735 'from the same store. "sensing_time" must be explicitly requested if needed. '
736 'Also accepts "field" as a synonym')),
737 QueryParamDef(
738 name='cal',
739 datatype=bool,
740 default='true',
741 description=(
742 'If specified and equal to "false", uncalibrated (raw) data is returned. By '
743 'default calibrated data is returned. For parameters with no calibration default '
744 'this option has no effect.')),
745 QueryParamDef(
746 name='region',
747 optional=True,
748 description=(
749 'Query statistics. The value of the parameter is the stats sampling region to '
750 'use. Available values depend on project and data source. Use the /meta/regions '
751 'endpoint to query for available regions. There\'s no guarantee all regions will '
752 'be available for all stores. Must be used in conjunction with "stat" parameter. '
753 'Use of stats is required to retrieve longer durations.')),
754 QueryParamDef(
755 name='stat',
756 choices=['min', 'max', 'avg', 'std'],
757 multiple=True,
758 description=(
759 'Select the statistic(s) from each region to retrieve. See /meta/project to '
760 'confirm available values for current project. Standard deviations \'std\' are '
761 'generally not available for PUS-based projects. Include multiple times to '
762 'retrieve multiple stats. This option is required for all stats retrievals.')),
763 QueryParamDef(
764 name='reverse',
765 datatype=bool,
766 default='false',
767 description='Retrieve rows in reverse time order'),
768 QueryParamDef(
769 name='limit',
770 datatype=int,
771 optional=True,
772 description='Retrieve only a fixed number of rows if specified'),
773 QueryParamDef(
774 name='filter',
775 optional=True,
776 description=(
777 'Allow for a limited set of filtering operations. SPID filtering for PUS tables '
778 'is implemented by setting "spid:12345"')),
779 QueryParamDef(
780 name='timefmt',
781 # choices=[\(str)</dt>
782 optional=True,
783 description=(
784 'Select format for sensing times. Allowed values at "python" (default, '
785 '2023-06-02 07:30:05), "iso8601" (2023-06-02T07:30:05), "excel" '
786 '(2023/06/02 07:30:05), "excel_fine" (2023/06/02 07:30:05,650000)')),
787 ],
788 return_def=('CSV document with lines delimited by newline characters. Cells containing or '
789 'potentially containing spaces are quoted with double quotes. Spaces inside a '
790 'quoted cell are backslashed.'),
791 url_name='api2:timeseries')
792def view_timeseries(request):
793 """Retrieve data for one or more fields from a table."""
794 sid = param_sid(request)
795 start, stop = param_start_stop(request)
796 store = param_store(sid, request)
797 request_param_strs = param_strs(request, ['param', 'field'])
798 cal = param_bool(request, 'calibrated', default=True)
799
800 time_format = DEFAULT_TIMEFORMAT
801 time_format_str = request.GET.get('timefmt')
802 if time_format_str is not None:
803 try:
804 time_format = TimeFormat(time_format_str)
805 except ValueError as e:
806 raise ParameterError('timefmt', 'Allowed values: {vs}'.format(vs=', '.join(
807 tf.value for tf in TimeFormat)))
808
809 stats_s = request.GET.getlist('stat')
810 if stats_s is not None:
811 stats = []
812 for s in stats_s:
813 s = s.upper()
814 if s not in ('MIN', 'MAX', 'AVG', 'STD', 'CNT'):
815 raise ParameterError('stat', 'Allowed values: MIN, MAX, AVG, STD, CNT')
816
817 stats.append(s)
818
819 else:
820 stats = None
821
822 region_s = request.GET.get('region')
823 if region_s is not None:
824 try:
825 region = sampling_from_name(region_s)
826 except NoSuchSampling as e:
827 raise ParameterError('region', e)
828
829 else:
830 region = None
831
832 # Decode parameter strings to FieldInfo or SensingTime objects
833 params = []
834 for f in request_param_strs:
835 if f.lower() == 'sensing_time':
836 params.append(SensingTime)
837
838 else:
839 try:
840 param = store.get_param(f)
841 if param.choice_name is not None:
842 # load choices, if not done already
843 choices = Choices(choice_id=param.choice_name)
844 param.choices = choices.load_xml(sid=sid)
845
846 params.append(param)
847
848 except NoSuchParameter:
849 raise ParameterError(f, 'Not found in store')
850
851 # check if the require violates time limits
852 if region is None:
853 # AP
854 if (stop - start) > MAX_TS_AP_DURATION:
855 raise TooLongError(
856 start=start,
857 stop=stop,
858 maximum=MAX_TS_AP_DURATION,
859 suggestion='Use a shorter timerange or retrieve using a statistics region')
860
861 else:
862 # Stats
863 est = (stop - start) // (region.nominal_duration if region.nominal_duration is not None\
864 else sid.satellite.orbit_duration)
865 if est > MAX_TS_STATS_ROWS:
866 raise TooManyError(
867 # start=start,
868 # stop=stop,
869 maximum=MAX_TS_STATS_ROWS,
870 estimated=est,
871 suggestion='Use a shorter duration or a longer statistics region')
872
873 # Check for reversed sensing time
874 ordering = None
875 if param_bool(request, 'reverse', default=False):
876 ordering = 'SENSING_TIME DESC'
877
878 # Allow limiting return rows
879 limit = param_int(request, 'limit', allow_none=True)
880
881 # Allow filtering by spid
882 where = None
883 filter_s = request.GET.get('filter')
884 if filter_s is not None:
885 field, _, value = filter_s.partition(':')
886 if value is None:
887 raise ParameterError('filter', 'Not in format field:value')
888
889 where = '{field}={value}'.format(field=field, value=value)
890
891 # AP retrieval
892 if region is None:
893 cursor = ts.select(sid=sid,
894 table=store,
895 fields=params,
896 sensing_start=start,
897 sensing_stop=stop,
898 calibrated=cal,
899 ordering=ordering,
900 where=where,
901 limit=limit)
902
903 # Stats retrieval
904 else:
905 subcursors = []
906 for param in params[1:]:
907 for stat in stats:
908 subcursors.append(ts.select(sid=sid,
909 table=store,
910 fields=[params[0], param],
911 sensing_start=start,
912 sensing_stop=stop,
913 calibrated=cal,
914 stat=stat,
915 region=region,
916 ordering=ordering,
917 limit=limit))
918
919 cursor = merge_ts_or(subcursors)
920
921 # Translate result cursor to response
922 pseudo_buffer = Echo()
923 writer = csv.writer(pseudo_buffer)
924 # Returning text/csv is more accurate but means Firefox always prompts to download the
925 # file instead of displaying in the browser (with or without Content-Disposition set)
926 # I'm not sure if there's a nice way around, but setting mime type to plain text works
927 if time_format is TimeFormat.PYTHON:
928 lines = (writer.writerow(row) for row in cursor)
929
930 elif time_format is TimeFormat.ISO8601:
931 lines = (
932 writer.writerow([
933 cell.isoformat() if isinstance(
934 cell, datetime) else cell for cell in row]) for row in cursor)
935
936 elif time_format is TimeFormat.EXCEL_FINE:
937 def excel_fine_row(row):
938 result = []
939 for cell in row:
940 if isinstance(cell, datetime):
941 result.append(cell.strftime(TimeFormat.EXCEL_FINE.format))
942 result.append(cell.microsecond / 1e6)
943
944 else:
945 result.append(cell)
946
947 return result
948
949 lines = (writer.writerow(excel_fine_row(row)) for row in cursor)
950
951 else:
952 lines = (
953 writer.writerow([
954 cell.strftime(time_format.format) if isinstance(
955 cell, datetime) else cell for cell in row]) for row in cursor)
956
957 response = StreamingHttpResponse(lines, content_type='text/plain')
958 # response['Content-Disposition'] = 'attachment; filename="somefilename.csv"'
959 return response
960
961
962@api_call(
963 description=('Retrieve events.'),
964 query_param_defs=[
965 QueryParamDef(
966 name='sid',
967 description=('Source ID (usually a spacecraft or ground station identifier). See '
968 '/meta/project endpoint for available values.')),
969 QueryParamDef(
970 name='start',
971 description=(
972 'Retrieve data starting with specified sensing time. Format is YYYYMMDDhhmmss. '
973 'Any missing digits are assumed to be zeroes dropped from the right hand side e.g. '
974 '20230401 is midnight, April 1 and 2023040101 is one hour later.'),
975 datatype='timestamp'),
976 QueryParamDef(
977 name='stop',
978 description=('Retrieve data up to but not including specified sensing time. Format as '
979 'above.'),
980 datatype='timestamp'),
981 QueryParamDef(
982 name='class',
983 multiple=True,
984 description=(
985 'Class name(s) to scan for. No wildcards or parent class names allowed.')),
986 ],
987 return_def=('JSON by row. Each row contains a separate JSON object encoding an event start, '
988 'stop times, classname and instance properties'),
989 url_name='api2:events')
990def view_events(request):
991 """Retrieve and interpolate geolocation information.
992
993 Query parameters:
994 `sid` (str):
995 `start` (datetime):
996 `stop` (datetime):
997 `class` (str, multiple)
998
999 Not implemented:
1000 - no control of which properties are retrieved
1001 - no option to do a simple count instead of full retrieval
1002 - no control over return format
1003 - no hierarchical selection - every class must be listed individually, no specifying of
1004 parent classes
1005
1006 Return value is a text document where each line contains an event as a JSON
1007 object with start, stop (optional), class (string) and JSON object with lat, lon (x, y, z, dx, dy, dz)
1008 """
1009 try:
1010 if 'sid' not in request.GET:
1011 # This function should have a decorator that converts certain exceptions into nice
1012 # json error messages
1013 raise RequiredParameterMissing('sid')
1014
1015 try:
1016 sid = SID.from_string(request.GET.get('sid'))
1017 except ValueError as e:
1018 raise ParameterError('sid', e)
1019
1020 if 'start' not in request.GET:
1021 raise RequiredParameterMissing('start')
1022
1023 try:
1024 # We try to decode the start time as a pure numerical format (20010506)
1025 # and drop back to the same decoder as the plot tool otherwise
1026 start_str = request.GET['start']
1027 if start_str.isdigit():
1028 start = digits_to_datetime(start_str)
1029
1030 else:
1031 start = str_to_datetime(request.GET['start'])
1032 except ValueError as e:
1033 raise ParameterError('start', e)
1034
1035 if 'stop' not in request.GET:
1036 raise RequiredParameterMissing('stop')
1037
1038 try:
1039 stop_str = request.GET.get('stop')
1040 if stop_str.isdigit():
1041 stop = digits_to_datetime(stop_str)
1042
1043 else:
1044 stop = str_to_datetime(stop_str)
1045 except ValueError as e:
1046 raise RequiredParameterMissing('stop', e)
1047
1048 classes_str = request.GET.getlist('class')
1049 classes = []
1050 for c in classes_str:
1051 try:
1052 classes.append(EventClass(c))
1053 except NoSuchEventClass as e:
1054 raise ParameterError('class', '{c} not found'.format(c=c))
1055
1056 if len(classes) == 0:
1057 raise RequiredParameterMissing('class')
1058
1059 # time limits
1060 if (stop - start) > MAX_EVENTS_DURATION:
1061 raise TooLongError(start=start,
1062 stop=stop,
1063 maximum=MAX_EVENTS_DURATION,
1064 suggestion='Use a shorter duration')
1065
1066 def imp():
1067 for event in find_events(sid=sid,
1068 start_time=start,
1069 stop_time=stop,
1070 event_class=classes):
1071 yield json.dumps({'start_time': datetime_to_xml(event.start_time, include_us=True),
1072 'stop_time': datetime_to_xml(event.stop_time, include_us=True) \
1073 if event.stop_time is not None else None,
1074 'classname': event.event_classname,
1075 'properties': event.instance_properties},
1076 default=json_default_encoder)
1077 yield NEWLINE
1078
1079 response = StreamingHttpResponse(imp(), content_type='text/plain')
1080 return response
1081
1082 except APIException as e:
1083 return json_exception(e)
1084
1085
1086@api_call(
1087 description=('Geolocate a timestamp for a satellite. This endpoint is only valid '
1088 'for LEO projects with geolocation information available'),
1089 query_param_defs=[
1090 QueryParamDef(
1091 name='sid',
1092 description=('Source ID (usually a spacecraft or ground station identifier). See '
1093 '/meta/project endpoint for available values.')),
1094 QueryParamDef(
1095 name='start',
1096 description=(
1097 'Retrieve data starting with specified sensing time. Format is YYYYMMDDhhmmss. '
1098 'Any missing digits are assumed to be zeroes dropped from the right hand side e.g. '
1099 '20230401 is midnight, April 1 and 2023040101 is one hour later.'),
1100 datatype='timestamp'),
1101 ],
1102 return_def='JSON object containing latitude, longitude in degrees.',
1103 url_name='api2:geolocate')
1104def view_geolocate(request):
1105 """Retrieve and interpolate geolocation information."""
1106 if 'sid' not in request.GET:
1107 # This function should have a decorator that converts certain exceptions into nice
1108 # json error messages
1109 raise RequiredParameterMissing('sid')
1110
1111 try:
1112 sid = SID.from_string(request.GET.get('sid'))
1113 except ValueError as e:
1114 raise ParameterError('sid', e)
1115
1116 if 'start' not in request.GET:
1117 raise RequiredParameterMissing('start')
1118
1119 try:
1120 # We try to decode the start time as a pure numerical format (20010506)
1121 # and drop back to the same decoder as the plot tool otherwise
1122 start_str = request.GET['start']
1123 if start_str.isdigit():
1124 start = digits_to_datetime(start_str)
1125
1126 else:
1127 start = str_to_datetime(request.GET['start'])
1128 except ValueError as e:
1129 raise ParameterError('start', e)
1130
1131 geoloc = Geoloc(sid, start, start)
1132 try:
1133 lat, lon = geoloc.lat_lon(start)
1134 except CannotGeolocate:
1135 raise DataError('Cannot geolocate for time')
1136
1137 return json_response({'latitude': lat, 'longitude': lon})
1138
1139
1140class BinaryHandler(json.JSONEncoder):
1141 """Allow structures containing memoryviews to be converted to json."""
1142 def default(self, obj):
1143 """Our special type handling."""
1144 if isinstance(obj, memoryview):
1145 return '0x{hex}'.format(hex=obj.hex())
1146
1147 return json.JSONEncoder.default(self, obj)
1148
1149
1150@api_call(
1151 description="""Retrieve PUS packets.
1152
1153Clients can request the "spid" parameter to read the packet identification value. If clients
1154require additional values which can be derived from "spid" (service, subservice, APID, param1,
1155param2) then these can be looked up via the "/meta/packet" endpoint.
1156
1157If the payload field is requested it is sent with minimal processing and clients must perform
1158calibration or lookup of parameter ID values via the "/calibrate" or "/meta/param" endpoint.
1159
1160The maximum time window (stop - start time) is {unfiltered} without SPID filtering, {filtered}
1161with SPID filtering.
1162
1163This endpoint is only valid for PUS based projects.""".format(
1164 unfiltered=MAX_PACKETS_UNFILTERED_DURATION, filtered=MAX_PACKETS_FILTERED_DURATION),
1165 query_param_defs=[
1166 QueryParamDef(
1167 name='sid',
1168 description=('Source ID (usually a spacecraft or ground station identifier). See '
1169 '/meta/project endpoint for available values.')),
1170 QueryParamDef(
1171 name='start',
1172 description=(
1173 'Retrieve data starting with specified sensing time. Format is YYYYMMDDhhmmss. '
1174 'Any missing digits are assumed to be zeroes dropped from the right hand side e.g. '
1175 '20230401 is midnight, April 1 and 2023040101 is one hour later.'),
1176 datatype='timestamp'),
1177 QueryParamDef(
1178 name='stop',
1179 description=('Retrieve data up to but not including specified sensing time. Format as '
1180 'above.'),
1181 datatype='timestamp'),
1182 QueryParamDef(
1183 name='domain',
1184 choices=['tm', 'tc', 'ev'],
1185 description='Domain of packets to retrieve from'),
1186 QueryParamDef(
1187 name='param',
1188 multiple=True,
1189 description=(
1190 'List of parameters to retireve. Available parameters for required `sid` can be '
1191 'found by calling /meta/store with `store` set to "tm_store" (for tm domain) or '
1192 '"tc_store" (for tc or ev domains)')),
1193 QueryParamDef(
1194 name='spid',
1195 datatype=int,
1196 multiple=True,
1197 description='Filter TM packets by SPID.'),
1198 QueryParamDef(
1199 name='reverse',
1200 datatype=bool,
1201 default='false',
1202 description='Retrieve rows in reverse time order',
1203 minor_version_added=5),
1204 QueryParamDef(
1205 name='limit',
1206 datatype=int,
1207 optional=True,
1208 description='Retrieve only a fixed number of rows if specified',
1209 minor_version_added=5),
1210 ],
1211 return_def=('JSON by row. Each row contains a separate JSON structure with the requested '
1212 'parameters.'),
1213 url_name='api2:packets')
1214def view_packets(request):
1215 """Retrieve data for packets."""
1216 sid = param_sid(request)
1217 start, stop = param_start_stop(request)
1218
1219 domain_s = request.GET.get('domain')
1220 if domain_s is None:
1221 raise RequiredParameterMissing('domain')
1222
1223 try:
1224 domain = PacketDomain(domain_s)
1225 except ValueError as e:
1226 raise ParameterError('domain', e)
1227
1228 # there must be a way to put these into the domain enum
1229 # Even if we have to read the first packetdef and pull out it's table
1230 if domain is PacketDomain.TM:
1231 table = StorageInfo('TM_STORE')
1232
1233 elif domain is PacketDomain.TC:
1234 table = StorageInfo('TC_STORE')
1235
1236 elif domain is PacketDomain.EV:
1237 raise NotImplementedError()
1238 table = StorageInfo('EV_STORE')
1239
1240 param_strs = request.GET.getlist('param')
1241 params = []
1242 for p in param_strs:
1243 if p.lower() == 'sensing_time':
1244 params.append(SensingTime)
1245
1246 else:
1247 try:
1248 params.append(table.get_param(p))
1249
1250 except NoSuchParameter:
1251 raise ParameterError(p, 'Not found in storage')
1252
1253 where = []
1254 bindvars = {}
1255 spid_filter = None
1256 spid_filter_s = request.GET.get('spid')
1257 if spid_filter_s is not None:
1258 spid_filter = int(spid_filter_s)
1259 where.append('SPID=:spid')
1260 bindvars['spid'] = spid_filter
1261
1262 # Time limits
1263 if spid_filter is None:
1264 if (stop - start) > MAX_PACKETS_UNFILTERED_DURATION:
1265 raise TooLongError(start=start,
1266 stop=stop,
1267 maximum=MAX_PACKETS_UNFILTERED_DURATION,
1268 suggestion='Use a shorter duration or add a SPID filter')
1269
1270 else:
1271 if (stop - start) > MAX_PACKETS_FILTERED_DURATION:
1272 raise TooLongError(start=start,
1273 stop=stop,
1274 maximum=MAX_PACKETS_FILTERED_DURATION,
1275 suggestion='Use a shorter duration')
1276
1277 # Check for reversed sensing time
1278 ordering = None
1279 if param_bool(request, 'reverse', default=False):
1280 ordering = 'SENSING_TIME DESC'
1281
1282 # Allow limiting return rows
1283 limit = param_int(request, 'limit', allow_none=True)
1284
1285 def imp():
1286 # cursor = ts.select(sid=sid,
1287 param_names = [p.name.lower() for p in params]
1288 for row in ts.select(sid=sid,
1289 sensing_start=start,
1290 sensing_stop=stop,
1291 table=table,
1292 fields=params,
1293 ordering=ordering,
1294 limit=limit,
1295 where=where,
1296 bindvars=bindvars):
1297 # packet = dict(zip(param_names, row))
1298 packet = {}
1299 for param_name, value in zip(param_names, row):
1300 if isinstance(value, datetime):
1301 packet[param_name] = datetime_to_xml(value, include_us=True)
1302
1303 else:
1304 packet[param_name] = value
1305
1306 try:
1307 yield json.dumps(packet, cls=BinaryHandler)
1308 except TypeError as e:
1309 # Normal error handling doesn't work here as the api_call
1310 # wrapper has already terminaled by the time this function is called
1311 # and we're down inside Django internals
1312 yield 'Data error: {e}'.format(e=e)
1313 return
1314
1315 yield NEWLINE
1316
1317 return StreamingHttpResponse(imp(), content_type='text/plain')
1318
1319
1320@api_call(
1321 description=('Calibrate a raw parameter value. Parameters with conditional calibrations '
1322 'are not currently handled properly and the value will simply calibrated using '
1323 'the first function in the SRDB. A future release may support conditional '
1324 'calibrations using an additional timestamp parameter'),
1325 query_param_defs=[
1326 param_sid.help,
1327 # QueryParamDef(
1328 # name='sid',
1329 # description=('Source ID (usually a spacecraft or ground station identifier). See '
1330 # '/meta/project endpoint for available values.')),
1331 QueryParamDef(
1332 name='store',
1333 description='Name of data store to look in'),
1334 QueryParamDef(
1335 name='param',
1336 description='Name of parameter inside store'),
1337 QueryParamDef(
1338 name='value',
1339 description='Value to calibrate'),
1340 ],
1341 return_def='Calibrated value',
1342 url_name='api2:calibrate')
1343def view_calibrate(request):
1344 """Calibrate a single value using a named calibration.
1345
1346 We always require SID and domain even if they may not be needed for a calibration, at least
1347 for now, because they will probably be required in future.
1348 """
1349 sid = param_sid(request)
1350 store = param_store(sid, request)
1351 param = param_param(store, request)
1352 # domain = param_packetdomain(request)
1353 value = param_numeric_value(request)
1354
1355 # Read the Calibration object
1356 cal_obj = param.cal[sid]
1357 if cal_obj is None:
1358 if param.choices is not None:
1359 # Return enum name instead of our calibration if available
1360 return json_response(param.choices.get_name(value))
1361
1362 else:
1363 return json_response(None)
1364
1365 cal_value = param.cal[sid].calibrate_value(value)
1366 return json_response(cal_value)
1367
1368
1369@api_call(
1370 description="""Retrieve the current satellite state at a point in time by scanning backwards
1371to find the most recent values for each given parameter.
1372
1373This endpoint is currently only valid for PUS based missions.""",
1374 query_param_defs=[
1375 QueryParamDef(
1376 name='sid',
1377 description=('Source ID (usually a spacecraft or ground station identifier). See '
1378 '/meta/project endpoint for available values.')),
1379 QueryParamDef(
1380 name='timestamp',
1381 description=(
1382 'Retrieve data starting with specified sensing time. Format is YYYYMMDDhhmmss. '
1383 'Any missing digits are assumed to be zeroes dropped from the right hand side e.g. '
1384 '20230401 is midnight, April 1 and 2023040101 is one hour later.'),
1385 datatype='timestamp'),
1386 QueryParamDef(
1387 name='store',
1388 description=('Data store to examine (in most projects the implementation will only '
1389 'work correctly on TM store')),
1390 QueryParamDef(
1391 name='param',
1392 multiple=True,
1393 description='List the parameters to retrieve current values for'),
1394 ],
1395 return_def='JSON object giving current values of each parameter',
1396 url_name='api2:state')
1397def view_state(request):
1398 """Retrieve satellite state at time point for PUS TM only."""
1399 sid = param_sid(request)
1400
1401 if 'timestamp' not in request.GET:
1402 raise RequiredParameterMissing('timestamp')
1403
1404 try:
1405 # We try to decode the time as a pure numerical format (20010506)
1406 # and drop back to the same decoder as the plot tool otherwise
1407 timestamp_str = request.GET['timestamp']
1408 if timestamp_str.isdigit():
1409 timestamp = digits_to_datetime(timestamp_str)
1410
1411 else:
1412 timestamp = str_to_datetime(request.GET['timestamp'])
1413 except ValueError as e:
1414 raise ParameterError('timestatmp', e)
1415
1416 if 'store' not in request.GET:
1417 raise RequiredParameterMissing('store')
1418
1419 store_str = request.GET.get('store')
1420
1421 try:
1422 store = StorageInfo(store_str)
1423 except NoSuchStore as e:
1424 raise ParameterError('store', e)
1425
1426 param_strs = request.GET.getlist('param')
1427 params = []
1428 for p in param_strs:
1429 if p.lower() == 'sensing_time':
1430 params.append(SensingTime)
1431
1432 else:
1433 try:
1434 params.append(store.get_param(p).name)
1435
1436 except NoSuchParameter:
1437 raise ParameterError(p, 'Not found in store')
1438
1439 state, times, spids = build_state(source_table=store,
1440 sid=sid,
1441 sensing_time=timestamp,
1442 params=params)
1443 # repack stats to allow for some space to record the SPID and timestamp of the packet
1444 # each value was received in
1445 result = {}
1446 for param, value in state.items():
1447 result[param] = {'value': value,
1448 'sensing_time': times[param],
1449 'spid': spids[param]}
1450 # return json_response({
1451 # k: {'value': v} for k, v in state.items()})
1452 return json_response(result)
1453
1454
1455@api_call(
1456 description=('Basic information about the ingestion of a product.'),
1457 query_param_defs=[
1458 QueryParamDef(
1459 name='id',
1460 datatype=int,
1461 optional=True,
1462 description='Product ID. If omitted then both filename and activity must be supplied.'),
1463 QueryParamDef(
1464 name='activity',
1465 description=(
1466 'Ingestion activity which handled the product.'),
1467 datatype='timestamp'),
1468 QueryParamDef(
1469 name='filename',
1470 description=(
1471 'Product filename to retrieve.'),
1472 datatype='timestamp'),
1473 ],
1474 return_def='JSON object giving product ingestion information',
1475 url_name='api2:product',
1476 minor_version_added=4)
1477def view_product(request):
1478 """Basic information about an ingested product, for supported missions."""
1479 product_id = param_int(request, 'id', allow_none=True)
1480 filename = None
1481 activity = None
1482 if product_id is None:
1483 filename = param_str(request, 'filename', allow_none=True)
1484 activity = param_activity(request, allow_none=True)
1485
1486 if filename is None or activity is None:
1487 raise ParameterError(
1488 'id activity filename',
1489 'Either id or activity and filename must be supplied')
1490
1491 product_row = find_products(
1492 fields=['id',
1493 'activity',
1494 'filename',
1495 'ingestion_time',
1496 'result',
1497 'sensing_start',
1498 'mtime',
1499 'filesize',
1500 'notes'] + SID.sql_sys_select('PRODUCTS'),
1501 product_id=product_id,
1502 activity=activity,
1503 filename=filename).fetchone()
1504
1505 if product_row is None:
1506 raise DataError('Product not found')
1507
1508 result = {
1509 'id': product_row[0],
1510 'activity': product_row[1],
1511 'filename': product_row[2],
1512 'ingestion_time': product_row[3],
1513 'result': product_row[4],
1514 'sensing_start': product_row[5],
1515 'mtime': product_row[6],
1516 'filesize': product_row[7],
1517 'notes': product_row[8],
1518 'sid': SID.from_sys_select('PRODUCTS', product_row[9:])}
1519 return json_response(result)
1520
1521@api_call(
1522 description=('Retrieve job statistics.'),
1523 query_param_defs=[
1524 QueryParamDef(
1525 name='category',
1526 description='Jobs category (usually "SCHEDULER")'),
1527 QueryParamDef(
1528 name='start',
1529 datatype=datetime,
1530 description='Sensing start to scan from'),
1531 QueryParamDef(
1532 name='stop',
1533 datatype=datetime,
1534 description='Sensing start to scan to'),
1535 QueryParamDef(
1536 name='gen_start',
1537 datatype=datetime,
1538 description='Generation start to scan from'),
1539 QueryParamDef(
1540 name='gen_stop',
1541 datatype=datetime,
1542 description='Generation start to scan to'),
1543 ],
1544 return_def='JSON object giving jobs statistics',
1545 url_name='api2:job/summary',
1546 minor_version_added=4)
1547def view_job_summary(request):
1548 """Retrieve job statistics."""
1549 category = param_category(request)
1550 start, stop = param_start_stop(request, allow_none_start=True, allow_none_stop=True)
1551 gen_start, gen_stop = param_start_stop(request,
1552 allow_none_start=True,
1553 allow_none_stop=True,
1554 param_start="gen_start",
1555 param_stop="gen_stop")
1556 result = []
1557 for row in find_jobs(
1558 fields=[
1559 'ACTIVITY',
1560 'STATUS',
1561 'count(*)',
1562 'max(GEN_TIME)',
1563 'max(EARLIEST_EXECUTION_TIME)',
1564 ] + SID.sql_sys_select('JOBS'),
1565 category=category,
1566 sensing_start_ge=start,
1567 sensing_start_lt=stop,
1568 gen_time_ge=gen_start,
1569 gen_time_lt=gen_stop,
1570 order_by=['ACTIVITY'] + SID.sql_sys_select('JOBS') + ['STATUS'],
1571 group_by=['ACTIVITY'] + SID.sql_sys_select('JOBS') + ['STATUS'],
1572 ):
1573 sid = SID.from_sys_select('JOBS', row[5:])
1574 result.append({
1575 'activity': row[0],
1576 'status': row[1] if row[1] is not None else JobStatus.PENDING.name,
1577 'count': row[2],
1578 'max_gen_time': row[3],
1579 'max_early': row[4],
1580 'sid': sid.name if sid is not None else None,
1581 })
1582
1583 return json_response(result)
1584
1585
1586@api_call(
1587 description=('Retrieve list of jobs.'),
1588 query_param_defs=[
1589 QueryParamDef(
1590 name='category',
1591 description='Jobs category (usually "SCHEDULER")'),
1592 QueryParamDef(
1593 name='start',
1594 datatype=datetime,
1595 description='Sensing start to scan from'),
1596 QueryParamDef(
1597 name='stop',
1598 datatype=datetime,
1599 description='Sensing start to scan to'),
1600 QueryParamDef(
1601 name='sid',
1602 description='Source ID'),
1603 ],
1604 return_def='JSON format jobs list',
1605 url_name='api2:job/list',
1606 minor_version_added=4)
1607def view_job_list(request):
1608 """Retrieve list of jobs."""
1609 category = param_category(request)
1610 sid = param_sid(request, allow_none=True)
1611 start, stop = param_start_stop(request, allow_none=True)
1612 activity = param_activity(request)
1613 status = param_jobstatus(request, allow_none=True)
1614
1615 # Put all the selection criteria into a dict because we use it twice below
1616 criteria=dict(category=category,
1617 sid=sid,
1618 sensing_start_ge=start,
1619 sensing_start_lt=stop,
1620 activity=activity,
1621 status=status)
1622
1623 # find_jobs doesnt accept smart database function objects
1624 count = find_jobs(fields=('count(*)',), **criteria).fetchone()[0]
1625 if count > MAX_JOBS_LIST:
1626 raise TooManyError(maximum=MAX_JOBS_LIST,
1627 actual=count,
1628 suggestion='Filter on more attributes or a shorter time range')
1629
1630 result = []
1631 for row in find_jobs(fields=['ID',
1632 'ACTIVITY',
1633 'STATUS',
1634 'FILENAME',
1635 'DIRNAME',
1636 'SENSING_START',
1637 'SENSING_STOP',
1638 'EARLIEST_EXECUTION_TIME',
1639 'GEN_TIME',
1640 'TABLENAME',
1641 ] + SID.sql_sys_select('JOBS'),
1642 **criteria):
1643 sid = SID.from_sys_select('JOBS', row[10:])
1644 if row[3] is not None:
1645 filename = '/'.join(row[3:5])
1646
1647 else:
1648 filename = None
1649
1650 result.append({
1651 'id': row[0],
1652 'activity': row[1],
1653 'status': row[2],
1654 'filename': filename,
1655 'sensing_start': row[5],
1656 'sensing_stop': row[6],
1657 'earliest_execution_time': row[7],
1658 'gen_time': row[8],
1659 'table': row[9],
1660 'sid': sid.name if sid is not None else None,
1661 })
1662
1663 return json_response(result)
1664
1665
1666@api_call(
1667 description=('Retrieve detailed info about a single job.'),
1668 query_param_defs=[
1669 QueryParamDef(
1670 name='id',
1671 datatype=int,
1672 description='Job ID'),
1673 ],
1674 return_def='JSON format job info',
1675 url_name='api2:job/info',
1676 minor_version_added=4)
1677def view_job_info(request):
1678 """Retrieve detailed info about a single job."""
1679 job_id = param_id(request)
1680 job_row = find_jobs(fields=['ID',
1681 'ACTIVITY',
1682 'STATUS',
1683 'FILENAME',
1684 'DIRNAME',
1685 'SENSING_START',
1686 'SENSING_STOP',
1687 'EARLIEST_EXECUTION_TIME',
1688 'GEN_TIME',
1689 'TABLENAME',
1690 'PROCESS_ID',
1691 ] + SID.sql_sys_select('JOBS'),
1692 job_id=job_id).fetchone()
1693 if job_row is None:
1694 raise ParameterError('id', 'No such job')
1695
1696 sid = SID.from_sys_select('JOBS', job_row[11:])
1697 if job_row[3] is not None:
1698 filename = '/'.join(job_row[3:5])
1699
1700 else:
1701 filename = None
1702
1703 result = {'job': {
1704 'id': job_row[0],
1705 'activity': job_row[1],
1706 'status': job_row[2],
1707 'filename': filename,
1708 'sensing_start': job_row[5],
1709 'sensing_stop': job_row[6],
1710 'earliest_execution_time': job_row[7],
1711 'gen_time': job_row[8],
1712 'table': job_row[9],
1713 'sid': sid.name if sid is not None else None,
1714 }}
1715
1716 process_id = job_row[10]
1717 process_row = find_processes(fields=('WORKER',
1718 'EXECUTE_START',
1719 'EXECUTE_STOP',
1720 'STATUS',
1721 'WORKING_DIR'),
1722 process_id=process_id).fetchone()
1723 work_dir = Path(process_row[4])
1724 result['process'] = {
1725 'id': process_id,
1726 'worker': process_row[0],
1727 'execute_start': process_row[1],
1728 'execute_stop': process_row[2],
1729 'status': process_row[3],
1730 'working_dir': work_dir}
1731
1732 files = []
1733 if work_dir.is_dir():
1734 for filename in work_dir.iterdir():
1735 files.append(filename.name)
1736
1737 result['files'] = files
1738
1739 return json_response(result)
1740
1741
1742@api_call(
1743 description=('Retrieve detailed info about a single job.'),
1744 query_param_defs=[
1745 QueryParamDef(
1746 name='id',
1747 datatype=int,
1748 description='Job ID'),
1749 QueryParamDef(
1750 name='filename',
1751 description='Work directory file to retrieve'),
1752 ],
1753 return_def='Binary file',
1754 url_name='api2:job/file',
1755 minor_version_added=4)
1756def view_job_file(request):
1757 """Retrieve a file from a job work directory."""
1758 job_id = param_id(request)
1759 filename = param_str(request, 'filename')
1760
1761 job_row = find_jobs(fields=['PROCESS_ID'], job_id=job_id).fetchone()
1762 if job_row is None:
1763 raise ParameterError('id', 'No such job')
1764
1765 process_id = job_row[0]
1766 process_row = find_processes(fields=['WORKING_DIR'], process_id=process_id).fetchone()
1767 if process_row is None:
1768 raise ParameterError('id', 'No process for job')
1769
1770 working_dir = Path(process_row[0])
1771 content = working_dir.joinpath(filename).open('rb').read()
1772 mime = mimetypes.guess_type(filename)
1773 return HttpResponse(content, content_type=mime)
1774
1775
1776@api_call(
1777 description='Retrieve basic information about a project.',
1778 return_def='JSON structure with various project values',
1779 url_name='api2:meta/project')
1780def view_meta_project(request):
1781 """Return general information about the app and datastore."""
1782 sids = []
1783 for s in SID.all(operational=None):
1784 new_item = {'name': s.name,
1785 'label': getattr(s, 'long_name', None),
1786 'description': getattr(s, 'description', None),
1787 'launch_date': None,
1788 'operational': getattr(s, 'operational', None),
1789 }
1790 if s.satellite is not None:
1791 if s.satellite.launch_date is not None:
1792 new_item['launch_date'] = datetime_to_xml(s.satellite.launch_date)
1793
1794 if new_item['label'] is None:
1795 new_item['label'] = s.satellite.name
1796
1797 if new_item['description'] is None:
1798 new_item['description'] = s.satellite.description
1799
1800 if new_item['operational'] is None:
1801 new_item['operational'] = s.satellite.operational
1802
1803 sids.append(new_item)
1804
1805 result = {'api_version': API_VERSION,
1806 'api_minor_version': API_MINOR_VERSION,
1807 'stats': ['min', 'max', 'avg'] + (['std'] if settings.STATS_HAVE_STDDEVS else []),
1808 'homepage': settings.PROJECT_HOMEPAGE_URL,
1809 'description': settings.PROJECT_DESCRIPTION,
1810 'name': settings.APPNAME,
1811 'sids': sids}
1812
1813 srdb_version = get_srdb_version()
1814 if srdb_version is not None:
1815 result['srdb_version'] = {
1816 'version': srdb_version,
1817 'date': get_srdb_date_implemented()}
1818
1819 if not settings.LOCKDOWN_RESTRICTED_ACCESS:
1820 # These values might be useful for internal users
1821 # However they reveal too much about the system design to allow for external acces
1822 result['application_server'] = {
1823 'hostname': socket.gethostname(),
1824 'port': settings.PORT,
1825 'prefix': settings.PREFIX}
1826 result['database_server'] = {
1827 'id': settings.DB_NAME,
1828 'engine': settings.DATABASES['default']['ENGINE'],
1829 'host': settings.DATABASES['default']['HOST'],
1830 'port': settings.DATABASES['default']['PORT'],
1831 'name': settings.DATABASES['default']['NAME'],
1832 'user': settings.DATABASES['default']['USER'],
1833 }
1834
1835 result['packaging'] = {}
1836 for prop in chart.version_info():
1837 prop_name, _, prop_value = prop.partition('=')
1838 result['packaging'][prop_name] = prop_value
1839
1840 return json_response(result)
1841
1842
1843@api_call(
1844 description=(
1845 'Retrieve a list of available data stores (also refered to as tables). The source-ID '
1846 'parameter is required since projects may have different stores for each source.'),
1847 query_param_defs=[
1848 QueryParamDef(
1849 name='sid',
1850 description=('Source ID (usually a spacecraft or ground station identifier). See '
1851 '/meta/project endpoint for available values.')),
1852 ],
1853 return_def='JSON structure with information for each store.',
1854 url_name='api2:meta/stores')
1855def view_meta_stores(request):
1856 """Return list of available tables, optionally per-SID."""
1857 if 'sid' not in request.GET: # Note the plot tool ajax calls work differently and
1858 # allow SIDs to be specified using a range parameters such as SCID, GSID, OGSID
1859 # but for the API2 we standardise and allow only a single string and only called sid
1860 raise RequiredParameterMissing('sid')
1861
1862 return json_response(
1863 sorted([
1864 {'name': t.name,
1865 'description': t.description,
1866 'period': t.period,
1867 'visible': t.visible,
1868 'has_stats': t.has_stats,
1869 'table_type': t.table_type.value}
1870 for t in StorageInfo.all()], key=itemgetter('name')))
1871
1872
1873def view_meta_tables(request):
1874 """Obsolete name."""
1875 if 'sid' in request.GET: # Note the plot tool ajax calls work differently and
1876 # allow SIDs to be specified using a range parameters such as SCID, GSID, OGSID
1877 # but for the API2 we standardise and allow only a single string and only called sid
1878 pass
1879
1880 return json_response({
1881 # sid.all_tables()
1882 'tables': sorted([t.name for t in StorageInfo.all()]),
1883 })
1884
1885
1886@api_call(
1887 description=('Retrieve a structure giving the available statistics regions and metadata about '
1888 'each. It\'s possible to have different regions available for different data '
1889 'sources and for different stores'),
1890 query_param_defs=[
1891 QueryParamDef(
1892 name='sid',
1893 description=('Source ID (usually a spacecraft or ground station identifier).')),
1894 QueryParamDef(
1895 name='store',
1896 description='Data store to view regions for'),
1897 ],
1898 return_def='JSON structure with information on each region.',
1899 url_name='api2:meta/regions')
1900def view_meta_regions(request):
1901 """Return available stats regions, potentially per-SID.
1902
1903 Some SIDs such as ground test data sources, lack some regions like orbital stats."""
1904 sid = param_sid(request)
1905
1906 if 'store' not in request.GET:
1907 # We don't actually have per-table stats regions yet
1908 # but to avoid future problems we force client to supply table name
1909 raise RequiredParameterMissing('store')
1910
1911 result = []
1912 for s in sid.sampling_options:
1913 if s.stats:
1914 # exclude AP and FIT
1915 result.append({
1916 'name': s.region,
1917 'description': s.description,
1918 'display_name': s.option,
1919 'duration': timedelta_to_xml(
1920 s.nominal_duration) if s.nominal_duration is not None else None,
1921 })
1922
1923 return json_response(result)
1924
1925
1926@api_call(
1927 description="""Retrieve information about a single store (also called a table) including
1928list of parameters.""",
1929 query_param_defs=[
1930 QueryParamDef(
1931 name='store',
1932 description='Store to retrieve information about.'),
1933 QueryParamDef(
1934 name='sid',
1935 description=('Source ID (usually a spacecraft or ground station identifier). See '
1936 '"/meta/project" endpoint for available values.'),
1937 minor_version_added=4),
1938 ],
1939 return_def='JSON structure with details on the store and parameters in it.',
1940 url_name='api2:meta/store')
1941def view_meta_single_store(request):
1942 """Return information about a single store."""
1943 if 'store' not in request.GET:
1944 raise RequiredParameterMissing('store')
1945
1946 if 'sid' not in request.GET: # Note the plot tool ajax calls work differently and
1947 # allow SIDs to be specified using a range parameters such as SCID, GSID, OGSID
1948 # but for the API2 we standardise and allow only a single string and only called sid
1949 raise RequiredParameterMissing('sid')
1950
1951 store_info = StorageInfo(request.GET['store'])
1952
1953 return json_response(
1954 {'name': store_info.name,
1955 'description': store_info.description,
1956 'has_stats': store_info.has_stats,
1957 'common_fields': ['SENSING_TIME'],
1958 'params': [f for f in sorted(store_info.fields.keys())],
1959 })
1960
1961
1962def view_meta_single_table(request):
1963 """Obsolete name."""
1964 if 'table' not in request.GET:
1965 raise ValueError('No table specified')
1966
1967 table_info = StorageInfo(request.GET['table'])
1968
1969 return json_response(
1970 {'name': table_info.name,
1971 'description': table_info.description,
1972 'has_stats': table_info.has_stats,
1973 'common_fields': ['SENSING_TIME'],
1974 'fields': [f for f in sorted(table_info.fields.keys())],
1975 })
1976
1977
1978@api_call(
1979 description='Retrieve information about a single parameter.',
1980 query_param_defs=[
1981 QueryParamDef(
1982 name='sid',
1983 description=('Source ID (usually a spacecraft or ground station identifier).')),
1984 QueryParamDef(
1985 name='store',
1986 description='Data store to retrieve information about.'),
1987 QueryParamDef(
1988 name='param',
1989 optional=True,
1990 description='Parameter name within store to retrieve information about.'),
1991 # QueryParamDef(
1992 # name='key',
1993 # optional=True,
1994 # datatype=int,
1995 # description="""For stores where parameter can also be refereed to by a unique integer
1996# such as key-value stores, or stores holding telecommand data from a PUS mission, this parameter
1997# is used to search by code or SPID.""",
1998 # nminor_version_added=4),
1999 ],
2000 return_def='JSON structure with parameter details.',
2001 url_name='api2:meta/param')
2002def view_meta_param(request):
2003 """Return information about a single parameter."""
2004 # We don't actually use SID but will in future multi-SID PUS code
2005 sid = param_sid(request)
2006
2007 store_name = param_str(request, 'store')
2008 param_name = param_str(request, 'param')#, allow_none=True)
2009
2010 # if param_name is not None:
2011 param_info = find_param_by_name(table=store_name, field=param_name)
2012
2013 # else:
2014 # only search by SPID is actually implemented
2015 # search by keyvalue store key code would also be possible here
2016 # spid = param_int('key', allow_none=True)
2017 # if spid is None:
2018 # raise RequiredParameterMissing('param or spid')
2019
2020 # if param_info is None:
2021 # raise ParameterError('param', 'No such parameter found')
2022
2023 result = {'name': param_info.name,
2024 'description': param_info.description,
2025 'datatype': param_info.datatype.value,
2026 'unit': param_info.unit,
2027 # 'limits': {
2028 # 'raw': None,
2029 # 'cal': None,
2030 # },
2031 'calibration': {
2032 'name': param_info.calibration_name,
2033 },
2034 'choices': None,
2035 'key': param_info.key,
2036 }
2037
2038 cal_info = param_info.cal[sid]
2039 if cal_info is not None:
2040 result['calibration']['type'] = cal_info.calibration_type
2041 if isinstance(cal_info, PolyCalibration):
2042 result['calibration']['coeffs'] = cal_info.coeffs
2043
2044 else:
2045 result['calibration']['type'] = None
2046
2047 if param_info.choices is not None:
2048 choices = []
2049 for choice in param_info.choices.items.values():
2050 item = {'value': choice.min_value,
2051 'name': choice.name}
2052 if choice.max_value != choice.min_value:
2053 item['max_value'] = choice.max_value
2054
2055 choices.append(item)
2056
2057 result['choices'] = choices
2058
2059 return json_response(result)
2060
2061
2062def view_meta_field(request):
2063 """Handle older name."""
2064 try:
2065 if 'table' not in request.GET:
2066 raise RequiredParameterMissing('table')
2067
2068 table_name = request.GET['table']
2069
2070 if 'field' not in request.GET:
2071 raise RequiredParameterMissing('field')
2072
2073 field_name = request.GET['field']
2074
2075 field_info = find_param_by_name(field=field_name, table=table_name)
2076
2077 return json_response(
2078 {'name': field_info.name,
2079 'description': field_info.description,
2080 'datatype': field_info.datatype.value,
2081 'calibration_name': field_info.calibration_name,
2082 })
2083
2084 except APIException as e:
2085 return json_exception(e)
2086
2087
2088@api_call(
2089 description='List all PUS packets for a packet domain.',
2090 query_param_defs=[
2091 QueryParamDef(
2092 name='sid',
2093 description=('Source ID (usually a spacecraft or ground station identifier).')),
2094 QueryParamDef(
2095 name='domain',
2096 choices=['tm', 'tc', 'ev'],
2097 description='Domain of packets to retrieve packets from'),
2098 ],
2099 return_def='JSON structure containing basic packet information.',
2100 url_name='api2:meta/packets')
2101def view_meta_packets(request):
2102 """Return all known identifiers for domain."""
2103 domain_s = request.GET.get('domain')
2104 if domain_s is None:
2105 raise RequiredParameterMissing('domain')
2106
2107 try:
2108 domain = PacketDomain(domain_s)
2109 except ValueError as e:
2110 raise ParameterError('domain', e)
2111
2112 # We don't really need a sid but will in future for multi-satellite PUS support
2113 if request.GET.get('sid') is None:
2114 raise RequiredParameterMissing('sid')
2115
2116 return json_response([
2117 {'name': packet_def.name,
2118 'spid': packet_def.spid}
2119 for packet_def in PacketDef.all(domain=domain)])
2120
2121
2122@api_call(
2123 description='Return data in a single packet.',
2124 query_param_defs=[
2125 param_sid.help,
2126 param_packetdomain.help,
2127 QueryParamDef(
2128 name='spid',
2129 datatype=int,
2130 description='Source Packet IDentifier to query for.'),
2131 ],
2132 return_def='JSON structure containing more detailed packet information.',
2133 url_name='api2:meta/packet')
2134def view_meta_single_packet(request):
2135 """Return full info on a single packetdef."""
2136 domain = param_packetdomain(request)
2137 if domain is PacketDomain.TM or domain is PacketDomain.EV:
2138 spid = param_int(request, 'spid')
2139 packetdef = PacketDef.find_one(domain=domain, spid=spid)
2140 if packetdef is None:
2141 raise ParameterError('spid', 'Parameter not known')
2142
2143 elif domain is PacketDomain.TC:
2144 name = param_str(request, 'name')
2145 packetdef = PacketDef.find_one(domain=domain, name=name)
2146 if packetdef is None:
2147 raise ParameterError('spid', 'Parameter not known')
2148
2149 else:
2150 raise ParameterError('domain', 'Cannot handle domain')
2151
2152 payload = []
2153 for p in packetdef.paramlists[0].params:
2154 paramdef = {'name': p.field_info.name}
2155 paramdef['byte'] = p.byte
2156 paramdef['bit'] = p.bit
2157 if p.group_size is not None:
2158 paramdef['group_sid'] = p.group_size
2159
2160 payload.append(paramdef)
2161
2162 result = {
2163 'name': packetdef.name,
2164 'description': packetdef.description,
2165 'service': packetdef.service,
2166 'subservice': packetdef.subservice,
2167 'apid': packetdef.apid,
2168 'param1': packetdef.param1,
2169 'param2': packetdef.param2,
2170 'store': packetdef.paramlists[0].table_info.name,
2171 'payload': payload,
2172 }
2173 return json_response(result)
2174
2175
2176@api_call(
2177 description='List available project event classes.',
2178 return_def='JSON list of class names.',
2179 url_name='api2:meta/events')
2180def view_meta_events(request):
2181 """List event class names."""
2182 return json_response([{"name": eventclass.name} for eventclass in EventClass.all()])
2183
2184
2185@api_call(
2186 description='Show detailed information about a single event class definition.',
2187 query_param_defs=[
2188 QueryParamDef(
2189 name='class',
2190 description='Event class name'),
2191 ],
2192 return_def='JSON structure of class information.',
2193 url_name='api2:meta/event')
2194def view_meta_single_event(request):
2195 """Info in a single event class definition."""
2196 if 'class' not in request.GET:
2197 raise RequiredParameterMissing('class')
2198
2199 cls = EventClass(request.GET['class'])
2200 result = {
2201 'name': cls.name,
2202 'description': cls.description,
2203 'renderer': name_of_thing(cls.renderer),
2204 'decoder': cls.decoder,
2205 'url': cls.url,
2206 'is_operator': cls.is_operator(),
2207 'instance_properties': [],
2208 }
2209 for prop in cls.instance_properties.values():
2210 new_prop = {}
2211 for prop_key, prop_value in prop.items():
2212 new_prop[prop_key] = prop_value
2213
2214 result['instance_properties'].append(new_prop)
2215
2216 return json_response(result)