1#!/usr/bin/env python3
2
3"""Implementation of RAPIDRecord class, and associated Header Classes.
4RAPID Records are collected in RAPID files, supplied by MOF (checK) and stored in the FBF
5
6See RAPID ICD, EGOS-MCS-S2K-ICD-1005 and
7 SCOS-2000 Packet Management ADD, EGOS-MCS-S2K-ADD-0024 for full description.
8"""
9
10import sys
11import logging
12
13from binascii import hexlify
14from enum import Enum
15from datetime import datetime, timedelta
16from collections import defaultdict
17
18from chart.common.binary import unpack_uint16_be
19from chart.common.binary import unpack_uint32_be
20from chart.common.binary import unpack_uint64_be
21from chart.products.pus.packetdef import PacketDef
22from chart.products.pus.packetdef import PacketDomain
23from chart.products.pus.ccsds import CCSDS
24from chart.products.pus.ccsds import CCSDSTM
25from chart.products.pus.ccsds import CCSDSSTDXPUSS
26from chart.products.pus.ccsds import CCSDSEV
27from chart.products.pus.ccsds import CCSDS_NO_HDR
28from chart.products.pus.ccsds import CCSDS_NIS_NO_HDR
29from chart.products.rapidfile.tdev import TDEVHeader
30from chart.products.rapidfile.tdev import TDEVTMHeader
31from chart.products.rapidfile.tdev import TDEVTCHeader
32from chart.products.rapidfile.tdev import TDEVTCVarHeader
33from chart.products.rapidfile.tdev import TDEV_TC_FILING_KEY_NORMAL
34from chart.products.rapidfile.tdev import TDEV_TC_FILING_KEY_ACK
35from chart.products.rapidfile.tdev import TDEVTCCevPtvDetails
36from chart.products.utils import Segment
37from chart.products.pus.packetdef import PacketHeaderType
38from chart.project import SID
39from chart.project import settings
40
41INDENTATION = ' '
42
43# the true length of a RAPID record is 4 bytes longer than its rapid_rec_len field
44RAPID_RECORD_OFFSET = 4
45
46logger = logging.getLogger()
47
48# Rapid Record Types
49RapidType = Enum('RapidType', 'CADU CCSDS')
50
51class TDEV_HEADER_TYPE(Enum):
52 """Interpretation of tdev_s2k_header_type field."""
53 TM = 1
54 TC = 2
55 EV = 3 # guessed
56
57class BadTCName(Exception):
58 def __init__(self, name):
59 self.name = name
60
61class UnknownTCName(BadTCName):
62 """Rapid header refers to an unknown command name."""
63 def __str__(self):
64 return 'Unknown TC name {name}'.format(name=self.name)
65
66class RestrictedTCName(BadTCName):
67 """Rapid header shows this is a TC name cover by security restrictions."""
68 def __str__(self):
69 return 'Restricted TC name {name}'.format(name=self.name)
70
71class UnknownTDEVHdr(BadTCName):
72 """Unknown TDEVHeader type read from Rapid TDEVHeader."""
73 def __str__(self):
74 return 'Unknown TDEV_Header_Type {name}'.format(name=self.name)
75
76class RAPIDRecord(Segment):
77 def __init__(self, buff, file_pos):
78 super(RAPIDRecord, self).__init__(buff)
79 self.file_pos = file_pos
80 self.rapid_header = None
81 self.tdev_header = None
82 self.nis_other_data = None
83 self.nis_header = None
84 self.tdev_tm_header = None
85 self.tdev_tc_header = None
86 self.tdev_tc_var_header = None
87 self.nis_cltu_header = None
88 # self.tdev_tc_dev_ptc_details = None
89 self.tdev_tc_cev_ptv_details = None
90 self.ccsds = None
91 self.tdev_data = None
92
93 # parts of the packet to be ingested into hex column of PUS tables
94 segments = [#,'rapid_header',
95 'tdev_header',
96 # 'nis_other_data',
97 'nis_header',
98 'tdev_tm_header',
99 'tdev_tc_header',
100 'tdev_tc_var_header',
101 'nis_cltu_header',
102 # 'tdev_tc_dev_ptc_details',
103 'tdev_tc_cev_ptv_details',
104 'ccsds',
105 'tdev_data']
106
107 @property
108 def sensing_time(self):
109 return self.tdev_header.timestamp
110
111 @property
112 def create_time(self):
113 return self.tdev_header.create_timestamp
114
115 @property
116 def length(self):
117 return self.rapid_header.record_length()
118
119 def __str__(self):
120 return 'RAPID({type} - {spid} @{len})'.format(
121 type=self.tdev_header.rapid_type,
122 spid=self.tdev_header.spid,
123 len=self.tdev_header.tdev_size)
124
125
126class RAPIDCADURecord(RAPIDRecord):
127 """A RAPIDFILE block containing a CADU."""
128
129 SPIDS_WITH_NIS_HEADER = None
130
131 @staticmethod
132 def init(sid=None):
133 SID(name=sid.name,sid=sid)
134 RAPIDCADURecord.sid = sid
135 # RAPIDFILE blocks that contain a NIS header
136 # This would be neater stored in PacketDef from a "<nis-header>" bool in the packets
137 # XML file
138 RAPIDCADURecord.SPIDS_WITH_NIS_HEADER = [
139 SID.special_packets.good_frame_spid,
140 SID.special_packets.bad_frame_spid,
141 SID.special_packets.idle_frame_spid,
142 SID.special_packets.time_couples_spid,
143 ]
144
145 def __init__(self, buff, file_pos, rapid_header, tdev_header, sid=None):
146 super(RAPIDCADURecord, self).__init__(buff, file_pos)
147 self.rapid_header = rapid_header
148 self.tdev_header = tdev_header
149
150 self.tdev_tm_header = TDEVTMHeader(
151 buff[RAPIDHeader.LENGTH + TDEVHeader.LENGTH:
152 RAPIDHeader.LENGTH + TDEVHeader.LENGTH + TDEVTMHeader.LENGTH])
153
154 # Note nis_header, where applicable, already included in byte offset
155 if self.tdev_header.spid in RAPIDCADURecord.SPIDS_WITH_NIS_HEADER:
156 to_NIS = RAPIDHeader.LENGTH + TDEVHeader.LENGTH + TDEVTMHeader.LENGTH
157 if settings.NIS_OTHER_DATA_BUFF_LEN > 0:
158 self.nis_other_data = NISOtherData(
159 buff[to_NIS: to_NIS + NISOtherData.LENGTH])
160
161 self.nis_header = NISHeader(
162 buff[to_NIS + NISOtherData.LENGTH: to_NIS + NISOtherData.LENGTH+ NISHeader.LENGTH])
163
164 self.ccsds = CCSDS_NIS_NO_HDR(buff[
165 RAPIDHeader.LENGTH + TDEVHeader.LENGTH + TDEVTMHeader.LENGTH + NISHeader.LENGTH:])
166
167 # TBD: Use named constant, or a value read from a config file instead of hard coded magic number
168 elif self.tdev_header.spid == 200:
169 # TBD: Inadequate warning message - should explain what the problem is
170 logger.warning('Dynamic UDC')
171 self.nis_header = None
172 # self.buff = self.buff
173 self.ccsds = self.buff
174
175 else:
176 self.nis_header = None
177 self.ccsds = CCSDSTM(buff[
178 RAPIDHeader.LENGTH + TDEVHeader.LENGTH + TDEVTMHeader.LENGTH:])
179
180 # self.tdev_data = buff[RAPIDHeader.LENGTH + TDEVHeader.LENGTH + TDEVTMHeader.LENGTH:]
181
182 self.packet_def = PacketDef.find_one(spid=self.tdev_header.spid, sid=sid)
183
184 def show_header(self, indent='', indentation=INDENTATION, target=sys.stdout):
185 target.write(self.rapid_header.show(indent, indentation))
186 target.write(self.tdev_header.show(indent, indentation))
187 if self.tdev_header.tdev_s2k_header_type is TDEV_HEADER_TYPE.TM:
188 if self.nis_header is not None and self.tdev_header.tdev_size > TDEVHeader.LENGTH + NISHeader.LENGTH:
189 if self.nis_other_data is not None:
190 target.write(self.nis_other_data.show(indent, indentation))
191 target.write(self.nis_header.show(indent, indentation))
192
193 def raw_hex(self):
194 """TDEV Packet in Hex String format, used for raw_hex jsonb block, to contain raw Hex CCSDS packet."""
195 content = {}
196 content['TDEV_HEADER'] = self.tdev_header.hex
197 content['TDEV_DATA'] = '0x' + hexlify(self.tdev_data).decode('latin-1').upper()
198
199 return content
200
201
202class RAPIDCCSDSRecord(RAPIDRecord):
203 """A RAPIDFILE block containing either a CCSDS packet or a TC ACK packet.
204
205 If a TC ACK, the `ccsds` member will be None.
206
207 This object can be passed around as though it were a plain CCSDS object."""
208
209 def __init__(self, file_pos, rapid_header, tdev_header, buff, sid=None):
210 super(RAPIDCCSDSRecord, self).__init__(buff, file_pos)
211 self.rapid_header = rapid_header
212 self.tdev_header = tdev_header
213 self.nis_header = None
214 self.tdev_tm_header = None
215 self.tdev_tc_header = None
216 self.tdev_tc_var_header = None
217 self.nis_cltu_header = None
218 self.tdev_tc_cev_ptv_details = None
219 self.ccsds = None
220 self.sid = sid
221
222 # test for TC / TM / EV
223 if self.tdev_header.tdev_s2k_header_type is TDEV_HEADER_TYPE.TC:
224 # it's a telecommand
225 tc_fixed_hdr = RAPIDHeader.LENGTH + TDEVHeader.LENGTH + TDEVTCHeader.LENGTH
226 self.tdev_tc_header = TDEVTCHeader(buff[
227 RAPIDHeader.LENGTH + TDEVHeader.LENGTH : tc_fixed_hdr])
228
229 # see if it contains a TC packet
230 if self.tdev_header.tdev_filing_key == TDEV_TC_FILING_KEY_NORMAL:
231 # it does
232 self.tdev_tc_var_header = TDEVTCVarHeader(buff[
233 tc_fixed_hdr:
234 tc_fixed_hdr + self.tdev_tc_header.variable_address_size],
235 self.tdev_tc_header)
236
237 if self.tdev_tc_var_header.command_name in settings.RESTRICTED_TC_PACKETS:
238 raise RestrictedTCName(self.tdev_tc_var_header.command_name)
239
240 self.packet_def = PacketDef.find_one(domain=PacketDomain.TC,
241 name=self.tdev_tc_var_header.command_name, sid=sid)
242
243 if self.packet_def is None:
244 logger.warning('Rapid Record contains unknown command_name {c}'.format(
245 c=self.tdev_tc_var_header.command_name))
246 # assert self.packet_def is not None
247 raise UnknownTCName(self.tdev_tc_var_header.command_name)
248
249 self.nis_cltu_header = NIS_CLTU_Header(buff[
250 tc_fixed_hdr + self.tdev_tc_header.variable_address_size:
251 tc_fixed_hdr + self.tdev_tc_header.variable_address_size + NIS_CLTU_Header.LENGTH])
252
253 ccsds_buff = buff[tc_fixed_hdr + self.tdev_tc_header.variable_address_size + NIS_CLTU_Header.LENGTH:]
254 buflen = len(buff)
255 ccsdsbuflen = len(ccsds_buff)
256 # make it a full sized TC packet - this is not always correct as there are packets
257 # in the SRDB that use the smaller CCSDSXCCSD packets
258 self.ccsds = CCSDS.make_packet(self.packet_def.header_type, ccsds_buff)
259 # if self.packet_def.header_type is PacketHeaderType.CCSDS_STDXCCSD:
260 # self.ccsds = CCSDSSTDXCCSD(ccsds_buff)
261
262 # elif self.packet_def.header_type is PacketHeaderType.CCSDS_STDXPUSS:
263 # self.ccsds = CCSDSSTDXPUSS(ccsds_buff)
264
265 # elif self.packet_def.header_type is PacketHeaderType.CCSDS_NO_HDR:
266 # self.ccsds = None
267
268 # else:
269 # raise ValueError('Unknown packet def header type {t}'.format(
270 # t=self.packet_def.header_type))
271 #self.ccsds = CCSDSTM(ccsds_buff)
272 # self.ccsds = None
273
274 elif self.tdev_header.tdev_filing_key == TDEV_TC_FILING_KEY_ACK:
275 self.packet_def = PacketDef.find_one(domain=PacketDomain.TC,
276 spid=TDEV_TC_FILING_KEY_ACK,
277 sid=sid)
278 # PVT / CEV
279 self.tdev_tc_cev_ptv_details = TDEVTCCevPtvDetails(buff[tc_fixed_hdr: tc_fixed_hdr +\
280 self.tdev_tc_header.variable_address_size])
281 self.ccsds = None
282
283 else:
284 # for other filing keys it's some kind of acknowledgment packet and we don't decode
285 logger.error('TC packet with unknown TDEV_TC_FILING_KEY {k}'.format(
286 k=self.tdev_header.tdev_filing_key))
287
288 elif self.tdev_header.tdev_s2k_header_type is TDEV_HEADER_TYPE.TM:
289 # it's a telemetry packet
290 self.packet_def = PacketDef.find_one(spid=self.tdev_header.spid, sid=sid)
291 self.tdev_tm_header = TDEVTMHeader(buff[
292 RAPIDHeader.LENGTH + TDEVHeader.LENGTH : RAPIDHeader.LENGTH +
293 TDEVHeader.LENGTH + TDEVTMHeader.LENGTH])
294
295 # this allows None packets through, which get handled in ccsds_read_tm.py ...
296 if self.packet_def is not None and self.packet_def.header_type is PacketHeaderType.CCSDS_NO_HDR:
297 self.ccsds = CCSDS_NO_HDR(buff[
298 RAPIDHeader.LENGTH + TDEVHeader.LENGTH + TDEVTMHeader.LENGTH:])
299
300 else:
301 self.ccsds = CCSDSTM(buff[
302 RAPIDHeader.LENGTH + TDEVHeader.LENGTH + TDEVTMHeader.LENGTH:])
303
304 elif self.tdev_header.tdev_s2k_header_type is TDEV_HEADER_TYPE.EV:
305 # it's an event packet
306 self.packet_def = PacketDef.find_one(domain=PacketDomain.EV,
307 spid=self.tdev_header.spid,
308 sid=sid)
309 self.ccsds = CCSDSEV(buff[
310 RAPIDHeader.LENGTH + TDEVHeader.LENGTH:])
311
312 # also load tdev_data
313 self.tdev_data = CCSDSEV(buff[
314 RAPIDHeader.LENGTH + TDEVHeader.LENGTH:])
315
316 else:
317 logger.warn('Unknown Packet type, tdev_s2k_header_type {s}'.format(s=self.tdev_header.tdev_s2k_header_type))
318 pass
319
320 def show_header(self, indent='', indentation=INDENTATION, target=sys.stdout):
321 target.write(self.rapid_header.show(indent, indentation))
322 target.write(self.tdev_header.show(indent, indentation))
323 if self.tdev_tc_header is not None:
324 target.write(self.tdev_tc_header.show(indent, indentation))
325
326 if self.tdev_tc_var_header is not None:
327 # TC Command
328 target.write(self.tdev_tc_var_header.show(indent, indentation))
329
330 if self.tdev_tc_cev_ptv_details is not None:
331 # TC Command CEV/PTV
332 target.write(self.tdev_tc_cev_ptv_details.show(indent, indentation))
333
334 if self.nis_header is not None:
335 target.write(self.nis_header.show(indent, indentation))
336
337
338class RAPIDHeader(Segment):
339 """Representation of a single Rapid File Record Fixed Header.
340
341 See RAPID ICD, EGOS-MCS-s2K-ICD-1005.
342 """
343
344 # this is a fixed length header
345 LENGTH = 60
346
347 @property
348 def rapid_rec_len(self):
349 """Get Rapid Record Len."""
350 return unpack_uint32_be(self.buff, 0)
351
352 def record_length(self):
353 """RAPID records begin with a 4-byte length value whose size is not included in rec_len."""
354 return self.rapid_rec_len + RAPID_RECORD_OFFSET
355
356 @property
357 def rapid_hdr_id(self):
358 """Get Rapid Record ID."""
359 return unpack_uint64_be(self.buff, 4)
360
361 @property
362 def rapid_hdr_start_time(self):
363 """Get Rapid Record start_time."""
364 return unpack_uint64_be(self.buff, 12)
365
366 @property
367 def rapid_hdr_end_time(self):
368 """Get Rapid Record end_time."""
369 return unpack_uint64_be(self.buff, 20)
370
371 @property
372 def rapid_hdr_action_time(self):
373 """Get Rapid Record action_time."""
374 # us since reftime
375 return unpack_uint64_be(self.buff, 28)
376
377 @property
378 def rapid_hdr_data_size(self):
379 """Get Rapid Record data_size."""
380 return int.from_bytes(self.buff[36:44], byteorder='big')
381 # return int(self.buff[36:44]) # unpack_uint64_be(self.buff, 36)
382 # return unpack_uint64_be(self.buff, 36)
383
384 @property
385 def rapid_hdr_action_type(self):
386 """Get Rapid Record action_type ."""
387 return unpack_uint32_be(self.buff, 44)
388
389 @property
390 def rapid_hdr_sync(self):
391 """Get Rapid Record synch marker."""
392 return self.buff[48:48+4]
393 # return unpack_uint32_be(self.buff, 48)
394
395 @property
396 def rapid_name_size(self):
397 """Get Rapid Record name size."""
398 return unpack_uint32_be(self.buff, 52)
399
400 @property
401 def rapid_filter_size(self):
402 """Get Rapid Record filter size."""
403 return unpack_uint32_be(self.buff, 56)
404
405 def show(self, indent, indentation):
406 """Display the Rapid Record format."""
407 return """{i}RAPID header:
408{i}{ii}rapid_rec_len: {rec_len}
409{i}{ii}rapid_hdr_id: {hdr_id}
410{i}{ii}rapid_hdr_start_time: {start_time}
411{i}{ii}rapid_hdr_end_time: {end_time}
412{i}{ii}rapid_hdr_action_time: {action_time} ({action_time_utc})
413{i}{ii}rapid_hdr_data_size: {data_size}
414{i}{ii}rapid_hdr_action_type: {action_type}
415{i}{ii}rapid_hdr_sync: 0x{sync}
416{i}{ii}rapid_name_size: {name_size}
417{i}{ii}rapid_filter_size: {filter_size}
418""".format(
419 i=indent,
420 ii=indentation,
421 rec_len=self.rapid_rec_len,
422 hdr_id=self.rapid_hdr_id,
423 start_time=self.rapid_hdr_start_time,
424 end_time=self.rapid_hdr_end_time,
425 action_time=self.rapid_hdr_action_time,
426 action_time_utc=datetime.utcfromtimestamp(self.rapid_hdr_action_time//1e6).replace(
427 microsecond=int(self.rapid_hdr_action_time%1e6)),
428 data_size=self.rapid_hdr_data_size,
429 action_type=self.rapid_hdr_action_type,
430 sync=self.rapid_hdr_sync.hex(),
431 name_size=self.rapid_name_size,
432 filter_size=self.rapid_filter_size
433)
434
435class NISOtherData(Segment):
436 """Project dependant NIS_other_data buffer."""
437
438 # JCS nis_other_data buffer = 16
439 # MTG nis_other_data buffer = 0
440
441 LENGTH = settings.NIS_OTHER_DATA_BUFF_LEN
442
443 @property
444 def nis_other_data(self):
445 return self.buff[0:NISOtherData.LENGTH].hex()
446
447 def show(self, indent, indentation):
448 """Display the Rapid Record NIS Other Data."""
449 if NISOtherData.LENGTH > 0:
450 return """{i}NIS Other Data Header:
451{i}{ii}nis_other_data: {other_data}
452""".format(
453 i=indent,
454 ii=indentation,
455 other_data=self.nis_other_data)
456
457 return ''
458
459
460class NISHeader(Segment):
461 """NIS Packet Header template as defined in EGOS-NIS-NCTR-ICD-0002, Issue: 4.0.2, Page 5."""
462
463 # From Romain Letor 31/07/2019
464 # in section 2.4.2, I can read that the NIS header is between 23 and 25 byte long depending on
465 # the ERT format. In our case it should be 23bytes for the following SPIDs 90010, 90020 and 90060
466 # (and not for 90030, 90040 and 90050 which are packets).
467 # fixed length block
468 LENGTH = 23
469
470 @property
471 def nis_du_version(self):
472 return self.buff[0]
473
474 @property
475 def nis_packet_size(self):
476 return unpack_uint32_be(self.buff, 1)
477
478 @property
479 def nis_sc_id(self):
480 return unpack_uint16_be(self.buff, 5)
481
482 @property
483 def nis_data_stream_type(self):
484 return self.buff[7]
485
486 def nis_vc_id(self):
487 return self.buff[8]
488
489 channel_id = property(nis_vc_id)
490
491 @property
492 def nis_route_id(self):
493 return unpack_uint16_be(self.buff, 9)
494
495 @property
496 def nis_seq_flag(self):
497 return self.buff[11]
498
499 @property
500 def nis_qual_update(self):
501 return self.buff[12]
502
503 @property
504 def nis_ert_fmt(self):
505 return self.buff[13]
506
507 @property
508 def nis_priv_ano_len(self):
509 return self.buff[14]
510
511 @property
512 def nis_ert(self):
513 return self.buff[15:23].hex()
514
515 def show(self, indent, indentation):
516 """Display the Rapid Record NIS Header format."""
517 return """{i}NIS header:
518{i}{ii}nis_du_version: {du_version}
519{i}{ii}nis_packet_size: {packet_size}
520{i}{ii}nis_sc_id: {sc_id}
521{i}{ii}nis_data_stream_type: {data_stream_type}
522{i}{ii}nis_vc_id: {vc_id}
523{i}{ii}nis_route_id: {route_id}
524{i}{ii}nis_seq_flag: {seq_flag}
525{i}{ii}nis_qual_update: {qual_update}
526{i}{ii}nis_ert_fmt: {ert_fmt}
527{i}{ii}nis_priv_ano_len: {priv_ano_len}
528{i}{ii}nis_ert: {ert}
529""".format(
530 i=indent,
531 ii=indentation,
532 du_version=self.nis_du_version,
533 packet_size=self.nis_packet_size,
534 sc_id=self.nis_sc_id,
535 data_stream_type=self.nis_data_stream_type,
536 vc_id=self.channel_id,
537 route_id=self.nis_route_id,
538 seq_flag=self.nis_seq_flag,
539 qual_update=self.nis_qual_update,
540 ert_fmt=self.nis_ert_fmt,
541 priv_ano_len=self.nis_priv_ano_len,
542 ert=self.nis_ert)
543
544
545class NIS_CLTU_Header(Segment):
546 """NIS CLTU Packet Header template as defined in EGOS-NIS-NCTR-ICD-0002, Issue: 4.0.2, Page 5.
547
548 ## TBD ##"""
549
550 # fixed length block
551 LENGTH = 47