1#!/usr/bin/env python3
2
3"""Handle conversions of native datatypes to and from text or xml.
4This module contains mostly legacy code using free functions to manipulate information
5about datatypes.
6New code should use the Trait class (which uses the Datatype enumeration to record basic types)
7which is also the base class of Field.
8"""
9
10import re
11import sys
12import json
13import logging
14from datetime import datetime, timedelta
15from enum import Enum
16from collections import namedtuple
17import collections.abc
18
19from chart.common.xml import SubElement
20from chart.common.path import Path
21from chart.common.exceptions import ConfigError
22from chart.common.exceptions import NoSuchField
23from chart.common.xml import parsechildstr
24from chart.common.xml import parsechildbool
25from chart.common.xml import parsechildint
26from chart.common.xml import timedelta_to_xml
27from chart.common.xml import XMLElement
28from chart.common.xml import EXT_XML
29from chart.common.xml import write_xml
30from chart.common.util import nvl
31
32# Normal decoding method to extract Python strings from unicode encoded binary buffers
33UNICODE_DECODING = 'utf-8'
34
35# fix this properly
36# (invalid argument name)
37# pylint: disable=C0103
38
39ELEM_ALLOW_NULL = 'allow-null'
40ELEM_DATATYPE = 'datatype'
41ELEM_DEFAULT = 'default'
42ELEM_DESCRIPTION = 'description'
43ELEM_LENGTH = 'length'
44ELEM_UNIT = 'unit'
45ELEM_LENGTH = 'length'
46ELEM_CHOICES = 'choices'
47ELEM_ITEM = 'item'
48ELEM_VALUE_MIN = 'min-value'
49ELEM_VALUE_MAX = 'max-value'
50ELEM_VALUE = 'value'
51ELEM_NAME = 'name'
52
53# record a single or range of values in a Choice selection
54# TBD: merge with WidgetOptionChoice, use in EventClass
55# description, unit, colour?
56Item = namedtuple('Item', 'min_value max_value name')
57
58logger = logging.getLogger()
59
60
61def name_of_thing(obj, value_if_none=''):
62 """Return either `obj` (if it is a string) or `obj.name` or `obj.__name__` otherwise."""
63 if obj is None:
64 return value_if_none
65
66 elif isinstance(obj, str):
67 # if obj is a Path we don't want to call anything on it
68 return obj
69
70 elif isinstance(obj, Path):
71 # filename.path strips directory
72 return str(obj)
73
74 elif hasattr(obj, 'name'):
75 return obj.name
76
77 elif hasattr(obj, '__name__'):
78 return obj.__name__
79
80 else:
81 return str(obj)
82
83
84def value_of_thing(obj, value_if_none=None):
85 """Return `obj.value` if it has one else `obj`, or `value_if_none` if it's None."""
86 if obj is None:
87 return value_if_none
88
89 return getattr(obj, 'value', obj)
90
91
92def id_of_thing(obj, value_if_none=None):
93 """Return either `obj` (if it is integer like) or `obj.id` otherwise."""
94 if obj is None:
95 return value_if_none
96
97 elif hasattr(obj, 'id'):
98 return obj.id
99
100 else:
101 return obj
102
103
104def is_listlike(obj):
105 """Test if `obj` is a list, tuple or similar array like iterable."""
106 # If `obj` is a normal iterable and not a string, it probably matches what the client wants
107 # we make a special extra exception to allow dict(...).values() to be classed as list-like
108 return (isinstance(obj, collections.abc.Sequence) and not isinstance(obj, str)) or\
109 type(obj).__name__ == 'dict_values'
110
111
112def list_of(obj):
113 """Convert `obj` into `[obj]` if not already a list, otherwise
114 just return `obj` unmodified."""
115 if is_listlike(obj):
116 return obj
117
118 else:
119 return [obj]
120
121
122class Datatype(Enum):
123 """Represent all the datatypes CHART reads from XML files.
124 (events files, report templates)."""
125
126 BOOL = 'boolean'
127 INT = 'int'
128 UINT = 'uint'
129 FLOAT = 'float'
130 # INTERVAL = 'interval'
131 DURATION = 'duration'
132 # TIMESTAMP = 'timestamp'
133 DATETIME = 'datetime' # millisecond accuracy assumed
134 STRING = 'string'
135 BINARY = 'binary'
136 XML = 'xml' # this and unicode only exist for <xml> and <unicode> DDL fields
137 UNICODE = 'unicode'
138 # the db/ts/*.xml definition files should be changed so only contain table definitions.
139 # in db/packets/*.xml should be the packet readers with more types available like mil1750,
140 # hirsunsigned, invertedbool etc.
141 EVENTCLASS = 'eventclass'
142 FIELD = 'field'
143 EVENTPROPERTY = 'eventproperty'
144 TABLE = 'table'
145 COLOUR = 'colour'
146 JSON = 'json'
147 JSONB = 'jsonb'
148 DEDUCED = 'deduced'
149
150
151# the ts modules use these to convert fieldinfos to types for database links
152Datatype.BOOL.python_type = bool
153Datatype.DATETIME.python_type = datetime
154Datatype.STRING.python_type = str
155Datatype.BINARY.python_type = bytearray
156Datatype.FLOAT.python_type = float
157Datatype.INT.python_type = int
158Datatype.UINT.python_type = int
159Datatype.DURATION.python_type = timedelta
160Datatype.JSONB.python_type = str
161
162
163class TimePrecision(Enum):
164 """Precision required for database storage of datatime fields.
165 Precision is normally specified in table XML files."""
166
167 MICROSECOND = 'us'
168 MILLISECOND = 'ms'
169 SECOND = 's'
170
171class NoSuchChoiceFile(Exception):
172 """No Such Choice File found."""
173
174 def __init__(self, name):
175 super(NoSuchChoiceFile, self).__init__()
176 self.name = name
177
178 def __str__(self):
179 return 'No such Choice file located: {bad}'.format(
180 bad=self.name)
181
182class Choices:
183 """Class to handle Textual Calibrations in template definition files.
184
185 This is used to handle the reading and writing of textual calibrations.
186 """
187
188 # TBD: store list of items and a dictionary cache separately for merge
189 # with WidgetOptionsChoice
190
191 cache = None
192
193 def __init__(self, elem=None, choice_id=None, description=None):
194 """Args:
195 `elem` (Element, optional): Parse a <choices> element
196 `choice_id` Id / Name of choices definition, name of file in which definition defined
197 `description` of choice definition
198 """
199 self.choice_id = choice_id
200 self.description = description
201 self.items = {}
202 if elem is not None:
203 self.parse(elem)
204
205 if Choices.cache is None:
206 Choices.cache = {}
207
208 def __str__(self):
209 return str(self.items)
210 # return 'Choices({c})'.format(c=','.join(str(i) for i in self.items))
211
212 def __len__(self):
213 """plot_utils.py needs to iterate through choice items for axis labelling."""
214
215 return len(self.items)
216
217 def values(self):
218 """Make ourselves look like a dictionary for plotting."""
219
220 return iter(self.items.values())
221
222 # def lookup()
223 # return Item object
224
225 def get_name(self, value):
226 """Convert a raw `value` to the name of the corresponding item.
227
228 Return None if no item found
229 """
230
231 name = None
232
233 if value in self.items:
234 name = self.items[value].name
235
236 else:
237 # look for the provided value in the available ranges
238 for key in sorted(iter(self.items.keys()), reverse=True):
239 if value <= key:
240 if value in self.items:
241 name = self.items[value].name
242
243 else:
244 name = None
245
246 break
247
248 return name
249
250 def get_values(self, name):
251 """Convert the name of an item to the corresponding raw `values`."""
252
253 min_value = None
254 max_value = None
255
256 for value in self.items:
257 if self.items[value].name == name:
258 min_value = self.items[value].min_value
259 max_value = self.items[value].max_value
260 break
261
262 return (min_value, max_value)
263
264 def parse(self, choices_elem):
265 """Create the choices attribute (textual calibrations) from the XML template definition."""
266
267 self.description = parsechildstr(choices_elem, ELEM_DESCRIPTION, None)
268
269 for elem_item in choices_elem.findall(ELEM_ITEM):
270 value = parsechildint(elem_item, ELEM_VALUE, None)
271 if value is None:
272 min_value = parsechildint(elem_item, ELEM_VALUE_MIN, None)
273 max_value = parsechildint(elem_item, ELEM_VALUE_MAX, None)
274
275 else:
276 min_value = value
277 max_value = value
278
279 # use 'min_value' as the key to access this calibration element
280 self.items[min_value] = Item(
281 name=parsechildstr(elem_item, ELEM_NAME, None),
282 min_value=min_value,
283 max_value=max_value)
284 # colour=parsechildstr(elem_item, ELEM_COLOUR, None))
285
286
287 def load_xml(self, sid=None, prefix=None):
288 """Populate our `choices` member.
289
290 Using either the `field_info` or `calibration_name` member."""
291
292 from chart.project import settings
293 if sid is None:
294 sid_str = ""
295 else:
296 sid_str = sid.db_path
297
298 if Choices.cache.get(sid_str+self.choice_id) is None:
299 # logger.debug('load_xml sid ' + sid_str + ' choice ' + self.choice_id)
300 xml_dir = sid.db_dir(settings.CHOICES_SUBDIR)
301
302 filename = xml_dir.joinpath(self.choice_id + EXT_XML)
303
304 if filename.exists():
305 root_node = XMLElement(filename=filename)
306 for choices_elem in root_node.findall(ELEM_CHOICES):
307 self.description = choices_elem.parse_str(ELEM_DESCRIPTION, None)
308
309 for elem_item in choices_elem.findall(ELEM_ITEM):
310 value = elem_item.parse_int(ELEM_VALUE, None)
311
312 if value is None:
313 min_value = elem_item.parse_int(ELEM_VALUE_MIN, None)
314 max_value = elem_item.parse_int(ELEM_VALUE_MAX, None)
315
316 else:
317 min_value = value
318 max_value = value
319
320 # use 'min_value' as the key to access this calibration element
321 self.items[min_value] = Item(
322 name=elem_item.parse_str(ELEM_NAME, None),
323 min_value=min_value,
324 max_value=max_value)
325
326 # store choice in cache
327 Choices.cache[sid_str+self.choice_id] = self
328
329 else:
330 raise NoSuchChoiceFile(str(filename))
331
332 return Choices.cache.get(sid_str+self.choice_id)
333
334
335 def def_to_xml(self, node):
336 """Store ourselves in child element <choices> of `elem`."""
337 choices_node = node.add(tag=ELEM_CHOICES)
338 # (re)order output elements and create choice subelements
339 if self.choice_id is not None:
340 choices_node.add(tag=ELEM_NAME, text=self.choice_id)
341
342
343 def write_definition(self, node):
344 """Prepare to write ourselves to disk."""
345
346 choices_node = node.add(tag=ELEM_CHOICES)
347 # (re)order output elements and create choice subelements
348 if self.description is not None:
349 choices_node.add(tag=ELEM_DESCRIPTION, text=self.description)
350
351 for key, item in sorted(self.items.items()):
352 item_node = choices_node.add(tag=ELEM_ITEM)
353 item_node.add(tag=ELEM_NAME, text=item.name)
354
355 # optimize the xml creation for the case where from/to are the same
356 if item.min_value != item.max_value:
357 item_node.add(tag=ELEM_VALUE_MIN, text=item.min_value)
358 item_node.add(tag=ELEM_VALUE_MAX, text=item.max_value)
359
360 else:
361 item_node.add(tag=ELEM_VALUE, text=item.min_value)
362
363class NoDefault:
364 """Weird little hack object. We want a global unique constant, Traits.NO_DEFAULT.
365
366 Normally, no problem .But a couple of widgets derived from Graph and call copy.deepcopy()
367 to duplicate their options before extending them.
368 That duplicates our "unique", causing subsequence "o.default is Traits.NO_DEFAULT" type
369 checks to fails.
370 There are other work arounds - something more subtle than deepcopy() - but this works."""
371
372 # def __init__(self):
373 # print('cons')
374 # def __str__(self):
375 # return 'NO_DEFAULT'
376 def this_is_no_default(self):
377 return None
378
379
380class Trait:
381 """Complete representation of a type of data which can be read from an XML file.
382 Includes base datatype plus supplemental and meta data."""
383
384 NO_DEFAULT = NoDefault()
385
386 def __init__(self,
387 node=None,
388 datatype=None,
389 time_precision=TimePrecision.MICROSECOND,
390 allow_null=False,
391 description=None,
392 length=None,
393 default=NO_DEFAULT,
394 unit=None,
395 choices=None):
396 """Set `elem` if instantiating from XML. Otherwise set individual fields.
397 If `elem` is set, `datatype`, `length`, 'time-precision' can still be non-None as an
398 override (also optimisation when constructed as FieldInfo).
399 """
400 if node is None:
401 # Use has supplied our attributes manually
402 self.datatype = datatype
403 self.time_precision = time_precision
404 self.allow_null = allow_null
405 self.description = description
406 self.length = length
407 self.choices = choices
408 self.default = default
409 self.unit = unit
410
411 else:
412 # read from XML
413 self.time_precision = None
414
415 # self.datatype = node.parse_str(ELEM_DATATYPE)
416 # self.allow_null = node.parse_bool(ELEM_ALLOW_NULL, False)
417 # self.description = node.parse_str(ELEM_DESCRIPTION, None)
418 # self.length = node.parse_int(ELEM_LENGTH, None)
419 # self.default = node.parse_str(ELEM_DEFAULT, Trait.NO_DEFAULT)
420 # self.unit = node.parse_str(ELEM_UNIT, None)
421 # choices_node = node.find(ELEM_CHOICES)
422 # if choices_node is not None:
423 # self.choices = Choices(choices_node.elem)
424
425 # else:
426 # self.choices = None
427
428 self.allow_null = False
429 self.description = None
430 self.length = None
431 self.default = Trait.NO_DEFAULT
432 self.unit = None
433 self.choices = None
434 # self.datatype = None # allowed for PUD deduced types
435 for child in node.elem.iterchildren():
436 if child.tag == ELEM_DATATYPE:
437 self.datatype = Datatype(child.text)
438
439 elif child.tag == ELEM_ALLOW_NULL and child.text == 'true':
440 self.allow_null = True
441
442 elif child.tag == ELEM_DESCRIPTION:
443 self.description = child.text
444
445 elif child.tag == ELEM_LENGTH:
446 self.length = int(child.text)
447
448 elif child.tag == ELEM_DEFAULT:
449 self.default = child.text
450
451 elif child.tag == ELEM_UNIT:
452 self.unit = child.text
453
454 elif child.tag == ELEM_CHOICES:
455 self.choices = Choices(child, description=self.description)
456
457 # Convert string datatypes ("str" etc) to Datatype objects
458 if isinstance(self.datatype, str):
459 try:
460 self.datatype = Datatype(self.datatype)
461 except ValueError:
462 pass
463
464 if isinstance(self.datatype, str):
465 raise ValueError('Unrecognised datatype {d} in XML file'.format(d=self.datatype))
466
467 elif self.datatype is int:
468 self.datatype = Datatype.INT
469
470 elif self.datatype is str:
471 self.datatype = Datatype.STRING
472
473 elif self.datatype is datetime:
474 self.datatype = Datatype.DATETIME
475
476 elif self.datatype is bool:
477 self.datatype = Datatype.BOOL
478
479 elif self.datatype is bytearray:
480 self.datatype = Datatype.BINARY
481
482 elif self.datatype is float:
483 self.datatype = Datatype.FLOAT
484
485 elif self.datatype is timedelta:
486 self.datatype = Datatype.DURATION
487
488 elif is_listlike(self.datatype):
489 # we do allow self.datatype to be a list for Widget nested properties.
490 # It's up to the client ot handle that case though, most Trait
491 # functions won't work
492 # raise ValueError('Cannot construct Trait from {d}'.format(d=self.datatype))
493 pass
494
495 elif self.datatype in Datatype:
496 # no error needed
497 pass
498
499 else:
500 raise ValueError('Unknown datatype {d} in XML file'.format(d=self.datatype))
501
502 def __str__(self):
503 return '{datatype}/{length}'.format(datatype=self.datatype.value, length=self.length)
504
505 def def_to_xml(self, node):
506 """Record ourselves inside XML `node` as a data description."""
507 # if self.datatype is not None:
508 # only PUS deduced parameters may have no datatype of their own
509 node.add(tag=ELEM_DATATYPE, text=self.datatype.value)
510
511 if self.length is not None:
512 node.add(tag=ELEM_LENGTH, text=self.length)
513
514 if self.allow_null:
515 node.add(tag=ELEM_ALLOW_NULL, text='true')
516
517 if self.choices is not None:
518 self.choices.def_to_xml(node)
519
520 if self.unit is not None:
521 node.add(tag=ELEM_UNIT, text=self.unit)
522
523 def from_str(self, in_str):
524 """Decode `in_str` into a native object."""
525 # maybe migrate these functions into member / static functions
526 # return from_str(self, in_str, self.allow_null)
527 if self.datatype is Datatype.STRING:
528 return in_str
529
530 elif self.datatype is Datatype.BINARY:
531 return str_to_binary(in_str)
532
533 elif self.datatype is Datatype.INT:
534 return str_to_int(in_str)
535
536 elif self.datatype is Datatype.UINT:
537 return str_to_uint(in_str)
538
539 elif self.datatype is Datatype.FLOAT:
540 return str_to_float(in_str)
541
542 elif self.datatype is Datatype.EVENTCLASS:
543 return str_to_eventclass(in_str)
544
545 elif self.datatype is Datatype.FIELD:
546 return str_to_field(in_str)
547
548 elif self.datatype is Datatype.COLOUR:
549 return in_str
550
551 elif self.datatype is Datatype.DATETIME:
552 return str_to_datetime(in_str)
553
554 elif self.datatype is Datatype.BOOL:
555 return str_to_boolean(in_str)
556
557 elif self.datatype is Datatype.TABLE:
558 return str_to_tableinfo(in_str)
559
560 elif self.datatype is Datatype.EVENTPROPERTY:
561 return str_to_eventproperty(in_str)
562
563 elif self.datatype is Datatype.DURATION:
564 return str_to_duration(in_str)
565
566 else:
567 raise ConfigError('Trait class cannot convert string to {t}'.format(
568 t=self.datatype))
569
570 def from_xml(self, elem):
571 """Decode the text of `elem` to a Python object."""
572
573 if elem.text is None:
574 return None
575
576 else:
577 return self.from_str(elem.text.strip())
578
579 def from_xml_child(self, elem, child_name):
580 """Decode the text of `child_name` of `elem` to a Python object."""
581
582 return self.from_str(parsechildstr(elem, child_name))
583
584
585# Begin of legacy code.
586# All following functions are obsolete and should not be used.
587# Use Trait class instead.
588
589
590def is_string(trait):
591 """Test if descriptor `trait` looks like a string."""
592 return trait in ('string', 'unicode', str, str)
593
594
595def is_str(trait):
596 """Test if descriptor `trait` is an ascii str."""
597 return trait in ('string', str)
598
599
600def is_unicode(trait):
601 """Test if descriptor `trait` looks like unicode."""
602 return trait in ('unicode', str)
603
604
605def is_filename(trait):
606 """Test if descriptor `trait` looks like a filename."""
607 return trait == 'filename'
608
609# def is_numeric(trait):
610 # pass
611
612
613def is_integer(trait):
614 """Test if `trait` describes an integer data type."""
615 return is_int(trait) or is_uint(trait)
616
617
618def is_int(trait):
619 """Test if descriptor `trait` looks like an integer."""
620 return trait in (int, 'int')
621
622
623def is_uint(trait):
624 """Test if descriptor `trait` looks like an unsigned integer."""
625 return trait == 'uint'
626
627
628def is_datetime(trait):
629 """Test if descriptor `trait` looks like a datetime."""
630 return trait in (datetime, 'datetime')
631
632
633def is_float(trait):
634 """Test if descriptor `trait` describes a floating point type."""
635 return trait in (float, 'float')
636
637def is_binary(trait):
638 """Test if descriptor `trait` describes a binary bytarray type."""
639 return trait in (bytearray, 'binary')
640
641
642def is_htmlcolour(trait):
643 """Test if descriptor `trait` describes a colour type."""
644 return trait == 'htmlcolour'
645
646
647def is_boolean(trait):
648 """Test if descriptor `trait` describes a boolean type."""
649 return trait in (bool, 'boolean')
650
651
652def is_datapoint(trait):
653 """Test if descriptor `trait` describes a CHART datapoint."""
654 return trait == 'datapoint'
655
656
657def is_duration(trait):
658 """Test if descriptor `trait` describes a duration (timedelta)."""
659 return trait in (timedelta, 'duration')
660
661
662def is_eventname(trait):
663 """Test if descriptor `trait` describes a CHART event classname."""
664 # obsolete, use eventclass instead
665
666 return trait == 'eventname'
667
668
669def is_eventclass(trait):
670 """Test if descriptor `trait` describes a CHART event classname."""
671 return trait == 'eventclass'
672
673
674def is_eventproperty(trait):
675 """Test if descriptor `trait` describes a CHART event classname and property."""
676 return trait == 'eventproperty'
677
678
679def is_table(trait):
680 """Test if descriptor `trait` describes a table name."""
681 return trait == 'table'
682
683
684def is_json(trait):
685 return trait == 'json'
686
687
688def traitname(trait):
689 """Return a user friendly name for this trait."""
690 for t in traits:
691 if t['test'](trait):
692 return t['name']
693
694 raise ConfigError('Unknown datatype: ' + str(trait))
695
696
697# def traittooltip(trait):
698 # """Return a tooltip for this trait describing what the acceptable syntax is."""
699 # for t in traits:
700 # if t['test'](trait):
701 # return t.get('tooltip')
702
703 # raise ConfigError('Unknown datatype: {trait}'.format(trait=trait))
704
705
706def from_str(trait, in_str, scid=None, allow_null=False): # (unused arg) pylint: disable=W0613
707 """Convert `s` to a native Python object assuming it is of type described by `trait`."""
708 if len(in_str) == 0:
709 if allow_null:
710 return None
711
712 else:
713 raise ValueError('Conversion from string to null value not allowed')
714
715 # logging.info('pre')
716 # if isinstance(s, unicode):
717 # s = s.encode('latin-1')
718 # logging.info('FROM_STR trait ' + str(trait) + ' s"' + s.encode('latin-1') + '"' +
719 # str(len(s)))
720
721 # try to match up `trait` against one of our known traits
722 for t in traits:
723 if t['test'](trait):
724 # ask the matching trait to decode `s` into a native Python type
725 return t['decode'](in_str)
726 # result = t['decode'](s)
727
728 # if 'choices' in trait:
729 # raise ConfigError('CHOICES ' + str(trait['choices']))
730
731 # return result
732
733 raise ConfigError('Unknown datatype: {trait}'.format(trait=trait))
734
735
736def trait_dtype(trait):
737 """Return underlying Python datatype for `trait`."""
738 for t in traits:
739 if t['test'](trait):
740 return t['dtype']
741
742
743def str_to_datetime(in_str):
744 """Convert string `s` to datetime."""
745 if in_str == 'now':
746 return datetime.utcnow()
747
748 elif in_str == 'today':
749 date = datetime.utcnow().date()
750 return datetime(date.year, date.month, date.day)
751
752 elif in_str == 'yesterday':
753 date = datetime.utcnow().date() - timedelta(days=1)
754 return datetime(date.year, date.month, date.day)
755
756 elif in_str == 'tomorrow':
757 date = datetime.utcnow().date() + timedelta(days=1)
758 return datetime(date.year, date.month, date.day)
759
760 elif in_str == 'launch': # and scid is not None:
761 # return get_launch_date(scid)
762 return 'launch'
763
764 matcher = re.compile( # (unused var) pylint: disable=W0612
765 r'(?P<year>\d{4})-(((?P<month>0[1-9]|1[0-2])-'
766 r'(?P<day>3[01]|[0-2]{0,1}\d))|(?P<yday>[0-9]{3}))'
767 r'(T(?P<hour>([0-2])\d):(?P<min>[0-5]\d):(?P<sec>[0-5]\d)(\.(?P<fine>\d{1,6}))?)?')
768
769 match_obj = matcher.search(in_str)
770 if match_obj is None:
771 raise ValueError('Cannot convert "{input}" to datetime'.format(input=in_str))
772
773 year = int(match_obj.group('year'))
774
775 # Check for subseconds
776 fine = match_obj.group('fine')
777 if fine is None:
778 us = 0
779
780 else:
781 # Convert fractional seconds part to count of microseconds, taking into account
782 # the number of significant digits the user passed in
783 us = int(1e6 * int(fine) / pow(10, len(fine)))
784
785 if match_obj.group('yday') is None:
786 month = int(match_obj.group('month'))
787 day = int(match_obj.group('day'))
788 if match_obj.group('hour') is None:
789 return datetime(year, month, day)
790
791 hour = int(match_obj.group('hour'))
792 minute = int(match_obj.group('min'))
793 sec = int(match_obj.group('sec'))
794 return datetime(year, month, day, hour, minute, sec, us)
795
796 else:
797 if match_obj.group('hour') is None:
798 return datetime.strptime(in_str, '%Y-%j')
799
800 if match_obj.group('fine') is None:
801 return datetime.strptime(in_str, '%Y-%jT%H:%M:%S')
802
803 return datetime.strptime(in_str[:17], '%Y-%jT%H:%M:%S').replace(
804 microsecond=us)
805
806
807def str_to_uint(in_str):
808 """Convert string `s` to unsigned integer."""
809 val = int(in_str)
810 if val < 0:
811 raise ValueError('Value must be greater than zero, not "{s}"'.format(s=in_str))
812
813 return val
814
815
816def str_to_int(s):
817 """Convert string `s` to integer."""
818 val = int(s)
819 return val
820
821
822def str_to_binary(s):
823 """Convert string `s` to binary."""
824 b = bytearray()
825 b.extend(s.encode())
826 return b
827
828
829def str_to_float(s):
830 """Convert string `s` to float."""
831 return float(s)
832
833
834def str_to_boolean(s, default=Trait.NO_DEFAULT):
835 """Convert string `s` to bool."""
836 if len(s) > 0:
837 if s[0] in ('f', 'F', '0'):
838 return False
839
840 elif s[0] in ('t', 'T', '1'):
841 return True
842
843 if default is Trait.NO_DEFAULT:
844 raise ValueError('Value cannot be converted to bool: "{s}"'.format(s=s))
845
846 return default
847
848
849def str_to_eventclass(s):
850 """Convert string `s` to an EventClass object.
851
852 >> str_to_eventclass('OPERATOR-AR') #doctest: +ELLIPSIS
853 <chart.events.eventclass._EventClass object at 0x...>
854 """
855
856 from chart.events.eventclass import EventClass
857 return EventClass(s)
858
859
860def str_to_eventproperty(s):
861 """Convert string `s` to an EventClass object.
862
863 >> str_to_eventproperty('GOME-HCL-ACTIVATOR.ignition_time')
864 {'eventclass': eventclass(<GOME-HCL-ACTIVATOR>), 'property': 'ignition_time'}
865 """
866
867 from chart.events.eventclass import EventClass
868
869 # See if the user has given an event name and property
870 pos = s.find('.')
871
872 if pos == -1:
873 # Assume its just a class name
874 return {'eventclass': EventClass(s),
875 'property': None}
876 # raise ValueError('Cannot decode event name and property from "{s}".'
877 # 'Syntax should be "EventClassName.property_name"'.format(s=s))
878
879 eventclass = EventClass(s[:pos])
880
881 prop = eventclass.instance_properties[s[pos + 1:]]
882
883 return {'eventclass': eventclass,
884 'property': prop}
885
886
887def str_to_field(s):
888 """Convert string `s` in format "table.field" or "field" to FieldInfo."""
889 from chart.db.model.table import find_param_by_name
890 from chart.db.model.table import TableInfo
891 from chart.db.model.field import RowcountFieldInfo
892
893 if s is None or len(s) == 0:
894 raise ValueError('Found empty string where field name expected')
895
896 parts = s.split('.')
897 if len(parts) == 1:
898 # User has specified only a fieldname
899 # This is fine as long as it's unique and only occurs in 1 table
900 res = find_param_by_name(s)
901 if res is None:
902 raise NoSuchField(s)
903
904 return res
905
906 if len(parts) == 2:
907 if parts[1].upper() == 'ROWCOUNT':
908 # User requests count of the total rows in a table
909 return RowcountFieldInfo(TableInfo(parts[0]), None)
910
911 # User specifies table.name
912 try:
913 return TableInfo(parts[0]).fields[parts[1]]
914 except KeyError:
915 raise ConfigError(f'Field/table {s} not found')
916
917 if len(parts) == 3:
918 if parts[2].upper() != 'ROWCOUNT':
919 raise ConfigError('Could not process {t}'.format(t=s))
920
921 table = TableInfo(parts[0])
922 field = table.fields[parts[1]]
923 return RowcountFieldInfo(table, field)
924
925 raise ConfigError('Could not process {t}'.format(t=s))
926
927
928def str_to_duration(s):
929 """Convert string `s` to timedelta."""
930 if s == '0':
931 return timedelta()
932
933 matcher = re.compile( # (unused var) pylint: disable=W0612
934 r'^(?P<minus>-)?'
935 r'P'
936 r'((?P<year>\d+)Y)?'
937 r'((?P<month>\d+)M)?'
938 r'((?P<day>\d+)D)?'
939 r'(T'
940 r'((?P<hour>\d+)H)?'
941 r'((?P<minute>\d+)M)?'
942 r'((?P<sec>[0-9.]+)S)?'
943 r')?$')
944
945 res = matcher.search(s)
946 if res is None:
947 # test if the problem was the user writing in lower case ...
948 if s != s.upper():
949 try:
950 str_to_duration(s.upper())
951 lower = True
952 except ValueError:
953 lower = False
954
955 if lower:
956 raise ValueError('Time duration given in lower case: {s}'.format(s=s))
957
958 raise ValueError('"{s}" did not match ISO8601 duration template'.format(s=s))
959
960 match = res.groupdict()
961 if match['year'] is not None:
962 raise ValueError('Use of year (PxY) in duration is not supported, use PxD instead')
963
964 if match['month'] is not None:
965 raise ValueError('Use of month (PxM) in duration is not supported, use PxD instead')
966
967 if match['day'] is None:
968 day = 0
969
970 else:
971 day = int(match['day'])
972
973 if match['hour'] is None:
974 hour = 0
975
976 else:
977 hour = int(match['hour'])
978
979 if match['minute'] is None:
980 minute = 0
981
982 else:
983 minute = int(match['minute'])
984
985 ssec = match.get('sec')
986 if ssec is None:
987 # no seconds specified
988 sec = 0
989 us = 0
990
991 else:
992 # look for fractional seconds
993 parts = ssec.split('.')
994 if len(parts) == 1:
995 # no, just whole seconds
996 sec = int(ssec)
997 us = 0
998
999 else:
1000 # partial seconds present
1001 sec = int(parts[0])
1002 us = float('.' + parts[1]) * 1000000
1003
1004 seconds = sec + minute * 60 + hour * 3600 + day * 86400
1005 res = timedelta(seconds=seconds, microseconds=us)
1006 if match['minus'] is not None:
1007 res = -res
1008
1009 return res
1010
1011
1012def bin_to_str(s): # unicode=True
1013 """Convert binary buffer `s` into a string assuming a UTF-8 Unicode encoding."""
1014 # (latin-1 for 8-bit ASCII encoding)
1015 return s.decode(UNICODE_DECODING)
1016
1017
1018def to_str(s):
1019 """Convert 's' into a plain ascii string.
1020 (If given unicode input any odd characters are converted to XML glyphs.)
1021 If given unicode input any mildly odd characters are converted to latin-1 character
1022 set.
1023 """
1024 if isinstance(s, str):
1025 if sys.version_info.major == 2:
1026 return s.encode('latin-1')
1027
1028 else:
1029 return s
1030
1031 elif isinstance(s, Enum):
1032 return s.value
1033
1034 elif isinstance(s, bool):
1035 return 'true' if s else 'false'
1036
1037 elif isinstance(s, timedelta):
1038 return timedelta_to_xml(s)
1039
1040 else:
1041 return str(s)
1042
1043
1044def str_filename(s):
1045 """Throw an exception if `s` is an invalid filename.
1046 For now, just complain if it contains spaces.
1047 This is to prevent report authors including spaces in filenames which cause problems
1048 later in a report viewer.
1049 """
1050 if ' ' in s:
1051 raise ValueError('Found space in filename "{filename}"'.format(filename=s))
1052
1053 return s
1054
1055
1056def str_to_tableinfo(s):
1057 """Return a TableInfo."""
1058 from chart.db.model.table import TableInfo
1059
1060 #logging.debug('MAKING a tableinfo from ' + str(s))
1061 return TableInfo(s)
1062
1063
1064def str_to_json(s):
1065 """Decode `s` to a Python json object."""
1066 # print('decoding', s)
1067 # print(json.loads(s))
1068 return json.loads(s)
1069
1070
1071traits = [
1072 {'test': is_unicode,
1073 'name': 'unicode',
1074 'decode': str,
1075 'dtype': str},
1076
1077 {'test': is_string,
1078 'name': 'string',
1079 'decode': to_str,
1080 'dtype': str},
1081
1082 {'test': is_filename,
1083 'name': 'filename',
1084 'decode': str_filename,
1085 'dtype': str},
1086
1087 {'test': is_htmlcolour,
1088 'name': 'htmlcolour',
1089 'decode': str,
1090 'tooltip': ('HTML style colour, either a named colour (i.e. "firebrick") or '
1091 'hex RGB in the form #rrggbb i.e. "#b22222"')},
1092
1093 {'test': is_eventname,
1094 'name': 'eventname',
1095 'decode': str,
1096 'tooltip': 'CHART event classname'},
1097
1098 {'test': is_eventclass,
1099 'name': 'eventclass',
1100 'decode': str_to_eventclass,
1101 'tooltip': 'CHART event class'},
1102
1103 {'test': is_eventproperty,
1104 'name': 'eventproperty',
1105 'decode': str_to_eventproperty,
1106 'tooltip': 'Name a property from an event as "Eventclass.property"'},
1107
1108 {'test': is_uint,
1109 'name': 'uint',
1110 'decode': str_to_uint,
1111 'dtype': int},
1112
1113 {'test': is_int,
1114 'name': 'int',
1115 'decode': str_to_int,
1116 'dtype': int},
1117
1118 {'test': is_float,
1119 'name': 'float',
1120 'decode': str_to_float,
1121 'dtype': float},
1122
1123 {'test': is_binary,
1124 'name': 'binary',
1125 'decode': str_to_binary,
1126 'dtype': bytearray},
1127
1128 {'test': is_boolean,
1129 'name': 'boolean',
1130 'decode': str_to_boolean,
1131 'tooltip': '"true" or "false"',
1132 'dtype': bool},
1133
1134 {'test': is_datetime,
1135 'name': 'datetime',
1136 'decode': str_to_datetime,
1137 'tooltip': 'ISO-8601 datetime, i.e. "2011-10-06T15:33:00"',
1138 'dtype': datetime},
1139
1140 {'test': is_duration,
1141 'name': 'duration',
1142 'decode': str_to_duration,
1143 'dtype': timedelta,
1144 'tooltip': 'ISO-8601 style time offset i.e. "P1D" or "PT6H10M"'},
1145 # 'tooltip': '<a http="http://en.wikipedia.org/wiki/ISO_8601">ISO-8601</a>
1146 # time offset i.e. "P1D" or "PT6H10M"'},
1147
1148 {'test': is_datapoint,
1149 'name': 'datapoint',
1150 'decode': str_to_field,
1151 'tooltip': 'Table and field name, colon-separated (i.e. "MHS.MHS_MODE")'},
1152
1153 {'test': is_table,
1154 'name': 'table',
1155 'decode': str_to_tableinfo,
1156 'tooltip': 'Tablename'},
1157
1158 {'test': is_json,
1159 'name': 'json',
1160 'decode': str_to_json,
1161 'tooltip': 'json'},
1162
1163 ]
1164
1165
1166def from_xml_child(trait,
1167 parent,
1168 name,
1169 scid=None, # (unused arg) pylint:disable=W0613
1170 allow_null=False):
1171 """Convert the text of child `name` of XML element `parent`."""
1172 return from_str(trait, parsechildstr(parent, name), allow_null=allow_null)
1173
1174
1175def from_xml(trait, elem, scid=None): # (unused arg) pylint: disable=W0613
1176 """Convert the text part of XML element `elem`."""
1177 # allow_null
1178 # empty_string_is_null
1179 if elem.text is None:
1180 return None
1181
1182 return from_str(trait, elem.text.strip())
1183 # res = from_str(trait, elem.text.strip())
1184 # from lxml import etree
1185 # print('Converted ' + etree.tostring(elem) + ' to ' + str(res) + ' using trait ' + str(trait))
1186 # return res
1187
1188
1189def to_htmlstr(s, trait=None):
1190 """Convert `s` into a scrap of text suitable for inserting to an HTML document.
1191 unicode glyphs will be converted to XML glyphs."""
1192 from chart.db.model.table import FieldInfo
1193 from chart.events.eventclass import EventClass
1194 from chart.common.prettyprint import Table
1195
1196 if isinstance(trait, dict):
1197 # guided conversion using trait
1198 res = to_htmlstr(s)
1199
1200 attrs = []
1201 if 'description' in trait:
1202 attrs.append('title="{desc}"'.format(desc=trait['description']))
1203
1204 if trait['type'] == 'htmlcolour':
1205 attrs.append('background-color="{col}"'.format(col=s))
1206
1207 if 'choices' in trait:
1208 for c in trait['choices']:
1209 if c.get('name') == s and 'description' in c:
1210 res += ' ({desc})'.format(desc=c['description'])
1211 break
1212
1213 return '<span {attrs}>{res}</span>'.format(attrs=' '.join(attrs), res=res)
1214
1215 else:
1216 # auto string -> html conversion
1217 if isinstance(s, str):
1218 return s
1219
1220 # elif isinstance(s, str):
1221 # print(u'CONVERTING ' + s + ' to ' + s.encode('ascii', 'xmlcharrefreplace'))
1222 # return s.encode('ascii', 'xmlcharrefreplace')
1223
1224 elif isinstance(s, Table):
1225 return s.to_html_str()
1226
1227 elif isinstance(s, FieldInfo):
1228 return '{table}.{field}'.format(field=s.name, table=s.table.name)
1229
1230 elif isinstance(s, EventClass):
1231 return s.name
1232
1233 else:
1234 return str(s)
1235
1236
1237def to_term(s):
1238 """Convert `s` into text that can be written to the current terminal."""
1239 if isinstance(s, str):
1240 return s
1241
1242 elif isinstance(s, str):
1243 return s.encode('utf-8')
1244
1245 else:
1246 return str(s)