1#!/usr/bin/env python3
2"""Common Utility/Helper Classes, functions and constants for APEX dat file ingestion."""
3
4from datetime import datetime, timedelta
5from lxml import etree as ET
6
7
8from chart.db.model.table import TableInfo
9from chart.db.connection import db_connect
10from chart.project import settings
11
12from chartepssg.alg.common import TIME_DECODER, is_sublist
13
14
15# text encodings
16UTF8 = 'UTF-8'
17CP1252 = 'cp1252'
18
19# table info constants
20TBL_APEX_INSTANCES = TableInfo('APEX_INSTANCES')
21TBL_APEX_LOG = TableInfo('APEX_LOG')
22
23# This is a margin when ingesting APEX_EVENTS as derived jobs
24# When a drived job is created, a start and stop time is set which derived
25# from the HL and PA ingester min and max time found in the ingested source
26# However, there seems to be some events that are not missing due possibly
27# to the start and stop not included in the retrieval,
28# hence adding an extra sec to ensure missing events are ingest into EVENTS table
29APEX_EV_ING_MARGIN = timedelta(seconds=1)
30
31# DB connection object
32db_conn = db_connect()
33
34
35def apextime_to_utc(str_apex_time):
36 """Convert an APEX timestamp seconds@millisecs into UTC date time."""
37 temp = str_apex_time.split('@')
38 coarse_time = int(temp[0])
39 fine_time = int(temp[1])
40 return datetime.utcfromtimestamp(coarse_time).replace(microsecond=fine_time * 1000)
41
42
43def str_to_datetime(str_date_time):
44 """Convert JSON datetime string to python datetime object."""
45 return datetime.strptime(str_date_time, TIME_DECODER)
46
47
48def datetime_to_str(date_time):
49 """Convert datetime object to JSON style datetime."""
50 return date_time.strftime(TIME_DECODER)
51
52
53def get_proc_attributes(sid_num, instance):
54 """Fetch from DB attributes of an instance and a given SID."""
55 sql = ('SELECT ATTRIBUTES FROM {table} WHERE '
56 'INSTANCE = :instance '
57 'AND SID_NUM = :sid_num '
58 'AND LEVEL = :level').format(table=TBL_APEX_INSTANCES.name)
59
60 res = db_conn.query(sql, instance=instance,
61 sid_num=sid_num,
62 level='PROCEDURE').fetchone()
63 if res is not None:
64 return res[0]
65
66 return None
67
68def delete_proc_instance(sid_num, instance):
69 """Delete proc instance from DB."""
70 sql = ('DELETE FROM {table} WHERE '
71 'INSTANCE = :instance '
72 'AND SID_NUM = :sid_num '
73 'AND LEVEL = :level').format(table=TBL_APEX_INSTANCES.name)
74
75 db_conn.execute(sql, instance=instance,
76 sid_num=sid_num,
77 level='PROCEDURE')
78
79 db_conn.commit()
80
81def update_instance(sid_num, instance, level, field, value):
82 """Update APEX instances table a given entry field value.
83
84 The entry identified by instance, SID and level.
85 """
86 sql = ('UPDATE {table} '
87 'SET {field} = :value WHERE '
88 'INSTANCE = :instance '
89 'AND SID_NUM = :sid_num '
90 'AND LEVEL = :level ').format(table=TBL_APEX_INSTANCES.name, field=field)
91
92 db_conn.execute(sql, value=value,
93 instance=instance,
94 sid_num=sid_num,
95 level=level)
96
97 db_conn.commit()
98
99def expand_proc_property(proc_props, prop, value, timestamp):
100 """Expand value list of a given prop in proc props."""
101 value_list = []
102 if prop in proc_props.keys():
103 value_list = proc_props[prop]
104
105 pair = [value, datetime_to_str(timestamp)]
106 if not is_sublist(pair, value_list):
107 value_list.append(pair)
108
109 proc_props[prop] = value_list
110
111
112def get_attributes_from_du(child_level, child_id, procedure):
113 """Retrieve from APEX DU the attributes of a child instance."""
114 du_file = settings.APEX_CONF_DIR.joinpath(procedure + '.xml')
115
116 if child_level == 'STEP':
117 child_level = 'Step'
118 elif child_level == 'THREAD':
119 child_level = 'PrimaryThread'
120 else:
121 return None
122 """
123 elif child_level == 'ARGUMENT':
124 child_level = 'PrimaryThread'
125 elif child_level == 'STEPLIST':
126 child_level = 'StepList'
127 """
128 xpath = ('//{child_level}[@Identifier=\'{id}\']').format(
129 child_level=child_level,
130 id=child_id)
131 parser = ET.XMLParser(recover=True, encoding=CP1252, resolve_entities=False)
132 tree = ET.parse(str(du_file), parser=parser)
133 elem = tree.find(xpath)
134 if elem is not None:
135 return dict(elem.attrib)
136 else:
137 raise Exception('Unable to find APEX object {id} attributes in DU.'.format(id=child_id))
138
139
140class ArchiveCursor:
141 """Helper class to navigate through APEX files for parsing purposes\
142 using a cursor alike approach."""
143
144 def __init__(self, lines, decoder_func):
145 # file = open(filename, encoding=UTF8)
146 self.lines = lines
147 self.index = -1
148 self.decoder_func = decoder_func
149
150 def current(self):
151 """Get current decoded line. Throws StopIteration exception when EOF reached."""
152 if self.index < len(self.lines):
153 line = self.lines[self.index]
154 return self.decoder_func(line)
155 else:
156 raise StopIteration
157
158 def next(self):
159 """Get next decoded line. Throws StopIteration exception when EOF reached."""
160 try:
161 self.index += 1
162 entry = self.current()
163
164 if entry is None:
165 return self.next()
166 else:
167 return entry
168
169 except IndexError:
170 raise StopIteration
171
172 def prev(self):
173 """Get previous decoded line. Throws StopIteration exception when EOF reached."""
174 self.index -= 1
175 if self.index < -1:
176 raise StopIteration
177 else:
178 return self.current()
179
180
181#
182# Excpetions thrown for the APEX ingesters
183#
184class UnexpectedAttribute(Exception):
185 """Thrown when an unexpected APEX file attribute encountered."""
186
187 pass
188
189
190class EmptyArchive(Exception):
191 """Thrown when an APEX file has no actual entries."""
192
193 pass
194
195
196class ProcedureNotFound(Exception):
197 """Thrown when an Argument or Variable attribute re-occurs within an instance."""
198
199 pass
200
201
202#
203# APEX History maps from code to label
204#
205# level
206MAP_LEVEL = {
207 '<H>': 'PROCEDURE',
208 '<L>1': 'STEP',
209 '<L>2': 'STEPLIST',
210 '<L>3': 'THREAD',
211 '<L>4': 'ARGUMENT',
212 '<L>5': 'VARIABLE',
213}
214
215# HL attributes
216MAP_H_ATTR = {
217 'S': 'STATE',
218 'ST': 'START_TIME',
219 'E': 'END_TIME',
220 'SC': 'SOURCE',
221 'I': 'INTERACTIONS',
222 'CS': 'CURRENT_STEP',
223 'ID': 'IDENTIFIER',
224 'V': 'VERSION',
225 'SP': 'PROCEDURE_SUSPENDED',
226 'EX': 'EXECUTION_MODE',
227 'T': 'TITLE',
228 'MET': 'MAXIMUM_EXECUTION_TIME',
229}
230
231# HL states
232MAP_H_STATES = {
233 '0': 'Inactive',
234 '1': 'Pending',
235 '2': 'Starting Up',
236 '3': 'Triggering',
237 '4': 'Triggering Paused',
238 '5': 'Triggering Failed',
239 '6': 'Executing',
240 '7': 'Pausing',
241 '8': 'Paused',
242 '9': 'Domain Paused',
243 '10': 'Domain Paused And Paused',
244 '11': 'Confirming',
245 '12': 'Confirming Paused',
246 '13': 'Aborted',
247 '14': 'Completed Not Confirmed',
248 '15': 'Completed Success',
249 '16': 'Completed Failed',
250 '17': 'System Closed',
251 '20': 'Expired',
252 '21': 'Monitoring',
253 '22': 'Verification In Error',
254 '23': 'Verification Unknown',
255 '24': 'Activation Error',
256 '25': 'Completed Verification in Error',
257 '26': 'Completed Verification Unknown',
258 '27': 'Completed Activation Error',
259 '28': 'Aborted Verification in Error',
260 '29': 'Aborted Verification Unknown',
261 '30': 'Aborted Activation Error',
262}
263
264# L1 attributes
265MAP_L1_ATTR = {
266 'S': 'STATE',
267 '0': 'FOR_INITIAL_VALUE',
268 '1': 'FOR_CURRENT_VALUE',
269 '2': 'FOR_INCREMENT',
270 '3': 'IF_EXPRESSION_RESULT',
271 '4': 'CASE_EXPRESSION_RESULT',
272 '5': 'INTERACTION_RESULT',
273 '6': 'SUB_PROCEDURE_STATUS',
274 '7': 'PROCEDURE_INSTANCE_IDENT',
275 '8': 'ALERT_STEP_RESULT',
276 '9': 'ARGUMENT_RESULT',
277 '10': 'PARAMETER_RESULT',
278 '11': 'PROCEDURE_VARIABLE_RESULT',
279 '12': 'AUTHORISED',
280 '13': 'TRANSMISSION_TIME',
281 '14': 'EXECUTION_TIME',
282 '15': 'INSTANCE_IDENTIFIER',
283 '16': 'COMMAND_COMPLETION_STATUS',
284 '17': 'COMMAND_DISABLE_PTV',
285 '18': 'GROUP_DISABLE_PTV',
286 '19': 'GROUP_DISABLE_CEV',
287 '20': 'CEV_EXPRESSION',
288 '21': 'LOG_STEP_TEXT',
289 '22': 'FAILURE_RECOVERY',
290 '23': 'ACTIVE',
291 '24': 'INACTIVE',
292 '25': 'COMPLETED',
293 '26': 'TRIGGER_FAILURE_TYPE',
294 '27': 'CONFIRMATION_FAILURE_TYPE',
295 '28': 'LOOP_EXPRESSION_RESULT',
296}
297
298# L1 states
299MAP_L1_STATES = {
300 '0': 'Inactive',
301 '1': 'Pending',
302 '2': 'Triggering Active',
303 '3': 'Triggering Suspended',
304 '4': 'Triggering Failed',
305 '5': 'Executing Active',
306 '6': 'Executing Suspended',
307 '7': 'Executing Failed',
308 '8': 'Confirming Active',
309 '9': 'Confirming Suspended',
310 '10': 'Confirming Failed',
311 '11': 'Completed',
312 '12': 'Aborted',
313 '13': 'Executing Timed Out',
314}
315
316# L2 states
317MAP_L2_STATES = {
318 '0': 'Inactive',
319 '1': 'Pending',
320 '2': 'Triggering Active',
321 '3': 'Triggering Suspended',
322 '4': 'Triggering Failed',
323 '5': 'Executing Active',
324 '6': 'Executing Suspended',
325 '7': 'Executing Failed',
326 '8': 'Confirming Active',
327 '9': 'Confirming Suspended',
328 '10': 'Confirming Failed',
329 '11': 'Completed',
330 '12': 'Aborted',
331 '13': 'Executing Timed Out',
332}
333
334# L3 attributes
335MAP_L3_ATTR = {
336 'S': 'STATE',
337 'R': 'REENTRANT',
338}
339
340# L3 states
341MAP_L3_STATES = {
342 '0': 'Inactive',
343 '1': 'Activating',
344 '2': 'Activating Suspending',
345 '3': 'Executing Auto',
346 '4': 'Executing Man Paused',
347 '5': 'Executing Man Exe',
348 '6': 'Suspending',
349 '7': 'Procedure Suspended',
350 '8': 'Procedure Suspended and Suspended',
351 '9': 'Suspended',
352 '10': 'Disabling',
353 '11': 'Disabled',
354 '12': 'Aborted',
355}