1#!/usr/bin/env python3
2
3"""Implementation of Job class."""
4
5import logging
6from enum import Enum
7
8from chart.common.decorators import memoized
9from chart.project import settings
10from chart.common.path import Path
11from chart.common.xml import xml_to_datetime
12from chart.db.model.table import TableInfo
13from chart.backend.activity import Activity
14from chart.common.traits import name_of_thing
15from chart.common.timerange import TimeRange
16from chart.project import SID
17from chart.db.func import ANY
18
19logger = logging.getLogger()
20
21# Allowed values for the JOBS table STATUS field
22JobStatus = Enum('JobStatus', 'COMPLETED FAILED RETRY TIMEOUT IN_PROGRESS PENDING')
23# class JobStatus(Enum):
24# COMPLETED = 'COMPLETED'
25# FAILED = 'FAILED'
26# RETRY = 'RETRY'
27# TIMEOUT = 'TIMEOUT'
28# IN_PROGRESS = 'IN_PROGRESS'
29# PENDING = 'PENDING'
30
31# JobStatus.COMPLETED.db_value = 'COMPLETED'
32# JobStatus.FAILED.db_value = 'FAILED'
33# JobStatus.RETRY.db_value = 'RETRY'
34# JobStatus.TIMEOUT.db_value = 'TIMEOUT'
35# JobStatus.IN_PROGRESS.db_value = 'IN_PROGRESS'
36# JobStatus.PENDING.db_value = None
37
38JOBS_ORBIT = 'orbit'
39ELEM_ACTIVITY = 'activity'
40ELEM_CATEGORY = 'category'
41ELEM_FILENAME = 'filename'
42ELEM_ID = 'id'
43ELEM_ORBIT = 'orbit'
44ELEM_SENSING_START = 'sensing-start'
45ELEM_SENSING_STOP = 'sensing-stop'
46ELEM_STATUS = 'status'
47ELEM_TABLENAME = 'tablename'
48
49@memoized
50def job_retrieve_fields():
51 """Standard set of fields to retrieve in order to build a Job object."""
52 return ['ID',
53 'CATEGORY',
54 'STATUS',
55 'GEN_TIME',
56 'ORBIT' if settings.ORBIT_IN_JOBS_TABLE else 'NULL',
57 'SENSING_START',
58 'SENSING_STOP',
59 'EARLIEST_EXECUTION_TIME',
60 'PROCESS_ID',
61 'TABLENAME',
62 'PARENT' if settings.DATABASE_JOBS_TABLE_PARENT else 'NULL',
63 'ACTIVITY',
64 'FILENAME',
65 'DIRNAME'] + SID.sql_sys_select('JOBS')
66
67
68class Job:
69 """Representation of a scheduler job, either a pending or a completed job.
70 It can be instantiated from the database via the `jobs` module or from XML by
71 `backend.workorder`.
72 """
73
74 def __init__(self,
75 row=None,
76 job_id=None,
77 category=None,
78 status=None,
79 gen_time=None,
80 activity=None,
81 filename=None,
82 orbit=None,
83 sensing_start=None,
84 sensing_stop=None,
85 tablename=None,
86 table=None,
87 earliest_execution_time=None,
88 process_id=None,
89 parent=None,
90 sid=None):
91 """Build a JOB. If coming from a database there will be a cursor retrieving
92 job_retrieve_fields() so the order or parameters must be consistent.
93 `sid` can be the text representation of a SID or a SID object."""
94 assert not isinstance(tablename, TableInfo)
95
96 if table is not None:
97 tablename = table.name
98
99 if row is not None:
100 self.job_id, self.category, self.status, self.gen_time, self.orbit, \
101 self.sensing_start, self.sensing_stop, self.earliest_execution_time, \
102 self.process_id, self.tablename, self.parent = row[:11]
103 self.activity = Activity(row[11])
104 if row[12] is None:
105 self.filename = None
106
107 else:
108 # remember it's filename, dirname
109 self.filename = Path(row[13]).joinpath(row[12])
110
111 self.sid = SID.from_sys_select('JOBS', row[14:])
112
113 if self.status is None:
114 self.status = JobStatus.PENDING
115
116 else:
117 self.status = JobStatus[self.status]
118
119 else:
120 self.job_id = job_id
121 self.category = category
122 if status is None or status is ANY:
123 self.status = JobStatus.PENDING
124
125 elif status in JobStatus:
126 self.status = status
127
128 else:
129 self.status = JobStatus[status]
130
131 self.gen_time = gen_time
132 self.filename = filename
133 self.sid = sid
134 self.orbit = orbit
135 self.sensing_start = sensing_start
136 self.sensing_stop = sensing_stop
137 self.earliest_execution_time = earliest_execution_time
138 self.process_id = process_id
139 # for performance we allow tablename to remain a string
140 self.tablename = tablename
141 self.parent = parent
142
143 # activity is converted to an Activity object though
144 if activity is None:
145 self.activity = None
146
147 elif isinstance(activity, Activity):
148 self.activity = activity
149
150 else:
151 self.activity = Activity(activity)
152
153 # additional information added by the result class. These things are
154 # never stored but used to create report manifest.xml files and
155 # derived jobs
156 self.tables = None
157 self.primary_output = None
158 self.aux_outputs = []
159 # assert self.sid is not None
160 # logger.debug('JOB ' + str(id(self)) +' cons sid is ' + str(self.sid))
161
162 # def get_sid(self):
163 # return self._sid
164
165 # def set_sid(self, value):
166 # assert value is not None
167 # self._sid = value
168
169 # sid = property(get_sid, set_sid)
170
171 def __str__(self):
172 things = []
173 for attr in ('job_id', 'category', 'status', 'gen_time', 'activity', 'filename',
174 'sid', 'orbit', 'sensing_start', 'sensing_stop', 'tablename',
175 'earliest_execution_time', 'process_id', 'primary_output', 'parent'):
176 val = getattr(self, attr)
177 if val is not None:
178 if attr == 'sensing_start':
179 attr = 'start'
180
181 elif attr == 'sensing_stop':
182 attr = 'stop'
183
184 elif attr == 'tablename':
185 attr = 'table'
186
187 things.append('{k}={v}'.format(k=attr, v=name_of_thing(val)))
188
189 if len(self.aux_outputs) > 0:
190 things.append('aux:{cc}'.format(cc=len(self.aux_outputs)))
191
192 if self.tables is not None:
193 things.append('tables:{t}'.format(t=','.join(tt.table.name for tt in self.tables)))
194
195 return 'Job({things})'.format(things=', '.join(things))
196
197 @staticmethod
198 def from_xml(elem):
199 """Convert a `job_elem` XML element into a job dictionary.
200 Remember that activity is an attribute of the wo file itself not of each
201 job."""
202
203 # We don't read `parent` here because it's set directly in the worker
204 # and not read from XML
205
206 # SID is special because the Job constructor fails without one
207 result = Job()
208 result.sid = SID.from_xml(elem)
209 # logging.debug('reading job from xml')
210
211 text_tags = (ELEM_STATUS, ELEM_CATEGORY)
212
213 for sub_elem in elem:
214 if sub_elem.tag == ELEM_ORBIT:
215 result.orbit = int(sub_elem.text)
216
217 elif sub_elem.tag == ELEM_SENSING_START:
218 result.sensing_start = xml_to_datetime(sub_elem.text)
219
220 elif sub_elem.tag == ELEM_SENSING_STOP:
221 result.sensing_stop = xml_to_datetime(sub_elem.text)
222
223 elif sub_elem.tag == ELEM_TABLENAME:
224 result.tablename = sub_elem.text
225
226 elif sub_elem.tag == ELEM_ID:
227 result.job_id = int(sub_elem.text)
228
229 elif sub_elem.tag == ELEM_FILENAME:
230 result.filename = Path(sub_elem.text)
231 # logging.debug('set filename to ' + str(type(result.filename)))
232
233 elif sub_elem.tag == ELEM_STATUS:
234 result.status = JobStatus[sub_elem.text]
235
236 elif sub_elem.tag in text_tags:
237 # catch everything else as plain text
238 setattr(result, sub_elem.tag, sub_elem.text)
239
240 return result
241
242 def __setitem__(self, name, value):
243 """Allow old style code to say job['status'] = 'COMPLETED'.
244 This shouldn't really be used now."""
245 if name == 'status':
246 if isinstance(value, str):
247 self.status = JobStatus[value]
248
249 else:
250 self.status = value
251
252 else:
253 raise ValueError('Do not use dictionary notation to write into Job objects')
254
255 def __getitem__(self, name):
256 """Allow old style code to read job attributes like it is still a dictionary."""
257 if name == 'id':
258 return self.job_id
259
260 elif name == 'sid':
261 return self.sid
262
263 elif name == 'scid': # scidfree
264 return self.scid # scidfree
265
266 elif name == 'sensing_start':
267 return self.sensing_start
268
269 elif name == 'sensing_stop':
270 return self.sensing_stop
271
272 elif name == 'table':
273 return TableInfo(self.tablename)
274
275 elif name == 'orbit':
276 return self.orbit
277
278 elif name == 'status':
279 return self.status
280
281 elif name == 'filename':
282 return self.filename
283
284 else:
285 # no test for `parent` because code should use job.parent
286 raise ValueError('Do not use dictionary notation to read from Job objects')
287
288 @property
289 def table(self):
290 """Retrieve a TableInfo for us."""
291 return TableInfo(self.tablename) if self.tablename is not None else None
292
293 @property
294 def timerange(self):
295 """Return a TimeRange object giving our start and stop times."""
296 return TimeRange(self.sensing_start, self.sensing_stop)
297
298 @property
299 def scid(self): # scidfree
300 """For old code; do not use."""
301 return self.sid.scid # scidfree