1#!/usr/bin/env python3
2
3"""Read jobcontrol a.k.a. scheduler files."""
4
5import logging
6import fnmatch
7import importlib
8from datetime import timedelta
9from datetime import datetime
10
11from chart.common.xml import load_xml
12from chart.common.xml import parsechildstr
13from chart.common.xml import parsechildstrs
14from chart.common.xml import parsechildint
15from chart.common.xml import parsechildbool
16from chart.common.xml import parsechildtimedelta
17from chart import settings
18from chart.common.exceptions import ConfigError
19from chart.backend.activity import Activity
20from chart.browse.make_url import make_url
21from chart.project import SID
22from chart.backend.activity import CallingConvention
23from chart.common.path import Path
24
25NULL_TIMEDELTA = timedelta()
26
27ELEM_RESPONSE = 'response'
28ELEM_ACTIVITY = 'activity'
29ELEM_ENABLED = 'enabled'
30ELEM_DIRECTORY = 'directory'
31ELEM_PATTERN = 'pattern'
32ELEM_PDU_DELAY_QUEUE = 'pdu-delay-queue'
33ELEM_TIMEOUT = 'timeout'
34ELEM_MONTH = 'month'
35ELEM_DAY = 'day'
36ELEM_HOUR = 'hour'
37ELEM_OFFSET = 'offset'
38ELEM_PARSER = 'parser'
39ELEM_MINUTE = 'minute'
40ELEM_DEPENDENCY = 'dependency'
41
42logger = logging.getLogger()
43
44
45class ProductAttributes:
46 """Basic information about a data file usually derived from filename."""
47
48 def __init__(self,
49 sid:SID,
50 sensing_start:datetime,
51 sensing_stop:datetime=None,
52 gen_time:datetime=None,
53 ground_segment:str=None):
54 """Only `sid` and `sensing_start` are needed by the scheduler. Other fields can be empty.
55
56 `sensing_stop` will be written to JOBS table if present and will be displayed in the job
57 viewer, but is not required.
58 `gen_time`, `ground_segment` and any other attributes may be set by an attributes function
59 if the code needs it elsewhere, but are not needed or shown anywhere.
60 """
61 self.sid = sid
62 self.sensing_start = sensing_start
63 self.sensing_stop = sensing_stop
64 self.gen_time = gen_time
65 self.ground_segment = ground_segment
66
67 def __str__(self):
68 # Just list non-None attributes
69 return 'ProductAttributes({attr})'.format(
70 attr=','.join('{k}:{v}'.format(k=k, v=v)
71 for k, v in self.__dict__.items() if v is not None))
72
73
74class JobCreator:
75 """Interpret a job creator XML file."""
76
77 def __init__(self, name):
78 name = name.upper()
79 self.name = name
80 self.elem = load_xml(settings.SCHEDULE_DIR.child(name + '.xml'))
81
82 @property
83 def description(self):
84 """Description of this file."""
85 return parsechildstr(self.elem, 'description', None)
86
87 @property
88 def trigger(self):
89 """Return a dictionary describing this job creator."""
90 trigger_elem = self.elem.find('trigger')
91 if trigger_elem is None:
92 raise ConfigError('Job creator definition file has no trigger')
93
94 if len(trigger_elem.getchildren()) != 1:
95 raise ConfigError('Job creator trigger element must have 1 child element')
96
97 trigger_child = trigger_elem.getchildren()[0]
98 trigger_type = trigger_child.tag
99
100 if trigger_type == 'daily':
101 return {'type': 'daily',
102 'hour': parsechildint(trigger_child, 'hour'),
103 'duration': parsechildtimedelta(trigger_child, 'duration', NULL_TIMEDELTA),
104 'offset': parsechildtimedelta(trigger_child, 'offset', NULL_TIMEDELTA)}
105
106 elif trigger_type == 'hourly':
107 return {'type': 'hourly',
108 'minute': parsechildint(trigger_child, ELEM_MINUTE, 0)}
109
110 elif trigger_type == 'weekly':
111 days = {'monday': 0,
112 'tuesday': 1,
113 'wednesday': 2,
114 'thursday': 3,
115 'friday': 4,
116 'saturday': 5,
117 'sunday': 6}
118 day = parsechildstr(trigger_child, ELEM_DAY, '0')
119 if day.isdigit():
120 day = int(day)
121
122 else:
123 day = days[day.lower()]
124
125 return {'type': 'weekly',
126 'day': day,
127 'hour': parsechildint(trigger_child, ELEM_HOUR, 0),
128 'offset': parsechildtimedelta(trigger_child, ELEM_OFFSET, None)}
129
130 elif trigger_type == 'monthly':
131 return {'type': 'monthly',
132 'day': parsechildint(trigger_child, ELEM_DAY, 1),
133 'hour': parsechildint(trigger_child, ELEM_HOUR, 0),
134 'offset': parsechildtimedelta(trigger_child, ELEM_OFFSET, None)}
135
136 elif trigger_type == 'annual':
137 return {'type': 'annual',
138 'month': parsechildint(trigger_child, ELEM_MONTH, 1),
139 'day': parsechildint(trigger_child, ELEM_DAY, 1),
140 'hour': parsechildint(trigger_child, ELEM_HOUR, 0),
141 'offset': parsechildtimedelta(trigger_child, ELEM_OFFSET, None)}
142
143 elif trigger_type == 'directory-monitor':
144 # don't use '-' chars here as it's difficult to read them from Django templates
145 if trigger_child.find(ELEM_DIRECTORY) is None:
146 # Handle scheduler entries with no directory that are written for the
147 # generic ingester to user
148 monitor = None
149
150 else:
151 monitor = Path(parsechildstr(trigger_child,
152 ELEM_DIRECTORY,
153 expand_settings=True,
154 expand_user=True))
155
156 parser_name = parsechildstr(trigger_child, ELEM_PARSER)
157
158 if parser_name is None or parser_name == '':
159 parser_fn = None
160
161 else:
162 module_name, _, fn_name = parser_name.rpartition('.')
163 module_obj = importlib.import_module(module_name, __package__)
164 parser_fn = getattr(module_obj, fn_name)
165
166 return {'type': 'directory-monitor',
167 'directory': monitor,
168 'pattern': parsechildstr(trigger_child, ELEM_PATTERN, expand_settings=True),
169 'pdu_delay_queue': parsechildint(trigger_child, ELEM_PDU_DELAY_QUEUE, None),
170 'timeout': parsechildtimedelta(trigger_child, ELEM_TIMEOUT, None),
171 'parser': parser_fn}
172
173 elif trigger_type == 'now':
174 return {'type': 'now',
175 'dependency': parsechildstr(trigger_child, ELEM_DEPENDENCY)}
176
177 else:
178 raise ConfigError('Unknown trigger type: {type}'.format(type=trigger_type))
179
180 def responses(self, # (dangerous default) pylint: disable=W0102
181 allowed_activities=None,
182 excluded_activities=[],
183 exclude_directory_monitoring=False):
184 """Yield a list of dictionaries describing the responses to this activity."""
185 for r in self.elem.findall(ELEM_RESPONSE):
186 sid = SID.from_xml(r, wildcard=True)
187 for a in parsechildstrs(r, ELEM_ACTIVITY, expand_settings=True):
188 if excluded_activities is not None and a in excluded_activities:
189 continue
190
191 activity = Activity(a)
192 if exclude_directory_monitoring and\
193 activity.convention is CallingConvention.FILENAME:
194 continue
195
196 if allowed_activities is not None:
197 allowed = False
198 for test in allowed_activities:
199 if fnmatch.fnmatch(a, test.upper()):
200 allowed = True
201
202 if not allowed:
203 continue
204
205 yield {'sids': [] if sid is None else sid.expand(),
206 'activity': activity}
207
208 def grouped_responses(self):
209 """Return a list of responses split into groups as in the original XML file."""
210 pass
211
212 @property
213 def enabled(self):
214 """False if <enabled>false</enabled> is present.
215 Deactivates the schedule entry completely."""
216 return parsechildbool(self.elem, ELEM_ENABLED, True)
217
218 def is_time_based(self):
219 """Test if our trigger is time based."""
220 return self.trigger['type'] in ('daily', 'hourly', 'weekly', 'monthly')
221
222 def is_file_based(self):
223 """Test if our trigger file based."""
224 return self.trigger['type'] == 'directory-monitor'
225
226 def browse_source_url(self):
227 """Return a URL which will display our source file."""
228 return make_url(settings.SCHEDULE_DIR.joinpath(self.name + '.xml'))
229
230 @staticmethod
231 def all():
232 """Yield all job creators except disabled ones."""
233 for filename in sorted(settings.SCHEDULE_DIR.glob('*.xml')):
234 yield JobCreator(filename.stem)
235
236 @staticmethod
237 def all_time_based():
238 """Yield all time based job creators."""
239 # print('all time based')
240 for filename in sorted(settings.SCHEDULE_DIR.iterdir()):
241 if filename.suffix == '.xml':
242 candidate = JobCreator(filename.stem)
243 # print('trying ' + candidate.name)
244 if candidate.is_time_based():
245 yield candidate
246
247 @staticmethod
248 def all_file_based():
249 """Yield all file based job creators."""
250 for filename in sorted(settings.SCHEDULE_DIR.iterdir()):
251 if filename.suffix == '.xml':
252 candidate = JobCreator(filename.stem)
253 if candidate.is_file_based():
254 yield candidate
255
256
257def list_all():
258 """List all job creator files."""
259 print('All job creator files:')
260 for c in JobCreator.all():
261 limit = 90 - len(c.name)
262 description = c.description
263 if len(description) > limit:
264 description = description[:limit] + '...'
265
266 print(' {name}: {desc}'.format(name=c.name, desc=description))
267
268
269def info(name):
270 """Show info about a single job creator."""
271 c = JobCreator(name)
272 from chart.common.prettyprint import Table
273 t = Table()
274 t.append(('Name', c.name))
275 t.append(('Description', c.description))
276 t.append(('Trigger', ', '.join('{k}:{v}'.format(k=k, v=v) for k, v in c.trigger.items())))
277 for a in c.responses():
278 t.append(('Activity', ', '.join('{k}:{v}'.format(k=k, v=v) for k, v in a.items())))
279
280 t.write()
281
282
283def main():
284 """Command line entry point."""
285 from chart.common.args import ArgumentParser
286 parser = ArgumentParser()
287 parser.add_argument('--list', '-l',
288 action='store_true',
289 help='List job creator files')
290 parser.add_argument('--info', '-i',
291 help='Show info on a single job creator')
292 args = parser.parse_args()
293 if args.list:
294 list_all()
295 parser.exit()
296
297 elif args.info:
298 info(args.info)
299 parser.exit()
300
301 else:
302 parser.error('No actions specified')
303
304if __name__ == '__main__':
305 main()