1#!/usr/bin/env python3
2
3"""Ingest report working directories into the archive storage area."""
4
5import logging
6import itertools
7
8from chart import settings
9from chart.backend.workorder import WorkOrder
10from chart.backend.result import Result
11import chart.alg.settings
12from chart.common.exceptions import ConfigError
13from chart.reports.archive import create_report
14from chart.reports.archive import count_reports
15from chart.reports.archive import get_report_relpath
16from chart.reports.archive import delete_report
17from chart.reports.manifest import Manifest
18
19def ingest_report(dirname,
20 sendmails=True,
21 export=None,
22 cutoff_duration=settings.EMAIL_CUTOFF): # (unused arg) pylint: disable=W0613
23 """Ingest a report working directory into the database
24 and archive area.
25
26 If `expprt` is set we write the archive to that directory instead.
27 The database is still updated as though a normal ingestion to the settings
28 archive area had occurred.
29 """
30 # Remove circular import from dispatcher.py
31 from chart.users.role import RoleManager
32
33 logging.info('Ingesting working dir {d}'.format(d=dirname.absolute()))
34
35 wo = WorkOrder(dirname.child(chart.alg.settings.WORKORDER_FILENAME))
36 resultfile = Result(filename=dirname.child(chart.alg.settings.RESULT_FILENAME),
37 mode='r')
38
39 if wo.activity.classname != 'report':
40 raise ConfigError('Working directory has activity {name} which is not contain '
41 'a report'.format(name=wo.activity.name))
42
43 # in the workorder each job has sid, sensing start and sensing stop
44 wo_job = next(wo.read_jobs())
45
46 # in the result file the sid and times are missing
47 job = next(resultfile.read_jobs())
48 job.sid = wo_job.sid
49 job.sensing_start = wo_job.sensing_start
50 job.sensing_stop = wo_job.sensing_stop
51
52 sid = job.sid
53 start = job.sensing_start
54 stop = job.sensing_stop
55 if start is None:
56 start = resultfile.execution_start
57
58 if stop is None:
59 stop = resultfile.execution_stop
60
61 # Create a manifest.xml file in both the working directories and report archive.
62 # This is used to a) create a proper dedicated meta-data file in our reports and
63 # b) used by the system job viewer to link from jobs to reports
64 manifest = Manifest(dirname, mode=Manifest.Mode.UPDATE)
65 manifest.set_creation_info()
66
67 # check if the REPORTs table already has an entry like this one and delete it
68 if count_reports(sid=sid, activity=wo.activity, start_time_eq=start) == 1:
69 archive_filename = get_report_relpath(wo.activity, sid, start)
70 delete_report(archive_filename=str(archive_filename))
71
72 # put an entry in REPORTs table and create a blank zip archive
73 archive = create_report(wo.activity,
74 sid,
75 start,
76 stop,
77 export=export,
78 gen_time=manifest.get_creation_time())
79
80 manifest.close()
81
82 # standard working direction files
83 standard_filenames = (chart.alg.settings.WORKORDER_FILENAME,
84 chart.alg.settings.RESULT_FILENAME,
85 chart.alg.settings.REPORT_FILENAME,
86 chart.alg.settings.MANIFEST_FILENAME)
87
88 # aux files
89 filenames = []
90 for aux_filename in job.aux_outputs:
91 if aux_filename in standard_filenames:
92 raise ValueError('{aux} is a reserved filename and should not '
93 'be included in the aux file list'.format(aux=aux_filename))
94
95 filenames.append(aux_filename)
96
97 # populate zip archive
98 for filename in itertools.chain(filenames, standard_filenames):
99 fullname = dirname.child(filename)
100 if fullname.exists():
101 # logging.debug('Deflating ' + filename)
102 fullname_str = str(fullname)
103 archive.write(fullname_str, str(filename))
104
105 # Sample manifest file:
106 # <manifest>
107 # <job-id>100</job-id>
108 # <report-id>1000<report-id>
109 # <scid>M02</scid> # scidfree
110 # <sensing-start>x<sensing-start>
111 # <sensing-stop>x<sensing-stop>
112 # </manifest>
113
114 archive.close()
115 RoleManager.instance().handle_report(activity=wo.activity,
116 sid=sid,
117 start=start,
118 stop=stop,
119 sendmails=sendmails)
120
121 return archive