1#!/usr/bin/env python3
 2
 3"""Clean up script for algorithms table and archive directory.
 4Results are purged completely if they are older than max-age and on the algorithms list.
 5Zip files in the archive directory that seem to be orphaned with no corresponding entry
 6in the algorithms table are deleted.
 7"""
 8
 9import logging
10from datetime import datetime
11
12from chart.common.path import Path
13from chart.alg import init_algorithm
14from chart.backend.processes import find_processes
15from chart.backend.processes import delete_processes
16from chart.backend.job import JobStatus
17from chart.backend.jobs import delete_jobs
18from chart.project import settings
19
20# we only examine one category
21CATEGORY = 'SCHEDULER'
22
23
24def purge_from(max_time):
25    """Delete JOBS, PROCESSES and related working directories older than
26    max_time. Also delete empty work area directories.
27    Does not detect orphaned work directories, but these should get GPFS-purged
28    after 30 days anyway.
29    """
30
31    # jobs table purge
32    del_count = delete_jobs(category=CATEGORY,
33                            gen_time_lt=max_time)
34    logging.info('Deleted {c} jobs'.format(c=del_count))
35
36    # work dir purge
37    logging.debug('Now purging work dirs')
38    for work_dir, in find_processes(fields=('WORKING_DIR',),
39                                    gen_time_lt=max_time):
40        if work_dir is None:
41            logging.error('Found process with working dir not set')
42            continue
43
44        work_dir = Path(work_dir)
45
46        if str(work_dir).startswith('/tcenas/home'):
47            logging.info('Not purging home directory {dir}'.format(dir=work_dir))
48            continue
49
50        logging.debug('Purging dir {path}'.format(path=work_dir))
51        if work_dir.exists():
52            logging.info('Purging work directory {dir}'.format(dir=work_dir))
53            try:
54                work_dir.rmtree()
55            except (IOError, OSError) as e:
56                logging.error('Cannot delete {dir}: {msg}'.format(
57                    dir=work_dir, msg=e))
58
59        work_area = work_dir.parent
60        if work_area.exists() and len(list(work_area.iterdir())) == 0:
61            logging.info('Purging work area {area}'.format(area=work_area))
62            try:
63                work_area.rmtree()
64            except IOError as e:
65                logging.error('Cannot delete empty directory {area}: {msg}'.format(
66                        area=work_area, msg=e))
67            except OSError as e:
68                if e.errno == 13:
69                    logging.error('Cannot delete protected directory {area}: {msg}'.format(
70                            area=work_area, msg=e))
71
72    logging.debug('Done with work dirs')
73    # processes + work dirs purge
74    del_count = delete_processes(gen_time_lt=max_time)
75    logging.info('Deleted {c} processes'.format(c=del_count))
76
77
78def main():
79    """Command line entry point."""
80    wo, resultfile, _ = init_algorithm()
81
82    first = True
83    for job in wo.read_jobs():
84        if first:
85            purge_from(datetime.utcnow().date() - settings.PURGE_INTERVAL)
86            first = False
87
88        resultfile.add_job(job, JobStatus.COMPLETED)
89
90if __name__ == '__main__':
91    main()