1#!/usr/bin/env python3
 2
 3"""Automatic ingestion of TC products.
 4
 5Conversion from raw TCs to CHART Events will either be done here or in a separate algorithm."""
 6
 7import re
 8import mmap
 9import logging
10from datetime import datetime
11
12from chart.alg import init_algorithm
13from chart.products.pus.ccsds_ingest import ingest_stream
14from chart.products.rapidfile.rapid_io import gen_rapid_from_raw
15from chart.alg import JobStatus
16from chart.project import SID
17from chart.project import settings
18from chart.products.products import insert_product
19from chart.products.utils import Segment
20from chart.products.products import ProductResult
21from chartepssg.alg.tm_ingester import fileattr
22
23logger = logging.getLogger()
24
25
26def ingest_tc(sid, filename, activity):
27    """Ingest a Telecommand file into the database."""
28    product_attributes = fileattr(filename)
29    start_time = None
30    if product_attributes is not None:
31        sid, start_time = product_attributes
32
33    # Make an entry into the PRODUCTS table, remembering the ID to stamp in the TC_STORE table
34    product_id = insert_product(activity=activity,
35                                filename=filename,
36                                result=ProductResult.IN_PROGRESS,
37                                sid=sid,
38                                filesize=filename.size(),
39                                mtime=filename.mtime(),
40                                commit=True)
41
42    # Open the file
43    handle = filename.open('rb')
44    # Fast memory mapped access
45    buff = mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ)
46    # Create our buffer wrapper class which keeps track of the product id and file position
47    data = Segment(buff=buff, file_pos=0, product_id=product_id)
48    # Make CCSDS packets
49    source = gen_rapid_from_raw(data, sid)
50    # Extract and write to database
51    tables = ingest_stream(sid, source, replace_existing=True)
52    # Mark the PRODUCT as completed
53    insert_product(activity=activity, filename=filename, result=ProductResult.OK)
54    return tables
55
56
57def main():
58    """Command line entry point"""
59    wo, resultfile, _ = init_algorithm()
60
61    for job in wo.read_jobs():
62        try:
63            tables = ingest_tc(job.sid, job.filename, job.activity)
64            resultfile.add_job(job, JobStatus.COMPLETED, tables)
65        except FileNotFoundError as e:
66            logger.error('Error processing {f}: {e}'.format(f=job.filename, e=e))
67            resultfile.add_job(job, JobStatus.FAILED)
68
69
70if __name__ == "__main__":
71    main()