1from pathlib import Path
2import sys
3
4
5def main():
6 indir, outdir = parse_args()
7 check_args(indir, outdir)
8 if not list(indir.glob("*.dat")):
9 print(f"No dat files found in {outdir}")
10 sys.exit(1)
11
12 for dat in indir.glob("*.dat"):
13 try:
14 csvname, csvlines = process(dat)
15 except UnknownTypeError:
16 # print(f"Skipping {dat} processing because of unknown type")
17 continue
18 with (outdir / csvname).open("w") as f:
19 f.write("\n".join(csvlines))
20 print(f"Created {outdir / csvname}")
21
22
23def process(dat):
24 # the known_types below have did have extra fields in EPSSG that are not specified
25 # in the SCOS2000 Database Import ICD. These extra fields were removed.
26 # The fields below resemble the SCOS2000 ICD.
27 known_types = {
28 "CAF": "CAF_NUMBR,CAF_DESCR,CAF_ENGFMT,CAF_RAWFMT,CAF_RADIX,CAF_UNIT,CAF_NCURVE,CAF_INTER",
29 "CAP": "CAP_NUMBR,CAP_XVALS,CAP_YVALS",
30 "CCA": "CCA_NUMBR,CCA_DESCR,CCA_ENGFMT,CCA_RAWFMT,CCA_RADIX,CCA_UNIT,CCA_NCURVE",
31 "CCF": "CCF_CNAME,CCF_DESCR,CCF_DESCR2,CCF_CTYPE,CCF_CRITICAL,CCF_PKTID,CCF_TYPE,CCF_STYPE,CCF_APID,CCF_NPARS,CCF_PLAN,CCF_EXEC,CCF_ILSCOPE,CCF_ILSTAGE,CCF_SUBSYS,CCF_HIPRI,CCF_MAPID,CCF_DEFSET,CCF_RAPID,CCF_ACK,CCF_SUBSCHEDID",
32 "CCS": "CCS_NUMBR,CCS_XVALS,CCS_YVALS",
33 "CDF": "CDF_CNAME,CDF_ELTYPE,CDF_DESCR,CDF_ELLEN,CDF_BIT,CDF_GRPSIZE,CDF_PNAME,CDF_INTER,CDF_VALUE,CDF_TMID",
34 "CPC": "CPC_PNAME,CPC_DESCR,CPC_PTC,CPC_PFC,CPC_DISPFMT,CPC_RADIX,CPC_UNIT,CPC_CATEG,CPC_PRFREF,CPC_CCAREF,CPC_PAREF,CPC_INTER,CPC_DEFVAL,CPC_CORR,CPC_OPTID",
35 "CPS": "CPS_NAME,CPS_PAR,CPS_BIT",
36 "CSF": "CSF_NAME,CSF_DESC,CSF_DESC2,CSF_IFTT,CSF_NFPARS",
37 "CSP": "CSP_SQNAME,CSP_FPNAME,CSP_FPNUM,CSP_DESCR,CSP_PTC,CSP_PFC,CSP_DISPFMT,CSP_RADIX,CSP_TYPE,CSP_VTYPE,CSP_DEFVAL,CSP_CATEG,CSP_PRFREF,CSP_CCAREF,CSP_PAFREF,CSP_UNIT",
38 "CSS": "CSS_SQNAME,CSS_COMM,CSS_ENTRY,CSS_TYPE,CSS_ELEMID,CSS_NPARS,CSS_MANDISP,CSS_RELTYPE,CSS_RELTIME,CSS_EXTIME,CSS_PREVREL,CSS_GROUP,CSS_BLOCK,CSS_ILSCOPE,CSS_ILSTAGE,CSS_DYNPTV,CSS_STAPTV,CSS_CEV",
39 "CUR": "CUR_PNAME,CUR_POS,CUR_RLCHK,CUR_VALPAR,CUR_SELECT",
40 "CVE": "CVE_CVSID,CVE_PARNAM,CVE_INTER,CVE_VAL,CVE_TOL,CVE_CHECK",
41 "CVP": "CVP_TASK,CVP_TYPE,CVP_CVSID",
42 "CVS": "CVS_ID,CVS_TYPE,CVS_SOURCE,CVS_START,CVS_INTERVAL,CVS_SPID,CVS_UNCERTAINTY",
43 "DPC": "DPC_NUMBE,DPC_NAME,DPC_FLDN,DPC_COMM,DPC_MODE,DPC_FORM,DPC_TEXT",
44 "DPF": "DPF_NUMBE,DPF_TYPE,DPF_HEAD",
45 "DST": "DST_APID,DST_ROUTE",
46 "GPC": "GPC_NUMBE,GPC_POS,GPC_WHERE,GPC_NAME,GPC_RAW,GPC_MINIM,GPC_MAXIM,GPC_PRCLR,GPC_SYMB0,GPC_LINE,GPC_DOMAIN",
47 "GPF": "GPF_NUMBE,GPF_TYPE,GPF_HEAD,GPF_SCROL,GPF_HCOPY,GPF_DAYS,GPF_HOURS,GPF_MINUT,GPF_AXCLR,GPF_XTICK,GPF_YTICK,GPF_XGRID,GPF_YGRID,GPF_UPUN",
48 "GRP": "GRP_NAME,GRP_DESCR,GRP_GTYPE",
49 "GRPA": "GRPA_GNAME,GRPA_PANAME",
50 "GRPK": "GRPK_GNAME,GRPK_PKSPID",
51 "LGF": "LGF_IDENT,LGF_DESCR,LGF_POL1,LGF_POL2,LGF_POL3,LGF_POL4,LGF_POL5",
52 "MCF": "MCF_IDENT,MCF_DESCR,MCF_POL1,MCF_POL2,MCF_POL3,MCF_POL4,MCF_POL5",
53 # mdf.dat (cannot parse this filetype currently)
54 "OCF": "OCF_NAME,OCF_NBCHK,OCF_NBOOL,OCF_INTER,OCF_CODIN",
55 "OCP": "OCP_NAME,OCP_POS,OCP_TYPE,OCP_LVALU,OCP_HVALU,OCP_RLCHK,OCP_VALPAR",
56 "PAF": "PAF_NUMBR,PAD_DESCR,PAF_RAWFMT,PAF_NALIAS",
57 "PAS": "PAS_NUMBR,PAS_ALTXT,PAS_ALVAL",
58 "PCDF": "PCDF_TCNAME,PCDF_DESC,PCDF_TYPE,PCDF_LEN,PCDF_BIT,PCDF_PNAME,PCDF_VALUE,PCDF_RADIX",
59 "PCF": "PCF_NAME,PCF_DESCR,PCF_PID,PCF_UNIT,PCF_PTC,PCF_PFC,PCF_WIDTH,PCF_VALID,PCF_RELATED,PCF_CATEG,PCF_NATUR,PCF_CURTX,PCF_INTER,PCF_USCON,PCF_DECIM,PCF_PARVAL,PCF_SUBSYS,PCF_VALPAR,PCF_SPTYPE,PCF_CORR,PCF_OBTID,PCF_DARC,PCF_ENDIAN",
60 "PCPC": "PCPC_PNAME,PCPC_DESC,PCPC_CODE",
61 "PIC": "PIC_TYPE,PIC_STYPE,PIC_PI1_OFF,PIC_PI1_WID,PIC_PI2_OFF,PIC_PI2_WID,PIC_APID",
62 "PID": "PID_TYPE,PID_STYPE,PID_APID,PID_PI1_VAL,PID_PI2_VAL,PID_SPID,PID_DESCR,PID_UNIT,PID_TPSD,PID_DFHSIZE,PID_TIME,PID_INTER,PID_VALID,PID_CHECK,PID_EVENT,PID_EVID",
63 "PLF": "PLF_NAME,PLF_SPID,PLF_OFFBY,PLF_OFFBI,PLF_NBOCC,PLF_LGOCC,PLF_TIME,PLF_TDOCC",
64 "PPC": "PPC_NUMBE,PPC_POS,PPC_NAME,PPC_FORM",
65 "PPF": "PPF_NUMBE,PPF_HEAD,PPF_NBPR",
66 "PRF": "PRF_NUMBR,PRF_DESCR,PRF_INTER,PRF_DSPFMT,PRF_RADIX,PRF_NRANGE,PRF_UNIT",
67 "PRV": "PRV_NUMBR,PRV_MINVAL,PRV_MAXVAL",
68 "PSM": "PSM_NAME,PSM_TYPE,PSM_PARSET",
69 "PST": "PST_NAME,PST_DESCR",
70 "PSV": "PSV_NAME,PSV_PVSID,PSV_DESCR",
71 "PTV": "PTV_CNAME,PTV_PARNAM,PTV_INTER,PTV_VAL",
72 "PVS": "PVS_ID,PVS_PSID,PVS_PNAME,PVS_INTER,PVS_VALS,PVS_BIT",
73 "SDF": "SDF_SQNAME,SDF_ENTRY,SDF_ELEMID,SDF_POS,SDF_PNAME,SDF_FTYPE,SDF_VTYPE,SDF_VALUE,SDF_VALSET,SDF_REPPOS",
74 "SPC": "SPC_NUMBE,SPC_POS,SPC_NAME,SPC_UPDT,SPC_MODE,SPC_FORM,SPC_BACK,SPC_FORE",
75 "SPF": "SPF_NUMBE,SPF_HEAD,SPF_NPAR,SPF_UPUN",
76 "TCP": "TCP_ID,TCP_DESC",
77 "TPCF": "TPCF_SPID,TPCF_NAME,TPCF_SIZE",
78 "TXF": "TXF_NUMBR,TXF_DESCR,TXF_RAWFMT,TXF_NALIAS",
79 "TXP": "TXP_NUMBR,TXP_FROM,TXP_TO,TXP_ALTXT",
80 "VDF": "VDF_NAME,VDF_COMMENT,VDF_DOMAINID,VDF_RELEASE,VDF_ISSUE",
81 "VPD": "VPD_TPSD,VPD_POS,VPD_NAME,VPD_GRPSIZE,VPD_FIXREP,VPD_CHOICE,VPD_PIDREF,VPD_DISDESC,VPD_WIDTH,VPD_JUSTIFY,VPD_NEWLINE,VPD_DCHAR,VPD_FORM,VPD_OFFSET",
82 }
83
84 typename = dat.stem.upper()
85 try:
86 csv_header = known_types[typename]
87 except KeyError:
88 raise UnknownTypeError(f"Unknown Type: {typename}")
89
90 print(f"Processing {dat}")
91 lines = [csv_header]
92 csv_header_items = csv_header.split(",")
93 expected_no_items = len(csv_header_items)
94 with dat.open("r") as f:
95 for lineno, line in enumerate(f, 1):
96 items = line.rstrip("\n").split("\t")
97 if len(items) != expected_no_items:
98 raise WrongNumberOfItemsError(
99 f"{dat}:{lineno}: Expected {expected_no_items} items, but got {len(items)}\n"
100 f"Expected items: {csv_header.split(',')}\n"
101 f"Extracted items: {items}"
102 )
103 items = quote_non_numerical_items(items)
104 lines.append(",".join(items))
105 return f"{typename}.csv", lines
106
107
108def quote_non_numerical_items(items):
109 ret = []
110 for item in items:
111 try:
112 float(item)
113 ret.append(item)
114 continue
115 except ValueError:
116 ret.append(f'"{item}"')
117 return ret
118
119
120def check_args(indir, outdir):
121 if not indir.is_dir():
122 raise OSError(f"{indir} is not a directory")
123 outdir.mkdir(parents=True, exist_ok=True)
124
125
126def parse_args():
127 """extract_sys.py <input_folder> <output_folder>"""
128 if len(sys.argv) != 3:
129 print(parse_args.__doc__)
130 exit(1)
131
132 indir = Path(sys.argv[1])
133 outdir = Path(sys.argv[2])
134 return indir, outdir
135
136
137class UnknownTypeError(Exception):
138 pass
139
140
141class WrongNumberOfItemsError(Exception):
142 pass
143
144
145if __name__ == "__main__":
146 main()