1#!/usr/bin/env python3
2
3"""Configuration for CHART-EPSSG project.
4This file is based on a Django settings.py and contains many Django-specific settings.
5It also includes project specific settings for projects re-using parts of CHART."""
6
7import os
8import re
9import runpy
10from collections import OrderedDict
11from datetime import timedelta
12
13
14from chart.common.path import Path
15<<<hidden due to potential security issue>>>
16from chart.settings import PROJECT_HOME_DIR
17from chart.settings import TEMPLATES
18from chart.settings import INSTALLED_APPS
19from chart.settings import MIDDLEWARE
20
21
22# allows "import chart.settings.project"
23# import chartepssg as project # (unused import) pylint: disable=W0611
24
25
26# section: Basic settings
27# -----------------------
28
29# APPNAME string is shown in the website
30APPNAME = 'CHART-EPSSG'
31PROJECT_DESCRIPTION = 'Analysis and Reporting of EPS-SG telemetry'
32ENV_PREFIX_ENV = 'CHARTEPSSG_ENV_PREFIX'
33
34# section: Email configuration
35# ----------------------------
36
37# use instance settings to override these
38DEBUG = False
39
40# Ending to our emails
41EMAIL_MESSAGE_SUFFIX = '\nbest regards, CHART-EPSSG'
42
43# For the email From field
44EMAIL_NAME = 'CHART-EPSSG-{0}'
45
46# Should be address our emails come from
47EMAIL_ADDRESS = 'chart-epssg-{0}@eumetsat.int'
48
49# Prefix to the email Re: subject field
50EMAIL_SUBJECT_PREFIX = 'CHART-EPSSG '
51
52#sender
53EMAIL_SENDER = 'CHART-EPSSG (no reply) <chartepssg-no-reply@eumetsat.int>'
54
55
56# section: Database configuration
57# -------------------------------
58
59# These databases are used for the timeseries and system tables
60# 'chart' is used by default, other DBs can be selected by setting the $CHART_DB
61# environment variable.
62# Databases marked with OPERATIONAL=False may be written to be automated tests.
63# If the CHART_TEST_DB environment variable is set, that database will be used
64# by tests.
65<<<hidden due to potential security issue>>>
66# options:
67# `string`
68<<<hidden due to potential security issue>>>
69# as this file is checked in to svn
70# `keyring='VAR'`
71<<<hidden due to potential security issue>>>
72# `gpg_keyring='VAR'`
73<<<hidden due to potential security issue>>>
74# `user_input='prompt'`
75<<<hidden due to potential security issue>>>
76#
77
78DATABASES = OrderedDict([
79 ('epp-ope-enduser', {
80 'DESCRIPTION': 'EPP_OPE database read only user',
81 'ENGINE': 'django.db.backends.postgresql',
82 'NAME': 'chartepssg',
83 'HOST': 'epp-fodbss17',
84 'PORT': '5432',
85 'USER': 'chartepssg_enduser',
86<<<hidden due to potential security issue>>>
87 }),
88 ('epp-ope-web', {
89 'DESCRIPTION': 'EPP OPE database website user',
90 'ENGINE': 'django.db.backends.postgresql',
91 'NAME': 'chartepssg',
92 'HOST': 'epp-fodbss17',
93 'PORT': '5432',
94 'USER': 'chartepssg_web',
95<<<hidden due to potential security issue>>>
96 }),
97 ('epp-ope-ingester', {
98 'DESCRIPTION': 'EPP OPE database website user',
99 'ENGINE': 'django.db.backends.postgresql',
100 'NAME': 'chartepssg',
101 'HOST': 'epp-fodbss17',
102 'PORT': '5432',
103 'USER': 'chartepssg_ingester',
104<<<hidden due to potential security issue>>>
105 }),
106 # no expert on the EPP
107 ('epp-val-enduser', {
108 'DESCRIPTION': 'EPP_VAL database read only user',
109 'ENGINE': 'django.db.backends.postgresql',
110 'NAME': 'chartepssg',
111 'HOST': 'epp-fvdbss17',
112 'PORT': '5432',
113 'USER': 'chartepssg_enduser',
114<<<hidden due to potential security issue>>>
115 }),
116 ('epp-val-web', {
117 'DESCRIPTION': 'EPP VAL database website user',
118 'ENGINE': 'django.db.backends.postgresql',
119 'NAME': 'chartepssg',
120 'HOST': 'epp-fvdbss17',
121 'PORT': '5432',
122 'USER': 'chartepssg_web',
123<<<hidden due to potential security issue>>>
124 }),
125 ('epp-val-ingester', {
126 'DESCRIPTION': 'EPP VAL database website user',
127 'ENGINE': 'django.db.backends.postgresql',
128 'NAME': 'chartepssg',
129 'HOST': 'epp-fvdbss17',
130 'PORT': '5432',
131 'USER': 'chartepssg_ingester',
132<<<hidden due to potential security issue>>>
133 }),
134 ('ope-enduser', {
135 'DESCRIPTION': 'OPE database read only user',
136 'ENGINE': 'django.db.backends.postgresql',
137 'NAME': 'chartepssg',
138 'HOST': 'fodbss17',
139 'PORT': '5432',
140 'USER': 'chartepssg_enduser',
141<<<hidden due to potential security issue>>>
142 }),
143 ('ope-expert', {
144 'DESCRIPTION': 'VAL database expert user',
145 'ENGINE': 'django.db.backends.postgresql',
146 'NAME': 'chartepssg',
147 'HOST': 'fodbss17',
148 'PORT': '5432',
149 'USER': 'chartepssg_expert',
150<<<hidden due to potential security issue>>>
151 }),
152 ('ope-web', {
153 'DESCRIPTION': 'OPE database website user',
154 'ENGINE': 'django.db.backends.postgresql',
155 'NAME': 'chartepssg',
156 'HOST': 'fodbss17',
157 'PORT': '5432',
158 'USER': 'chartepssg_web',
159<<<hidden due to potential security issue>>>
160 }),
161 ('ope-ingester', {
162 'DESCRIPTION': 'OPE database write anywhere user',
163 'ENGINE': 'django.db.backends.postgresql',
164 'NAME': 'chartepssg',
165 'HOST': 'fodbss17',
166 'PORT': '5432',
167 'USER': 'chartepssg_ingester',
168<<<hidden due to potential security issue>>>
169 }),
170 ('val-enduser', {
171 'DESCRIPTION': 'VAL database read only user',
172 'ENGINE': 'django.db.backends.postgresql',
173 'NAME': 'chartepssg',
174 'HOST': 'fvdbss17',
175 'PORT': '5432',
176 'USER': 'chartepssg_enduser',
177<<<hidden due to potential security issue>>>
178 }),
179 ('val-expert', {
180 'DESCRIPTION': 'VAL database expert user',
181 'ENGINE': 'django.db.backends.postgresql',
182 'NAME': 'chartepssg',
183 'HOST': 'fvdbss17',
184 'PORT': '5432',
185 'USER': 'chartepssg_expert',
186<<<hidden due to potential security issue>>>
187 }),
188 ('val-web', {
189 'DESCRIPTION': 'VAL database website user',
190 'ENGINE': 'django.db.backends.postgresql',
191 'NAME': 'chartepssg',
192 'HOST': 'fvdbss17',
193 'PORT': '5432',
194 'USER': 'chartepssg_web',
195<<<hidden due to potential security issue>>>
196 }),
197 ('val-ingester', {
198 'DESCRIPTION': 'VAL database write anywhere user',
199 'ENGINE': 'django.db.backends.postgresql',
200 'NAME': 'chartepssg',
201 'HOST': 'fvdbss17',
202 'PORT': '5432',
203 'USER': 'chartepssg_ingester',
204<<<hidden due to potential security issue>>>
205 }),
206 ('tst-enduser', {
207 'DESCRIPTION': 'TST database read only user',
208 'ENGINE': 'django.db.backends.postgresql',
209 'NAME': 'chartepssg',
210 'HOST': 'epssgchartdb1',
211 'PORT': '5432',
212 'USER': 'chartepssg_enduser',
213<<<hidden due to potential security issue>>>
214 'WRITABLE': False, # avoid running automated tests against this
215 'TESTABLE': False, # not a test database
216 }),
217 ('tst-web', {
218 'DESCRIPTION': 'TST database for web user',
219 'ENGINE': 'django.db.backends.postgresql',
220 'NAME': 'chartepssg',
221 'HOST': 'epssgchartdb1',
222 'PORT': '5432',
223 'USER': 'chartepssg_web',
224<<<hidden due to potential security issue>>>
225 'WRITABLE': False, # avoid running automated tests against this
226 'TESTABLE': False, # not a test database
227 }),
228 ('tst-expert', {
229 'DESCRIPTION':
230 'TST database with write to derived and stats tables',
231 'ENGINE': 'django.db.backends.postgresql',
232 'NAME': 'chartepssg',
233 'HOST': 'epssgchartdb1',
234 'PORT': '5432',
235 'USER': 'chartepssg_expert',
236<<<hidden due to potential security issue>>>
237 'WRITABLE': False, # avoid running automated tests against this
238 'TESTABLE': False, # not a test database
239 }),
240 ('tst-ingester', {
241 'DESCRIPTION': 'TST database with write to all tables',
242 'ENGINE': 'django.db.backends.postgresql',
243 'NAME': 'chartepssg',
244 'HOST': 'epssgchartdb1',
245 'PORT': '5432',
246 'USER': 'chartepssg_ingester',
247<<<hidden due to potential security issue>>>
248 'WRITABLE': True, # avoid running automated tests against this
249 'TESTABLE': False, # not a test database
250 }),
251 ('tst-admin', {
252 'DESCRIPTION': 'TST database administrator access',
253 'ENGINE': 'django.db.backends.postgresql',
254 'NAME': 'chartepssg',
255 'HOST': 'epssgchartdb1',
256 'PORT': '5432',
257 'USER': 'chartepssg_admin',
258<<<hidden due to potential security issue>>>
259 'WRITABLE': False, # avoid running automated tests against this
260 'TESTABLE': False, # not a test database
261 }),
262 # DEV dbs
263 (
264 "slx-dev",
265 {
266 "DESCRIPTION": "SLX Dev database",
267 "ENGINE": "django.db.backends.postgresql",
268 "NAME": "CHARTEPSSG",
269 "HOST": "localhost",
270 "PORT": "5132",
271 "USER": "dev",
272<<<hidden due to potential security issue>>>
273 },
274 ),
275 (
276 "integrationtest",
277 {
278 "DESCRIPTION": "SLX Test database",
279 "ENGINE": "django.db.backends.postgresql",
280 "NAME": "integrationtest",
281 "HOST": "localhost",
282 "PORT": "5123",
283 "USER": "dev",
284<<<hidden due to potential security issue>>>
285 },
286 ),
287])
288
289# Default database name for the project
290DB_NAME = 'ope-enduser'
291
292<<<hidden due to potential security issue>>>
293
294# CONN_MAX_AGE = None
295
296# Source ID for project timeseries data
297SID_CLASS = 'chart.sids.sid_pus.SID_PUS'
298
299# Experimental super-SID code
300ALL_SIDS = 'chartepssg.config.sids.SIDS'
301
302# default database for running auto tsts
303TEST_DB_NAME = 'dev'
304
305# SRDB Database type
306TABLE_PATTERN = 'PUS'
307
308# For EPSSG, set TRUE
309PUS_INGEST_RAW_HEX = True
310# PUS_EXTENDED_BUFFER_OFFSET = 3
311# PUS_SHOW_EMPTY_ARRAYS = True
312PUS_DROP_DUPLICATES = ('TM_STORE', 'EV_STORE')
313
314# MTG RAPID TM Header Length, usually 16 but for EPS-SG data = 20
315# Is it the padding feature? (+4Bytes if padding=1)
316TDEV_TM_HEADER_LENGTH = 20
317
318
319# section: Project directories and URLs
320# -------------------------------------
321
322# Directory to hold all notification statefiles
323STATEFILE_DIR = Path('~/tmp/chartepssg/statefiles').expand()
324
325# Standard report archive
326ARCHIVE_DIR = Path("~/tmp/archive").expand()
327
328# Our wiki page
329PROJ_WIKI_URL = 'https://confluence.eumetsat.int/display/SAE/CHART+EPS-SG'
330
331EMAIL_CSS_BASENAME = "email.css"
332
333CHART_WEB = "http://chart/epssg"
334# Location of extracted, separated CSV or DAT files containing SCOS2000 SRDB database
335SRDB_DIR = PROJECT_HOME_DIR.parent.joinpath('SRDB', 'current')
336
337SCOS_USES_PARAM2 = True
338# SCOS_USES_PARAM2 = False
339
340# Location to read all input products from
341DATALAKE_PRIMARY = None
342
343# Location of the 'ingested' WIMPY files. They are copied on the local facility
344# and its metadata is ingested in the db instead
345AUX_ARCHIVE_DIR = PROJECT_ROOT_DIR.joinpath('pfs', 'aux')
346
347WORKER_JOB_CHAIN = "chartepssg.backend.worker_chains"
348##WORKER_JOB_CHAIN = 'chart.backend.worker_job_chains'
349
350# JCS / EPS-SG specific PUS ingestion configuration
351PUS_EXTENDED_BUFFER_OFFSET = 3
352
353###TBD TBD Find correct Conversion for EPS_SG
354# For ops data ops angle conversion factor - S6 satellite specific
355OPS_ANGLE_FACTOR = 0.005729577951 # Degrees (radians * 180 / pi ) / 10000
356###TBD TBD Find correct Conversion for EPS_SG
357
358# For Polar Orbiting programs: JCS, S3, EPS_SG
359OPS_DATA_FIELDS = ['ops_angle', 'ops_orbit', 'is_MRC', 'is_reinserted']
360
361# OPS data applicable for EPS_SG
362OPS_DATA_APPLICABLE = True
363
364# TM table contains TDEV_GS_ID field
365TM_PACKET_TDEV_GS_ID = True
366
367# TM table contains RECEPTION_TIME field
368TM_RECEPTION_TIME = True
369
370
371
372
373# db definitions directories
374
375# description: Location of our deployment Data Unit files
376# best practive to have the du inside the project folder
377DU_DIR = PROJECT_HOME_DIR.parent.joinpath('chartepssg_du')
378
379# Reports
380REPORT_TEMPLATE_DIR = DU_DIR.joinpath('reports')
381REPORT_GROUP_FILE = DU_DIR.joinpath('report_groups.xml')
382
383# Source-ID configuration file
384SOURCES = DU_DIR.joinpath('sources.xml')
385
386# Schedule
387SCHEDULE_DIR = DU_DIR.joinpath('schedule')
388
389
390# Location of non-SID-specific db xml definitions NOT built using srdb_tool...
391# description: Location of fixed non-satellite specific timeseries table definitions
392TS_TABLE_DIRS = [PROJECT_HOME_DIR.joinpath('db', 'ts')]
393
394# description: Location of fixed non-satellite specific event packet xmls
395EV_PACKET_DIR = PROJECT_HOME_DIR.joinpath('db', 'packets', 'ev')
396
397
398# Location of non-SID-specific db files built using srdb_tool...
399# description: Location of our deployment srdb database files built using srdb_tool
400SRDB_DU_DIR = PROJECT_HOME_DIR.parent.joinpath('chartepssg_srdb_du')
401
402# Change to sat_specific to we can have multiple SIDs referring to the same satellite
403SID_SPECIFIC_DIR = SRDB_DU_DIR.joinpath('db', 'sid_specific')
404
405# Files and subdirectories within the SID_SPECIFIC_DIR for PUS-based projects
406# using satellite-specific databases
407TS_TABLE_SUBDIR = Path('ts')
408TC_PACKET_SUBDIR = Path('packets/tc')
409TM_PACKET_SUBDIR = Path('packets/tm')
410
411TM_CRITERIA_FILENAME = Path('packets/tm_criteria.xml')
412PARAM1_PARAM2_INFO_FILENAME = Path('param1_param2_info.xml')
413PARAM_SPIDS_INFO_FILENAME = Path('param_spids_info.xml')
414SRDB_VERSION_INFO_FILENAME = Path('srdb_version.xml')
415
416CALIBRATION_SUBDIR = Path('named_cal')
417CHOICES_SUBDIR = Path('choices')
418LIMITS_SUBDIR = Path('limits')
419
420
421# Enable Operator Event upload functionality:
422# When enabling this, do not forget to add an XML-definition
423# for this file as `chart{mission}/events/operator_events.xml` and be sure to
424# reference it in `chart{mission}/events/event_classes.xml`. Check ChartJCS for
425# a reference implementation.
426EVENTVIEWER_OPERATOR_EVENTS_UPLOAD = True
427
428
429# As we will be using the alg.constants module, it has to be configured first
430# CONSTANTS_MODULE = 'chartmtg.alg.constants'
431CONSTANTS_FILE = PROJECT_HOME_DIR.joinpath('alg', 'constants.py')
432
433# Allow Project widgets inside DU
434WIDGET_DIRS.append({'name': 'Project widgets',
435 'dir': PROJECT_HOME_DIR.joinpath('widgets'),
436 'show-in-gallery': True})
437
438TOOLS_DIRS.append({'name': 'Project tools', 'dir': PROJECT_HOME_DIR.joinpath('cmd')})
439
440
441# section: Django website configuration
442# -------------------------------------
443
444# unless there is a local override, show templates in release mode
445# TEMPLATE_DEBUG = False
446
447# Location of top level urls.py
448ROOT_URLCONF = 'chartepssg.urls'
449
450TEMPLATES[0]['DIRS'] += [
451 PROJECT_HOME_DIR.joinpath('homepage', 'templates'),
452 PROJECT_HOME_DIR.joinpath('alg', 'templates'),
453 PROJECT_HOME_DIR.joinpath('api', 'templates'),
454 ]
455
456# add addition project directories which include Django urls.py files
457# to get the index and system pages so staticfiles picks up thumbnails for
458# project widgets
459INSTALLED_APPS += [
460 "chartepssg.homepage",
461 # 'chartepssg.widgets',
462 # 'django.contrib.admin',
463]
464
465# Directory to contain consolidated static files.
466STATIC_ROOT = str(Path('~/tmp/epssg/static').expand())
467
468MIDDLEWARE += [
469 'chart.web.lockdown.LockdownMiddleWare',
470]
471
472
473# section: Plot tool configuration
474# --------------------------------
475
476# Filters for the plotting tool data tree
477# PLOT_TABLE_FILTERS = ['PLM', 'MHS', 'IASI', 'SEM', 'GOME', 'GRAS', 'ASCAT', 'ADCS', 'AMSU',
478 # 'AOCS', 'AVHRR', 'HIRS', 'SCAO', 'TMFIXE', 'TMVAR', 'SSR', 'SVM']
479
480# Do we show the bar chart side-by-side and overlay options in the plot tool?
481PLOT_INCLUDE_BARCHART = False
482
483# Do we show the Daily Total check box in the plot tool?
484PLOT_INCLUDE_DAILY_TOTAL = False
485
486# GEOLOC_CONSTRUCTOR = "chartepssg.alg.epssg_geoloc.Geoloc"
487
488CHOICE_YAXIS_RESCALING = True
489
490# section: Eventviewer configuration
491# ----------------------------------
492
493# Enable TC type dropdown
494EVENTVIEWER_TIME_FILTER_OPTIONS = True
495
496# We use TM and TC multitable events
497EVENTVIEWER_MULTITABLE_SUPPORT = True
498
499# maximum number of events for table. Note this can be increased to 500000 for projects
500# which have implemented db performance improvements
501EVENTVIEWER_MAX_TABLE_EVENTS = 250000
502
503# section: Reports
504# ----------------
505
506DEFAULT_SINGLE_DATAPOINT_APPEARANCE = "min-max-avg"
507DEFAULT_MULTIPLE_DATAPOINT_APPEARANCE = "line"
508
509# worker priority
510WORKER_JOB_PRIORITY = 'chart.backend.worker_next_jobs'
511
512# Ignore any files older than this when looking for files to ingest
513FILE_SCAN_CUTOFF = timedelta(weeks=200)
514
515# Sleep duration for scheduler
516SCHEDULER_SLEEP = timedelta(seconds=3 * 60)
517
518# Sleep duration for worker in seconds
519WORKER_SLEEP = 3 * 60
520
521# Parameters for using the wkhtmltopdf HTML to PDF converter command
522PDF_CONVERTER_COMMAND = (
523 'wkhtmltopdf',
524 '--print-media-type',
525 '--footer-right',
526 '"Page [page] of [toPage]"',
527 '--footer-spacing',
528 '5',
529 '--footer-font-size',
530 '8')
531
532# Distribution points for FTP'ing reports
533FTP_RECIPIENT = OrderedDict([
534 ('TEST_EPSSG_FTP', {
535 'location': 'ftp.opscloud.eumetsat.int',
536 'username': 'chartepssg',
537<<<hidden due to potential security issue>>>
538 'destination_dir': 'ftp_test_dir',
539 'passive_mode': True,
540 }),
541 ('IVV_DIS_FTP', {
542 'location': 'ifids.eumetsat.int',
543 'username': 'epssgmona',
544<<<hidden due to potential security issue>>>
545 'destination_dir': '/out/rep',
546 'passive_mode': False,
547 }),
548 ('VAL_DIS_FTP', {
549 'location': 'vfids.eumetsat.int',
550 'username': 'epssgmona',
551<<<hidden due to potential security issue>>>
552 'destination_dir': '/out/rep',
553 'passive_mode': False,
554 }),
555 ('OPE_DIS_FTP', {
556 'location': 'ofids.eumetsat.int',
557 'username': 'epssgmona',
558<<<hidden due to potential security issue>>>
559 'destination_dir': '/out/rep',
560 'passive_mode': False,
561 }),
562])
563
564INGEST_GS_ENV = None
565
566# Users and Roles config
567ROLES_FILE = PROJECT_HOME_DIR.parent.joinpath('roles.xml')
568EMAIL_CUTOFF = None
569
570# event viewer table support
571EVENTVIEWER_MULTITABLE_SUPPORT = True
572
573# syslog
574LOG_TO_SYSLOG_PREFIX = 'chartepssg: '
575
576
577# Project specific environment variables
578ENV_OVERRIDES.extend([
579 {'env': '{ENV_PREFIX}DATALAKE_PRIMARY', 'setting': 'DATALAKE_PRIMARY', 'datatype': 'path'},
580 {'env': '{ENV_PREFIX}FD_ATT_MON_LOCAL_CONFIG', 'setting': 'FD_ATT_MON_LOCAL_CONFIG', 'datatype': 'path'},
581])