1#!/usr/bin/env python3
2
3"""Top level admin tasks for a CHART-based project.
4
5This is configuration file for the `invoke` tool. To use run the following commands in the top
6level of the project:
7
8 `invoke -l`
9 Print a list of available commands with descriptions.
10
11 `invoke <command> --help`
12 Print detailed help for a single function.
13
14 `invoke [-e] <command> [<options>]`
15 Run a command, optionally echoing each step to terminal before executing it, with optional
16 command parameters.
17"""
18
19import os
20import sys
21import stat
22from pathlib import Path
23import importlib
24
25from invoke import task
26import dotenv
27
28output = sys.stdout
29
30# configuration file used to fix sub repositories
31DOCKER_CONF_FILE=Path('docker/project.env')
32
33# for setting up completion in activate scripts
34PROJECT = 'chartepssg'
35
36# ssh connection string for TST docker deployment
37SSH_TST = 'chartepssg@chartepssgap1'
38
39RELOAD_TST_CMD = 'NO_ANSI=1 docker-chartepssg-TST/start.sh --reload'
40
41# For creating database
42DEFAULT_POSTGRES_HOST = 'localhost'
43
44# For creating database
45DEFAULT_POSTGRES_PORT = 5432
46
47# For creating database
48DEFAULT_PSQL = 'psql'
49
50# User to log in as to create users and databases
51DEFAULT_POSTGRES_CREATOR = 'creator'
52
53def core_dir():
54 """Return location of CHART-framework top directory.
55
56 It must either be on PYTHONPATH or as a subdirectory of this directory.
57
58 Also set sys.path if needed so it can be imported.
59
60 And PYTHONPATH so child processes can find it
61 """
62 import chart
63 # print('imported', chart.__file__)
64 # if you import a directory that's not a module, in python 3.6 (current OPE) there
65 # is not __file__ attribute. Under python 3.7+ it exists but is None
66 if not hasattr(chart, '__file__') or chart.__file__ is None:
67 del sys.modules['chart']
68 here = Path(__file__).parent
69 # print('here', here)
70 chart_dir = here.joinpath('chart')
71 if chart_dir.joinpath('chart').is_dir():
72 # print('inserting', chart_dir)
73 # Yes, this is what PEP8 says don't do. But otherwise we have to rely on users
74 # setting PYTHONPATH before running anything else
75 sys.path.insert(0, str(chart_dir))
76 # Also what PEP8 says don't do
77 os.environ['PYTHONPATH'] = os.environ.get('PYTHONPATH', '') + ':' + str(chart_dir)
78 # print(sys.path)
79 import chart
80 # print('reimport fie', chart.__file__)
81
82 else:
83 raise IOError('Cannot find CHART core - try setting PYTHONPATH or placing it in this '
84 'directory')
85
86 return Path(chart.__file__).parent.parent
87
88
89def dev_repo(c, repo):
90 """Fix sub repositories."""
91 config = dotenv.dotenv_values(DOCKER_CONF_FILE)
92 # Assume we have to fix 2 sub repositories defined by the config file entries
93 # GIT_[repo]_[DIR|REPO|REMOTE|BRANCH]
94 # local directory name of the repository
95 repo_dir = Path(config['GIT_{repo}_DIR'.format(repo=repo)])
96 # remote repository URL
97 repo_repo = config['GIT_{repo}_REPO'.format(repo=repo)]
98 # git server name of the remote repository
99 repo_remote = config['GIT_{repo}_REMOTE'.format(repo=repo)]
100 # branch to clone/switch to
101 repo_branch = config['GIT_{repo}_BRANCH'.format(repo=repo)]
102 if not repo_dir.exists():
103 print('Cloning {repo}/{branch}'.format(repo=repo_repo, branch=repo_branch))
104 c.run('git clone {repo} --origin {remote} --branch {branch} {dir}'.format(
105 repo=repo_repo, remote=repo_remote, branch=repo_branch, dir=repo_dir))
106
107 else:
108 with c.cd(repo_dir):
109 print('In {dir} fetching from {repo}'.format(
110 dir=repo_dir, repo=repo_repo))
111 c.run('git fetch {origin}'.format(origin=repo_remote))
112 print('Switching to {branch}'.format(branch=repo_branch))
113 # git switch is the newer command but we have some Centos 7.9 servers
114 # with git 1.8 with no switch command
115 c.run('git checkout {branch}'.format(branch=repo_branch))
116 c.run('git branch {branch} --set-upstream-to {origin}/{branch}'.format(
117 origin=repo_remote, branch=repo_branch))
118
119
120def dev_activate(c):
121 """Write activate script."""
122 # Make an `activate` script for the project
123 # Includes sourcing 'env/bin/activate' virtual environment if directory present
124 here = Path(__file__).parent
125 activate_path = Path('activate')
126 completion_dir = here.joinpath('.completion')
127 completion_bash_dir = completion_dir.joinpath('bash')
128 completion_zsh_dir = completion_dir.joinpath('zsh')
129 with activate_path.open('w') as h:
130 h.write("""#!/bin/bash
131# Our launcher location
132export PATH={here}/bin:$PATH
133
134# not needed to normal operation but makes the automated tests simpler to run
135# and maybe nice for standalone project tool scripts
136export PYTHONPATH={here}:$PYTHONPATH
137
138# set up autocompletion (out of date, see JCS for better)
139if [ -n "$ZSH_VERSION" ]; then
140 fpath=({here}/.completion/zsh $fpath)
141 autoload -U compinit
142 compinit -u
143elif [ -n "$BASH_VERSION" ]; then
144 . {here}/.completion/bash/_{project}
145 complete -F _{project} -o default {project}
146fi
147""".format(project=PROJECT, here=here))
148 activate_path.chmod(activate_path.stat().st_mode | stat.S_IEXEC)
149 output.write('Wrote {script}\n'.format(script=activate_path))
150
151 import chartepssg.project
152 chartepssg.project.init()
153 core_dir()
154 from chart.cmd.launcher import scan_tools
155 tools = list(tool.name for tool in scan_tools())
156
157 c.run('mkdir -p {d}'.format(d=completion_zsh_dir))
158 zsh_completion_script = completion_zsh_dir.joinpath('_{p}'.format(p=PROJECT))
159 with zsh_completion_script.open('w') as h:
160 h.write("""#compdef {project}
161_arguments "1:Tool:({tools})" "*:Filename:_files"
162""".format(project=PROJECT, tools=' '.join(tools)))
163
164 zsh_completion_script.chmod(zsh_completion_script.stat().st_mode | stat.S_IEXEC)
165 output.write('Wrote {comp}\n'.format(comp=zsh_completion_script))
166
167 c.run('mkdir -p {d}'.format(d=completion_bash_dir))
168 bash_completion_script = completion_bash_dir.joinpath('_{p}'.format(p=PROJECT))
169 with bash_completion_script.open('w') as h:
170 h.write("""#!/bin/bash
171_{project} ()
172{{
173 local cur
174 cur=${{COMP_WORDS[COMP_CWORD]}}
175 if [ "$COMP_CWORD" == "1" ]; then
176 COMPREPLY=( $( compgen -W '{tools}' -- "$cur") )
177 fi
178 return 0
179}}
180""".format(project=PROJECT, tools=' '.join(tools)))
181
182 bash_completion_script.chmod(bash_completion_script.stat().st_mode | stat.S_IEXEC)
183 output.write('Wrote {comp}\n'.format(comp=bash_completion_script))
184
185
186@task(help={'all_repos': 'Clone or update framework and all DU repositories',
187 'repo_framework': 'Clone or update framework',
188 'repo_du': 'Clone or update main DU',
189 # 'repo_srdb_du': 'Clone or update SRDB DU',
190 'activate': 'Write activate script',
191 })
192def dev(c,
193 all_repos=False,
194 repo_framework=False,
195 repo_du=False,
196 # repo_srdb_du=False,
197 activate=False):
198 """Set up development environment.
199
200 1. If framework and DU are missing, clone them (set CHART_GITLAB_REPO to use
201 non-EUM gitlab) (set CHART_GITLAB_KEY to use remote access key)
202 2. If framework and/or DU exist bugt are set to the wrong branch, attempt
203 to switch them
204 3. Create activate file
205 """
206 if all_repos:
207 repo_framework = True
208 repo_du = True
209 # repo_srdb_du = True
210
211 if activate:
212 dev_activate(c)
213
214 if repo_framework:
215 dev_repo(c, 'FRAMEWORK')
216
217 if repo_du:
218 dev_repo(c, 'DU')
219
220 # if repo_srdb_du:
221 # dev_repo(c, 'DU_SRDB')
222
223
224@task(help={
225 'all': '(re)build all autogenerated content. Activates all the other options',
226 'force-framework': 'Force rebuilding of all generated content in the framework',
227 'force-proj': 'Force rebuilding of all content in the project',
228 'force-thirdparty': 'Rebuild web 3rdparty js modules',
229 'force-schemas': 'Rebuild all XML schemas',
230 'force-params': 'Rebuild SRDB versions and parameter store XML files',
231 'force-packets': 'Rebuild all PUS packet definition files and param1_param2_info',
232 'force-cal': 'Rebuild all calibration files',
233 'force-choices': 'Rebuild choices definition files',
234 'force-limits': 'Rebuild limits information',
235 })
236def build(c,
237 all=False,
238 force_framework=False,
239 force_proj=False,
240 force_thirdparty=False,
241 force_schemas=False,
242 force_params=False, # ts and srdb_version
243 force_packets=False, # includes param1_param2_info and param_spids
244 force_limits=False,
245 force_cal=False,
246 force_choices=False,
247 ):
248 """Build auto-generatable files required to run some or all tools."""
249 # For now the user has to manually force things to be built.
250 # A nice enhancement would be to check what needs to be built and just do those automatically
251 if all:
252 force_framework = True
253 force_proj = True
254
255 if force_framework:
256 force_thirdparty = True
257 force_schemas = True
258
259 if force_proj:
260 force_params = True
261 force_packets = True
262 force_limits = True
263 force_cal = True
264 force_choices = True
265
266 # Unpack third party web code if not present or if --force-thirdparty passed
267 build_thirdparty = force_thirdparty
268 third_party_dir = core_dir().joinpath('chart', 'web', 'static', '3rdparty')
269 if not third_party_dir.exists():
270 build_thirdparty = True
271
272 if build_thirdparty:
273 core = core_dir()
274 with c.cd(str(core)):
275 c.run('chart/web/3rdparty.packages/unpack.sh')
276
277 if force_schemas:
278 core = core_dir()
279 with c.cd(str(core)):
280 c.run('chart/schemas/build.sh')
281
282 else:
283 # this is the only option where we actually check if the work needs doing
284 from chart.schemas.schema_tool import convert_all_schemas
285 convert_all_schemas()
286
287 sats = [
288 {'srdb': 'SRDB/EPSSGA', 'sid': 'sga1',},
289 {'srdb': 'SRDB/EPSSGB', 'sid': 'sgb1',},
290 ]
291
292 if force_params:
293 for sat in sats:
294 c.run('{project} srdb --srdb {srdb} --sid {sid} --gen-version-info'.format(
295 project=PROJECT, srdb=sat['srdb'], sid=sat['sid']))
296 c.run(('{project} srdb --srdb {srdb} --sid {sid} --gen-table-xml TM.xml '
297 '--storage-table TM_STORE').format(
298 project=PROJECT,
299 srdb=sat['srdb'],
300 sid=sat['sid'],))
301 c.run(('{project} srdb --srdb {srdb} --sid {sid} --gen-table-xml TC.xml '
302 '--tc --storage-table TC_STORE').format(
303 project=PROJECT,
304 srdb=sat['srdb'],
305 sid=sat['sid'],))
306
307 if force_packets:
308 for sat in sats:
309 # param1-param2 file
310 c.run('{project} srdb --srdb {srdb} --sid {sid} --gen-param1-param2-xml'.format(
311 project=PROJECT, srdb=sat['srdb'], sid=sat['sid']))
312 # param-spids fast lookup file
313 c.run('{project} srdb --srdb {srdb} --sid {sid} --gen-param-spid-xml'.format(
314 project=PROJECT, srdb=sat['srdb'], sid=sat['sid']))
315 # TM packets
316 c.run('{project} srdb --srdb {srdb} --sid {sid} --gen-packets-xml'.format(
317 project=PROJECT, srdb=sat['srdb'], sid=sat['sid']))
318 # TC packets
319 c.run('{project} srdb --srdb {srdb} --sid {sid} --gen-packets-xml --tc'.format(
320 project=PROJECT, srdb=sat['srdb'], sid=sat['sid']))
321
322 if force_limits:
323 for sat in sats:
324 c.run('{project} srdb --srdb {srdb} --sid {sid} --gen-limits-xml'.format(
325 project=PROJECT, srdb=sat['srdb'], sid=sat['sid']))
326
327 if force_cal:
328 for sat in sats:
329 c.run('{project} srdb --srdb {srdb} --sid {sid} --gen-cal-xml'.format(
330 project=PROJECT, srdb=sat['srdb'], sid=sat['sid']))
331 c.run('{project} srdb --srdb {srdb} --sid {sid} --gen-cal-xml --tc'.format(
332 project=PROJECT, srdb=sat['srdb'], sid=sat['sid']))
333
334 if force_choices:
335 for sat in sats:
336 c.run('{project} srdb --srdb {srdb} --sid {sid} --gen-choices-xml'.format(
337 project=PROJECT, srdb=sat['srdb'], sid=sat['sid']))
338 c.run('{project} srdb --srdb {srdb} --sid {sid} --gen-choices-xml --tc'.format(
339 project=PROJECT, srdb=sat['srdb'], sid=sat['sid']))
340
341
342
343@task(help={'python': 'Disable all Python checks',
344 'css': 'Disable all CSS checks',
345 'js': 'Disable all Javascript checks'})
346def lint(c,
347 all=True,
348 python=False,
349 python_pylint=False,
350 python_pycodestyle=False,
351 python_pydocstyle=False,
352 python_mypy=False,
353 css=False,
354 css_csslint=False,
355 js=False,
356 js_jslint=False):
357 """Run static checks against source files."""
358 # Missing but doable:
359 # - HTML lint (plain and Django templates)
360 # - Prose (spelling, style and grammar), including prose in docstrings and HTML files
361 # - Shell script lint
362 # - Django lint
363 # - PostgreSQL schema lint
364 # - Filesystem lint (dupes, weird permissions)
365 # - RPM lint
366 # - restructuredtext lint
367 # - XML syntax and schema validation
368 # - RNC lint (by running trang)
369 if all:
370 python = True
371 css = True
372 js = True
373
374 if python:
375 python_pylint = True
376 python_pycodestyle = True
377 python_pydocstyle = True
378 # for some reason it can't find core directory which makes most of the outputs
379 # useless because it swamps with false errors
380 # python_mypy = True
381
382 if css:
383 # This probably works but csslint is too hard to install on most EUM machines
384 # css_csslint = True
385 pass
386
387 if js:
388 # jslint is hard to install on most EUM machines
389 # js_jslint = True
390 pass
391
392 if python_pylint:
393 c.run('pylint {project}'.format(project=PROJECT))
394
395 if python_pycodestyle:
396 # The docs claim the tool will find setup.cfg automatically but it doesn't work
397 # for me
398 c.run('pycodestyle --config=setup.cfg charteps')
399 # 'grep -v chart/web/static')
400
401 if python_pydocstyle:
402 c.run('pydocstyle charteps')
403
404 if python_mypy:
405 c.run('PYTHONPATH={core} mypy charteps'.format(core=core_dir()))
406
407 if css_csslint:
408 c.run('find chart -name \'*.css\' -type f | grep -v 3rdparty | grep -v includes | '
409 'PYTHONPATH=$PWD CHART_SETTINGS_MODULE=charteps.settings xargs chart/tools/check.py')
410
411 if js_jslint:
412 c.run('find chart -name \'*.js\' -type f | grep -v 3rdparty | grep -v includes | xargs jslint '
413 '--report xml')
414
415
416@task
417def doc(c):
418 """Automatic documentation."""
419 # We have lots of scope to make a big auto-generated document with lots
420 # of information about the project extracted from source and XML files and wiki pages
421
422 # extract --help pages for all command line tools
423 # doc_exe(c) # usage for all executables
424 # doc_algs(c)
425 # doc_widgets(c)
426 # doc_reports(c)
427 # doc_schedule(c)
428 # doc_wiki(c)
429 # doc_sphinx(c)
430 # doc_source_uml(c)
431 # scan source code for line counts per language
432 c.run('PYTHONPATH=$PWD:$PWD/chart CHART_SETTINGS_MODULE=tests.settings '
433 '../chart/dist/cloc/cloc.py dist {project}'.format(project=PROJECT))
434 # doc_lint(c) # coding style violations
435 # doc_combine(c) # build a big pdf/word/html site
436
437
438class PgCursor:
439 """Very basic postgres cursor wrapper with some logging."""
440 def __init__(self, cursor, user):
441 self.cursor = cursor
442 self.user = user
443
444 def execute(self, *args, **kwargs):
445 """Run a query."""
446 self.cursor.execute(*args, **kwargs)
447 if len(kwargs) == 0:
448 print_kwargs = ''
449
450 else:
451 print_kwargs = str(kwargs)
452
453 print('Executed as {user} {args} {kwargs}'.format(
454 user=self.user, args=', '.join(args), kwargs=kwargs if len(kwargs) > 0 else ''))
455 return self.cursor
456
457
458class PgConnection:
459 """Very basic postgres connection wrapper with some logging."""
460<<<hidden due to potential security issue>>>
461 import psycopg2
462 self.user = user
463 self.connection = psycopg2.connect(host=host,
464 port=port,
465 dbname=dbname,
466 user=user,
467<<<hidden due to potential security issue>>>
468 self.connection.autocommit = True
469 print('Connected to postgres database {db} on {host} as {user}'.format(
470 db=dbname, host=host, user=user))
471
472 def cursor(self):
473 """Create a blank cursor."""
474 return PgCursor(self.connection.cursor(), self.user)
475
476
477@task(help={
478 'host': 'Address or name of database server',
479 'port': 'Server port',
480 'create': (
481 'Postgres name of database to create. Database users will be created with this name as a '
482 'prefix. If omitted no database will be created'),
483 'psql': 'Location of psql tool',
484 'creator': (
485<<<hidden due to potential security issue>>>
486 'must already be set up'),
487 'init': (
488 'Project database connection to use when initialising objects. Must already be added to '
489 'project_settings.py or local_settings.py. If omitted no database objects will be created'),
490})
491def db(c,
492 host=DEFAULT_POSTGRES_HOST,
493 port=DEFAULT_POSTGRES_PORT,
494 create=None,
495 psql=DEFAULT_PSQL,
496 creator=DEFAULT_POSTGRES_CREATOR,
497 init=None):
498 """Create and initialise a database, users, tables, views and functions.
499
500 If the `create` option is used we build a new, locked down database and create 5 users:
501 - `create`_admin
502 - `create`_enduser
503 - `create`_expert
504 - `create`_web
505 - `create`_ingester
506
507<<<hidden due to potential security issue>>>
508 The ingester user has write access to all tables; the others have only read access.
509
510 If the `init` option is used then all the blank tables, views, sequences and stored procedures
511 for the project are created. This relies on a CHART database connection being present.
512 """
513 if create:
514 # Create a new database and users and set up lockdown and security
515
516 # Make local administrator
517 print('Logging in as {u}@{h}:{p}/{db}'.format(
518 u=creator, h=host, p=port, db='postgres'))
519 creator_conn = PgConnection(host=host, port=port, dbname='postgres', user=creator)
520 creator_cursor = creator_conn.cursor()
521 admin_username = create + '_admin'
522 print('Creating admin user {a}'.format(a=admin_username))
523<<<hidden due to potential security issue>>>
524 admin=admin_username))
525
526 # Make database, owned by admin user
527 print('Creating database {d}'.format(d=create))
528 creator_cursor.execute('CREATE DATABASE {name} OWNER {admin}'.format(name=create,
529 admin=admin_username))
530
531 # Don't let anyone log in unless we say they can
532 print('Revoking normal privileges')
533 creator_cursor.execute('REVOKE ALL PRIVILEGES ON DATABASE {name} FROM public'.format(
534 name=create))
535
536 # New schema with limited permissions
537 admin_conn = PgConnection(
538<<<hidden due to potential security issue>>>
539 admin_cursor = admin_conn.cursor()
540 print('Creating protected schema')
541 admin_cursor.execute('CREATE SCHEMA protected')
542 print('Making protected schema the default')
543 admin_cursor.execute('ALTER DATABASE {name} SET SEARCH_PATH TO protected'.format(
544 name=create))
545
546<<<hidden due to potential security issue>>>
547 """Create a user with basic read permissions."""
548 # Make the user object
549 print('Creating user {u}'.format(u=username))
550<<<hidden due to potential security issue>>>
551<<<hidden due to potential security issue>>>
552 # Allow them to log in
553 print('Allowing {u} to connect'.format(u=username))
554 admin_cursor.execute('GRANT CONNECT ON DATABASE {name} TO {user}'.format(
555 name=create, user=username))
556 # Let them see the tables
557 print('Allow {u} to read tables'.format(u=username))
558 admin_cursor.execute('GRANT USAGE ON SCHEMA protected TO {user}'.format(user=username))
559 # Let them select all current and future tables
560 print('Allow {u} to read future tables'.format(u=username))
561 admin_cursor.execute(('ALTER DEFAULT PRIVILEGES IN SCHEMA protected '
562 'GRANT SELECT ON TABLES TO {user}').format(user=username))
563
564 # Make basic user
565 make_user(create + '_enduser', 'enduser')
566 # Make expert user
567 make_user(create + '_expert', 'expert')
568 # Make web user
569 make_user(create + '_web', 'web')
570 # Make ingester user
571 ingester = create + '_ingester'
572 make_user(ingester, 'ingester')
573 # Allow ingester to write to all tables
574 print('Allow ingester user to write all tables')
575 admin_cursor.execute((
576 'ALTER DEFAULT PRIVILEGES IN SCHEMA protected '
577 'GRANT INSERT, DELETE, UPDATE ON TABLES TO {user}').format(user=ingester))
578
579 if init:
580 # Create tables, views, sequences, functions in the database
581 c.run('{project} django_manage migrate --database {db}'.format(
582 db=init, project=PROJECT), warn=True)
583 c.run('{project} ddl --db {db} --all-ts --execute'.format(
584 db=init, project=PROJECT), warn=True)
585 c.run('{project} ddl --db {db} --all-sys --execute'.format(
586 db=init, project=PROJECT), warn=True)
587 c.run('{project} ddl --db {db} --all-cal --execute'.format(
588 db=init, project=PROJECT), warn=True)
589 c.run('{project} ddl --db {db} --all-functions --execute'.format(
590 db=init, project=PROJECT), warn=True)
591
592
593@task(help={'environment': 'Run Python environment tests from core',
594 'framework': 'Run core application tests, using this project as the launcher',
595 'application': ('Run this project application tests. Tests using database and website '
596 'will be skipped unless a database is configured'),
597 'doctest': 'Run source code doctests',
598 'web': ('Run web tests. For local tests a database is required. If --url is specified '
599 'the tests will be run against an existing running web server'),
600 'database': 'Test database contents. Requires start, stop and optional sid',
601 'start': 'Start time for database tests',
602 'stop': 'Stop time for database tests',
603 'sid': 'Optionally restrict database tests to certain sid(s)',
604 'db': 'Database connection to use for application or database tests',
605 'url': ('If specified, website tests will be run against a remote web server not '
606 'internal web server'),
607})
608def test(c,
609 environment=True,
610 framework=True,
611 application=True,
612 web=False,
613 doctest=False,
614 database=None,
615 start=None,
616 stop=None,
617 sid=None,
618 db=None,
619 url=None,
620 xunit=None,
621 html=None,
622 verbose=False):
623 """Run automated tests.
624
625 Application checks may require a readable or writeable database but must supply any needed
626 test data.
627 Server tests can be done against a local server (requires a populated database and configuration
628 of time range) or against a remote server (default satellite and time range exists but can
629 be overridden).
630 Database checks are read only and require a database configuration, populated database,
631 and time range specified.
632
633 Use cases:
634 - A new developer wants to see some basic results
635 - During coding, a developer wants to perform a thorough check of their changes
636 - To routinely monitor an OPE website
637 - To routinely monitor database correctness
638 - To generate the test section in automatic documentation generation
639 """
640 params = []
641 # Don't use --xunit or --html unless you're only running one set of tests
642 # This would probably all work better if we squashed everything to a single call
643 # to pytest
644 if xunit is not None:
645 params.append('--junit-xml={f}'.format(f=xunit))
646
647 if html is not None:
648 params.append('--html={f} --self-contained-html'.format(f=html))
649
650 if verbose:
651 params.append('-v')
652
653 if application:
654 # project first otherwise the settings module gets skewie
655 # c.run('PYTHONPATH=$PWD:$PYTHONPATH CHART_SETTINGS_MODULE={project}.project_settings '
656 params.append('tests/application')
657
658 if environment:
659 params.append('{core}/tests/environment'.format(core=core_dir()))
660
661 if application:
662 params.append('--launcher={project} {core}/tests/application'.format(
663 project=PROJECT, core=core_dir()))
664
665 if doctest:
666 # c.run('python -m doctest -v {project}'.format(project=project))
667 # c.run('pytest --doctest-modules --disable-warnings{flags} chart', pty=True, flags=flags)
668 # --doctest-continue-on-failure --ignore-glob="project_settings.py"
669 if core or application:
670 params.append('--doctest-modules')
671
672 if application:
673 params.append(('--ignore={project}/project_settings.py '
674 '{project}').format(project=PROJECT))
675
676 if core:
677 params.append(('{core}/chart --ignore-glob={core}/chart/web '
678 '--ignore={core}t/chart/settings.py').format(core=core_dir()))
679
680 if db:
681 params.append('--db {db}'.format(db=db))
682
683 if url:
684 pass
685
686 if web:
687 # test_server(c)
688 pass
689
690 if database:
691 # test_database(c)
692 pass
693
694 c.run('pytest ' + ' '.join(params), pty=True)
695
696@task(help={
697 'info': 'Show location of all files and directories',
698 'build-framework': 'Build framework docker image',
699 'build-project': 'Build project docker image',
700 'save-image': 'Save image to temporary file',
701 'deploy-tst': 'Deploy image to TST environment',
702 'reload-tst': '(re)start services on TST',
703 'dockerdir-tst': 'Copy the docker directory to the TST environment'})
704def deploy(c,
705 info=False,
706 build_framework=False,
707 build_project=False,
708 save_image=False,
709 deploy_tst=False,
710 reload_tst=False,
711 dockerdir_tst=False):
712 """Package and deploy docker images."""
713 done_something = False
714 config = dotenv.dotenv_values(DOCKER_CONF_FILE)
715 image_name = config['CHART_PROJECT_IMAGE_NAME']
716 tmp_image_filename = Path('tmp/{project}.image'.format(project=PROJECT))
717
718 if info:
719 print('Framework dir: {core}'.format(core=core_dir()))
720 print('Docker image name: {image}'.format(image=image_name))
721 print('Temp image filename: {tmp}'.format(tmp=tmp_image_filename))
722 print('SSH connection for TST: {tst}'.format(tst=SSH_TST))
723 return
724
725 if build_framework:
726 with c.cd(core_dir()):
727 # Set CHART_NO_PROMPT_BUILD to build without waiting for user to press enter.
728 # Set BUILDKIT_PROGRESS to switch off the clever control codes that buildx uses
729 # to make a more entertaining build progress page normally
730 c.run('CHART_NO_PROMPT_BUILD=1 BUILDKIT_PROGRESS=plain docker/build.sh')
731
732 done_something = True
733
734 if build_project:
735 c.run('CHART_NO_PROMPT_BUILD=1 BUILDKIT_PROGRESS=plain docker/build.sh')
736 done_something = True
737
738 if save_image:
739 tmp_image_filename.parent.mkdir(exist_ok=True)
740 print('Saving {image} to {filen}'.format(image=image_name, filen=tmp_image_filename))
741 c.run('docker image save {image} > {filename}'.format(
742 image=image_name, filename=tmp_image_filename))
743 done_something = True
744
745 if deploy_tst:
746 c.run('ssh {tst} "docker image load < {abs_image_file}"'.format(
747 tst=SSH_TST, abs_image_file=tmp_image_filename.absolute()))
748 done_something = True
749
750 if dockerdir_tst:
751 print('Copying docker to {tst}'.format(tst=SSH_TST))
752 import sys
753 c.run('rsync --rsh ssh --verbose --update --delete --recursive docker {tst}:'.format(
754 tst=SSH_TST))
755 # echo=True,
756 # out_stream=sys.stdout,
757 # err_stream=sys.stdout)
758 done_something = True
759
760 if reload_tst:
761 c.run('ssh {server} "{cmd}"'.format(server=SSH_TST, cmd=RELOAD_TST_CMD))
762 #, pty=True)
763 done_something = True
764
765 if not done_something:
766 raise ValueError('No actions given')