1#!/usr/bin/env python3
2
3"""Manage the publishing/unpublishing of reports."""
4
5from typing import Optional
6from datetime import datetime
7import logging
8
9try:
10 import paramiko
11except ImportError:
12 paramiko = None
13
14from chart import settings
15import chart.alg.settings
16from chart.db.connection import db_connect
17from chart.reports.archive import get_report_abspath
18from chart.reports.archive import get_report_relpath
19from chart.common.path import Path
20from chart.reports.manifest import Manifest
21from chart.reports.archive import retrieve_file
22from chart.reports.archive import update_manifest
23
24logger = logging.getLogger()
25
26class PublishError(Exception):
27 pass
28
29# def get_staged():
30# """Return the list of staged reports, with spacecraft status last."
31# Returned value is a list of names only.
32# """
33# # we have a specific list for status reports so that they can be later appended at the end
34# # of the original list. (SC Status reports should always be dealt with at the end)
35# staged_reports = []
36# staged_status_report = []
37
38# # create the staging directory if it does not already exists
39# ensure_dir_exists(settings.REPORT_STAGING_AREA)
40
41# if settings.REPORT_STAGING_AREA.is_dir():
42# for filename in settings.REPORT_STAGING_AREA.iterdir():
43# # we must only try to commit report (zip) files
44# if filename.suffix == ZIPFILE_EXT:
45# if 'EPS_SC_STATUS_WEEKLY_REPORT' in str(filename):
46# staged_status_report.append(filename.name)
47
48# else:
49# staged_reports.append(filename.name)
50
51# else:
52# logging.info('The staging directory {dir} is not a directory'.format(
53# dir=settings.REPORT_STAGING_AREA))
54
55# # list the results alphabetically...
56# staged_status_report.sort()
57# staged_reports.sort()
58
59# # ...and move STATUS Reports to the end
60# staged_reports.extend(staged_status_report)
61# return staged_reports
62
63
64def publish_report(report_name, user):
65 """Publish a report.
66
67 1. Transfer it via scp to the external reports archive
68 2. Update REPORTS.LAST_PUBLISH_TIME to current time
69
70 If settings.REPORT_PUBLISH_HOST is not set then we just update REPORTS table
71 and don't actually publish anywhere.
72 """
73 logging.info('Publishing {name}'.format(name=report_name))
74
75 # Update manifest in archive
76 manifest = Manifest(buff=retrieve_file(report_name, chart.alg.settings.MANIFEST_FILENAME))
77 manifest.add_publish_time(datetime.utcnow(), user)
78 update_manifest(report_name, manifest)
79
80 # Get absolute filename to local file
81 full_path = get_report_abspath(report_name)
82 if not full_path.exists():
83 logging.error('{rep} does not exist.'.format(rep=full_path))
84 return
85
86 # Get relative path to required remote file within archive
87 rel_path = get_report_relpath(report_name=report_name)
88
89 # Open SSH connection
90 client = connect_remote()
91 if client is not None:
92 sftp = client.open_sftp()
93
94 # Push the .zip file over
95 remote_file = Path(settings.REPORT_PUBLISH_DIR).joinpath(rel_path)
96 mkdir_remote(sftp, remote_file.parent)
97 sftp.put(full_path, str(remote_file))
98 logger.info('Transferred local {src} to remote {dst}'.format(src=full_path, dst=remote_file))
99
100 # update local REPORTS table, which already includes this report,
101 # to include published date
102 db_conn = db_connect('REPORTS')
103 db_conn.execute('UPDATE REPORTS SET LAST_PUBLISH_TIME=:last_publish_time '
104 'WHERE ARCHIVE_FILENAME=:rel_path',
105 rel_path=str(rel_path),
106 last_publish_time=datetime.utcnow())
107 db_conn.commit()
108
109 # update remote REPORTS table
110 if client is not None:
111 resync_external_reports(client)
112
113
114def unpublish_report(report_name):
115 """Undo publish of a report.
116
117 1. The local REPORTS table is updated to clear the LAST_PUBLISH_TIME field.
118 2. The .zip archive is deleted from the remove reports file system archive.
119 3. The remote database REPORTS table is updated to delete the entry for the report.
120 """
121 logging.info('Unpublishing {name}'.format(name=report_name))
122
123 # Determine report filename
124 rel_path = get_report_relpath(report_name=report_name)
125
126 # Open SSH connection
127 client = connect_remote()
128
129 if client is not None:
130 sftp = client.open_sftp()
131
132 # Remove remote file
133 remote_file = Path(settings.REPORT_PUBLISH_DIR).joinpath(rel_path)
134 sftp.remove(str(remote_file))
135
136 # Remove LAST_PUBLISH_TIME from local REPORTS table
137 db_conn = db_connect('REPORTS')
138 db_conn.execute('UPDATE REPORTS SET LAST_PUBLISH_TIME=null '
139 'WHERE ARCHIVE_FILENAME=:rel_path',
140 rel_path=str(rel_path))
141 db_conn.commit()
142
143 # update remote REPORTS table
144 if client is not None:
145 resync_external_reports(client)
146
147
148def resync_external_reports(client):
149 """Update REPORTS table on the remote system to reflect any new report .zip files."""
150 logger.info('Running sync command {cmd}'.format(cmd=settings.REPORT_PUBLISH_DB_SYNC_CMD))
151 _, stdout, stderr = client.exec_command(settings.REPORT_PUBLISH_DB_SYNC_CMD)
152 errors = stderr.read()
153 if len(errors) > 0:
154 logger.error('Remote sync error: {e} stdout: {o}'.format(e=errors, o=stdout.read()))
155
156
157def connect_remote() -> Optional[paramiko.SSHClient]:
158 """Connect to remote server.
159
160 If settings.REPORT_PUBLISH_HOST is None then we return None.
161 Otherwise any problem raises an exception."""
162 if paramiko is None:
163 raise PublishError('paramiko library not installed')
164
165 if settings.REPORT_PUBLISH_HOST is None:
166 return None
167 # raise PublishError('No publish server host specified')
168
169 if settings.REPORT_PUBLISH_USER is None:
170 raise PublishError('No publish remote user specified')
171
172 if settings.REPORT_PUBLISH_DIR is None:
173 raise PublishError('No publish remote directory specified')
174
175 client = paramiko.SSHClient()
176 client.load_system_host_keys()
177 try:
178 client.connect(settings.REPORT_PUBLISH_HOST,
179 username=settings.REPORT_PUBLISH_USER,
180<<<hidden due to potential security issue>>>
181 except OSError as e:
182 raise PublishError('Connecting to {target}: {err}'.format(
183 target=settings.REPORT_PUBLISH_TARGET, err=e))
184
185 return client
186
187def list_remote(client):
188 """Retrieve list of report zipfiles on remote server."""
189 sftp = client.open_sftp()
190 try:
191 result = sftp.listdir(settings.REPORT_PUBLISH_DIR)
192 except OSError as e:
193 raise PublishError('Cannot access remote dir {dir}: {err}'.format(
194 dir=settings.REPORT_PUBLISH_DIR,
195 err=e))
196
197 return result
198
199
200def mkdir_remote(sftp_client, remote_dir):
201 """Recursively create directory `remote_dir`."""
202 logger.debug('mkdir {d}'.format(d=remote_dir))
203 # see if it's already there
204 try:
205 sftp_client.stat(str(remote_dir))
206 except FileNotFoundError:
207 # not exists so we have work to do
208 logger.debug(' it does not exist')
209 pass
210 else:
211 # exists so nothing to do
212 logger.debug(' it exists')
213 return
214
215 # otherwise see if the parent exists
216 parent_dir = remote_dir.parent
217 try:
218 sftp_client.stat(str(parent_dir))
219 except FileNotFoundError:
220 # if not, try to create parent with recursion
221 mkdir_remote(sftp_client, parent_dir)
222
223 # and if the parent does exist, create our directory
224 sftp_client.mkdir(str(remote_dir))
225 logger.debug(' made {d}'.format(d=remote_dir))
226
227
228def test_publish_connection():
229 """Verify that a remote report archive is configured and contactable."""
230 client = connect_remote()
231 list_remote(client)