1#!/usr/bin/env python3
2
3"""Generate detailed CHART docs using Sphinx.
4
5* Find all executables in ``chart/commands`` and run them with ``--help`` option, capturing
6 output.
7* Recursively walk the ``chart`` directory, find all python code and generate documentation with.
8 Sphinx autodoc extension.
9
10.. note:: Before running this program, make sure that environment is set to allow access to CHART
11 database.
12"""
13
14import os
15import sys
16import logging
17import subprocess
18import shutil
19
20from chart.project import settings
21from chart.common.args import ArgumentParser
22
23base_dir = os.path.dirname(settings.CORE_HOME_DIR)
24doc_dir = os.path.join(base_dir, 'doc')
25# result_dir = os.path.join(base_dir, 'sphinx_html')
26result_dir = settings.SPHINX_DIR
27
28# sys.path.append(base_dir)
29
30excluded = ('runserver', 'runserverp', 'doctest', 'test', 'find_orphaned_archives',
31 'db_size_multiplier', 'xmlgen', 'amsua_report', 'aocs_scao17',
32 'mhs_long_term_report', '_charm')
33
34excluded_imports = []
35
36excluded_dirs = ('tests', '.svn', 'static', '.git', '.hg')
37
38
39def get_help(executable):
40 """Capture executables' help by running it with --help."""
41
42 # logging.info('Reading '+executable)
43 os.environ['CHART_BRIEF_HELP'] = '1'
44 try:
45 child = subprocess.Popen((sys.argv[0], executable, '--help'),
46 stdout=subprocess.PIPE,
47 stderr=subprocess.STDOUT)
48 except OSError:
49 return False
50
51 retcode = child.wait()
52
53 if retcode != 0:
54 logging.info('No help available for {exe}'.format(exe=executable))
55 return False
56
57 output = child.stdout.read()
58 out_file_name = os.path.join(doc_dir, 'command_help', executable + '.txt')
59
60 f = open(out_file_name, 'w')
61 f.write(output)
62 f.close()
63
64
65def process_commands():
66 """Put help for all executables in ``chart/command`` directory into separate files in
67 ``doc/command_help`` directory and import these files with ``.. literalinclude::``
68 from the main file ``doc/commands.rst``.
69 """
70
71 command_help_dir = os.path.join(doc_dir, 'command_help')
72 shutil.rmtree(command_help_dir, ignore_errors=True)
73 os.mkdir(command_help_dir)
74
75 f = open(os.path.join(doc_dir, 'commands.rst'), 'w')
76 f.write("""**********************************
77{APPNAME} Command-line Tools Reference
78**********************************
79
80""".format(APPNAME=settings.APPNAME))
81
82 for tool_dir in (settings.CORE_TOOLS_DIR, settings.PROJ_TOOLS_DIR):
83 for exe in sorted(os.listdir(tool_dir)):
84 abspath = os.path.realpath(exe) # .abspath?
85 basename = os.path.basename(exe)
86
87 # print rp
88
89 if basename == '__init__.py' or not basename.endswith('.py'):
90 continue
91
92 # module = '.'.join([:-3].split('/')[-3:])
93
94 # print module, bn, rp,
95
96 if exe in excluded:
97 logging.info('Skipped {exe}'.format(exe=exe))
98 continue
99
100 # logging.info('Processing ' + ex)
101 f.write(basename)
102 f.write('\n' + '-' * len(basename) + '\n\n')
103 f.write('.. automodule:: ' + abspath + '\n\n')
104
105 include_path = os.path.join('command_help', basename + '.txt3')
106 if get_help(exe) == 0:
107 f.write('.. literalinclude:: ' + include_path + '\n\n')
108
109 else:
110 f.write(""".. warning::
111 Failed to extract help by running this module.
112
113 """)
114
115
116def process_modules():
117 """Walk ``chart`` in search of all python modules and extract docstrings
118 with Shpinx ``..automodule::`` extension.
119
120 Make links to modules' source code in trac/subversion.
121 """
122
123 modules_dir = os.path.join(doc_dir, 'modules')
124 shutil.rmtree(modules_dir, ignore_errors=True)
125 os.mkdir(modules_dir)
126
127 browse_root_url = os.path.dirname(settings.PROJ_BROWSE_URL)
128 top_dirname = os.path.basename(settings.PROJ_BROWSE_URL)
129
130 for root, dirs, files in os.walk(settings.CORE_HOME_DIR):
131
132 # skip __init__.py of zero size
133 mods = [f[:-3] for f in files
134 if (f[-3:] == '.py' and
135 f[:-3] not in excluded and not f.startswith('.#') and
136 os.path.getsize(os.path.join(root, f)) > 0)]
137
138 if len(mods) == 0:
139 continue
140
141 mods.sort()
142
143 chart_ix = root.rfind(top_dirname)
144
145 prefix = '.'.join(root[chart_ix:].split('/'))
146
147 # print root, prefix, mods
148
149 out_file_name = os.path.join(modules_dir, prefix + '.rst')
150 f = open(out_file_name, 'w')
151 f.write('\n' + '*' * len(prefix) + '\n' + prefix + '\n' + '*' * len(prefix) + '\n\n')
152
153 for m in mods:
154 import_name = prefix + '.' + m
155 ln = ':mod:`' + prefix + '.' + m + '`'
156 f.write(ln + '\n' + '-' * len(ln) + '\n\n')
157
158 if import_name not in excluded_imports:
159 f.write(""".. automodule:: %s
160 :members:
161 :undoc-members:
162
163""" % (import_name,))
164
165 f.write("""`View source: {text} <{url}>`_
166
167""".format(text=import_name + '.py',
168 url=browse_root_url + '/' + root[chart_ix:] + '/' + m + '.py'))
169
170 f.close()
171
172 for skip in excluded_dirs:
173 if skip in dirs:
174 dirs.remove(skip)
175
176 return 0
177
178
179def check_imports():
180 """Walk ``chart`` in search of all python modules and try to import them."""
181 # global excluded_imports
182
183 # sys.path.append('/homespace/mikhails/chart')
184 # sys.path.append('/homespace/mikhails/chart/chart')
185
186 for root, dirs, files in os.walk(settings.CORE_HOME_DIR):
187 # Collect list of suitable Python modules for documentation in this directory.
188 # only scan Python code,
189 # ignore manually excluded files,
190 # ignore zero-length files (__init__.py),
191 # ignore emacs temp files (.#*)
192 mods = [f[:-3] for f in files
193 if (f[-3:] == '.py' and
194 f[:-3] not in excluded and
195 not f.startswith('.#') and
196 os.path.getsize(os.path.join(root, f)) > 0)]
197
198 if len(mods) == 0:
199 continue
200
201 mods.sort()
202
203 chart_ix = root.rfind('chart')
204
205 prefix = '.'.join(root[chart_ix:].split('/'))
206
207 for m in mods:
208 import_name = prefix + '.' + m
209 logging.debug('Importing ' + import_name)
210 try:
211 m = __import__(import_name)
212
213 except Exception:
214 logging.info('Failed to import: ' + import_name)
215 excluded_imports.append(import_name)
216 continue
217
218 for skip in excluded_dirs:
219 if skip in dirs:
220 dirs.remove(skip)
221
222 return 0
223
224
225def main():
226 """Command line entry point."""
227
228 parser = ArgumentParser(__doc__)
229 parser.parse_args()
230
231 logging.info("Checking imports...")
232 check_imports()
233
234 logging.info('Processing executable help')
235 process_commands()
236
237 logging.info('Processing CHART modules')
238 process_modules()
239
240 shutil.rmtree(result_dir, ignore_errors=True)
241
242 run_sphinx = ('sphinx-build', '-b', 'html', doc_dir, result_dir)
243
244 logging.info('Excluded modules:' + str(excluded))
245 logging.info('Excluded imports:' + str(excluded_imports))
246
247 logging.info('...done')
248 logging.info('Running sphinx as ' + ' '.join(str(c) for c in run_sphinx))
249
250 try:
251 child = subprocess.Popen(run_sphinx)
252 except OSError:
253 raise Exception('Cannot find sphinx-build executable, '
254 'check that Sphinx is installed correctly')
255
256 child.wait()
257
258 modules_dir = os.path.join(doc_dir, 'modules')
259 command_help_dir = os.path.join(doc_dir, 'command_help')
260
261 logging.info('Deleting auto-generated files')
262 shutil.rmtree(modules_dir, ignore_errors=True)
263 shutil.rmtree(command_help_dir, ignore_errors=True)
264 os.unlink(os.path.join(doc_dir, 'commands.rst'))
265
266 logging.info('All done')
267
268if __name__ == '__main__':
269 main()