ref: 2620f4bcfd9bdc382918679bdb2077fd05501156
dir: /docs/doxygen2rst.py/
from __future__ import print_function
import re, sys, os, time, glob, errno, tempfile, binascii, subprocess, shutil
from lxml import etree
from optparse import OptionParser
import textwrap
import string
VERSION = '0.1'
__all__ = ['DoxyGen2RST']
LINE_BREAKER = "\n"
MAX_COLUMN = 80
def is_valid_uuid(uuid_string):
uuid4hex = re.compile('[0-9a-f]{32}\Z', re.I)
return uuid4hex.match(uuid_string) != None
def get_page(refid):
fields = refid.split("_")
if(is_valid_uuid(fields[-1][-32:])):
return ["_".join(fields[0:-1]), fields[-1]]
return [refid, None]
def mkdir_p(path):
try:
os.makedirs(path)
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
raise
def _glob(path, *exts):
path = os.path.join(path, "*") if os.path.isdir(path) else path + "*"
return [f for files in [glob.glob(path + ext) for ext in exts] for f in files]
class DoxyGen2RST(object):
"""
Customize the Doxygen XML output into RST format, then it can
be translated into all formats with the unified user interface.
The Doxygen output itself is too verbose and not hard to be
organized for a good documentation.
"""
def __init__(self,
src,
dst,
missing_filename = "missing.rst",
is_github = False,
enable_uml = True,
github_ext = ""):
self.doxy_output_dir = os.path.join(src, "_doxygen", "xml")
self.output_dir = dst
self.rst_dir = src
self.enable_uml = enable_uml
mkdir_p(dst)
self.is_github = is_github
if(is_github):
self.page_ext = github_ext
self.anchor_prefix = "wiki-"
else:
self.anchor_prefix = ""
self.page_ext = ".html"
self.filter = ["*.rst", "*.rest"]
self.re_doxy = "<doxygen2rst\s(\S*)=(\S*)>(.*?)</doxygen2rst>"
self.index_root = etree.parse(os.path.join(self.doxy_output_dir, "index.xml")).getroot()
self.references = {}
self.missed_types_structs = {}
self.name_refid_map = {}
self.build_references()
self.page_references = {}
self.missing_filename = missing_filename
self.temp_uml_path = os.path.join(tempfile.gettempdir(), "uml_" + binascii.b2a_hex(os.urandom(15)))
if os.path.exists(self.temp_uml_path):
shutil.rmtree(self.temp_uml_path)
os.mkdir(self.temp_uml_path)
def _find_ref_id(self, kind, name):
#print("_find_ref_id, %s - %s" %(kind, name))
if(kind == "function"):
for comp in self.index_root.iter("member"):
if(comp.attrib["kind"].lower() == kind.lower() and
comp.findtext("name").lower() == name.lower()):
return (comp.attrib["refid"])
pass
else:
for comp in self.index_root.iter("compound"):
if(comp.attrib["kind"].lower() == kind.lower() and
comp.findtext("name").lower() == name.lower()):
return comp.attrib["refid"]
return None
def strip_title_ref(self, text):
table = string.maketrans("","")
retstr = text.translate(table, string.punctuation)
words = retstr.split()
retstr = "-".join(words)
return retstr.lower()
def build_references(self):
for file in _glob(self.rst_dir, *self.filter):
filename = os.path.basename(file)
fin = open(file,'r')
content = fin.read()
it = re.finditer(self.re_doxy, content, re.DOTALL)
for m in it:
ref_id = self._find_ref_id(m.groups()[0], m.groups()[1])
if(ref_id is None):
#print("Reference is NOT found for: %s=%s" % (m.groups()[0], m.groups()[1]))
continue
page_name = os.path.splitext(filename)[0]
title_ref = self.strip_title_ref(m.groups()[2])
self.references[ref_id] = [m.groups()[0], m.groups()[1], page_name, filename, title_ref]
self.name_refid_map[m.groups()[1]] = ref_id
fin.close()
#print(self.references)
def call_plantuml(self):
if(not self.enable_uml):
return
java_bin = os.path.join(os.environ['JAVA_HOME'], "bin", "java")
output_path = os.path.abspath(os.path.join(self.output_dir, "images"))
cmds = ["\"" + java_bin + "\"", "-jar", "plantuml.jar", self.temp_uml_path + "/", "-o", output_path]
print(" ".join(cmds))
os.system(" ".join(cmds))
shutil.rmtree(self.temp_uml_path)
def _build_uml(self, uml_name, content):
uml_path = os.path.join(self.temp_uml_path, uml_name + ".txt")
fuml = open(uml_path, "w+")
fuml.write("@startuml\n")
fuml.write(content)
fuml.write("\n@enduml\n")
fuml.close()
return ".. image:: images/" + uml_name + ".png" + LINE_BREAKER
def _build(self, m):
retstr = ""
if(m.groups()[0] == "uml"):
retstr = self._build_uml(m.groups()[1], m.groups()[2])
elif(m.groups()[0] == "link"):
link = m.groups()[1] + self.page_ext
retstr = ("`%s <%s>`_" % (m.groups()[2], link))
else:
if(m.groups()[0] != "function"):
retstr += self._build_title(m.groups()[2])
retstr += self.convert_doxy(m.groups()[0], m.groups()[1])
return retstr
def generate(self):
for file in _glob(self.rst_dir, *self.filter):
filename = os.path.basename(file)
fin = open(file,'r')
input_txt = fin.read()
fin.close()
output_txt = re.sub(self.re_doxy, self._build, input_txt, 0, re.DOTALL)
output_txt += self._build_page_ref_notes()
fout = open(os.path.join(self.output_dir, filename), 'w+')
fout.write(output_txt)
fout.close()
#print("%s --- %s" %( file, os.path.join(self.output_dir, filename)))
self._build_missed_types_and_structs()
self.call_plantuml()
def make_para_title(self, title, indent = 4):
retstr = LINE_BREAKER
if(title):
retstr += "".ljust(indent, " ") + "| **" + title + "**" + LINE_BREAKER
return retstr
def _build_title(self, title, flag = '=', ref = None):
retstr = LINE_BREAKER
if(ref):
retstr += ".. _ref-" + ref + ":" + LINE_BREAKER + LINE_BREAKER
retstr += title + LINE_BREAKER
retstr += "".ljust(20, flag) + LINE_BREAKER
retstr += LINE_BREAKER
return retstr
def _build_ref(self, node):
text = node.text.strip()
retstr = ""
target = '`' + text + '`'
retstr += target + "_ "
if target in self.page_references:
reflink = self.page_references[target]
print("Link already added: %s == %s" % (reflink[0], node.attrib["refid"]))
assert(reflink[0] == node.attrib["refid"])
pass
else:
self.page_references[target] = (node.attrib["refid"], node.attrib["kindref"], text)
return retstr
def _build_code_block(self, node):
retstr = "::" + LINE_BREAKER + LINE_BREAKER
for codeline in node.iter("codeline"):
retstr += " "
for phrases in codeline.iter("highlight"):
if(phrases.text):
retstr += phrases.text.strip()
for child in phrases:
if(child.text):
retstr += child.text.strip()
if(child.tag == "sp"):
retstr += " "
if(child.tag == "ref" and child.text):
#escape the reference in the code block
retstr += "" # self._build_ref(child)
if(child.tail):
retstr += child.tail.strip()
retstr += LINE_BREAKER
return retstr
def _build_itemlist(self, node):
retstr = ""
for para in node:
if(para.tag != "para"):
continue
if(para.text):
retstr += para.text.strip()
for child in para:
if(child.tag == "ref" and child.text):
retstr += self._build_ref(child)
if(child.tail):
retstr += child.tail.strip()
return retstr
def _build_itemizedlist(self, node):
retstr = LINE_BREAKER
if(node == None):
return ""
for item in node:
if(item.tag != "listitem"):
continue
retstr += " - " + self._build_itemlist(item)
retstr += LINE_BREAKER
return retstr
def _build_verbatim(self, node):
retstr = LINE_BREAKER
if(node.text):
lines = node.text.splitlines()
print(lines[0])
m = re.search("{plantuml}\s(\S*)", lines[0])
if(m):
uml_name = "uml_" + m.groups()[0]
retstr += self._build_uml(uml_name, "\n".join(lines[1:]))
else:
retstr += "::" + LINE_BREAKER + LINE_BREAKER
retstr += node.text
return retstr
def _build_para(self, para):
retstr = ""
no_new_line = False
if(para.text):
retstr += textwrap.fill(para.text.strip(), MAX_COLUMN) + LINE_BREAKER + LINE_BREAKER
for child in para:
no_new_line = False
if(child.tag == "simplesect"):
for child_para in child:
if(child.attrib["kind"] == "return"):
return_str = self._build_para(child_para)
retstr += "".ljust(4, " ") + "| Return:" + LINE_BREAKER
for line in return_str.splitlines():
retstr += "".ljust(4, " ") + "| " + line + LINE_BREAKER
elif(child_para.tag == "title" and child_para.text):
lf.make_para_title(child_para.text.strip(), 4)
elif(child_para.tag == "para"): #for @see
retstr += self._build_para(child_para)
elif(child_para.text):
retstr += "".ljust(4, " ") + "| " + child_para.text.strip() + LINE_BREAKER
if(child.tag == "preformatted"):
retstr += "::" + LINE_BREAKER + LINE_BREAKER
if(child.text):
for line in child.text.splitlines():
retstr += " " + line + LINE_BREAKER
if(child.tag == "ref" and child.text):
retstr = retstr.rstrip('\n')
retstr += " " + self._build_ref(child)
no_new_line = True
if(child.tag == "programlisting"):
retstr += self._build_code_block(child)
if(child.tag == "itemizedlist"):
retstr += self._build_itemizedlist(child)
if(child.tag == "verbatim"):
retstr += self._build_verbatim(child)
if(not no_new_line):
retstr += LINE_BREAKER
if(child.tail):
retstr += textwrap.fill(child.tail.strip(), MAX_COLUMN) + LINE_BREAKER + LINE_BREAKER
return retstr
def get_text(self, node):
retstr = ""
if(node == None):
return ""
for para in node:
if(para.tag != "para"):
continue
retstr += self._build_para(para)
return retstr
def _find_text_ref(self, node):
retstr = ""
if(node.text):
retstr += node.text.strip()
for child in node:
if(child.tag == "ref"):
retstr += " " + self._build_ref(child) + " "
if(child.tail):
retstr += child.tail.strip()
return retstr
def _build_row_breaker(self, columns):
retstr = "+"
for column in columns:
retstr += "".ljust(column, "-") + "+"
return retstr + LINE_BREAKER
def _wrap_cell(self, text, length = 30):
newlines = []
for line in text.splitlines():
newlines.extend(textwrap.wrap(line, length))
return newlines
def _build_row(self, row, columns):
retstr = ""
row_lines = []
max_line = 0
for i in range(3):
row_lines.append(row[i].splitlines())
if(max_line < len(row_lines[i])):
max_line = len(row_lines[i])
for i in range(max_line):
for j in range(3):
retstr += "|"
if(len(row_lines[j]) > i):
retstr += row_lines[j][i]
retstr += "".ljust(columns[j] - len(row_lines[j][i]), " ")
else:
retstr += "".ljust(columns[j], " ")
retstr += "|" + LINE_BREAKER
return retstr
def _build_table(self, rows):
retstr = ""
columns = [0, 0, 0]
for row in rows:
for i in range(3):
for rowline in row[i].splitlines():
if(columns[i] < len(rowline) + 2):
columns[i] = len(rowline) + 2
#columns[0] = 40 if(columns[0] > 40) else columns[0]
#columns[1] = 40 if(columns[1] > 40) else columns[1]
#columns[2] = MAX_COLUMN - columns[0] - columns[1]
retstr += self._build_row_breaker(columns)
for row in rows:
retstr += self._build_row(row, columns)
retstr += self._build_row_breaker(columns)
return retstr;
def build_param_list(self, params, paramdescs):
retstr = ""
param_descriptions = []
for desc in paramdescs:
param_descriptions.append(desc)
rows = []
rows.append(("Name", "Type", "Descritpion"))
for param in params:
declname = param.findtext("declname")
paramdesc = None
for desc in param_descriptions:
paramname = desc.findtext("parameternamelist/parametername")
if(paramname.lower() == declname.lower()):
paramdesc = desc.find("parameterdescription")
break
decltype = self._find_text_ref(param.find("type"))
rows.append((declname, decltype, self.get_text(paramdesc)))
if(len(rows) > 1):
retstr += self._build_table(rows)
return retstr
def _build_enum(self, member):
enum_id = member.attrib["id"]
file, tag = get_page(enum_id)
retstr = self._build_title(member.findtext("name"), ref = tag)
detail_node = self.get_desc_node(member)
if(detail_node is not None):
retstr += LINE_BREAKER
retstr += self.get_text(detail_node)
rows = []
rows.append(("Name", "Initializer", "Descritpion"))
for enumvalue in member.iter("enumvalue"):
name = enumvalue.findtext("name")
initializer = enumvalue.findtext("initializer")
if(not initializer):
initializer = ""
desc = self.get_text(enumvalue.find("briefdescription"))
desc += self.get_text(enumvalue.find("detaileddescription"))
if(not desc):
desc = ""
rows.append((name, initializer, desc))
if(len(rows) > 1):
retstr += self._build_table(rows)
return retstr
def _build_struct(self, node):
retstr = ""
detail_node = self.get_desc_node(node)
if(detail_node is not None):
retstr += self.get_text(detail_node) + LINE_BREAKER
rows = []
rows.append(("Name", "Type", "Descritpion"))
for member in node.iter("memberdef"):
if(member.attrib["kind"] == "variable"):
name = member.findtext("name")
type = self._find_text_ref(member.find("type"))
desc = self.get_text(member.find("briefdescription"))
desc += self.get_text(member.find("detaileddescription"))
desc += self.get_text(member.find("inbodydescription"))
if(not desc):
desc = ""
rows.append((name, type, desc))
if(len(rows) > 1):
retstr += self._build_table(rows)
return retstr
def _build_class(self, node):
retstr = ""
for member in node.iter("memberdef"):
if(member.attrib["kind"] == "function"):
retstr += self.build_function(member)
return retstr
def get_desc_node(self, member):
detail_node = member.find("detaileddescription")
brief_node = member.find("briefdescription")
detail_txt = ""
if(detail_node == None and brief_node == None):
return None
if(detail_node is not None):
detail_txt = detail_node.findtext("para")
if(not detail_txt and brief_node != None):
detail_txt = brief_node.findtext("para")
detail_node = brief_node
return detail_node
def build_function(self, member):
retstr = ""
desc_node = self.get_desc_node(member)
if(desc_node is None):
return ""
detail_txt = desc_node.findtext("para")
if(not detail_txt or detail_txt.strip() == "{ignore}"):
return ""
func_id = member.attrib["id"]
page_id, ref_id = get_page(func_id)
retstr += self._build_title(member.findtext("name"), '-', ref = ref_id)
retstr += self.get_text(desc_node)
retstr += LINE_BREAKER
detail_node = member.find("detaileddescription")
if(desc_node != detail_node):
retstr += self.get_text(detail_node)
retstr += self.build_param_list(member.iter("param"), detail_node.iter("parameteritem"))
return retstr
def _build_missed_types_and_structs(self):
fout = open(os.path.join(self.output_dir, self.missing_filename), 'w+')
fout.write(".. contents:: " + LINE_BREAKER)
fout.write(" :local:" + LINE_BREAKER)
fout.write(" :depth: 2" + LINE_BREAKER + LINE_BREAKER)
footnote = ""
while (len(self.missed_types_structs) > 0):
for key, value in self.missed_types_structs.iteritems():
fout.write(self.covert_item(value[0], key, value[1]))
#print(value)
self.missed_types_structs = {}
footnote += self._build_page_ref_notes()
fout.write(footnote)
fout.close()
def _build_page_ref_notes(self):
retstr = LINE_BREAKER
#TODO
for key, value in self.page_references.iteritems():
page, tag = get_page(value[0])
m = re.search("_8h_", page)
if(m):
continue;
rstname = None
anchor = value[2].lower()
if not page in self.references:
self.missed_types_structs[value[0]] = (page, tag)
rstname = os.path.splitext(self.missing_filename)[0]
else:
rstname = self.references[page][2]
anchor = self.references[page][4]
#if(tag and not self.is_github):
# anchor = self.anchor_prefix + "ref-" + tag
retstr += ".. _" + key + ": " + rstname + self.page_ext + "#" + anchor
retstr += LINE_BREAKER + LINE_BREAKER
self.page_references = {}
return retstr
def _build_item_by_id(self, node, id):
retstr = ""
for member in node.iter("memberdef"):
if(member.attrib["id"] != id):
continue
if(member.attrib["kind"] == "enum"):
retstr += self._build_enum(member)
return retstr
def covert_item(self, compound, id, tag):
xml_path = os.path.join(self.doxy_output_dir, "%s.xml" % compound)
print("covert_item: id=%s, name=%s" % (id, xml_path))
obj_root = etree.parse(xml_path).getroot()
retstr = ""
compound = obj_root.find("compounddef")
compound_kind = compound.attrib["kind"]
if(not tag):
retstr += self._build_title(compound.findtext("compoundname"))
if(compound_kind == "class"):
retstr += self._build_class(compound)
elif(compound_kind == "struct"):
retstr += self._build_struct(compound)
else:
retstr += self._build_item_by_id(compound, id)
return retstr
def _build_page(self, compound):
retstr = ""
retstr += self.get_text(compound.find("detaileddescription"))
return retstr
def _build_file(self, compound, type, ref_id, name):
retstr = ""
for member in compound.iter("memberdef"):
if(member.attrib["kind"] == "function" and member.attrib["id"] == ref_id):
retstr += self.build_function(member)
return retstr
def convert_doxy(self, type, name):
#print(name)
file = ref_id = self.name_refid_map[name]
dst_kind = type
if(type == "function"):
file, tag = get_page(ref_id)
dst_kind = "file"
xml_path = os.path.join(self.doxy_output_dir, "%s.xml" % file)
print("convert_doxy: type=%s, name=%s" % (type, xml_path))
obj_root = etree.parse(xml_path).getroot()
compound = obj_root.find("compounddef")
compound_kind = compound.attrib["kind"]
assert(dst_kind == compound_kind)
retstr = ""
if(compound_kind == "class"):
retstr += self._build_class(compound)
elif(compound_kind == "struct"):
retstr += self._build_struct(compound)
elif(compound_kind == "page"):
retstr += self._build_page(compound)
elif(compound_kind == "group"):
retstr += self._build_page(compound)
elif(compound_kind == "file"):
retstr += self._build_file(compound, type, ref_id, name)
return retstr
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("-g", "--github", action="store_true", help="Render the link in format of github wiki.")
parser.add_argument("-e", "--ext", default="", help="extension for github wiki")
parser.add_argument("-i", "--input", default="doxygen", help="Input file path of doxygen output and source rst file.")
parser.add_argument("-o", "--output", default="wikipage", help="Output converted restructured text files to path.")
parser.add_argument("-s", "--struct", default="TypesAndStructures.rest", help="Output of auto generated enum and structures.")
parser.add_argument("-u", "--uml", action="store_true", help="Enable UML, you need to download plantuml.jar from Plantuml and put it to here. http://plantuml.sourceforge.net/")
args = parser.parse_args()
ext = ""
if(len(args.ext) > 0):
ext = ("." + args.ext)
agent = DoxyGen2RST(args.input,
args.output,
args.struct,
is_github = True,
enable_uml = args.uml,
github_ext = ext)
agent.generate()