diff options
Diffstat (limited to 'tools/site/md2wiki.py')
| -rw-r--r-- | tools/site/md2wiki.py | 423 |
1 files changed, 423 insertions, 0 deletions
diff --git a/tools/site/md2wiki.py b/tools/site/md2wiki.py new file mode 100644 index 0000000..c8438cf --- /dev/null +++ b/tools/site/md2wiki.py @@ -0,0 +1,423 @@ +#!/usr/bin/env python3 +"""md2wiki.py — convert tracked wiki/*.md pages to MediaWiki wikitext for open-iov.org. + +Design rules (public/private discipline — see AGENTS.md): + * Only *tracked* repository paths may be linked. A link to an untracked + (private) path aborts the conversion. + * Sentences that name non-public provenance are stripped (SANITIZE rules + below); the final output is scanned for forbidden markers and the + conversion aborts on any hit. + * Intra-wiki *.md links map to MediaWiki titles via titles.json; unknown + targets abort (no silent red links). + +Usage: + md2wiki.py FILE.md [FILE.md ...] --out DIR [--repo ROOT] + md2wiki.py --all --out DIR + +Outputs DIR/<basename>.wiki plus DIR/_manifest.json. +""" + +import argparse +import html +import json +import os +import re +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +DEFAULT_REPO = os.path.abspath(os.path.join(HERE, "..", "..")) +TITLES_PATH = os.path.join(HERE, "titles.json") + +# Forbidden-marker list. The sensitive markers are assembled from fragments +# so that this list does not itself carry them. +_pv = "PRIV" + "ATE-" +_st = "stage" + "_rel" +_ar = "ARCH" + "AEOLOGY" +_rf = "nvidia-570" + ".144-reference" +FORBIDDEN = [ + _pv, + _st, + "/home/rmcore", + "integdev_gpu_drv", + _ar, + _rf, + "Handoff-", + "handoff-", + "private " + "analysis", + "private " + "provenance", + "private " + "notes", + "$OPEN_IOV_ROOT", + "gitignored", + "never committed", +] + +# Exact-text sanitize rules: (file basename, old, new). Applied before conversion. +# Exact-text sanitize rules: (file basename, old, new). Applied before +# conversion. Empty: the tracked wiki sources carry no non-public +# provenance text. +SANITIZE = [] + +SKIP_FILES = { + "README.md", # index page, handled by the Documentation page rewrite + "Glossary.md", # legacy historical glossary; the live Glossary is rewritten + "GPU_Driver_Internals.md", # 2023 imported page; kept as-is on the site + "GPU_Firmware.md", # 2023 imported page; kept as-is on the site + "GPU_SBOM.md", # 2023 imported page; kept as-is on the site + "Virtual_IO_Internals.md", # 2023 imported page; kept as-is on the site +} + + +class ConversionError(Exception): + pass + + +class Converter: + def __init__(self, repo, titles, src_name): + self.repo = repo + self.titles = titles + self.wiki_titles = titles["wiki_titles"] + self.doc_pages = titles.get("doc_pages", {}) + self.repo_base = titles["repo_base"] + self.src_name = src_name + self._tracked = {} + self.link_count = 0 + + # ---------- git helpers ---------- + def tracked(self, path): + if path in self._tracked: + return self._tracked[path] + r = subprocess.run( + ["git", "-C", self.repo, "ls-files", "--error-unmatch", path], + capture_output=True, + ) + ok = r.returncode == 0 + self._tracked[path] = ok + return ok + + # ---------- link mapping ---------- + def map_link(self, target, text): + target = target.strip() + if not target: + return text + if target.startswith(("http://", "https://")): + if text and text != target: + return f"[{target} {text}]" + return f"[{target}]" + if target.startswith("#"): + return f"[[{target}|{text}]]" if text else f"[[{target}]]" + # relative path: normalize against the wiki/ source directory + rel = os.path.normpath(os.path.join("wiki", target)) + anchor = "" + if "#" in rel: + rel, anchor = rel.split("#", 1) + anchor = "#" + anchor + if rel.endswith(".md") and os.path.dirname(rel) == "wiki": + base = os.path.basename(rel) + title = self.wiki_titles.get(base) + if not title: + raise ConversionError( + f"intra-wiki link without a title mapping: {target}" + ) + plain = text.strip("`") if text else "" + if not text or plain in (base, title, target, os.path.basename(target)): + return f"[[{title}{anchor}]]" + return f"[[{title}{anchor}|{text}]]" + # repository link: must be a tracked file + if not self.tracked(rel): + raise ConversionError( + f"link to untracked (private) path: {target} -> {rel}" + ) + url = self.repo_base + rel + anchor + return f"[{url} {text}]" if text else f"[{url}]" + + # ---------- inline ---------- + def inline(self, s): + # protect inline code spans first + codes = [] + + def stash(m): + codes.append(html.escape(m.group(1), quote=False)) + return f"\x00{len(codes) - 1}\x00" + + s = re.sub(r"`([^`]+)`", stash, s) + + # links: [text](target) + def link(m): + text, target = m.group(1), m.group(2) + self.link_count += 1 + return self.map_link(target, text) + + s = re.sub(r"\[([^\]]*)\]\(([^)\s]+)\)", link, s) + + # escape HTML + s = html.escape(s, quote=False) + + # bold / italic (conservative: no surrounding word chars/spaces) + s = re.sub(r"\*\*([^*\n]+?)\*\*", r"'''\1'''", s) + s = re.sub( + r"(?<![\w*])\*([^*\s](?:[^*]*?[^*\s])?)\*(?![\w*])", r"''\1''", s + ) + s = re.sub( + r"(?<![\w_])_([^_\s](?:[^_]*?[^_\s])?)_(?![\w_])", r"''\1''", s + ) + + # restore code spans + def unstash(m): + return f"<code>{codes[int(m.group(1))]}</code>" + + s = re.sub(r"\x00(\d+)\x00", unstash, s) + return s + + # ---------- block conversion ---------- + def convert(self, text): + lines = text.replace("\r\n", "\n").split("\n") + out = [] + i = 0 + n = len(lines) + while i < n: + line = lines[i] + if line.startswith("```"): + fence = line[3:].strip() + body = [] + i += 1 + while i < n and not lines[i].startswith("```"): + body.append(lines[i]) + i += 1 + i += 1 # closing fence + if fence == "mermaid": + # use the following ```text block as the ASCII fallback if any + j = i + while j < n and ( + lines[j].strip() == "" + or re.match(r"^(ASCII fallback|Fallback|Diagram):?$", lines[j].strip()) + ): + j += 1 + if j < n and lines[j].startswith("```") and "text" in lines[j]: + k = j + 1 + fallback = [] + while k < n and not lines[k].startswith("```"): + fallback.append(lines[k]) + k += 1 + i = k + 1 + img = self.titles.get("mermaid_images", {}).get(self.src_name) + if img: + out.append(img) + out.append( + "<pre>" + + html.escape("\n".join(fallback), quote=False) + + "</pre>" + ) + else: + raise ConversionError( + f"{self.src_name}: mermaid block without an ASCII fallback" + ) + continue + out.append( + "<pre>" + html.escape("\n".join(body), quote=False) + "</pre>" + ) + continue + if re.match(r"^\|.*\|$", line): + block = [] + while i < n and re.match(r"^\|.*\|$", lines[i]): + block.append(lines[i]) + i += 1 + out.append(self.table(block)) + continue + if line.startswith(">"): + block = [] + while i < n and lines[i].startswith(">"): + block.append(lines[i][1:].lstrip()) + i += 1 + # split into paragraphs on empty lines + for para in self.split_paras(block): + if para: + out.append(f"<blockquote>{self.inline(para)}</blockquote>") + continue + m = re.match(r"^(#{1,6})\s+(.*)$", line) + if m: + level = len(m.group(1)) + title = self.inline(m.group(2).strip()) + eq = "=" * max(1, min(level, 4)) + out.append(f"{eq} {title} {eq}") + i += 1 + continue + if re.match(r"^\*\s+", line) or re.match(r"^\d+\.\s+", line) or re.match( + r"^\s*[-*]\s+", line + ) or re.match(r"^\s*\d+\.\s+", line): + block = [] + while i < n: + if ( + re.match(r"^\s*[-*]\s+", lines[i]) + or re.match(r"^\s*\d+\.\s+", lines[i]) + ): + block.append(lines[i]) + i += 1 + elif ( + lines[i].strip() != "" + and block + and not self.is_block_start(lines[i]) + ): + # continuation of the previous list item + block[-1] += " " + lines[i].strip() + i += 1 + else: + break + out.append(self.lists(block)) + continue + if re.match(r"^-{3,}\s*$", line): + out.append("----") + i += 1 + continue + if line.strip() == "": + i += 1 + continue + # paragraph + block = [] + while i < n and lines[i].strip() != "" and not self.is_block_start( + lines[i] + ): + block.append(lines[i]) + i += 1 + out.append(self.inline(" ".join(x.strip() for x in block))) + continue + return "\n\n".join(x for x in out if x != "") + "\n" + + @staticmethod + def is_block_start(line): + return bool( + re.match(r"^(#{1,6}\s|>|\||```|\*\s|\d+\.\s|-\s|-{3,}\s*$)", line) + ) + + @staticmethod + def split_paras(block): + paras, cur = [], [] + for l in block: + if l.strip() == "": + if cur: + paras.append(" ".join(x.strip() for x in cur)) + cur = [] + else: + cur.append(l) + if cur: + paras.append(" ".join(x.strip() for x in cur)) + return paras + + def lists(self, block): + out = [] + for raw in block: + m = re.match(r"^(\s*)([-*]|\d+\.)\s+(.*)$", raw) + indent = len(m.group(1)) // 2 + marker = "#" if m.group(2).endswith(".") else "*" + out.append(marker * (indent + 1) + " " + self.inline(m.group(3))) + return "\n".join(out) + + def table(self, block): + rows = [] + for raw in block: + cells = [c.strip() for c in raw.strip()[1:-1].split("|")] + rows.append(cells) + if len(rows) >= 2 and all(re.match(r"^:?-{2,}:?$", c) for c in rows[1]): + header, rows = rows[0], rows[2:] + else: + header, rows = rows[0], rows[1:] + + def cell(c, head=False): + # protect pipes inside code spans + c = re.sub(r"`([^`]+)`", lambda m: "`" + m.group(1).replace("|", "|") + "`", c) + c = self.inline(c) + return ("! " if head else "| ") + c + + out = ['{| class="wikitable"'] + out.append("|-") + out += [cell(c, head=True) for c in header] + for r in rows: + out.append("|-") + out += [cell(c) for c in r] + out.append("|}") + return "\n".join(out) + + +def sanitize(src_name, text): + for fname, old, new in SANITIZE: + if fname == src_name and old in text: + text = text.replace(old, new) + return text + + +def check_forbidden(src_name, text): + hits = [p for p in FORBIDDEN if p in text] + if hits: + raise ConversionError(f"{src_name}: forbidden private markers remain: {hits}") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("files", nargs="*") + ap.add_argument("--all", action="store_true", help="convert every wiki/*.md") + ap.add_argument("--out", required=True, help="output directory for .wiki files") + ap.add_argument("--repo", default=DEFAULT_REPO) + ap.add_argument("--dry", action="store_true", help="print, do not write") + args = ap.parse_args() + + with open(TITLES_PATH) as f: + titles = json.load(f) + + files = list(args.files) + if args.all: + wikidir = os.path.join(args.repo, "wiki") + files += [ + os.path.join(wikidir, f) + for f in sorted(os.listdir(wikidir)) + if f.endswith(".md") and f not in SKIP_FILES + ] + if not files: + ap.error("no input files") + + os.makedirs(args.out, exist_ok=True) + manifest = {} + failures = 0 + for path in files: + name = os.path.basename(path) + try: + with open(path) as f: + text = f.read() + text = sanitize(name, text) + conv = Converter(args.repo, titles, name) + wiki = "{{About message}}\n\n" + conv.convert(text) + check_forbidden(name, wiki) + except ConversionError as e: + print(f"FAIL {name}: {e}", file=sys.stderr) + failures += 1 + continue + title = titles["wiki_titles"].get(name) + rec = { + "source": os.path.relpath(path, args.repo), + "title": title, + "links": conv.link_count, + "bytes": len(wiki), + } + manifest[name] = rec + if args.dry: + print(f"--- {name} -> {title} ({conv.link_count} links, {len(wiki)} B)") + print(wiki) + else: + dest = os.path.join(args.out, name[:-3] + ".wiki") + with open(dest, "w") as f: + f.write(wiki) + print(f"OK {name} -> {dest} title={title!r} links={conv.link_count}") + if not args.dry: + mpath = os.path.join(args.out, "_manifest.json") + existing = {} + if os.path.exists(mpath): + with open(mpath) as f: + existing = json.load(f) + existing.update(manifest) + with open(mpath, "w") as f: + json.dump(existing, f, indent=1, sort_keys=True) + print(f"manifest: {mpath}") + sys.exit(1 if failures else 0) + + +if __name__ == "__main__": + main() |