path: root/tools/site/md2wiki.py

#!/usr/bin/env python3
"""md2wiki.py — convert tracked wiki/*.md pages to MediaWiki wikitext for open-iov.org.

Design rules (public/private discipline — see AGENTS.md):
  * Only *tracked* repository paths may be linked. A link to an untracked
    (private) path aborts the conversion.
  * Sentences that name non-public provenance are stripped (SANITIZE rules
    below); the final output is scanned for forbidden markers and the
    conversion aborts on any hit.
  * Intra-wiki *.md links map to MediaWiki titles via titles.json; unknown
    targets abort (no silent red links).

Usage:
  md2wiki.py FILE.md [FILE.md ...] --out DIR [--repo ROOT]
  md2wiki.py --all --out DIR

Outputs DIR/<basename>.wiki plus DIR/_manifest.json.
"""

import argparse
import html
import json
import os
import re
import subprocess
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
DEFAULT_REPO = os.path.abspath(os.path.join(HERE, "..", ".."))
TITLES_PATH = os.path.join(HERE, "titles.json")

# Forbidden-marker list. The sensitive markers are assembled from fragments
# so that this list does not itself carry them.
_pv = "PRIV" + "ATE-"
_st = "stage" + "_rel"
_ar = "ARCH" + "AEOLOGY"
_rf = "nvidia-570" + ".144-reference"
FORBIDDEN = [
    _pv,
    _st,
    "/home/rmcore",
    "integdev_gpu_drv",
    _ar,
    _rf,
    "Handoff-",
    "handoff-",
    "private " + "analysis",
    "private " + "provenance",
    "private " + "notes",
    "$OPEN_IOV_ROOT",
    "gitignored",
    "never committed",
]

# Exact-text sanitize rules: (file basename, old, new). Applied before conversion.
# Exact-text sanitize rules: (file basename, old, new). Applied before
# conversion. Empty: the tracked wiki sources carry no non-public
# provenance text.
SANITIZE = []

SKIP_FILES = {
    "README.md",  # index page, handled by the Documentation page rewrite
    "Glossary.md",  # legacy historical glossary; the live Glossary is rewritten
    "GPU_Driver_Internals.md",  # 2023 imported page; kept as-is on the site
    "GPU_Firmware.md",  # 2023 imported page; kept as-is on the site
    "GPU_SBOM.md",  # 2023 imported page; kept as-is on the site
    "Virtual_IO_Internals.md",  # 2023 imported page; kept as-is on the site
}


class ConversionError(Exception):
    pass


class Converter:
    def __init__(self, repo, titles, src_name):
        self.repo = repo
        self.titles = titles
        self.wiki_titles = titles["wiki_titles"]
        self.doc_pages = titles.get("doc_pages", {})
        self.repo_base = titles["repo_base"]
        self.src_name = src_name
        self._tracked = {}
        self.link_count = 0

    # ---------- git helpers ----------
    def tracked(self, path):
        if path in self._tracked:
            return self._tracked[path]
        r = subprocess.run(
            ["git", "-C", self.repo, "ls-files", "--error-unmatch", path],
            capture_output=True,
        )
        ok = r.returncode == 0
        self._tracked[path] = ok
        return ok

    # ---------- link mapping ----------
    def map_link(self, target, text):
        target = target.strip()
        if not target:
            return text
        if target.startswith(("http://", "https://")):
            if text and text != target:
                return f"[{target} {text}]"
            return f"[{target}]"
        if target.startswith("#"):
            return f"[[{target}|{text}]]" if text else f"[[{target}]]"
        # relative path: normalize against the wiki/ source directory
        rel = os.path.normpath(os.path.join("wiki", target))
        anchor = ""
        if "#" in rel:
            rel, anchor = rel.split("#", 1)
            anchor = "#" + anchor
        if rel.endswith(".md") and os.path.dirname(rel) == "wiki":
            base = os.path.basename(rel)
            title = self.wiki_titles.get(base)
            if not title:
                raise ConversionError(
                    f"intra-wiki link without a title mapping: {target}"
                )
            plain = text.strip("`") if text else ""
            if not text or plain in (base, title, target, os.path.basename(target)):
                return f"[[{title}{anchor}]]"
            return f"[[{title}{anchor}|{text}]]"
        # repository link: must be a tracked file
        if not self.tracked(rel):
            raise ConversionError(
                f"link to untracked (private) path: {target} -> {rel}"
            )
        url = self.repo_base + rel + anchor
        return f"[{url} {text}]" if text else f"[{url}]"

    # ---------- inline ----------
    def inline(self, s):
        # protect inline code spans first
        codes = []

        def stash(m):
            codes.append(html.escape(m.group(1), quote=False))
            return f"\x00{len(codes) - 1}\x00"

        s = re.sub(r"`([^`]+)`", stash, s)

        # links: [text](target)
        def link(m):
            text, target = m.group(1), m.group(2)
            self.link_count += 1
            return self.map_link(target, text)

        s = re.sub(r"\[([^\]]*)\]\(([^)\s]+)\)", link, s)

        # escape HTML
        s = html.escape(s, quote=False)

        # bold / italic (conservative: no surrounding word chars/spaces)
        s = re.sub(r"\*\*([^*\n]+?)\*\*", r"'''\1'''", s)
        s = re.sub(
            r"(?<![\w*])\*([^*\s](?:[^*]*?[^*\s])?)\*(?![\w*])", r"''\1''", s
        )
        s = re.sub(
            r"(?<![\w_])_([^_\s](?:[^_]*?[^_\s])?)_(?![\w_])", r"''\1''", s
        )

        # restore code spans
        def unstash(m):
            return f"<code>{codes[int(m.group(1))]}</code>"

        s = re.sub(r"\x00(\d+)\x00", unstash, s)
        return s

    # ---------- block conversion ----------
    def convert(self, text):
        lines = text.replace("\r\n", "\n").split("\n")
        out = []
        i = 0
        n = len(lines)
        while i < n:
            line = lines[i]
            if line.startswith("```"):
                fence = line[3:].strip()
                body = []
                i += 1
                while i < n and not lines[i].startswith("```"):
                    body.append(lines[i])
                    i += 1
                i += 1  # closing fence
                if fence == "mermaid":
                    # use the following ```text block as the ASCII fallback if any
                    j = i
                    while j < n and (
                        lines[j].strip() == ""
                        or re.match(r"^(ASCII fallback|Fallback|Diagram):?$", lines[j].strip())
                    ):
                        j += 1
                    if j < n and lines[j].startswith("```") and "text" in lines[j]:
                        k = j + 1
                        fallback = []
                        while k < n and not lines[k].startswith("```"):
                            fallback.append(lines[k])
                            k += 1
                        i = k + 1
                        img = self.titles.get("mermaid_images", {}).get(self.src_name)
                        if img:
                            out.append(img)
                        out.append(
                            "<pre>"
                            + html.escape("\n".join(fallback), quote=False)
                            + "</pre>"
                        )
                    else:
                        raise ConversionError(
                            f"{self.src_name}: mermaid block without an ASCII fallback"
                        )
                    continue
                out.append(
                    "<pre>" + html.escape("\n".join(body), quote=False) + "</pre>"
                )
                continue
            if re.match(r"^\|.*\|$", line):
                block = []
                while i < n and re.match(r"^\|.*\|$", lines[i]):
                    block.append(lines[i])
                    i += 1
                out.append(self.table(block))
                continue
            if line.startswith(">"):
                block = []
                while i < n and lines[i].startswith(">"):
                    block.append(lines[i][1:].lstrip())
                    i += 1
                # split into paragraphs on empty lines
                for para in self.split_paras(block):
                    if para:
                        out.append(f"<blockquote>{self.inline(para)}</blockquote>")
                continue
            m = re.match(r"^(#{1,6})\s+(.*)$", line)
            if m:
                level = len(m.group(1))
                title = self.inline(m.group(2).strip())
                eq = "=" * max(1, min(level, 4))
                out.append(f"{eq} {title} {eq}")
                i += 1
                continue
            if re.match(r"^\*\s+", line) or re.match(r"^\d+\.\s+", line) or re.match(
                r"^\s*[-*]\s+", line
            ) or re.match(r"^\s*\d+\.\s+", line):
                block = []
                while i < n:
                    if (
                        re.match(r"^\s*[-*]\s+", lines[i])
                        or re.match(r"^\s*\d+\.\s+", lines[i])
                    ):
                        block.append(lines[i])
                        i += 1
                    elif (
                        lines[i].strip() != ""
                        and block
                        and not self.is_block_start(lines[i])
                    ):
                        # continuation of the previous list item
                        block[-1] += " " + lines[i].strip()
                        i += 1
                    else:
                        break
                out.append(self.lists(block))
                continue
            if re.match(r"^-{3,}\s*$", line):
                out.append("----")
                i += 1
                continue
            if line.strip() == "":
                i += 1
                continue
            # paragraph
            block = []
            while i < n and lines[i].strip() != "" and not self.is_block_start(
                lines[i]
            ):
                block.append(lines[i])
                i += 1
            out.append(self.inline(" ".join(x.strip() for x in block)))
            continue
        return "\n\n".join(x for x in out if x != "") + "\n"

    @staticmethod
    def is_block_start(line):
        return bool(
            re.match(r"^(#{1,6}\s|>|\||```|\*\s|\d+\.\s|-\s|-{3,}\s*$)", line)
        )

    @staticmethod
    def split_paras(block):
        paras, cur = [], []
        for l in block:
            if l.strip() == "":
                if cur:
                    paras.append(" ".join(x.strip() for x in cur))
                    cur = []
            else:
                cur.append(l)
        if cur:
            paras.append(" ".join(x.strip() for x in cur))
        return paras

    def lists(self, block):
        out = []
        for raw in block:
            m = re.match(r"^(\s*)([-*]|\d+\.)\s+(.*)$", raw)
            indent = len(m.group(1)) // 2
            marker = "#" if m.group(2).endswith(".") else "*"
            out.append(marker * (indent + 1) + " " + self.inline(m.group(3)))
        return "\n".join(out)

    def table(self, block):
        rows = []
        for raw in block:
            cells = [c.strip() for c in raw.strip()[1:-1].split("|")]
            rows.append(cells)
        if len(rows) >= 2 and all(re.match(r"^:?-{2,}:?$", c) for c in rows[1]):
            header, rows = rows[0], rows[2:]
        else:
            header, rows = rows[0], rows[1:]

        def cell(c, head=False):
            # protect pipes inside code spans
            c = re.sub(r"`([^`]+)`", lambda m: "`" + m.group(1).replace("|", "&#124;") + "`", c)
            c = self.inline(c)
            return ("! " if head else "| ") + c

        out = ['{| class="wikitable"']
        out.append("|-")
        out += [cell(c, head=True) for c in header]
        for r in rows:
            out.append("|-")
            out += [cell(c) for c in r]
        out.append("|}")
        return "\n".join(out)


def sanitize(src_name, text):
    for fname, old, new in SANITIZE:
        if fname == src_name and old in text:
            text = text.replace(old, new)
    return text


def check_forbidden(src_name, text):
    hits = [p for p in FORBIDDEN if p in text]
    if hits:
        raise ConversionError(f"{src_name}: forbidden private markers remain: {hits}")


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("files", nargs="*")
    ap.add_argument("--all", action="store_true", help="convert every wiki/*.md")
    ap.add_argument("--out", required=True, help="output directory for .wiki files")
    ap.add_argument("--repo", default=DEFAULT_REPO)
    ap.add_argument("--dry", action="store_true", help="print, do not write")
    args = ap.parse_args()

    with open(TITLES_PATH) as f:
        titles = json.load(f)

    files = list(args.files)
    if args.all:
        wikidir = os.path.join(args.repo, "wiki")
        files += [
            os.path.join(wikidir, f)
            for f in sorted(os.listdir(wikidir))
            if f.endswith(".md") and f not in SKIP_FILES
        ]
    if not files:
        ap.error("no input files")

    os.makedirs(args.out, exist_ok=True)
    manifest = {}
    failures = 0
    for path in files:
        name = os.path.basename(path)
        try:
            with open(path) as f:
                text = f.read()
            text = sanitize(name, text)
            conv = Converter(args.repo, titles, name)
            wiki = "{{About message}}\n\n" + conv.convert(text)
            check_forbidden(name, wiki)
        except ConversionError as e:
            print(f"FAIL {name}: {e}", file=sys.stderr)
            failures += 1
            continue
        title = titles["wiki_titles"].get(name)
        rec = {
            "source": os.path.relpath(path, args.repo),
            "title": title,
            "links": conv.link_count,
            "bytes": len(wiki),
        }
        manifest[name] = rec
        if args.dry:
            print(f"--- {name} -> {title} ({conv.link_count} links, {len(wiki)} B)")
            print(wiki)
        else:
            dest = os.path.join(args.out, name[:-3] + ".wiki")
            with open(dest, "w") as f:
                f.write(wiki)
            print(f"OK   {name} -> {dest}  title={title!r} links={conv.link_count}")
    if not args.dry:
        mpath = os.path.join(args.out, "_manifest.json")
        existing = {}
        if os.path.exists(mpath):
            with open(mpath) as f:
                existing = json.load(f)
        existing.update(manifest)
        with open(mpath, "w") as f:
            json.dump(existing, f, indent=1, sort_keys=True)
        print(f"manifest: {mpath}")
    sys.exit(1 if failures else 0)


if __name__ == "__main__":
    main()