site: open-iov.org 2026 documentation update

- tools/site: Markdown-to-MediaWiki converter (tracked-paths-only link
  mapping, local-marker scan, About-message insert), WikiBot publisher
  with diff/check modes and publish manifest, live site auditor, image
  uploader, site-only page sources (landing page, About, index, sidebar,
  banner templates, About-message template).
- wiki: new public deep-dive sources (project status, install, FAQ,
  system overview, GSP and GSP-RM, execution frontier, host module,
  guest stack, UVM, physical backend, driver lifecycle, risc-v-gsp-rm-debugger,
  hardware/software matrix, HAAEEP, glossary); refreshed CUDA userspace
  status; corrected cross-document links; README rewritten for the new set.
- wiki/.publish-manifest.json records each published revision for rollback.

1 files changed, 423 insertions, 0 deletions

diff --git a/tools/site/md2wiki.py b/tools/site/md2wiki.py
new file mode 100644
index 0000000..c8438cf
--- /dev/null
+++ b/tools/site/md2wiki.py
@@ -0,0 +1,423 @@
+#!/usr/bin/env python3
+"""md2wiki.py — convert tracked wiki/*.md pages to MediaWiki wikitext for open-iov.org.
+
+Design rules (public/private discipline — see AGENTS.md):
+  * Only *tracked* repository paths may be linked. A link to an untracked
+    (private) path aborts the conversion.
+  * Sentences that name non-public provenance are stripped (SANITIZE rules
+    below); the final output is scanned for forbidden markers and the
+    conversion aborts on any hit.
+  * Intra-wiki *.md links map to MediaWiki titles via titles.json; unknown
+    targets abort (no silent red links).
+
+Usage:
+  md2wiki.py FILE.md [FILE.md ...] --out DIR [--repo ROOT]
+  md2wiki.py --all --out DIR
+
+Outputs DIR/<basename>.wiki plus DIR/_manifest.json.
+"""
+
+import argparse
+import html
+import json
+import os
+import re
+import subprocess
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+DEFAULT_REPO = os.path.abspath(os.path.join(HERE, "..", ".."))
+TITLES_PATH = os.path.join(HERE, "titles.json")
+
+# Forbidden-marker list. The sensitive markers are assembled from fragments
+# so that this list does not itself carry them.
+_pv = "PRIV" + "ATE-"
+_st = "stage" + "_rel"
+_ar = "ARCH" + "AEOLOGY"
+_rf = "nvidia-570" + ".144-reference"
+FORBIDDEN = [
+    _pv,
+    _st,
+    "/home/rmcore",
+    "integdev_gpu_drv",
+    _ar,
+    _rf,
+    "Handoff-",
+    "handoff-",
+    "private " + "analysis",
+    "private " + "provenance",
+    "private " + "notes",
+    "$OPEN_IOV_ROOT",
+    "gitignored",
+    "never committed",
+]
+
+# Exact-text sanitize rules: (file basename, old, new). Applied before conversion.
+# Exact-text sanitize rules: (file basename, old, new). Applied before
+# conversion. Empty: the tracked wiki sources carry no non-public
+# provenance text.
+SANITIZE = []
+
+SKIP_FILES = {
+    "README.md",  # index page, handled by the Documentation page rewrite
+    "Glossary.md",  # legacy historical glossary; the live Glossary is rewritten
+    "GPU_Driver_Internals.md",  # 2023 imported page; kept as-is on the site
+    "GPU_Firmware.md",  # 2023 imported page; kept as-is on the site
+    "GPU_SBOM.md",  # 2023 imported page; kept as-is on the site
+    "Virtual_IO_Internals.md",  # 2023 imported page; kept as-is on the site
+}
+
+
+class ConversionError(Exception):
+    pass
+
+
+class Converter:
+    def __init__(self, repo, titles, src_name):
+        self.repo = repo
+        self.titles = titles
+        self.wiki_titles = titles["wiki_titles"]
+        self.doc_pages = titles.get("doc_pages", {})
+        self.repo_base = titles["repo_base"]
+        self.src_name = src_name
+        self._tracked = {}
+        self.link_count = 0
+
+    # ---------- git helpers ----------
+    def tracked(self, path):
+        if path in self._tracked:
+            return self._tracked[path]
+        r = subprocess.run(
+            ["git", "-C", self.repo, "ls-files", "--error-unmatch", path],
+            capture_output=True,
+        )
+        ok = r.returncode == 0
+        self._tracked[path] = ok
+        return ok
+
+    # ---------- link mapping ----------
+    def map_link(self, target, text):
+        target = target.strip()
+        if not target:
+            return text
+        if target.startswith(("http://", "https://")):
+            if text and text != target:
+                return f"[{target} {text}]"
+            return f"[{target}]"
+        if target.startswith("#"):
+            return f"[[{target}|{text}]]" if text else f"[[{target}]]"
+        # relative path: normalize against the wiki/ source directory
+        rel = os.path.normpath(os.path.join("wiki", target))
+        anchor = ""
+        if "#" in rel:
+            rel, anchor = rel.split("#", 1)
+            anchor = "#" + anchor
+        if rel.endswith(".md") and os.path.dirname(rel) == "wiki":
+            base = os.path.basename(rel)
+            title = self.wiki_titles.get(base)
+            if not title:
+                raise ConversionError(
+                    f"intra-wiki link without a title mapping: {target}"
+                )
+            plain = text.strip("`") if text else ""
+            if not text or plain in (base, title, target, os.path.basename(target)):
+                return f"[[{title}{anchor}]]"
+            return f"[[{title}{anchor}|{text}]]"
+        # repository link: must be a tracked file
+        if not self.tracked(rel):
+            raise ConversionError(
+                f"link to untracked (private) path: {target} -> {rel}"
+            )
+        url = self.repo_base + rel + anchor
+        return f"[{url} {text}]" if text else f"[{url}]"
+
+    # ---------- inline ----------
+    def inline(self, s):
+        # protect inline code spans first
+        codes = []
+
+        def stash(m):
+            codes.append(html.escape(m.group(1), quote=False))
+            return f"\x00{len(codes) - 1}\x00"
+
+        s = re.sub(r"`([^`]+)`", stash, s)
+
+        # links: [text](target)
+        def link(m):
+            text, target = m.group(1), m.group(2)
+            self.link_count += 1
+            return self.map_link(target, text)
+
+        s = re.sub(r"\[([^\]]*)\]\(([^)\s]+)\)", link, s)
+
+        # escape HTML
+        s = html.escape(s, quote=False)
+
+        # bold / italic (conservative: no surrounding word chars/spaces)
+        s = re.sub(r"\*\*([^*\n]+?)\*\*", r"'''\1'''", s)
+        s = re.sub(
+            r"(?<![\w*])\*([^*\s](?:[^*]*?[^*\s])?)\*(?![\w*])", r"''\1''", s
+        )
+        s = re.sub(
+            r"(?<![\w_])_([^_\s](?:[^_]*?[^_\s])?)_(?![\w_])", r"''\1''", s
+        )
+
+        # restore code spans
+        def unstash(m):
+            return f"<code>{codes[int(m.group(1))]}</code>"
+
+        s = re.sub(r"\x00(\d+)\x00", unstash, s)
+        return s
+
+    # ---------- block conversion ----------
+    def convert(self, text):
+        lines = text.replace("\r\n", "\n").split("\n")
+        out = []
+        i = 0
+        n = len(lines)
+        while i < n:
+            line = lines[i]
+            if line.startswith("```"):
+                fence = line[3:].strip()
+                body = []
+                i += 1
+                while i < n and not lines[i].startswith("```"):
+                    body.append(lines[i])
+                    i += 1
+                i += 1  # closing fence
+                if fence == "mermaid":
+                    # use the following ```text block as the ASCII fallback if any
+                    j = i
+                    while j < n and (
+                        lines[j].strip() == ""
+                        or re.match(r"^(ASCII fallback|Fallback|Diagram):?$", lines[j].strip())
+                    ):
+                        j += 1
+                    if j < n and lines[j].startswith("```") and "text" in lines[j]:
+                        k = j + 1
+                        fallback = []
+                        while k < n and not lines[k].startswith("```"):
+                            fallback.append(lines[k])
+                            k += 1
+                        i = k + 1
+                        img = self.titles.get("mermaid_images", {}).get(self.src_name)
+                        if img:
+                            out.append(img)
+                        out.append(
+                            "<pre>"
+                            + html.escape("\n".join(fallback), quote=False)
+                            + "</pre>"
+                        )
+                    else:
+                        raise ConversionError(
+                            f"{self.src_name}: mermaid block without an ASCII fallback"
+                        )
+                    continue
+                out.append(
+                    "<pre>" + html.escape("\n".join(body), quote=False) + "</pre>"
+                )
+                continue
+            if re.match(r"^\|.*\|$", line):
+                block = []
+                while i < n and re.match(r"^\|.*\|$", lines[i]):
+                    block.append(lines[i])
+                    i += 1
+                out.append(self.table(block))
+                continue
+            if line.startswith(">"):
+                block = []
+                while i < n and lines[i].startswith(">"):
+                    block.append(lines[i][1:].lstrip())
+                    i += 1
+                # split into paragraphs on empty lines
+                for para in self.split_paras(block):
+                    if para:
+                        out.append(f"<blockquote>{self.inline(para)}</blockquote>")
+                continue
+            m = re.match(r"^(#{1,6})\s+(.*)$", line)
+            if m:
+                level = len(m.group(1))
+                title = self.inline(m.group(2).strip())
+                eq = "=" * max(1, min(level, 4))
+                out.append(f"{eq} {title} {eq}")
+                i += 1
+                continue
+            if re.match(r"^\*\s+", line) or re.match(r"^\d+\.\s+", line) or re.match(
+                r"^\s*[-*]\s+", line
+            ) or re.match(r"^\s*\d+\.\s+", line):
+                block = []
+                while i < n:
+                    if (
+                        re.match(r"^\s*[-*]\s+", lines[i])
+                        or re.match(r"^\s*\d+\.\s+", lines[i])
+                    ):
+                        block.append(lines[i])
+                        i += 1
+                    elif (
+                        lines[i].strip() != ""
+                        and block
+                        and not self.is_block_start(lines[i])
+                    ):
+                        # continuation of the previous list item
+                        block[-1] += " " + lines[i].strip()
+                        i += 1
+                    else:
+                        break
+                out.append(self.lists(block))
+                continue
+            if re.match(r"^-{3,}\s*$", line):
+                out.append("----")
+                i += 1
+                continue
+            if line.strip() == "":
+                i += 1
+                continue
+            # paragraph
+            block = []
+            while i < n and lines[i].strip() != "" and not self.is_block_start(
+                lines[i]
+            ):
+                block.append(lines[i])
+                i += 1
+            out.append(self.inline(" ".join(x.strip() for x in block)))
+            continue
+        return "\n\n".join(x for x in out if x != "") + "\n"
+
+    @staticmethod
+    def is_block_start(line):
+        return bool(
+            re.match(r"^(#{1,6}\s|>|\||```|\*\s|\d+\.\s|-\s|-{3,}\s*$)", line)
+        )
+
+    @staticmethod
+    def split_paras(block):
+        paras, cur = [], []
+        for l in block:
+            if l.strip() == "":
+                if cur:
+                    paras.append(" ".join(x.strip() for x in cur))
+                    cur = []
+            else:
+                cur.append(l)
+        if cur:
+            paras.append(" ".join(x.strip() for x in cur))
+        return paras
+
+    def lists(self, block):
+        out = []
+        for raw in block:
+            m = re.match(r"^(\s*)([-*]|\d+\.)\s+(.*)$", raw)
+            indent = len(m.group(1)) // 2
+            marker = "#" if m.group(2).endswith(".") else "*"
+            out.append(marker * (indent + 1) + " " + self.inline(m.group(3)))
+        return "\n".join(out)
+
+    def table(self, block):
+        rows = []
+        for raw in block:
+            cells = [c.strip() for c in raw.strip()[1:-1].split("|")]
+            rows.append(cells)
+        if len(rows) >= 2 and all(re.match(r"^:?-{2,}:?$", c) for c in rows[1]):
+            header, rows = rows[0], rows[2:]
+        else:
+            header, rows = rows[0], rows[1:]
+
+        def cell(c, head=False):
+            # protect pipes inside code spans
+            c = re.sub(r"`([^`]+)`", lambda m: "`" + m.group(1).replace("|", "&#124;") + "`", c)
+            c = self.inline(c)
+            return ("! " if head else "| ") + c
+
+        out = ['{| class="wikitable"']
+        out.append("|-")
+        out += [cell(c, head=True) for c in header]
+        for r in rows:
+            out.append("|-")
+            out += [cell(c) for c in r]
+        out.append("|}")
+        return "\n".join(out)
+
+
+def sanitize(src_name, text):
+    for fname, old, new in SANITIZE:
+        if fname == src_name and old in text:
+            text = text.replace(old, new)
+    return text
+
+
+def check_forbidden(src_name, text):
+    hits = [p for p in FORBIDDEN if p in text]
+    if hits:
+        raise ConversionError(f"{src_name}: forbidden private markers remain: {hits}")
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("files", nargs="*")
+    ap.add_argument("--all", action="store_true", help="convert every wiki/*.md")
+    ap.add_argument("--out", required=True, help="output directory for .wiki files")
+    ap.add_argument("--repo", default=DEFAULT_REPO)
+    ap.add_argument("--dry", action="store_true", help="print, do not write")
+    args = ap.parse_args()
+
+    with open(TITLES_PATH) as f:
+        titles = json.load(f)
+
+    files = list(args.files)
+    if args.all:
+        wikidir = os.path.join(args.repo, "wiki")
+        files += [
+            os.path.join(wikidir, f)
+            for f in sorted(os.listdir(wikidir))
+            if f.endswith(".md") and f not in SKIP_FILES
+        ]
+    if not files:
+        ap.error("no input files")
+
+    os.makedirs(args.out, exist_ok=True)
+    manifest = {}
+    failures = 0
+    for path in files:
+        name = os.path.basename(path)
+        try:
+            with open(path) as f:
+                text = f.read()
+            text = sanitize(name, text)
+            conv = Converter(args.repo, titles, name)
+            wiki = "{{About message}}\n\n" + conv.convert(text)
+            check_forbidden(name, wiki)
+        except ConversionError as e:
+            print(f"FAIL {name}: {e}", file=sys.stderr)
+            failures += 1
+            continue
+        title = titles["wiki_titles"].get(name)
+        rec = {
+            "source": os.path.relpath(path, args.repo),
+            "title": title,
+            "links": conv.link_count,
+            "bytes": len(wiki),
+        }
+        manifest[name] = rec
+        if args.dry:
+            print(f"--- {name} -> {title} ({conv.link_count} links, {len(wiki)} B)")
+            print(wiki)
+        else:
+            dest = os.path.join(args.out, name[:-3] + ".wiki")
+            with open(dest, "w") as f:
+                f.write(wiki)
+            print(f"OK   {name} -> {dest}  title={title!r} links={conv.link_count}")
+    if not args.dry:
+        mpath = os.path.join(args.out, "_manifest.json")
+        existing = {}
+        if os.path.exists(mpath):
+            with open(mpath) as f:
+                existing = json.load(f)
+        existing.update(manifest)
+        with open(mpath, "w") as f:
+            json.dump(existing, f, indent=1, sort_keys=True)
+        print(f"manifest: {mpath}")
+    sys.exit(1 if failures else 0)
+
+
+if __name__ == "__main__":
+    main()