From cff1461ef459e98dea063abc72ae0819fea7b5ce Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Mon, 12 Oct 2026 00:32:00 -0400 Subject: site: open-iov.org 2026 documentation update - tools/site: Markdown-to-MediaWiki converter (tracked-paths-only link mapping, local-marker scan, About-message insert), WikiBot publisher with diff/check modes and publish manifest, live site auditor, image uploader, site-only page sources (landing page, About, index, sidebar, banner templates, About-message template). - wiki: new public deep-dive sources (project status, install, FAQ, system overview, GSP and GSP-RM, execution frontier, host module, guest stack, UVM, physical backend, driver lifecycle, risc-v-gsp-rm-debugger, hardware/software matrix, HAAEEP, glossary); refreshed CUDA userspace status; corrected cross-document links; README rewritten for the new set. - wiki/.publish-manifest.json records each published revision for rollback. --- tools/site/README.md | 54 ++++ tools/site/check_site.py | 133 +++++++++ tools/site/md2wiki.py | 423 +++++++++++++++++++++++++++ tools/site/pages/Community.wiki | 15 + tools/site/pages/Documentation.wiki | 50 ++++ tools/site/pages/Main-Page.wiki | 78 +++++ tools/site/pages/MediaWiki-Sidebar.wiki | 31 ++ tools/site/pages/Open-IOV-About.wiki | 29 ++ tools/site/pages/Template-About-message.wiki | 11 + tools/site/pages/Template-Historical.wiki | 7 + tools/site/pages/Template-Superseded.wiki | 7 + tools/site/publish.py | 402 +++++++++++++++++++++++++ tools/site/titles.json | 59 ++++ tools/site/upload_file.py | 132 +++++++++ 14 files changed, 1431 insertions(+) create mode 100644 tools/site/README.md create mode 100644 tools/site/check_site.py create mode 100644 tools/site/md2wiki.py create mode 100644 tools/site/pages/Community.wiki create mode 100644 tools/site/pages/Documentation.wiki create mode 100644 tools/site/pages/Main-Page.wiki create mode 100644 tools/site/pages/MediaWiki-Sidebar.wiki create mode 100644 tools/site/pages/Open-IOV-About.wiki create mode 100644 tools/site/pages/Template-About-message.wiki create mode 100644 tools/site/pages/Template-Historical.wiki create mode 100644 tools/site/pages/Template-Superseded.wiki create mode 100644 tools/site/publish.py create mode 100644 tools/site/titles.json create mode 100644 tools/site/upload_file.py (limited to 'tools') diff --git a/tools/site/README.md b/tools/site/README.md new file mode 100644 index 0000000..2e9de5d --- /dev/null +++ b/tools/site/README.md @@ -0,0 +1,54 @@ +# tools/site — open-iov.org publishing toolchain + +Converts the tracked `wiki/*.md` sources to MediaWiki wikitext and publishes +them to https://open-iov.org. No secrets live in this directory. + +## Files + +| Tool | Purpose | +|---|---| +| `md2wiki.py` | Markdown → MediaWiki converter. Maps `../docs/...` links to the public repository (tracked files only — untracked/private paths abort), maps intra-wiki `*.md` links to page titles via `titles.json`, renders diagram image references, strips non-public-provenance sentences, and refuses output containing forbidden markers. | +| `publish.py` | WikiBot client: `login`, `check --all`, `diff`, `publish`, `publish-all`, `delete`, `move`. Scans before writing; records every revision in `wiki/.publish-manifest.json`. | +| `check_site.py` | Live audit: forbidden markers, raw links, unresolved internal links across every page. | +| `upload_file.py` | Upload an image (e.g. a rendered diagram) as WikiBot. | +| `titles.json` | Page title map (wiki source → wiki page), canonical-doc → site-page map, diagram image map. | +| `pages/` | Site-only pages (Main Page, About, Community, sidebar, templates) as tracked wikitext. | + +## Credentials + +`~/.config/open-iov/wiki-bot.env` (mode 0600, outside the repository): + +``` +WIKI_BOT_USER=... +WIKI_BOT_PASS=... +``` + +## Typical flow + +```bash +# edit wiki/.md, then: +python3 tools/site/md2wiki.py --all --out /tmp/open-iov-site +python3 tools/site/publish.py diff --title "FAQ" --file /tmp/open-iov-site/FAQ.wiki +python3 tools/site/publish.py publish --title "FAQ" --file /tmp/open-iov-site/FAQ.wiki \ + --summary "Update from tracked wiki source" +python3 tools/site/check_site.py +``` + +Site-only pages: + +```bash +python3 tools/site/publish.py publish --title "Main Page" --file tools/site/pages/Main-Page.wiki --summary "..." +``` + +Diagram images: render with `mermaid-cli` (`mmdc`, see `titles.json` +`mermaid_images` for the mapping), upload with `upload_file.py`. + +## Rules + +* Only tracked repository paths are ever linked; the converter fails loudly on + anything else. +* Non-public provenance (notes files, study sources) must never appear in + site output; both `md2wiki.py` and `publish.py` enforce a forbidden-marker + list. +* Every edit goes through the bot with a descriptive summary and is recorded + with its old/new revision ids for rollback. diff --git a/tools/site/check_site.py b/tools/site/check_site.py new file mode 100644 index 0000000..e56d0c0 --- /dev/null +++ b/tools/site/check_site.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""check_site.py — audit the live open-iov.org content. + +Checks, for every page: + * forbidden private markers (same list as publish.py) + * raw relative/.md links and raw mermaid blocks + * internal [[links]] that do not resolve to an existing page + +Usage: check_site.py # full audit + check_site.py --pages "Main Page" "Documentation" +""" + +import argparse +import json +import re +import sys +import time +import urllib.parse +import urllib.request + +API = "https://open-iov.org/api.php" +UA = {"User-Agent": "OpenIOV-WikiBot/1.0"} + +# Forbidden-marker list. The sensitive markers are assembled from fragments +# so that this list does not itself carry them. +_pv = "PRIV" + "ATE-" +_st = "stage" + "_rel" +_ar = "ARCH" + "AEOLOGY" +_rf = "nvidia-570" + ".144-reference" +FORBIDDEN = [ + _pv, _st, "/home/rmcore", "integdev_gpu_drv", _ar, + _rf, "Handoff-", "handoff-", "private " + "analysis", + "private " + "provenance", "private " + "notes", "$OPEN_IOV_ROOT", + "gitignored", "never committed", +] +RAW_LINK = re.compile(r"\[\[\.\./|\[\[[^\]]*\.md(\||\]\])") +MERMAID = 'class="mermaid"' + + +def api(params, retries=6): + params = dict(params) + params.setdefault("format", "json") + qs = urllib.parse.urlencode(params) + last = None + for attempt in range(retries): + try: + req = urllib.request.Request(API + "?" + qs, headers=UA) + with urllib.request.urlopen(req, timeout=90) as r: + raw = r.read() + return json.loads(raw) + except Exception as e: # noqa: BLE001 + last = e + time.sleep(2 * (attempt + 1)) + raise RuntimeError(f"api failed: {last}") + + +def all_titles(): + titles, cont = [], {} + while True: + p = {"action": "query", "list": "allpages", "aplimit": "500"} + p.update(cont) + d = api(p) + titles += [x["title"] for x in d["query"]["allpages"]] + if "continue" in d: + cont = d["continue"] + else: + break + # include the namespaces we use (project pages, templates) + for ns in (4, 10): + d = api({"action": "query", "list": "allpages", "apnamespace": ns, "aplimit": "500"}) + titles += [x["title"] for x in d["query"]["allpages"]] + return titles + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--pages", nargs="*", default=[]) + args = ap.parse_args() + + titles = args.pages or all_titles() + existing = {t.lower(): t for t in all_titles()} + + problems = 0 + for title in titles: + d = api( + { + "action": "query", + "prop": "revisions", + "rvprop": "content", + "rvslots": "main", + "titles": title, + } + ) + page = list(d["query"]["pages"].values())[0] + if page.get("missing") is not None: + print(f"BAD {title}: missing") + problems += 1 + continue + c = page["revisions"][0]["slots"]["main"]["*"] + issues = [] + is_redirect = c.strip().upper().startswith("#REDIRECT") + is_template = title.startswith("Template:") + if not (is_redirect or is_template) and "{{About message}}" not in c and "An absence of critical technical documentation" not in c: + issues.append("missing {{About message}} site-wide message") + for p in FORBIDDEN: + if p in c: + issues.append(f"forbidden marker {p!r}") + if RAW_LINK.search(c): + issues.append("raw relative/.md link") + if MERMAID in c: + issues.append("raw mermaid block") + for m in re.finditer(r"\[\[([^\]]+)\]\]", c): + tgt = m.group(1).split("|")[0].split("#")[0].strip() + if not tgt or tgt.startswith( + ("File:", "Category:", "http", "wikipedia:", "Special:", "MediaWiki:") + ): + continue + if tgt.lower() not in existing: + issues.append(f"unresolved link [[{tgt}]]") + if issues: + problems += 1 + print(f"BAD {title}") + for i in sorted(set(issues)): + print(f" - {i}") + else: + print(f"ok {title}") + time.sleep(0.4) + print(f"checked {len(titles)} pages, {problems} with problems") + sys.exit(1 if problems else 0) + + +if __name__ == "__main__": + main() diff --git a/tools/site/md2wiki.py b/tools/site/md2wiki.py new file mode 100644 index 0000000..c8438cf --- /dev/null +++ b/tools/site/md2wiki.py @@ -0,0 +1,423 @@ +#!/usr/bin/env python3 +"""md2wiki.py — convert tracked wiki/*.md pages to MediaWiki wikitext for open-iov.org. + +Design rules (public/private discipline — see AGENTS.md): + * Only *tracked* repository paths may be linked. A link to an untracked + (private) path aborts the conversion. + * Sentences that name non-public provenance are stripped (SANITIZE rules + below); the final output is scanned for forbidden markers and the + conversion aborts on any hit. + * Intra-wiki *.md links map to MediaWiki titles via titles.json; unknown + targets abort (no silent red links). + +Usage: + md2wiki.py FILE.md [FILE.md ...] --out DIR [--repo ROOT] + md2wiki.py --all --out DIR + +Outputs DIR/.wiki plus DIR/_manifest.json. +""" + +import argparse +import html +import json +import os +import re +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +DEFAULT_REPO = os.path.abspath(os.path.join(HERE, "..", "..")) +TITLES_PATH = os.path.join(HERE, "titles.json") + +# Forbidden-marker list. The sensitive markers are assembled from fragments +# so that this list does not itself carry them. +_pv = "PRIV" + "ATE-" +_st = "stage" + "_rel" +_ar = "ARCH" + "AEOLOGY" +_rf = "nvidia-570" + ".144-reference" +FORBIDDEN = [ + _pv, + _st, + "/home/rmcore", + "integdev_gpu_drv", + _ar, + _rf, + "Handoff-", + "handoff-", + "private " + "analysis", + "private " + "provenance", + "private " + "notes", + "$OPEN_IOV_ROOT", + "gitignored", + "never committed", +] + +# Exact-text sanitize rules: (file basename, old, new). Applied before conversion. +# Exact-text sanitize rules: (file basename, old, new). Applied before +# conversion. Empty: the tracked wiki sources carry no non-public +# provenance text. +SANITIZE = [] + +SKIP_FILES = { + "README.md", # index page, handled by the Documentation page rewrite + "Glossary.md", # legacy historical glossary; the live Glossary is rewritten + "GPU_Driver_Internals.md", # 2023 imported page; kept as-is on the site + "GPU_Firmware.md", # 2023 imported page; kept as-is on the site + "GPU_SBOM.md", # 2023 imported page; kept as-is on the site + "Virtual_IO_Internals.md", # 2023 imported page; kept as-is on the site +} + + +class ConversionError(Exception): + pass + + +class Converter: + def __init__(self, repo, titles, src_name): + self.repo = repo + self.titles = titles + self.wiki_titles = titles["wiki_titles"] + self.doc_pages = titles.get("doc_pages", {}) + self.repo_base = titles["repo_base"] + self.src_name = src_name + self._tracked = {} + self.link_count = 0 + + # ---------- git helpers ---------- + def tracked(self, path): + if path in self._tracked: + return self._tracked[path] + r = subprocess.run( + ["git", "-C", self.repo, "ls-files", "--error-unmatch", path], + capture_output=True, + ) + ok = r.returncode == 0 + self._tracked[path] = ok + return ok + + # ---------- link mapping ---------- + def map_link(self, target, text): + target = target.strip() + if not target: + return text + if target.startswith(("http://", "https://")): + if text and text != target: + return f"[{target} {text}]" + return f"[{target}]" + if target.startswith("#"): + return f"[[{target}|{text}]]" if text else f"[[{target}]]" + # relative path: normalize against the wiki/ source directory + rel = os.path.normpath(os.path.join("wiki", target)) + anchor = "" + if "#" in rel: + rel, anchor = rel.split("#", 1) + anchor = "#" + anchor + if rel.endswith(".md") and os.path.dirname(rel) == "wiki": + base = os.path.basename(rel) + title = self.wiki_titles.get(base) + if not title: + raise ConversionError( + f"intra-wiki link without a title mapping: {target}" + ) + plain = text.strip("`") if text else "" + if not text or plain in (base, title, target, os.path.basename(target)): + return f"[[{title}{anchor}]]" + return f"[[{title}{anchor}|{text}]]" + # repository link: must be a tracked file + if not self.tracked(rel): + raise ConversionError( + f"link to untracked (private) path: {target} -> {rel}" + ) + url = self.repo_base + rel + anchor + return f"[{url} {text}]" if text else f"[{url}]" + + # ---------- inline ---------- + def inline(self, s): + # protect inline code spans first + codes = [] + + def stash(m): + codes.append(html.escape(m.group(1), quote=False)) + return f"\x00{len(codes) - 1}\x00" + + s = re.sub(r"`([^`]+)`", stash, s) + + # links: [text](target) + def link(m): + text, target = m.group(1), m.group(2) + self.link_count += 1 + return self.map_link(target, text) + + s = re.sub(r"\[([^\]]*)\]\(([^)\s]+)\)", link, s) + + # escape HTML + s = html.escape(s, quote=False) + + # bold / italic (conservative: no surrounding word chars/spaces) + s = re.sub(r"\*\*([^*\n]+?)\*\*", r"'''\1'''", s) + s = re.sub( + r"(?{codes[int(m.group(1))]}" + + s = re.sub(r"\x00(\d+)\x00", unstash, s) + return s + + # ---------- block conversion ---------- + def convert(self, text): + lines = text.replace("\r\n", "\n").split("\n") + out = [] + i = 0 + n = len(lines) + while i < n: + line = lines[i] + if line.startswith("```"): + fence = line[3:].strip() + body = [] + i += 1 + while i < n and not lines[i].startswith("```"): + body.append(lines[i]) + i += 1 + i += 1 # closing fence + if fence == "mermaid": + # use the following ```text block as the ASCII fallback if any + j = i + while j < n and ( + lines[j].strip() == "" + or re.match(r"^(ASCII fallback|Fallback|Diagram):?$", lines[j].strip()) + ): + j += 1 + if j < n and lines[j].startswith("```") and "text" in lines[j]: + k = j + 1 + fallback = [] + while k < n and not lines[k].startswith("```"): + fallback.append(lines[k]) + k += 1 + i = k + 1 + img = self.titles.get("mermaid_images", {}).get(self.src_name) + if img: + out.append(img) + out.append( + "
"
+                            + html.escape("\n".join(fallback), quote=False)
+                            + "
" + ) + else: + raise ConversionError( + f"{self.src_name}: mermaid block without an ASCII fallback" + ) + continue + out.append( + "
" + html.escape("\n".join(body), quote=False) + "
" + ) + continue + if re.match(r"^\|.*\|$", line): + block = [] + while i < n and re.match(r"^\|.*\|$", lines[i]): + block.append(lines[i]) + i += 1 + out.append(self.table(block)) + continue + if line.startswith(">"): + block = [] + while i < n and lines[i].startswith(">"): + block.append(lines[i][1:].lstrip()) + i += 1 + # split into paragraphs on empty lines + for para in self.split_paras(block): + if para: + out.append(f"
{self.inline(para)}
") + continue + m = re.match(r"^(#{1,6})\s+(.*)$", line) + if m: + level = len(m.group(1)) + title = self.inline(m.group(2).strip()) + eq = "=" * max(1, min(level, 4)) + out.append(f"{eq} {title} {eq}") + i += 1 + continue + if re.match(r"^\*\s+", line) or re.match(r"^\d+\.\s+", line) or re.match( + r"^\s*[-*]\s+", line + ) or re.match(r"^\s*\d+\.\s+", line): + block = [] + while i < n: + if ( + re.match(r"^\s*[-*]\s+", lines[i]) + or re.match(r"^\s*\d+\.\s+", lines[i]) + ): + block.append(lines[i]) + i += 1 + elif ( + lines[i].strip() != "" + and block + and not self.is_block_start(lines[i]) + ): + # continuation of the previous list item + block[-1] += " " + lines[i].strip() + i += 1 + else: + break + out.append(self.lists(block)) + continue + if re.match(r"^-{3,}\s*$", line): + out.append("----") + i += 1 + continue + if line.strip() == "": + i += 1 + continue + # paragraph + block = [] + while i < n and lines[i].strip() != "" and not self.is_block_start( + lines[i] + ): + block.append(lines[i]) + i += 1 + out.append(self.inline(" ".join(x.strip() for x in block))) + continue + return "\n\n".join(x for x in out if x != "") + "\n" + + @staticmethod + def is_block_start(line): + return bool( + re.match(r"^(#{1,6}\s|>|\||```|\*\s|\d+\.\s|-\s|-{3,}\s*$)", line) + ) + + @staticmethod + def split_paras(block): + paras, cur = [], [] + for l in block: + if l.strip() == "": + if cur: + paras.append(" ".join(x.strip() for x in cur)) + cur = [] + else: + cur.append(l) + if cur: + paras.append(" ".join(x.strip() for x in cur)) + return paras + + def lists(self, block): + out = [] + for raw in block: + m = re.match(r"^(\s*)([-*]|\d+\.)\s+(.*)$", raw) + indent = len(m.group(1)) // 2 + marker = "#" if m.group(2).endswith(".") else "*" + out.append(marker * (indent + 1) + " " + self.inline(m.group(3))) + return "\n".join(out) + + def table(self, block): + rows = [] + for raw in block: + cells = [c.strip() for c in raw.strip()[1:-1].split("|")] + rows.append(cells) + if len(rows) >= 2 and all(re.match(r"^:?-{2,}:?$", c) for c in rows[1]): + header, rows = rows[0], rows[2:] + else: + header, rows = rows[0], rows[1:] + + def cell(c, head=False): + # protect pipes inside code spans + c = re.sub(r"`([^`]+)`", lambda m: "`" + m.group(1).replace("|", "|") + "`", c) + c = self.inline(c) + return ("! " if head else "| ") + c + + out = ['{| class="wikitable"'] + out.append("|-") + out += [cell(c, head=True) for c in header] + for r in rows: + out.append("|-") + out += [cell(c) for c in r] + out.append("|}") + return "\n".join(out) + + +def sanitize(src_name, text): + for fname, old, new in SANITIZE: + if fname == src_name and old in text: + text = text.replace(old, new) + return text + + +def check_forbidden(src_name, text): + hits = [p for p in FORBIDDEN if p in text] + if hits: + raise ConversionError(f"{src_name}: forbidden private markers remain: {hits}") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("files", nargs="*") + ap.add_argument("--all", action="store_true", help="convert every wiki/*.md") + ap.add_argument("--out", required=True, help="output directory for .wiki files") + ap.add_argument("--repo", default=DEFAULT_REPO) + ap.add_argument("--dry", action="store_true", help="print, do not write") + args = ap.parse_args() + + with open(TITLES_PATH) as f: + titles = json.load(f) + + files = list(args.files) + if args.all: + wikidir = os.path.join(args.repo, "wiki") + files += [ + os.path.join(wikidir, f) + for f in sorted(os.listdir(wikidir)) + if f.endswith(".md") and f not in SKIP_FILES + ] + if not files: + ap.error("no input files") + + os.makedirs(args.out, exist_ok=True) + manifest = {} + failures = 0 + for path in files: + name = os.path.basename(path) + try: + with open(path) as f: + text = f.read() + text = sanitize(name, text) + conv = Converter(args.repo, titles, name) + wiki = "{{About message}}\n\n" + conv.convert(text) + check_forbidden(name, wiki) + except ConversionError as e: + print(f"FAIL {name}: {e}", file=sys.stderr) + failures += 1 + continue + title = titles["wiki_titles"].get(name) + rec = { + "source": os.path.relpath(path, args.repo), + "title": title, + "links": conv.link_count, + "bytes": len(wiki), + } + manifest[name] = rec + if args.dry: + print(f"--- {name} -> {title} ({conv.link_count} links, {len(wiki)} B)") + print(wiki) + else: + dest = os.path.join(args.out, name[:-3] + ".wiki") + with open(dest, "w") as f: + f.write(wiki) + print(f"OK {name} -> {dest} title={title!r} links={conv.link_count}") + if not args.dry: + mpath = os.path.join(args.out, "_manifest.json") + existing = {} + if os.path.exists(mpath): + with open(mpath) as f: + existing = json.load(f) + existing.update(manifest) + with open(mpath, "w") as f: + json.dump(existing, f, indent=1, sort_keys=True) + print(f"manifest: {mpath}") + sys.exit(1 if failures else 0) + + +if __name__ == "__main__": + main() diff --git a/tools/site/pages/Community.wiki b/tools/site/pages/Community.wiki new file mode 100644 index 0000000..1516783 --- /dev/null +++ b/tools/site/pages/Community.wiki @@ -0,0 +1,15 @@ +{{About message}} + +== Community == +'''Chat:''' [https://discord.gg/Rb9K9DYxKK Open-IOV Discord] + +'''Source code and issues:''' [https://git.vgpu.io/open-iov git.vgpu.io/open-iov] + +== Contributing == +The project welcomes contributors of every kind. The most valuable contribution is evidence: a new hardware or software combination tested with the project's methodology, with the captures listed in the repository's debugging document. + +* Start from [[Hardware and Software Matrix]] to see which combinations are Reference, Target or Experimental. +* The repository's developer onboarding document describes the environment, the coding standards and how a change is landed. +* For the firmware side, [[risc-v-gsp-rm-debugger]] is the host-side workbench; the tool documentation is in the repository. + +'''Editorial policy for this wiki:''' account creation and editing are closed to prevent spam. The wiki is maintained from the project's tracked documentation by the project's publishing bot; corrections and additions are made through the repository. diff --git a/tools/site/pages/Documentation.wiki b/tools/site/pages/Documentation.wiki new file mode 100644 index 0000000..00d40bc --- /dev/null +++ b/tools/site/pages/Documentation.wiki @@ -0,0 +1,50 @@ +{{About message}} + +This is the index of the Open-IOV wiki. If you are new, read '''[[Project Status]]''' first, then '''[[System Overview]]'''. The wiki is a curated public view of the project's maintained documentation; each page names its canonical document in the [https://git.vgpu.io/open-iov repository]. + +== Start here == +* [[Project Status]] — what works today, what is in progress, what is not started +* [[Install and First Run]] — build, load, and a stock guest driver in about half an hour +* [[System Overview]] — the architecture in one page +* [[FAQ]] — common questions, including "why is nvidia-smi working but CUDA not?" + +== Architecture == +* [[System Overview]] — principle, layer map, data flows +* [[GSP and GSP-RM]] — the firmware interface the project emulates +* [[The Host Module (vgsp.ko)]] — virtual device, emulation core, memory model, forwarding +* [[The Guest Stack and CUDASW]] — what runs unmodified inside the guest +* [[The vGSP Execution Plane (vEP)]] — the execution and completion layer +* [[Introduction to GMMU]] — GPU address translation from first principles +* [[UVM in the Guest]] — unified memory, semaphores, page trees +* [[GMMU Multi-Tenant Protection]] — isolation design for multiple VMs and tenants +* [[Physical GPU Backend and Guest Wiring]] — how guest work reaches real hardware +* [[Guest Driver Lifecycle: A Field Guide]] — boot to CUDA, with real control IDs and log signatures + +== CUDA == +* [[CUDA Userspace Driver]] — the libcuda.so deep dive +* [[The Execution Frontier]] — the GR compute-launch chain and its current wall + +== Firmware interoperability == +* [[GSP 570.144W Firmware Analysis]] — how the 28.5 MB image was decompiled and mapped +* [[Address Reference]] — firmware-side address tables +* [[risc-v-gsp-rm-debugger]] — the firmware runtime workbench (hooks, watchpoints, sessions) +* Historical protocol analyses, kept for the record: [[RPC Message Format]] · [[BAR Emulation Specification]] · [[WPR Management]] · [[Migration Protocol]] · [[GSP RPC Emulation]] · [[Error Codes]] · [[vGPU-IO Integration Guide]] + +== Portability and ecosystem == +* [[Hardware and Software Matrix]] — Reference / Target / Experimental, honestly labelled +* [[HAAEEP and Collaboration]] — the agent-edge use case, collaboration and licensing models + +== Reference == +* [[Glossary]] +* [[FAQ]] +* [[Error Codes]] — historical naming, superseded by the wire status values + +== History == +This wiki previously hosted the OpenMdev / LibVF.IO project documentation (2021–2023). The pages below are archived for reference; they describe earlier projects and architectures, not Open-IOV. + +* [[Introduction]] · [[Virtualization Fundamentals]] · [[Virtual I/O Internals]] · [[GPU Driver Internals]] · [[GPU Firmware]] · [[GPU Support]] · [[CPU Support]] · [[Hypervisor Support]] · [[Merged Drivers]] · [[OpenRM]] · [[AMDGPU]] · [[Intel]] · [[Mdev-GPU]] · [[GVM]] · [[Hyperborea]] · [[LIME Is Mediated Emulation]] · [[LibVF.IO Setup Guide]] · [[Looking Glass KVMFR]] · [[VGPU Unlock]] · [[GPU Software Bill Of Materials (SBOM)]] + +== Meta == +* [[Open-IOV:About|About Open-IOV]] +* [[Community]] +* Wiki architecture pages name their canonical source in the repository; when a page and its source disagree, the repository wins. diff --git a/tools/site/pages/Main-Page.wiki b/tools/site/pages/Main-Page.wiki new file mode 100644 index 0000000..fb08f81 --- /dev/null +++ b/tools/site/pages/Main-Page.wiki @@ -0,0 +1,78 @@ +{{About message}} + +__NOTOC__ +
+'''Open-IOV''' is an open-source virtualization layer for NVIDIA GPUs: it runs '''stock, unmodified NVIDIA drivers inside virtual machines''' — Linux and Windows — with CUDA on hardware you own, '''without proprietary vGPU licenses''', without patching the guest driver, the host driver, the hypervisor or the firmware. + +'''''Emulate the interface. Pass through the work. Never patch anyone.''''' + +The only new component is one host kernel module (vgsp.ko), which implements the interface the guest driver expects to talk to — the GPU System Processor and its resource-manager firmware — and arranges for the guest's work to run on real GPU hardware. + +'''[[Open-IOV:About|About Open-IOV]]''' · '''[[Community|Community]]''' · '''[[Documentation|Documentation]]''' +
+ +== Start here == +* '''[[Project Status]]''' — the honest, dated state of the project. +* '''[[Install and First Run]]''' — fresh host to a stock guest driver in about half an hour. +* '''[[System Overview]]''' — how Open-IOV works, in one page. +* '''[[Documentation]]''' — the full index of this wiki. +* '''[[FAQ]]''' — the questions we get asked most. + +== Project status (2026-10-12) == +{| class="wikitable" +! Area !! State +|- +| Emulated vGPU device (PCI, BARs, MSI-X, VFIO mdev) || Working +|- +| GSP boot emulation and GSP-RM message queues / RPC || Working +|- +| Stock guest driver probe, nvidia-smi (TU104) || Working +|- +| CUDA userspace init and the driver-API smoke (ALL OK, 64-byte payload round-trips) || Working +|- +| Guest CUDA kernel execution on real hardware || In progress — a forwarded launch reaches the engine queue; engine execution is the frontier +|- +| OpenGL / Vulkan, Windows guests, multi-tenancy, migration, performance || Not yet +|} + +The full story, with evidence and the current wall: [[Project Status]] and [[The Execution Frontier]]. + +== Deep dives == +* [[System Overview]] — the architecture before any subsystem +* [[GSP and GSP-RM]] — the firmware interface we emulate +* [[The vGSP Execution Plane (vEP)]] — how guest work becomes execution and completion +* [[The Host Module (vgsp.ko)]] — the module internals +* [[The Guest Stack and CUDASW]] — what runs unmodified in the guest +* [[CUDA Userspace Driver]] — the libcuda.so analysis +* [[Introduction to GMMU]] — GPU address translation from first principles +* [[UVM in the Guest]] — unified memory as a correctness oracle +* [[Physical GPU Backend and Guest Wiring]] — how work reaches real hardware +* [[The Execution Frontier]] — the leading edge, evidence-led +* [[Guest Driver Lifecycle: A Field Guide]] — boot to CUDA with real IDs and log signatures + +== Firmware interoperability == +* [[GSP 570.144W Firmware Analysis]] +* [[Address Reference]] +* [[RPC Message Format]] · [[BAR Emulation Specification]] · [[WPR Management]] · [[Migration Protocol]] · [[GSP RPC Emulation]] (historical, superseded) +* [[risc-v-gsp-rm-debugger]] — the firmware runtime workbench + +== Project and ecosystem == +* [[Hardware and Software Matrix]] — supported combinations, honestly labelled +* [[HAAEEP and Collaboration]] — the agent-edge use case and how to work with us +* [[FAQ]] · [[Glossary]] + +== Community == +* Source and issues: [https://git.vgpu.io/open-iov git.vgpu.io/open-iov] +* Chat: [https://discord.gg/Rb9K9DYxKK Open-IOV Discord] + +== About Open-IOV == +Open-IOV runs unmodified NVIDIA driver stacks inside virtual machines with GPU compute on hardware you own — no proprietary vGPU license, and no patches to the guest driver, the host driver, the hypervisor or the firmware. The project adds one host kernel module (vgsp.ko) that implements the interface the guest driver expects to talk to — the GPU System Processor and its resource-manager firmware — and arranges for the guest's work to run on real GPU hardware. + +* '''Mission:''' make NVIDIA GPUs fully virtualizable, by anyone, for any hypervisor, without modifying NVIDIA software, firmware or hardware. +* '''Principles:''' no patches anywhere; honest status carried with dated evidence; an independent, self-contained IP block; durability by owning the interface rather than a snapshot of it. +* '''Licensing:''' all documentation on this site is [https://creativecommons.org/licenses/by/4.0/ CC-BY-4.0]; the project's code is BSD-3-Clause (GPL-2.0 for the kernel modules); NVIDIA software and firmware are used unmodified and are not redistributed. + +'''[[Open-IOV:About|Read more about Open-IOV →]]''' + +== About this wiki == +This wiki previously hosted documentation from the earlier project era (2021–2023). Those pages are retained for reference under [[Documentation|History]]; current documentation starts at [[Documentation]]. diff --git a/tools/site/pages/MediaWiki-Sidebar.wiki b/tools/site/pages/MediaWiki-Sidebar.wiki new file mode 100644 index 0000000..499f954 --- /dev/null +++ b/tools/site/pages/MediaWiki-Sidebar.wiki @@ -0,0 +1,31 @@ +* navigation +** mainpage|mainpage-description +** Project Status +** Documentation +** FAQ +** recentchanges-url|recentchanges +* Deep dives +** System Overview +** GSP and GSP-RM +** The vGSP Execution Plane (vEP) +** The Host Module (vgsp.ko) +** The Guest Stack and CUDASW +** CUDA Userspace Driver +** Introduction to GMMU +** UVM in the Guest +** Physical GPU Backend and Guest Wiring +** The Execution Frontier +* Project +** Open-IOV:About|About Open-IOV +** Install and First Run +** Hardware and Software Matrix +** HAAEEP and Collaboration +** risc-v-gsp-rm-debugger +** Glossary +* Community +** Community +** https://git.vgpu.io/open-iov|Source repository +** https://discord.gg/Rb9K9DYxKK|Discord +* SEARCH +* TOOLBOX +* LANGUAGES diff --git a/tools/site/pages/Open-IOV-About.wiki b/tools/site/pages/Open-IOV-About.wiki new file mode 100644 index 0000000..1084ae1 --- /dev/null +++ b/tools/site/pages/Open-IOV-About.wiki @@ -0,0 +1,29 @@ +{{About message}} + +'''Open-IOV''' is an open-source virtualization layer for NVIDIA GPUs. It runs '''stock, unmodified NVIDIA drivers inside virtual machines''' — Linux and Windows — with GPU compute (CUDA), and eventually 3D graphics and video, on hardware you already own: without a proprietary vGPU license, without patching the guest driver, the host driver, the hypervisor or the GPU firmware. + +== Mission == +Make NVIDIA GPUs fully virtualizable, by anyone, for any hypervisor, without modifying NVIDIA software, firmware or hardware. + +== How it works == +The only component the project adds is a host kernel module (vgsp.ko). It implements the interface the guest driver expects to talk to — the GPU System Processor (GSP) and its resource-manager firmware — and arranges for the guest's work to execute on real GPU hardware. The guiding sentence is: + +::'''''Emulate the interface. Pass through the work. Never patch anyone.''''' + +See [[System Overview]] for the architecture and [[Project Status]] for the honest current state. + +== Principles == +* '''No patches anywhere.''' Guest drivers, host drivers, firmware and hypervisors are all used unmodified; interoperability is the goal, not counterfeiting. +* '''Honest status.''' Claims carry dates and evidence; the project publishes what does not work as clearly as what does. +* '''Independent IP block.''' Open-IOV is self-contained so KVM, Xen, Argo/OpenXT, Qubes OS Air and others can carry it without forks. +* '''Durability by owning the interface.''' The protocol, not a snapshot of it, is reimplemented — so new driver versions, firmware and hardware can be carried forward. + +== Licensing == +'''All documentation on this site is licensed [https://creativecommons.org/licenses/by/4.0/ CC-BY-4.0]''' — the statement carried at the top of every page. It may be reused with attribution, or used as inspiration for new documentation, including by GPU vendors for public commercial distribution as developer documentation. + +The project's source code is licensed separately: '''BSD 3-Clause''' for everything reusable (userspace libraries, tools, scripts, configuration) and '''GPL-2.0''' for the kernel modules that must legally link against GPL-only kernel exports (VFIO, mdev, IOMMU); per-file SPDX identifiers govern. NVIDIA's software and firmware are used unmodified and are not redistributed. See the repository's LICENSE and docs/09-reference/legal-and-licensing.md for the full statement. + +== Community == +* Source and issues: [https://git.vgpu.io/open-iov git.vgpu.io/open-iov] +* Chat: [https://discord.gg/Rb9K9DYxKK Open-IOV Discord] +* See [[Community]] and [[HAAEEP and Collaboration]]. diff --git a/tools/site/pages/Template-About-message.wiki b/tools/site/pages/Template-About-message.wiki new file mode 100644 index 0000000..893e3b1 --- /dev/null +++ b/tools/site/pages/Template-About-message.wiki @@ -0,0 +1,11 @@ +
+''An absence of critical technical documentation has historically slowed growth and adoption of developer ecosystems for GPU virtualization.'' + +''This [https://creativecommons.org/licenses/by/4.0/ CC-BY-4.0] licensed content can either be used with attribution, or used as inspiration for new documentation, created by GPU vendors for public commercial distribution as developer documentation.'' + +''Where possible, this documentation will clearly label dates and versions of observed-but-not-guaranteed behaviour vs. vendor-documented stable interfaces/behaviour with guarantees of forward or backward compatibility.'' +
+ +== Usage == +Every content page on this wiki carries this message: {{About message}} + diff --git a/tools/site/pages/Template-Historical.wiki b/tools/site/pages/Template-Historical.wiki new file mode 100644 index 0000000..8473b47 --- /dev/null +++ b/tools/site/pages/Template-Historical.wiki @@ -0,0 +1,7 @@ +
+'''Historical page.''' This page documents an earlier project or architecture (the OpenMdev / LibVF.IO era, 2021–2023). It is retained for reference and is not part of the current Open-IOV stack. Current documentation starts at [[Documentation]]. +
[[Category:History]] + +== Usage == +Place at the top of archived pages: {{Historical}} + diff --git a/tools/site/pages/Template-Superseded.wiki b/tools/site/pages/Template-Superseded.wiki new file mode 100644 index 0000000..2fb64db --- /dev/null +++ b/tools/site/pages/Template-Superseded.wiki @@ -0,0 +1,7 @@ +
+'''Superseded document.''' This page records an earlier analysis that the current implementation no longer follows. It is kept for the record; current documentation is linked at the top of the page or at [[Documentation]]. +
[[Category:Superseded]] + +== Usage == +Place at the top of superseded analysis pages: {{Superseded}} + diff --git a/tools/site/publish.py b/tools/site/publish.py new file mode 100644 index 0000000..e70b3de --- /dev/null +++ b/tools/site/publish.py @@ -0,0 +1,402 @@ +#!/usr/bin/env python3 +"""publish.py — publish converted wiki pages to open-iov.org via the MediaWiki API. + +Credentials: WIKI_BOT_USER / WIKI_BOT_PASS from the environment or from +~/.config/open-iov/wiki-bot.env (never from the repository). + +Examples: + publish.py login + publish.py check --all + publish.py diff --title "FAQ" --file out/FAQ.wiki + publish.py publish --title "FAQ" --file out/FAQ.wiki --summary "Add FAQ" + publish.py publish-all --dir out --only "FAQ,Glossary" + publish.py delete --title "Dopefish" --reason "joke page, retired" + publish.py move --from "Articles" --to "Documentation" --reason "new IA" + +Every publish is verified by re-reading the stored revision and recorded in +wiki/.publish-manifest.json (source hash -> revision ids). +""" + +import argparse +import difflib +import hashlib +import json +import os +import re +import sys +import urllib.parse +import urllib.request +import http.cookiejar + +API = "https://open-iov.org/api.php" +UA = "OpenIOV-WikiBot/1.0" +ENVFILE = os.path.expanduser("~/.config/open-iov/wiki-bot.env") +HERE = os.path.dirname(os.path.abspath(__file__)) +MANIFEST = os.path.abspath(os.path.join(HERE, "..", "..", "wiki", ".publish-manifest.json")) + +# Forbidden-marker list. The sensitive markers are assembled from fragments +# so that this list does not itself carry them. +_pv = "PRIV" + "ATE-" +_st = "stage" + "_rel" +_ar = "ARCH" + "AEOLOGY" +_rf = "nvidia-570" + ".144-reference" +FORBIDDEN = [ + _pv, + _st, + "/home/rmcore", + "integdev_gpu_drv", + _ar, + _rf, + "Handoff-", + "handoff-", + "private " + "analysis", + "private " + "provenance", + "private " + "notes", + "$OPEN_IOV_ROOT", + "gitignored", + "never committed", +] +RAW_LINK = re.compile(r"\[\[\.\./|\[\[[^\]]*\.md(\||\]\])") +MERMAID = "class=\"mermaid\"" + + +def load_creds(): + user = os.environ.get("WIKI_BOT_USER") + password = os.environ.get("WIKI_BOT_PASS") + if not (user and password) and os.path.exists(ENVFILE): + with open(ENVFILE) as f: + for line in f: + line = line.strip() + if line.startswith("WIKI_BOT_USER="): + user = line.split("=", 1)[1] + elif line.startswith("WIKI_BOT_PASS="): + password = line.split("=", 1)[1] + if not (user and password): + sys.exit(f"no credentials (set WIKI_BOT_* or create {ENVFILE})") + return user, password + + +class Bot: + def __init__(self): + self.cj = http.cookiejar.CookieJar() + self.opener = urllib.request.build_opener( + urllib.request.HTTPCookieProcessor(self.cj) + ) + self.csrf = None + + def api(self, params, post=False, retries=5): + params = dict(params) + params.setdefault("format", "json") + qs = urllib.parse.urlencode(params) + import time + + last = None + for attempt in range(retries): + try: + if post: + req = urllib.request.Request( + API, data=qs.encode(), headers={"User-Agent": UA} + ) + else: + req = urllib.request.Request( + API + "?" + qs, headers={"User-Agent": UA} + ) + with self.opener.open(req, timeout=60) as r: + raw = r.read() + return json.loads(raw) + except Exception as e: # network / cloudflare hiccups + last = e + time.sleep(3 * (attempt + 1)) + raise RuntimeError(f"api failed: {last}") + + def login(self): + user, password = load_creds() + tok = self.api({"action": "query", "meta": "tokens", "type": "login"})[ + "query" + ]["tokens"]["logintoken"] + res = self.api( + {"action": "login", "lgname": user, "lgpassword": password, "lgtoken": tok}, + post=True, + ) + result = res.get("login", {}).get("result") + if result != "Success": + sys.exit(f"login failed: {res}") + self.csrf = self.api({"action": "query", "meta": "tokens", "type": "csrf"})[ + "query" + ]["tokens"]["csrftoken"] + info = self.api( + {"action": "query", "meta": "userinfo", "uiprop": "rights|groups"} + )["query"]["userinfo"] + print(f"login: Success as {info['name']} groups={info['groups']}") + return info + + def fetch(self, title): + d = self.api( + { + "action": "query", + "prop": "revisions", + "rvprop": "content|timestamp|ids", + "rvslots": "main", + "titles": title, + } + ) + page = list(d["query"]["pages"].values())[0] + if page.get("missing") is not None: + return None + rev = page["revisions"][0] + return { + "revid": rev["revid"], + "timestamp": rev["timestamp"], + "content": rev["slots"]["main"]["*"], + } + + def edit(self, title, text, summary, minor=False): + res = self.api( + { + "action": "edit", + "title": title, + "text": text, + "token": self.csrf, + "bot": "1", + "minor": "1" if minor else "0", + "summary": summary, + }, + post=True, + ) + e = res.get("edit", {}) + if e.get("result") != "Success": + raise RuntimeError(f"edit failed for {title}: {res}") + return e + + def delete(self, title, reason): + res = self.api( + { + "action": "delete", + "title": title, + "token": self.csrf, + "reason": reason, + }, + post=True, + ) + return res.get("delete", res) + + def move(self, frm, to, reason, noredirect=False): + res = self.api( + { + "action": "move", + "from": frm, + "to": to, + "token": self.csrf, + "reason": reason, + "noredirect": "1" if noredirect else "0", + "movetalk": "1", + }, + post=True, + ) + return res.get("move", res) + + def all_pages(self): + titles, cont = [], {} + while True: + p = {"action": "query", "list": "allpages", "aplimit": "500"} + p.update(cont) + d = self.api(p) + titles += [r["title"] for r in d["query"]["allpages"]] + if "continue" in d: + cont = d["continue"] + else: + break + return titles + + +def scan(name, text): + problems = [] + for pat in FORBIDDEN: + if pat in text: + problems.append(f"forbidden marker: {pat!r}") + if RAW_LINK.search(text): + problems.append("raw relative/.md link remains") + if MERMAID in text: + problems.append("raw mermaid block remains") + return problems + + +def record(entry): + data = {} + if os.path.exists(MANIFEST): + with open(MANIFEST) as f: + data = json.load(f) + data[entry["title"]] = entry + os.makedirs(os.path.dirname(MANIFEST), exist_ok=True) + with open(MANIFEST, "w") as f: + json.dump(data, f, indent=1, sort_keys=True) + + +def sha(text): + return hashlib.sha256(text.encode()).hexdigest()[:16] + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + sub = ap.add_subparsers(dest="cmd", required=True) + + sub.add_parser("login") + + p = sub.add_parser("check") + p.add_argument("--all", action="store_true") + p.add_argument("--title", action="append", default=[]) + + p = sub.add_parser("diff") + p.add_argument("--title", required=True) + p.add_argument("--file", required=True) + + p = sub.add_parser("publish") + p.add_argument("--title", required=True) + p.add_argument("--file", required=True) + p.add_argument("--summary", required=True) + p.add_argument("--force", action="store_true") + p.add_argument("--minor", action="store_true") + + p = sub.add_parser("publish-all") + p.add_argument("--dir", required=True) + p.add_argument("--summary", default="Update from tracked wiki/ source") + p.add_argument("--only", default="") + p.add_argument("--force", action="store_true") + + p = sub.add_parser("delete") + p.add_argument("--title", required=True) + p.add_argument("--reason", required=True) + + p = sub.add_parser("move") + p.add_argument("--from", dest="frm", required=True) + p.add_argument("--to", required=True) + p.add_argument("--reason", required=True) + p.add_argument("--noredirect", action="store_true") + + args = ap.parse_args() + + bot = Bot() + bot.login() + + if args.cmd == "check": + titles = args.title + if args.all: + titles = bot.all_pages() + bad = 0 + for t in titles: + page = bot.fetch(t) + if page is None: + print(f"MISSING {t}") + continue + problems = scan(t, page["content"]) + if problems: + bad += 1 + print(f"BAD {t}") + for pr in problems: + print(f" - {pr}") + else: + print(f"ok {t}") + print(f"checked {len(titles)} pages, {bad} with problems") + return + + if args.cmd == "diff": + with open(args.file) as f: + new = f.read() + cur = bot.fetch(args.title) + old = cur["content"] if cur else "" + d = difflib.unified_diff( + old.splitlines(), new.splitlines(), lineterm="", n=2, + fromfile=f"live:{args.title}", tofile=args.file, + ) + print("\n".join(d) or "(identical)") + + if args.cmd == "publish": + with open(args.file) as f: + new = f.read() + problems = scan(args.title, new) + if problems: + sys.exit(f"refusing to publish {args.title}: {problems}") + cur = bot.fetch(args.title) + old = cur["content"] if cur else "" + if old == new and not args.force: + print(f"SKIP {args.title} (unchanged, rev {cur['revid']})") + return + res = bot.edit(args.title, new, args.summary, minor=args.minor) + check = bot.fetch(args.title) + entry = { + "title": args.title, + "file": os.path.basename(args.file), + "source_sha": sha(new), + "old_revid": cur["revid"] if cur else None, + "new_revid": check["revid"], + "timestamp": check["timestamp"], + "summary": args.summary, + } + record(entry) + print( + f"EDIT {args.title}: rev {entry['old_revid']} -> {entry['new_revid']}" + f" (new={res.get('new')}, sha={entry['source_sha']})" + ) + return + + if args.cmd == "publish-all": + only = [x for x in args.only.split(",") if x] + for name in sorted(os.listdir(args.dir)): + if not name.endswith(".wiki"): + continue + path = os.path.join(args.dir, name) + with open(path) as f: + new = f.read() + # title is the first level-2 heading or from manifest + mpath = os.path.join(args.dir, "_manifest.json") + title = None + if os.path.exists(mpath): + with open(mpath) as f: + man = json.load(f) + title = man.get(name[:-5] + ".md", {}).get("title") + if not title: + m = re.search(r"^= (.+?) =$", new, re.M) + title = m.group(1) if m else name[:-5].replace("-", " ") + if only and title not in only and name[:-5] not in only: + continue + problems = scan(title, new) + if problems: + print(f"FAIL {title}: {problems}") + continue + cur = bot.fetch(title) + old = cur["content"] if cur else "" + if old == new and not args.force: + print(f"SKIP {title} (unchanged, rev {cur['revid']})") + continue + res = bot.edit(title, new, args.summary) + check = bot.fetch(title) + record( + { + "title": title, + "file": name, + "source_sha": sha(new), + "old_revid": cur["revid"] if cur else None, + "new_revid": check["revid"], + "timestamp": check["timestamp"], + "summary": args.summary, + } + ) + print( + f"EDIT {title}: rev {(cur or {}).get('revid', '-')} -> {check['revid']}" + f" (new={res.get('new')})" + ) + return + + if args.cmd == "delete": + res = bot.delete(args.title, args.reason) + print(f"DELETE {args.title}: {res}") + return + + if args.cmd == "move": + res = bot.move(args.frm, args.to, args.reason, noredirect=args.noredirect) + print(f"MOVE {args.frm} -> {args.to}: {res}") + return + + +if __name__ == "__main__": + main() diff --git a/tools/site/titles.json b/tools/site/titles.json new file mode 100644 index 0000000..cffa52b --- /dev/null +++ b/tools/site/titles.json @@ -0,0 +1,59 @@ +{ + "repo_base": "https://git.vgpu.io/open-iov/blob/master/", + "wiki_titles": { + "README.md": "Documentation", + "Address-Reference.md": "Address Reference", + "BAR-Emulation-Specification.md": "BAR Emulation Specification", + "CUDA-Userspace-Driver.md": "CUDA Userspace Driver", + "Error-Codes.md": "Error Codes", + "Execution-Plane.md": "The vGSP Execution Plane (vEP)", + "GMMU-Multi-Tenant-Protection.md": "GMMU Multi-Tenant Protection", + "GSP-570.144W-Firmware-Analysis.md": "GSP 570.144W Firmware Analysis", + "GSP-RPC-Emulation.md": "GSP RPC Emulation", + "Introduction-to-GMMU.md": "Introduction to GMMU", + "Migration-Protocol.md": "Migration Protocol", + "RPC-Message-Format.md": "RPC Message Format", + "vGPU-IO-Integration-Guide.md": "VGPU-IO Integration Guide", + "WPR-Management.md": "WPR Management", + "Project-Status.md": "Project Status", + "Install-and-First-Run.md": "Install and First Run", + "FAQ.md": "FAQ", + "System-Overview.md": "System Overview", + "GSP-and-GSP-RM.md": "GSP and GSP-RM", + "Glossary.md": "Glossary", + "Reference-Glossary.md": "Glossary", + "Execution-Frontier.md": "The Execution Frontier", + "Host-Module.md": "The Host Module (vgsp.ko)", + "Guest-Stack-and-CUDASW.md": "The Guest Stack and CUDASW", + "UVM-in-the-Guest.md": "UVM in the Guest", + "Physical-GPU-Backend.md": "Physical GPU Backend and Guest Wiring", + "Driver-Lifecycle-Field-Guide.md": "Guest Driver Lifecycle: A Field Guide", + "Riscv-GSP-RM-Debugger.md": "risc-v-gsp-rm-debugger", + "Hardware-Software-Matrix.md": "Hardware and Software Matrix", + "HAAEEP-and-Collaboration.md": "HAAEEP and Collaboration" + }, + "mermaid_images": { + "Execution-Plane.md": "[[File:The-vGSP-Execution-Plane-diagram.png|frameless|center|upright=2.2|The vGSP execution plane (vEP): one CUDA launch, end to end.]]" + }, + "doc_pages": { "docs/README.md": "Documentation", + "docs/03-architecture/01-system-overview.md": "System Overview", + "docs/03-architecture/03-gsp-and-gsp-rm.md": "GSP and GSP-RM", + "docs/03-architecture/05-gmmu-amap-hal-gr.md": "Introduction to GMMU", + "docs/03-architecture/06-host-module-and-hybrid-model.md": "The Host Module (vgsp.ko)", + "docs/03-architecture/09-full-system-picture.md": "System Overview", + "docs/03-architecture/10-interaction-and-scheduling.md": "System Overview", + "docs/03-architecture/13-vexecution-plane.md": "The vGSP Execution Plane (vEP)", + "docs/03-architecture/14-multi-tenant-gmmu-protection.md": "GMMU Multi-Tenant Protection", + "docs/02-getting-started/01-install-and-first-run.md": "Install and First Run", + "docs/03-architecture/02-guest-stack-and-cudasw.md": "The Guest Stack and CUDASW", + "docs/03-architecture/04-uvm.md": "UVM in the Guest", + "docs/03-architecture/07-driver-lifecycle-field-guide.md": "Guest Driver Lifecycle: A Field Guide", + "docs/03-architecture/08-physical-gpu-integration.md": "Physical GPU Backend and Guest Wiring", + "docs/03-architecture/11-physical-backend-guest-wiring.md": "Physical GPU Backend and Guest Wiring", + "docs/04-implementation/03-execution-frontier.md": "The Execution Frontier", + "docs/05-operations/03-risc-v-gsp-rm-vm.md": "risc-v-gsp-rm-debugger", + "docs/05-operations/04-risc-v-gsp-rm-debugger-user-guide.md": "risc-v-gsp-rm-debugger", + "docs/06-portability/hardware-software-matrix.md": "Hardware and Software Matrix", + "docs/08-ecosystem/haaeep-and-collaboration.md": "HAAEEP and Collaboration" + } +} diff --git a/tools/site/upload_file.py b/tools/site/upload_file.py new file mode 100644 index 0000000..8d347ba --- /dev/null +++ b/tools/site/upload_file.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""upload_file.py — upload an image to open-iov.org as WikiBot. + +Usage: + upload_file.py --file PATH --name "File:Example.png" --text "wikitext description" +""" + +import argparse +import http.cookiejar +import json +import mimetypes +import os +import sys +import urllib.parse +import urllib.request +import uuid + +API = "https://open-iov.org/api.php" +ENVFILE = os.path.expanduser("~/.config/open-iov/wiki-bot.env") +UA = "OpenIOV-WikiBot/1.0" + + +def load_creds(): + user = os.environ.get("WIKI_BOT_USER") + password = os.environ.get("WIKI_BOT_PASS") + if not (user and password) and os.path.exists(ENVFILE): + with open(ENVFILE) as f: + for line in f: + line = line.strip() + if line.startswith("WIKI_BOT_USER="): + user = line.split("=", 1)[1] + elif line.startswith("WIKI_BOT_PASS="): + password = line.split("=", 1)[1] + if not (user and password): + sys.exit("no credentials") + return user, password + + +class Multipart: + def __init__(self): + self.boundary = uuid.uuid4().hex + self.parts = [] + + def field(self, name, value): + self.parts.append( + ( + f"--{self.boundary}\r\n" + f'Content-Disposition: form-data; name="{name}"\r\n\r\n' + f"{value}\r\n" + ).encode() + ) + + def file(self, name, filename, data, ctype): + self.parts.append( + ( + f"--{self.boundary}\r\n" + f'Content-Disposition: form-data; name="{name}"; filename="{filename}"\r\n' + f"Content-Type: {ctype}\r\n\r\n" + ).encode() + + data + + b"\r\n" + ) + + def body(self): + return b"".join(self.parts) + f"--{self.boundary}--\r\n".encode() + + def content_type(self): + return f"multipart/form-data; boundary={self.boundary}" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--file", required=True) + ap.add_argument("--name", required=True, help='wiki file title, e.g. "File:Example.png"') + ap.add_argument("--text", default="") + ap.add_argument("--summary", default="Upload image from tracked analysis") + args = ap.parse_args() + + user, password = load_creds() + cj = http.cookiejar.CookieJar() + opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj)) + + def api(params, post=False): + params.setdefault("format", "json") + qs = urllib.parse.urlencode(params) + if post: + req = urllib.request.Request(API, data=qs.encode(), headers={"User-Agent": UA}) + else: + req = urllib.request.Request(API + "?" + qs, headers={"User-Agent": UA}) + with opener.open(req, timeout=60) as r: + return json.loads(r.read()) + + tok = api({"action": "query", "meta": "tokens", "type": "login"})["query"]["tokens"]["logintoken"] + res = api({"action": "login", "lgname": user, "lgpassword": password, "lgtoken": tok}, post=True) + if res.get("login", {}).get("result") != "Success": + sys.exit(f"login failed: {res}") + csrf = api({"action": "query", "meta": "tokens", "type": "csrf"})["query"]["tokens"]["csrftoken"] + + with open(args.file, "rb") as f: + data = f.read() + ctype = mimetypes.guess_type(args.file)[0] or "application/octet-stream" + mp = Multipart() + mp.field("action", "upload") + mp.field("format", "json") + mp.field("token", csrf) + mp.field("filename", args.name) + mp.field("comment", args.summary) + mp.field("text", args.text) + mp.field("ignorewarnings", "1") + mp.file("file", os.path.basename(args.file), data, ctype) + req = urllib.request.Request( + API, + data=mp.body(), + headers={"User-Agent": UA, "Content-Type": mp.content_type()}, + ) + with opener.open(req, timeout=120) as r: + raw = r.read() + try: + result = json.loads(raw) + except Exception: + print(raw[:2000]) + raise + upload = result.get("upload", {}) + if not upload: + print("RAW RESPONSE:", raw[:2000]) + print(json.dumps(upload, indent=1)) + if upload.get("result") != "Success": + sys.exit(1) + + +if __name__ == "__main__": + main() -- cgit v1.2.3