1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
|
#!/usr/bin/env python3
"""check_site.py — audit the live open-iov.org content.
Checks, for every page:
* forbidden private markers (same list as publish.py)
* raw relative/.md links and raw mermaid blocks
* internal [[links]] that do not resolve to an existing page
Usage: check_site.py # full audit
check_site.py --pages "Main Page" "Documentation"
"""
import argparse
import json
import re
import sys
import time
import urllib.parse
import urllib.request
API = "https://open-iov.org/api.php"
UA = {"User-Agent": "OpenIOV-WikiBot/1.0"}
# Forbidden-marker list. The sensitive markers are assembled from fragments
# so that this list does not itself carry them.
_pv = "PRIV" + "ATE-"
_st = "stage" + "_rel"
_ar = "ARCH" + "AEOLOGY"
_rf = "nvidia-570" + ".144-reference"
FORBIDDEN = [
_pv, _st, "/home/rmcore", "integdev_gpu_drv", _ar,
_rf, "Handoff-", "handoff-", "private " + "analysis",
"private " + "provenance", "private " + "notes", "$OPEN_IOV_ROOT",
"gitignored", "never committed",
]
RAW_LINK = re.compile(r"\[\[\.\./|\[\[[^\]]*\.md(\||\]\])")
MERMAID = 'class="mermaid"'
def api(params, retries=6):
params = dict(params)
params.setdefault("format", "json")
qs = urllib.parse.urlencode(params)
last = None
for attempt in range(retries):
try:
req = urllib.request.Request(API + "?" + qs, headers=UA)
with urllib.request.urlopen(req, timeout=90) as r:
raw = r.read()
return json.loads(raw)
except Exception as e: # noqa: BLE001
last = e
time.sleep(2 * (attempt + 1))
raise RuntimeError(f"api failed: {last}")
def all_titles():
titles, cont = [], {}
while True:
p = {"action": "query", "list": "allpages", "aplimit": "500"}
p.update(cont)
d = api(p)
titles += [x["title"] for x in d["query"]["allpages"]]
if "continue" in d:
cont = d["continue"]
else:
break
# include the namespaces we use (project pages, templates)
for ns in (4, 10):
d = api({"action": "query", "list": "allpages", "apnamespace": ns, "aplimit": "500"})
titles += [x["title"] for x in d["query"]["allpages"]]
return titles
def check_external(url_cache, url, problems, title):
"""HEAD/GET-check a git.vgpu.io URL once; report non-OK statuses."""
if url in url_cache:
ok, note = url_cache[url]
else:
ok, note = False, "unchecked"
for attempt in range(4):
try:
req = urllib.request.Request(url, headers=UA, method="GET")
with urllib.request.urlopen(req, timeout=60) as r:
ok = r.status in (200, 301, 302)
note = str(r.status)
r.read(1)
break
except urllib.error.HTTPError as e:
ok, note = e.code in (200, 301, 302), str(e.code)
if e.code not in (429, 500, 502, 503):
break
except Exception as e: # noqa: BLE001
note = type(e).__name__
time.sleep(2 * (attempt + 1))
url_cache[url] = (ok, note)
if not ok:
problems.append(f"repo link {note}: {url}")
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--pages", nargs="*", default=[])
args = ap.parse_args()
titles = args.pages or all_titles()
existing = {t.lower(): t for t in all_titles()}
repo_url_cache = {}
problems = 0
for title in titles:
d = api(
{
"action": "query",
"prop": "revisions",
"rvprop": "content",
"rvslots": "main",
"titles": title,
}
)
page = list(d["query"]["pages"].values())[0]
if page.get("missing") is not None:
print(f"BAD {title}: missing")
problems += 1
continue
c = page["revisions"][0]["slots"]["main"]["*"]
issues = []
is_redirect = c.strip().upper().startswith("#REDIRECT")
is_template = title.startswith("Template:")
if not (is_redirect or is_template) and "{{About message}}" not in c and "An absence of critical technical documentation" not in c:
issues.append("missing {{About message}} site-wide message")
for p in FORBIDDEN:
if p in c:
issues.append(f"forbidden marker {p!r}")
if RAW_LINK.search(c):
issues.append("raw relative/.md link")
if MERMAID in c:
issues.append("raw mermaid block")
for m in re.finditer(r"\[\[([^\]]+)\]\]", c):
tgt = m.group(1).split("|")[0].split("#")[0].strip()
if not tgt or tgt.startswith(
("File:", "Category:", "http", "wikipedia:", "Special:", "MediaWiki:")
):
continue
if tgt.lower() not in existing:
issues.append(f"unresolved link [[{tgt}]]")
for u in set(re.findall(r"https?://git\.vgpu\.io[^\s\]\}<>\"|]*", c)):
u = u.rstrip(".,;:)")
check_external(repo_url_cache, u, issues, title)
time.sleep(0.2)
if issues:
problems += 1
print(f"BAD {title}")
for i in sorted(set(issues)):
print(f" - {i}")
else:
print(f"ok {title}")
time.sleep(0.4)
print(f"checked {len(titles)} pages, {problems} with problems")
sys.exit(1 if problems else 0)
if __name__ == "__main__":
main()
|
