1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
|
#!/usr/bin/env python3
"""Check the project's Markdown files for broken internal links.
What it checks, offline (`make docs` runs without network):
* relative links to files: the target must exist in the repository;
* anchors (`#section`), in the same file or another Markdown file: a heading
with that GitHub-style anchor must exist.
External links (http, https, mailto) are not fetched; the network is off.
Usage: check_docs.py [ROOT] (ROOT defaults to the current directory)
Exit: 0 when every link resolves, 1 otherwise. Each problem is printed as
`file:line: message`.
"""
from __future__ import annotations
import re
import sys
from collections.abc import Iterator
from dataclasses import dataclass
from pathlib import Path
# Folders that hold generated or third-party files, never project docs.
SKIP_DIRS = {".git", ".cache", "build", "node_modules"}
# [text](target) or [text](target "title"); the target may be wrapped in <...>.
LINK_RE = re.compile(r"\[(?:[^\]\\]|\\.)*\]\(\s*(<[^>]*>|[^)\s]+)(?:\s+\"[^\"]*\")?\s*\)")
HEADING_RE = re.compile(r"^(#{1,6})\s+(.*?)\s*#*\s*$")
FENCE_RE = re.compile(r"^\s*(```|~~~)")
INLINE_CODE_RE = re.compile(r"`[^`]*`")
EXTERNAL_PREFIXES = ("http://", "https://", "mailto:")
@dataclass(frozen=True)
class Problem:
"""One broken link, reported as `file:line: message`."""
path: Path
line: int
message: str
def __str__(self) -> str:
return f"{self.path}:{self.line}: {self.message}"
def slugify(heading: str) -> str:
"""Return the anchor GitHub generates for a heading.
GitHub lowercases the text, drops everything except letters, digits,
spaces, hyphens and underscores, then turns spaces into hyphens. Inline
markup is removed first: `code` keeps its text, [text](url) keeps `text`.
"""
text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", heading)
text = text.replace("`", "").replace("*", "").strip().lower()
text = re.sub(r"[^\w\- ]", "", text)
return text.replace(" ", "-")
def iter_lines_outside_code(text: str) -> Iterator[tuple[int, str]]:
"""Yield (line number, line) for lines outside fenced code blocks."""
in_fence = False
for number, line in enumerate(text.splitlines(), start=1):
if FENCE_RE.match(line):
in_fence = not in_fence
continue
if not in_fence:
yield number, line
def anchors_of(path: Path) -> set[str]:
"""All heading anchors of a Markdown file, with GitHub's -1, -2 suffixes
for repeated headings."""
seen: dict[str, int] = {}
anchors: set[str] = set()
for _, line in iter_lines_outside_code(path.read_text(encoding="utf-8")):
match = HEADING_RE.match(line)
if not match:
continue
slug = slugify(match.group(2))
count = seen.get(slug, 0)
anchors.add(slug if count == 0 else f"{slug}-{count}")
seen[slug] = count + 1
return anchors
def markdown_files(root: Path) -> list[Path]:
"""Every Markdown file of the project, skipping generated folders."""
return sorted(p for p in root.rglob("*.md") if not any(part in SKIP_DIRS for part in p.relative_to(root).parts))
def check_file(path: Path, root: Path, cache: dict[Path, set[str]]) -> list[Problem]:
"""Check every link of one Markdown file."""
problems: list[Problem] = []
rel = path.relative_to(root)
for number, line in iter_lines_outside_code(path.read_text(encoding="utf-8")):
for match in LINK_RE.finditer(INLINE_CODE_RE.sub("", line)):
target = match.group(1).strip("<>")
if target.startswith(EXTERNAL_PREFIXES):
continue
file_part, _, anchor = target.partition("#")
dest = path if not file_part else (path.parent / file_part).resolve()
if not dest.exists():
problems.append(Problem(rel, number, f"missing file '{file_part}'"))
continue
if anchor and dest.suffix == ".md":
if dest not in cache:
cache[dest] = anchors_of(dest)
if anchor not in cache[dest]:
problems.append(Problem(rel, number, f"no heading for anchor '#{anchor}' in {dest.name}"))
return problems
def main(argv: list[str]) -> int:
root = Path(argv[1] if len(argv) > 1 else ".").resolve()
cache: dict[Path, set[str]] = {}
files = markdown_files(root)
problems = [p for f in files for p in check_file(f, root, cache)]
for problem in problems:
print(problem)
print(f"check_docs: {len(files)} Markdown files, {len(problems)} broken links")
return 1 if problems else 0
if __name__ == "__main__":
sys.exit(main(sys.argv))
|
