Coverage for src/coursebook/markdown.py: 99%

1"""Tools for reading and adjusting the markdown in source files. 

2 

3Contents: 

4 extract_images: returns the images in markdown text. 

5 namespace_footnotes: adds a prefix to the footnote labels in markdown text. 

6 normalize_name: converts a name to lower case words joined by underscores. 

7 shift_headings: changes the level of every heading in markdown text. 

8 split_front_matter: separates the front matter from markdown text. 

9 split_sections: divides markdown text into its marked sections. 

10 _DIVIDER: matches a line that opens or closes a section (such as 

11 '<slides>'). 

12 _FENCE: matches the opening or closing line of a fenced code block. 

13 _FOOTNOTE: matches a footnote reference or definition label. 

14 _FRONT_MATTER_DELIMITERS: the lines that may open and close front matter. 

15 _HEADING: matches the '#' characters at the start of an ATX heading. 

16 _IMAGE: matches a markdown image and any pandoc attributes after it. 

17 _MAXIMUM_HEADING_LEVEL: the deepest heading level that markdown supports. 

18 

19""" 

20 

21from __future__ import annotations 

22 

23import re 

24from typing import TYPE_CHECKING, Any 

25 

26import yaml 

27 

28if TYPE_CHECKING: 

29 from collections.abc import Iterator 

30 

31_FENCE: re.Pattern[str] = re.compile(r'^ {0,3}(`{3,}|~{3,})') 

32# Front matter normally opens with '---'. The `'''` delimiter is also allowed 

33# because the original `coursebook` examples used it. 

34_FRONT_MATTER_DELIMITERS: dict[str, tuple[str, ...]] = { 

35 '---': ('---', '...'), 

36 "'''": ("'''",)} 

37_DIVIDER: re.Pattern[str] = re.compile( 

38 r'^\s*<\s*(/?)\s*([A-Za-z_]+)\s*>\s*$') 

39_FOOTNOTE: re.Pattern[str] = re.compile(r'\[\^([^\]\s]+)\]') 

40_HEADING: re.Pattern[str] = re.compile(r'^(#{1,6})(?=\s|$)') 

41_IMAGE: re.Pattern[str] = re.compile( 

42 r'!\[[^\]]*\]\([^)]+\)(?:\{[^}]*\})?') 

43_MAXIMUM_HEADING_LEVEL: int = 6 

44 

45 

46""" Public Functions """ 

47 

48 

49def extract_images(text: str) -> list[str]: 

50 """Returns the images in markdown `text`. 

51 

52 Images inside fenced code blocks are ignored. 

53 

54 Args: 

55 text: markdown text. 

56 

57 Returns: 

58 The markdown for each image (including any pandoc attributes, such as 

59 `{width=50%}`), in the order in which they appear. 

60 

61 """ 

62 images: list[str] = [] 

63 for line, in_code in _iterate_lines(text): 

64 if not in_code: 

65 images.extend(match.group(0) for match in _IMAGE.finditer(line)) 

66 return images 

67 

68 

69def namespace_footnotes(text: str, prefix: str) -> str: 

70 """Adds `prefix` to the footnote labels in markdown `text`. 

71 

72 Pandoc treats footnote labels as belonging to the whole document. So, if two 

73 source files each use `[^1]`, the second footnote would be replaced by the 

74 first when the files are combined. Adding a prefix that is unique to each 

75 source file keeps the footnotes separate. 

76 

77 Args: 

78 text: markdown text. 

79 prefix: text to add to the start of each footnote label. 

80 

81 Returns: 

82 `text` with each footnote label (in both the references and the 

83 definitions) changed from `[^label]` to `[^prefix-label]`. 

84 

85 """ 

86 lines = [] 

87 for line, in_code in _iterate_lines(text): 

88 lines.append( 

89 line if in_code else _FOOTNOTE.sub(rf'[^{prefix}-\1]', line)) 

90 return '\n'.join(lines) 

91 

92 

93def normalize_name(name: str) -> str: 

94 """Converts `name` to lower case words joined by underscores. 

95 

96 Every run of characters that are not letters or numbers is replaced with a 

97 single underscore. For example, 'Law & Economics' becomes 'law_economics' 

98 and 'Law Review' becomes 'law_review'. 

99 

100 Args: 

101 name: the name to convert. 

102 

103 Returns: 

104 The normalized name. 

105 

106 """ 

107 return re.sub(r'[^0-9a-z]+', '_', name.lower()).strip('_') 

108 

109 

110def shift_headings(text: str, levels: int) -> str: 

111 """Changes the level of every heading in markdown `text`. 

112 

113 Only ATX headings (headings that start with '#') are changed. Headings 

114 inside fenced code blocks are left alone. Markdown does not support more 

115 than six levels of headings, so deeper headings are kept at level six. 

116 

117 Args: 

118 text: markdown text. 

119 levels: number of levels to add to each heading. A negative number 

120 raises the headings, but never above level one. 

121 

122 Returns: 

123 `text` with its headings shifted. 

124 

125 """ 

126 lines = [] 

127 for line, in_code in _iterate_lines(text): 

128 match = None if in_code else _HEADING.match(line) 

129 if match: 

130 level = len(match.group(1)) + levels 

131 level = max(1, min(level, _MAXIMUM_HEADING_LEVEL)) 

132 lines.append('#' * level + line[match.end():]) 

133 else: 

134 lines.append(line) 

135 return '\n'.join(lines) 

136 

137 

138def split_front_matter(text: str) -> tuple[dict[str, Any], str]: 

139 """Separates the YAML front matter from markdown `text`. 

140 

141 Front matter must start on the first line that is not blank. It opens with 

142 '---' (and closes with '---' or '...') or opens and closes with "'''". 

143 

144 Args: 

145 text: markdown text that may start with front matter. 

146 

147 Returns: 

148 The front matter as a `dict` (which is empty if there is no front 

149 matter) and the rest of `text`. 

150 

151 Raises: 

152 ValueError: if the front matter is never closed or is not a mapping of 

153 keys to values. 

154 

155 """ 

156 lines = text.lstrip('').splitlines() 

157 start = 0 

158 while start < len(lines) and not lines[start].strip(): 

159 start += 1 

160 if start == len(lines): 160 ↛ 161line 160 didn't jump to line 161 because the condition on line 160 was never true

161 return {}, '' 

162 opener = lines[start].strip() 

163 if opener not in _FRONT_MATTER_DELIMITERS: 

164 return {}, text 

165 closers = _FRONT_MATTER_DELIMITERS[opener] 

166 for end in range(start + 1, len(lines)): 

167 if lines[end].strip() in closers: 

168 break 

169 else: 

170 message = f'front matter opened with "{opener}" is never closed' 

171 raise ValueError(message) 

172 metadata = yaml.safe_load('\n'.join(lines[start + 1:end])) 

173 if metadata is None: 

174 metadata = {} 

175 if not isinstance(metadata, dict): 

176 message = 'front matter must be a mapping of keys to values' 

177 raise ValueError(message) # noqa: TRY004 

178 return metadata, '\n'.join(lines[end + 1:]) 

179 

180 

181def split_sections( 

182 text: str, 

183 names: dict[str, str], 

184 default: str) -> dict[str, str]: 

185 """Divides markdown `text` into the sections marked by dividers. 

186 

187 A divider is a line with only a section name inside '<' and '>' (such as 

188 '<slides>'). The text after a divider, up to the next divider, belongs to 

189 that section. Text before the first divider belongs to the `default` 

190 section. A closing divider (such as '</slides>') returns to the `default` 

191 section. Dividers inside fenced code blocks are ignored. If a section 

192 appears more than once, its parts are joined in order. 

193 

194 Args: 

195 text: markdown text without front matter. 

196 names: `dict` with keys that are the names that may appear in dividers 

197 (in lower case) and values that are the sections they mark. This 

198 allows more than one name for a section (such as 'manuscript' for 

199 'text'). 

200 default: the section for text that is not after a divider. 

201 

202 Returns: 

203 A `dict` with keys that are section names and values that are the 

204 stripped text of each section. Only sections with text are included. 

205 

206 Raises: 

207 ValueError: if a divider names a section that is not in `names`. 

208 

209 """ 

210 parts: dict[str, list[str]] = {} 

211 current = default 

212 for line, in_code in _iterate_lines(text): 

213 match = None if in_code else _DIVIDER.match(line) 

214 if match: 

215 closing, name = match.groups() 

216 if name.lower() not in names: 

217 known = ', '.join(f'<{key}>' for key in names) 

218 message = f'unknown divider "{line.strip()}" (use {known})' 

219 raise ValueError(message) 

220 current = default if closing else names[name.lower()] 

221 parts.setdefault(current, []).append('') 

222 continue 

223 parts.setdefault(current, []).append(line) 

224 sections = {} 

225 for name, lines in parts.items(): 

226 section = '\n'.join(lines).strip('\n') 

227 if section.strip(): 

228 sections[name] = section 

229 return sections 

230 

231 

232""" Private Functions """ 

233 

234 

235def _iterate_lines(text: str) -> Iterator[tuple[str, bool]]: 

236 """Yields each line of `text` and whether it is in a fenced code block. 

237 

238 The opening and closing lines of a fenced code block count as being in the 

239 code block. 

240 

241 Args: 

242 text: markdown text. 

243 

244 Yields: 

245 Each line of `text` and whether that line is part of a fenced code 

246 block. 

247 

248 """ 

249 fence: str | None = None 

250 for line in text.split('\n'): 

251 match = _FENCE.match(line) 

252 if fence is None and match: 

253 fence = match.group(1) 

254 yield line, True 

255 elif fence is not None: 

256 if ( 

257 match 

258 and match.group(1)[0] == fence[0] 

259 and len(match.group(1)) >= len(fence) 

260 and not line.strip()[len(match.group(1)):].strip()): 

261 fence = None 

262 yield line, True 

263 else: 

264 yield line, False