Coverage for src/fjc_biographies/rules.py: 93%
240 statements
« prev ^ index » next coverage.py v7.17.0, created at 2026-10-22 22:47 +0000
1"""Rules that make new fields from the text of judges' biographies.
3The rules are written in CSV files (spreadsheets), so they can be read and
4changed without knowing Python. They are the rules of courtpy, whose "judges"
5rulebook (in fjc_biographies' "instructions" folder) codes each judge's
6professional career. Each row is one rule. The columns are:
8| Column | Meaning |
9| --- | --- |
10| `target` | The section of text to search, such as "career". Several sections can be listed, separated by commas, and are searched in order. |
11| `variable` | The name of the column that the rule makes (or adds to). |
12| `kind` | What the rule does (see below). |
13| `pattern` | A regular expression to search for. |
14| `value` | For a `label` rule, the value to record. Otherwise blank. |
15| `ignorecase` | TRUE to ignore the difference between capital and lower-case letters. |
16| `dotall` | TRUE for "." in the pattern to also match line breaks. |
17| `note` | Anything you want to say about the rule. It is not used. |
19The kinds of rules:
21| Kind | Makes |
22| --- | --- |
23| `flag` | TRUE if the pattern appears, and otherwise FALSE. |
24| `count` | The number of times the pattern appears. |
25| `matches` | A list of every match. |
26| `label` | The rule's `value` if the pattern appears. The first rule for a variable that matches wins, so it can turn text into categories (such as kinds of positions into numbers). |
27| `names` | A list of names, found by splitting the text wherever the pattern matches. The pattern should match the words and punctuation between names. |
28| `section` | A new section of text: the first match. Later rules can search it. |
29| `excerpts` | A new section of text: every match, one per line. |
30| `split` | Two new sections, named in `variable` (such as "before, after"): the text before and after the first match. |
31| `remove` | Nothing. It deletes every match from the target for the rules after it. |
33Rules are applied from top to bottom, so a rule can search a section made by
34a rule above it. Rules with the same `variable` are combined: flags are TRUE if
35any rule matches, counts are added, lists of matches and names are joined, and
36the first `label` or `section` that matches is used.
38Contents:
39 KINDS: the kinds of rules.
40 Rule: one rule from a CSV file.
41 Rulebook: an ordered collection of rules.
42 find_instructions: returns the CSV files for a built-in or custom rulebook.
43 split_names: splits text into names wherever a pattern matches.
45"""
47from __future__ import annotations
49import csv
50import dataclasses
51import importlib.resources
52import pathlib
53import re
54from collections.abc import (
55 Iterable,
56 Iterator,
57 Mapping,
58 MutableMapping,
59 Sequence,
60)
61from typing import Any
63KINDS: tuple[str, ...] = (
64 'count', 'excerpts', 'flag', 'label', 'matches', 'names', 'remove',
65 'section', 'split')
66_SECTION_KINDS: frozenset[str] = frozenset({'excerpts', 'section', 'split'})
67# The curly apostrophe (U+2019) is added with `chr` so that the source file
68# has only plain characters.
69_PUNCTUATION: re.Pattern[str] = re.compile("[.'" + chr(0x2019) + ']')
70# Columns that every rules file must have. "value" and "note" are optional.
71_REQUIRED: tuple[str, ...] = (
72 'target', 'variable', 'kind', 'pattern', 'ignorecase', 'dotall')
75@dataclasses.dataclass(frozen = True)
76class Rule:
77 """One rule from a CSV file.
79 Args:
80 targets: names of the sections of text to search, in order.
81 variables: names of the columns (or, for `section`, `excerpts`, and
82 `split` rules, the sections) that the rule makes. Only a `split`
83 rule has more than one.
84 kind: one of `KINDS`.
85 pattern: the compiled regular expression.
86 value: for a `label` rule, the value to record. Defaults to None.
87 note: a description of the rule. Defaults to an empty `str`.
89 """
91 targets: tuple[str, ...]
92 variables: tuple[str, ...]
93 kind: str
94 pattern: re.Pattern[str]
95 value: Any = None
96 note: str = ''
98 """ Class Methods """
100 @classmethod
101 def create(cls, row: Mapping[str, Any]) -> Rule:
102 """Returns a rule made from a row of a CSV file.
104 Args:
105 row: the cells of the row, by the names of the columns.
107 Returns:
108 The rule.
110 Raises:
111 ValueError: if the kind is not one of `KINDS`, the pattern is not
112 a valid regular expression, a cell that is needed is blank, or
113 a `split` rule does not name two variables.
115 """
116 kind = str(row.get('kind') or '').strip().lower()
117 if kind not in KINDS:
118 message = f'the kind {kind!r} is not one of {list(KINDS)}'
119 raise ValueError(message)
120 targets = tuple(_listify(row.get('target')))
121 variables = tuple(_listify(row.get('variable')))
122 if not targets:
123 message = 'the target is blank'
124 raise ValueError(message)
125 if kind == 'split' and len(variables) != 2:
126 message = (
127 'a split rule needs two variables, separated by a comma (such '
128 'as "before, after")'
129 )
130 raise ValueError(message)
131 if kind != 'remove' and not variables: 131 ↛ 132line 131 didn't jump to line 132 because the condition on line 131 was never true
132 message = 'the variable is blank'
133 raise ValueError(message)
134 source = str(row.get('pattern') or '')
135 if not source:
136 message = 'the pattern is blank'
137 raise ValueError(message)
138 flags = 0
139 if _to_bool(row.get('ignorecase')):
140 flags |= re.IGNORECASE
141 if _to_bool(row.get('dotall')): 141 ↛ 142line 141 didn't jump to line 142 because the condition on line 141 was never true
142 flags |= re.DOTALL
143 try:
144 pattern = re.compile(source, flags)
145 except re.error as error:
146 message = f'the pattern {source!r} is not valid: {error}'
147 raise ValueError(message) from error
148 value = row.get('value')
149 return cls(
150 targets = targets,
151 variables = variables,
152 kind = kind,
153 pattern = pattern,
154 value = _convert(value) if kind == 'label' else None,
155 note = str(row.get('note') or ''))
157 """ Properties """
159 @property
160 def variable(self) -> str:
161 """Returns the name of the (first) variable that the rule makes.
163 Returns:
164 The first of `variables`, or an empty `str` for a `remove` rule,
165 which makes none.
167 """
168 return self.variables[0] if self.variables else ''
170 """ Public Methods """
172 def apply(
173 self,
174 sections: MutableMapping[str, str],
175 results: MutableMapping[str, Any]) -> None:
176 """Applies the rule to `sections`, recording what it finds in `results`.
178 Args:
179 sections: the sections of text, by name. `remove`, `section`,
180 `excerpts`, and `split` rules change it.
181 results: the variables found so far, by name. It is changed in
182 place.
184 """
185 texts = [sections[t] for t in self.targets if sections.get(t)]
186 if self.kind == 'remove':
187 for target in self.targets:
188 if sections.get(target): 188 ↛ 187line 188 didn't jump to line 187 because the condition on line 188 was always true
189 sections[target] = self.pattern.sub(' ', sections[target])
190 elif self.kind == 'split':
191 self._split(texts, sections, results)
192 elif self.kind == 'flag':
193 flagged = any(self.pattern.search(text) for text in texts)
194 results[self.variable] = bool(results.get(self.variable)) or flagged
195 elif self.kind == 'count':
196 counted = sum(
197 1 for text in texts for _ in self.pattern.finditer(text))
198 results[self.variable] = results.get(self.variable, 0) + counted
199 elif self.kind == 'matches':
200 matches = results.setdefault(self.variable, [])
201 matches.extend(
202 m.group(0).strip() for t in texts
203 for m in self.pattern.finditer(t))
204 elif self.kind == 'label':
205 if results.get(self.variable) is None:
206 matched = any(self.pattern.search(text) for text in texts)
207 results[self.variable] = self.value if matched else None
208 elif self.kind == 'names':
209 names = results.setdefault(self.variable, [])
210 for text in texts:
211 names.extend(n for n in split_names(text, self.pattern)
212 if n not in names)
213 elif self.kind == 'section':
214 if not results.get(self.variable): 214 ↛ exitline 214 didn't return from function 'apply' because the condition on line 214 was always true
215 match = next(
216 (m for t in texts if (m := self.pattern.search(t))), None)
217 section = match.group(0).strip() if match else None
218 results[self.variable] = section
219 if section:
220 sections[self.variable] = section
221 elif self.kind == 'excerpts': 221 ↛ exitline 221 didn't return from function 'apply' because the condition on line 221 was always true
222 excerpts = [
223 m.group(0).strip() for t in texts
224 for m in self.pattern.finditer(t) if m.group(0).strip()]
225 existing = results.get(self.variable)
226 joined = '\n'.join(([existing] if existing else []) + excerpts)
227 results[self.variable] = joined or None
228 if joined:
229 sections[self.variable] = joined
231 """ Private Methods """
233 def _split(
234 self,
235 texts: Sequence[str],
236 sections: MutableMapping[str, str],
237 results: MutableMapping[str, Any]) -> None:
238 """Divides the first target at the first match of the pattern.
240 Args:
241 texts: the texts of the targets that are not blank.
242 sections: the sections of text, by name.
243 results: the variables found so far, by name.
245 """
246 first, second = self.variables
247 if not texts: 247 ↛ 248line 247 didn't jump to line 248 because the condition on line 247 was never true
248 results.setdefault(first, None)
249 results.setdefault(second, None)
250 return
251 text = texts[0]
252 match = self.pattern.search(text)
253 if match is None:
254 before, after = text, ''
255 else:
256 before, after = text[:match.start()], text[match.end():]
257 for name, part in ((first, before.strip()), (second, after.strip())):
258 results[name] = part or None
259 if part:
260 sections[name] = part
263@dataclasses.dataclass
264class Rulebook:
265 """An ordered collection of rules.
267 Args:
268 rules: the rules, in the order they are applied. Defaults to an empty
269 `list`.
270 name: a name for the rulebook, such as the file it came from. Defaults
271 to an empty `str`.
273 """
275 rules: list[Rule] = dataclasses.field(default_factory = list)
276 name: str = ''
278 """ Class Methods """
280 @classmethod
281 def create(cls, source: Rulebook | pathlib.Path | str) -> Rulebook:
282 """Returns a rulebook made from `source`.
284 Args:
285 source: a rulebook, or the name of a built-in rulebook or the path
286 of a CSV file or a folder of them (see `load`).
288 Returns:
289 The rulebook.
291 """
292 return source if isinstance(source, Rulebook) else cls.load(source)
294 @classmethod
295 def load(cls, *sources: str | pathlib.Path) -> Rulebook:
296 """Loads rules from CSV files.
298 Args:
299 *sources: names of built-in rulebooks (such as "judges"), or paths
300 to CSV files or to folders of them. The rules are applied in
301 the order they are given (and, in a folder, in the
302 alphabetical order of the files). A file that two sources
303 name is read only the first time.
305 Returns:
306 A rulebook with the rules of every source.
308 Raises:
309 FileNotFoundError: if a source is not a built-in rulebook or a file
310 or folder.
311 ValueError: if a file is missing a column or a row is not a valid
312 rule. The message names the file and row.
314 """
315 rules: list[Rule] = []
316 names: list[str] = []
317 read: set[pathlib.Path] = set()
318 for source in sources:
319 for path in find_instructions(source):
320 if path.resolve() in read:
321 continue
322 read.add(path.resolve())
323 rules.extend(_read(path))
324 names.append(path.stem)
325 return cls(rules = rules, name = ', '.join(names))
327 """ Properties """
329 @property
330 def sections(self) -> list[str]:
331 """Returns the names of the sections of text that the rules make.
333 Returns:
334 The variables of the `section`, `excerpts`, and `split` rules, in
335 order, without repeats.
337 """
338 return _unique(
339 v for r in self.rules if r.kind in _SECTION_KINDS
340 for v in r.variables)
342 @property
343 def targets(self) -> list[str]:
344 """Returns the names of the sections of text that the rules search.
346 Returns:
347 The targets of every rule, in order, without repeats.
349 """
350 return _unique(t for rule in self.rules for t in rule.targets)
352 @property
353 def variables(self) -> list[str]:
354 """Returns the names of the columns that the rules make.
356 Returns:
357 The variables of every rule except `remove` rules (which make
358 none), in order, without repeats.
360 """
361 return _unique(
362 v for r in self.rules if r.kind != 'remove' for v in r.variables)
364 """ Public Methods """
366 def apply(self, sections: MutableMapping[str, str]) -> dict[str, Any]:
367 """Applies every rule, in order, to `sections`.
369 Args:
370 sections: the sections of text, by name. Rules that make or change
371 sections change it in place.
373 Returns:
374 The value of each variable, by name. A variable whose rules found
375 nothing is False (flags), 0 (counts), an empty `list`
376 (matches and names), or None (labels and sections).
378 """
379 results: dict[str, Any] = {}
380 for rule in self.rules:
381 rule.apply(sections, results)
382 return results
384 """ Dunder Methods """
386 def __add__(self, other: Rulebook) -> Rulebook:
387 """Returns a rulebook with the rules of both rulebooks.
389 Args:
390 other: the rulebook whose rules come second.
392 Returns:
393 A new rulebook with this rulebook's rules followed by those of
394 `other`, named for both.
396 """
397 name = ', '.join(n for n in (self.name, other.name) if n)
398 return Rulebook(rules = [*self.rules, *other.rules], name = name)
400 def __iter__(self) -> Iterator[Rule]:
401 """Returns an iterator of the rules.
403 Returns:
404 An iterator of `rules`, in the order they are applied.
406 """
407 return iter(self.rules)
409 def __len__(self) -> int:
410 """Returns the number of rules.
412 Returns:
413 The length of `rules`.
415 """
416 return len(self.rules)
419def find_instructions(source: str | pathlib.Path) -> list[pathlib.Path]:
420 """Returns the CSV files for a built-in or custom rulebook.
422 Args:
423 source: the name of a built-in rulebook (a file in fjc_biographies'
424 "instructions" folder, such as "judges"), or the path to a CSV
425 file or a folder of them.
427 Returns:
428 The paths of the CSV files, in the order they are applied.
430 Raises:
431 FileNotFoundError: if `source` is not a built-in rulebook, a CSV file,
432 or a folder with CSV files.
434 """
435 path = pathlib.Path(source)
436 if not path.exists():
437 folder = pathlib.Path(
438 str(importlib.resources.files('fjc_biographies'))) / 'instructions'
439 name = str(source)
440 for candidate in (folder / name, folder / f'{name}.csv'):
441 if candidate.exists():
442 path = candidate
443 break
444 else:
445 built_in = sorted(p.stem for p in folder.glob('*.csv'))
446 message = (
447 f'{source!r} is not a CSV file, a folder, or a built-in '
448 f'rulebook ({built_in})'
449 )
450 raise FileNotFoundError(message)
451 if path.is_dir():
452 files = sorted(path.glob('*.csv'))
453 if not files:
454 message = f'there are no CSV files in {path}'
455 raise FileNotFoundError(message)
456 return files
457 return [path]
460def split_names(text: str, separators: re.Pattern[str]) -> list[str]:
461 """Splits `text` into names wherever `separators` matches.
463 The text is made upper-case and periods and apostrophes are removed first
464 (so "O'Scannlain" becomes "OSCANNLAIN"), and line breaks always separate
465 names. Pieces without at least two letters are dropped.
467 Args:
468 text: text with names, such as "Hon. SMITH, JONES, and BROWN".
469 separators: a pattern that matches the words and punctuation between
470 the names. It is matched against the upper-case text.
472 Returns:
473 The names, in order, without repeats.
475 """
476 cleaned = _PUNCTUATION.sub('', text.upper())
477 names: list[str] = []
478 for line in cleaned.splitlines():
479 # The text between matches is used (rather than `re.split`) so that
480 # groups in the pattern are not mistaken for names.
481 pieces, start = [], 0
482 for match in separators.finditer(line):
483 pieces.append(line[start:match.start()])
484 start = match.end()
485 pieces.append(line[start:])
486 for piece in pieces:
487 name = ' '.join(piece.split())
488 if sum(c.isalpha() for c in name) >= 2 and name not in names:
489 names.append(name)
490 return names
493""" Private Functions """
496def _convert(value: Any) -> Any:
497 """Returns a cell of a CSV file as a number if it is one.
499 Args:
500 value: the text of the cell.
502 Returns:
503 An `int` or `float` if the text is a number, the text otherwise, or
504 None if the cell is blank.
506 """
507 if value is None: 507 ↛ 508line 507 didn't jump to line 508 because the condition on line 507 was never true
508 return None
509 text = str(value).strip()
510 if not text: 510 ↛ 511line 510 didn't jump to line 511 because the condition on line 510 was never true
511 return None
512 for kind in (int, float):
513 try:
514 return kind(text)
515 except ValueError:
516 continue
517 return text
520def _listify(item: Any) -> list[str]:
521 """Returns a cell of a CSV file as a list of names.
523 Args:
524 item: None, or a `str` that may list names separated by commas.
526 Returns:
527 The names, with surrounding spaces removed and blank names dropped.
529 """
530 if item is None: 530 ↛ 531line 530 didn't jump to line 531 because the condition on line 530 was never true
531 return []
532 return [name.strip() for name in str(item).split(',') if name.strip()]
535def _read(path: pathlib.Path) -> list[Rule]:
536 """Returns the rules in the CSV file at `path`.
538 Args:
539 path: path to a CSV file of rules, in UTF-8 (with or without a byte
540 order mark, as Excel saves them).
542 Returns:
543 The rules, in order.
545 Raises:
546 ValueError: if the file is missing a column or a row is not a valid
547 rule.
549 """
550 with path.open(encoding = 'utf-8-sig', newline = '') as file:
551 reader = csv.DictReader(file)
552 columns = [c.strip().lower() for c in reader.fieldnames or []]
553 missing = [c for c in _REQUIRED if c not in columns]
554 if missing:
555 message = f'{path} is missing the columns {missing}'
556 raise ValueError(message)
557 rules = []
558 # Row 1 is the header, so the first rule is on row 2.
559 for number, row in enumerate(reader, start = 2):
560 cells = {
561 str(k).strip().lower(): v for k, v in row.items()
562 if k is not None}
563 if not any(str(v or '').strip() for v in cells.values()): 563 ↛ 564line 563 didn't jump to line 564 because the condition on line 563 was never true
564 continue
565 try:
566 rules.append(Rule.create(cells))
567 except ValueError as error:
568 message = f'{path}, row {number}: {error}'
569 raise ValueError(message) from error
570 return rules
573def _to_bool(value: Any) -> bool:
574 """Converts a cell of a CSV file to a boolean.
576 Args:
577 value: a `bool`, or text such as "TRUE", "yes", "1", "t", "FALSE",
578 "no", "0", or "f" (in any case). Blank text is False.
580 Returns:
581 The boolean value.
583 Raises:
584 ValueError: if `value` is text that is not one of those.
586 """
587 if isinstance(value, bool): 587 ↛ 588line 587 didn't jump to line 588 because the condition on line 587 was never true
588 return value
589 text = '' if value is None else str(value).strip().lower()
590 if text in {'true', 't', 'yes', 'y', '1'}:
591 return True
592 if text in {'false', 'f', 'no', 'n', '0', ''}:
593 return False
594 message = f'{value!r} is not TRUE or FALSE'
595 raise ValueError(message)
598def _unique(items: Iterable[str]) -> list[str]:
599 """Returns `items` in order, without repeats.
601 Args:
602 items: names.
604 Returns:
605 The names, in the order first seen.
607 """
608 return list(dict.fromkeys(items))