Coverage for src/fjc_biographies/parser.py: 98%
155 statements
« prev ^ index » next coverage.py v7.17.0, created at 2026-10-22 22:47 +0000
1"""Parses the FJC's Biographical Directory of Article III Federal Judges.
3Each of the FJC's six files (see `downloader`) becomes a tidy table with
4`load`: its columns get short, lower case names (such as "commission_date"
5for "Commission Date"), blank cells are missing, the FJC's numbers ("nid",
6"sequence", and the years) are whole numbers, and its dates are dates.
8`build_roster` combines the files of federal judicial service,
9demographics, and professional careers into one table, with a row for each
10court that each judge has served on (each appointment), that codes what researchers usually
11want to know about a judge's appointment (such as the party of the
12appointing president and the share of senators who voted to confirm the
13judge). It is the table that courtpy uses to identify the judges of cases:
15```python
16import fjc_biographies
18tables = fjc_biographies.load_all()
19roster = fjc_biographies.build_roster(
20 tables['service'], tables['demographics'], tables['career'])
21by_judge = fjc_biographies.roster_by_judge(roster)
22```
24`roster_by_judge` organizes the same roster with a row for each judge.
26The fields that a roster has about judges' careers are made by a rulebook
27(see `rules`): the "judges" rulebook in fjc_biographies' "instructions"
28folder, unless another is named. It is a CSV file, so fields can be added or
29changed in a spreadsheet, without changing fjc_biographies.
31Contents:
32 build_roster: makes a table with a row for each court that each judge has
33 served on.
34 code_careers: makes fields from judges' professional careers with a
35 rulebook.
36 load: returns the tidy table of one of the FJC's files.
37 load_all: returns the tidy tables of all of the FJC's files.
38 read: returns the tidy table of a CSV file of the directory.
39 roster_by_judge: returns a roster with one row for each judge.
40 tidy: returns a table of the directory with names, missing values, and
41 types that are easy to use.
43"""
45from __future__ import annotations
47import pathlib
48import re
49from collections.abc import Iterable
50from typing import Any
52import numpy as np
53import pandas as pd
55from . import downloader, names, options, rules
57_APPEALS: str = 'U.S. Court of Appeals'
58# The columns of an appointment, which a judge who was reassigned to another
59# court keeps from the judge's last appointment.
60_APPOINTMENT: tuple[str, ...] = (
61 'appointing_president', 'party_of_appointing_president', 'aba_rating',
62 'recess_appointment_date', 'senate_vote_type', 'ayes_nays')
63# The columns of a roster that describe one of a judge's appointments (and
64# so can differ between a judge's rows), in the order that `roster_by_judge`
65# numbers them. Every other column describes the judge.
66_BY_APPOINTMENT: tuple[str, ...] = (
67 'sequence', 'court', 'court_type', 'circuit_num', 'start_date',
68 'end_date', 'senior_date', 'start_year', 'end_year', 'senior_year',
69 'president', 'party', 'aba_rating', 'recess', 'senate_vote', 'jcs',
70 'mq_first', 'mq_last', 'mq_mean', 'mq_terms')
71_CIRCUIT: re.Pattern[str] = re.compile(r'for the (.+) Circuit\s*$')
72# "Middle District of Georgia" and "Districts of North Carolina" both end
73# with the state, which any note in parentheses follows.
74_DISTRICT: re.Pattern[str] = re.compile(
75 r'Districts? of ([A-Za-z ]+?)\s*(?:\(|$)')
76_DISTRICT_COURT: str = 'U.S. District Court'
77_NUMBERS: frozenset[str] = frozenset({
78 'nid', 'jid', 'sequence', 'birth_month', 'birth_day', 'death_month',
79 'death_day'})
80_PERSONAL: tuple[str, ...] = (
81 'nid', 'last_name', 'first_name', 'middle_name', 'suffix', 'birth_year',
82 'gender', 'race_or_ethnicity')
83_SERVICE: tuple[str, ...] = (
84 'nid', 'judge_name', 'court_type', 'court_name', 'commission_date',
85 'senior_status_date', 'termination_date', *_APPOINTMENT)
86_VOTES: re.Pattern[str] = re.compile(r'(\d+)\s*/\s*(\d+)')
87_YEARS: frozenset[str] = frozenset({
88 'birth_year', 'death_year', 'degree_year', 'chief_judge_begin',
89 'chief_judge_end', 'second_chief_judge_begin', 'second_chief_judge_end'})
92def build_roster(
93 service: pd.DataFrame | pathlib.Path | str,
94 demographics: pd.DataFrame | pathlib.Path | str,
95 career: pd.DataFrame | pathlib.Path | str | None = None,
96 *,
97 rulebook: rules.Rulebook | pathlib.Path | str = 'judges') -> pd.DataFrame:
98 """Makes a table with a row for each court that each judge has served on.
100 The table has these columns:
102 | Column | Meaning |
103 | --- | --- |
104 | `nid` | The FJC's number for the judge. |
105 | `judge` | The judge's name, as the FJC reports it. |
106 | `last_name`, `first_name`, `middle_name`, `suffix` | Its parts. |
107 | `name_first_middle_last`, ... | The name written in each of the ways that `names.permutations` writes it, for matching it against other sources: with and without the middle name or its initial and the suffix, and with the letters that the FJC puts in square brackets kept or replaced with a period (see `names`). |
108 | `sequence` | The FJC's number for the order of the appointment among the judge's, which identifies it with `nid`. |
109 | `court`, `court_type` | The court's name and its type. |
110 | `circuit_num` | The number of the court's circuit: its own for a court of appeals (12 for the D.C. Circuit and 13 for the Federal Circuit), or for a district court, that of its state now. Missing for other courts. |
111 | `start_date`, `end_date`, `senior_date` | The dates the judge's service on the court began (with a commission or a recess appointment) and ended, and the judge took senior status. `end_date` is missing for a judge who is still serving. |
112 | `start_year`, `end_year`, `senior_year` | Their years. |
113 | `birth_year` | The year the judge was born. |
114 | `president` | The president who appointed the judge. |
115 | `party` | That president's party: -1 for Democratic and 1 for Republican. Missing for other parties. |
116 | `aba_rating` | The American Bar Association's rating, from 1 (not qualified) to 4 (exceptionally well qualified). Missing if there was none, or it was split ("Qualified/Well Qualified"). |
117 | `recess` | Whether the judge first took the seat by a recess appointment. |
118 | `senate_vote` | The share of senators who voted to confirm the judge (1 for a voice vote). Missing if the vote is not recorded. |
119 | `woman`, `minority` | Whether the judge is a woman, and whether the judge's race or ethnicity is anything other than white. Missing if the FJC does not report it. |
121 A judge who was reassigned to another court without a new appointment
122 (as when the Eleventh Circuit was divided from the Fifth) keeps the
123 president, party, rating, recess appointment, and vote of the judge's
124 last appointment.
126 If `career` is passed, the judge's professional career is the column
127 "career", with each position on its own line, in order. It is the text
128 that the rules search, so the fields made from it can be checked
129 against it. Each variable that `rulebook` makes from the career (see
130 `code_careers`) is a column after it. The "judges" rulebook makes
131 "prosecutor", "public_defender", "law_clerk", "supreme_court_clerk",
132 "solicitor_general", and "law_professor". "career" is missing for a
133 judge with no career in the file.
135 Args:
136 service: the file of federal judicial service, as a path, the table
137 that was read from it, or its tidy table.
138 demographics: the file of demographics, in any of those forms.
139 career: the file of professional careers, in any of those forms.
140 Defaults to None, in which case no careers are coded.
141 rulebook: the rules that code the careers: a rulebook, or the name of
142 a built-in rulebook or the path of a CSV file. Defaults to
143 "judges".
145 Returns:
146 The table, sorted by judge and then by the order of the judge's
147 service.
149 Raises:
150 KeyError: if a file lacks a column that is needed.
151 ValueError: if the file of demographics has more than one row for a
152 judge, or a variable of `rulebook` has the name of a column of
153 the table.
155 """
156 served = _table(service)
157 _require(served, _SERVICE, 'the file of federal judicial service')
158 people = _table(demographics)
159 _require(people, _PERSONAL, 'the file of demographics')
160 order = ['nid', 'sequence'] if 'sequence' in served.columns else ['nid']
161 served = served.sort_values(order, kind = 'stable').reset_index(drop = True)
162 # A reassigned judge has "None (reassignment)" in place of a president,
163 # so the columns of the appointment are taken from the judge's last row
164 # that is not a reassignment.
165 reassigned = (
166 served['appointing_president'].fillna('').str.startswith('None')
167 .to_numpy())
168 own = pd.Series(np.arange(len(served)), dtype = 'float64').mask(reassigned)
169 source = (
170 own.groupby(served['nid'], dropna = False).ffill()
171 .fillna(-1).astype(int))
172 columns = list(_APPOINTMENT)
173 appointed = served[columns].reindex(source.to_numpy())
174 appointed.index = served.index
175 served[columns] = appointed
176 # Each row of a judge's service gets the judge's demographics. A judge
177 # who is not in the file of demographics has missing ones.
178 personal = served[['nid']].merge(
179 people[list(_PERSONAL)], on = 'nid', how = 'left',
180 validate = 'many_to_one')
181 started = served['commission_date'].fillna(
182 served['recess_appointment_date'])
183 gender = personal['gender'].map({'Female': True, 'Male': False})
184 race = personal['race_or_ethnicity']
185 roster = pd.DataFrame({
186 'nid': served['nid'],
187 'judge': served['judge_name'],
188 'last_name': personal['last_name'],
189 'first_name': personal['first_name'],
190 'middle_name': personal['middle_name'],
191 'suffix': personal['suffix'],
192 'sequence': (
193 served['sequence'] if 'sequence' in served.columns
194 else served.groupby('nid').cumcount() + 1).astype('Int64'),
195 'court': served['court_name'],
196 'court_type': served['court_type'],
197 'circuit_num': pd.array(
198 [_circuit(kind, court) for kind, court in zip(
199 served['court_type'], served['court_name'], strict = True)],
200 dtype = 'Int64'),
201 'start_date': started,
202 'end_date': served['termination_date'],
203 'senior_date': served['senior_status_date'],
204 'start_year': _year(started),
205 'end_year': _year(served['termination_date']),
206 'senior_year': _year(served['senior_status_date']),
207 'birth_year': personal['birth_year'].astype('Int64'),
208 'president': served['appointing_president'],
209 'party': served['party_of_appointing_president'].map(
210 options._PARTIES).astype('Int64'),
211 'aba_rating': served['aba_rating'].map(
212 options._ABA_RATINGS).astype('Int64'),
213 'recess': served['recess_appointment_date'].notna(),
214 'senate_vote': pd.array(
215 [_vote_share(kind, votes) for kind, votes in zip(
216 served['senate_vote_type'], served['ayes_nays'],
217 strict = True)],
218 dtype = 'Float64'),
219 'woman': gender.astype('boolean'),
220 'minority': (race != 'White').astype('boolean').mask(race.isna())})
221 roster = names.add_permutations(roster)
222 if career is not None:
223 positions = _table(career)
224 _require(
225 positions, ['nid', 'position'], 'the file of professional careers')
226 roster['career'] = roster['nid'].map(_histories(positions)).astype(
227 'str')
228 coded = code_careers(positions, roster['nid'], rulebook = rulebook)
229 clashes = [c for c in coded.columns if c in roster.columns]
230 if clashes:
231 message = (
232 f'the rulebook makes {clashes}, which are columns of the '
233 f'roster already: rename them in the rulebook')
234 raise ValueError(message)
235 roster = roster.merge(
236 coded, left_on = 'nid', right_index = True, how = 'left')
237 return roster
240def code_careers(
241 career: pd.DataFrame | pathlib.Path | str,
242 nids: Iterable[int] | None = None,
243 *,
244 rulebook: rules.Rulebook | pathlib.Path | str = 'judges') -> pd.DataFrame:
245 """Makes fields from judges' professional careers with a rulebook.
247 Each judge's positions are put in order, one on each line, as the section
248 of text "career", which the rules of `rulebook` search (see `rules`).
249 Each variable that the rules make is a field. The "judges" rulebook (in
250 fjc_biographies' "instructions" folder) makes a flag for each of six
251 kinds of positions: "prosecutor", "public_defender", "law_clerk",
252 "supreme_court_clerk", "solicitor_general", and "law_professor".
254 Args:
255 career: the file of professional careers, as a path, the table that
256 was read from it, or its tidy table.
257 nids: the numbers of the judges to code. A judge with no career in
258 the file gets what the rules make of no text (a flag is False, a
259 count is 0, and a label is missing). Defaults to None, in which
260 case the judges in `career` are coded.
261 rulebook: the rules: a rulebook, or the name of a built-in rulebook
262 or the path of a CSV file or a folder of them. Defaults to
263 "judges".
265 Returns:
266 A table with a column for each variable of the rulebook, in order,
267 labeled by the judges' numbers ("nid").
269 Raises:
270 KeyError: if `career` lacks a column that is needed.
272 """
273 positions = _table(career)
274 _require(positions, ['nid', 'position'], 'the file of professional careers')
275 book = rules.Rulebook.create(rulebook)
276 histories = _histories(positions)
277 judges = (
278 list(histories.index) if nids is None
279 else list(dict.fromkeys(int(nid) for nid in nids)))
280 coded = [
281 book.apply({'career': histories.get(nid, '')}) for nid in judges]
282 return pd.DataFrame(
283 coded, index = pd.Index(judges, name = 'nid', dtype = 'Int64'),
284 columns = book.variables)
287def load(
288 category: str,
289 folder: pathlib.Path | str | None = None,
290 *,
291 update: bool = False) -> pd.DataFrame:
292 """Returns the tidy table of one of the FJC's files.
294 The file is downloaded first if it is not in the "raw" folder of
295 `folder` (see `downloader.download`).
297 Args:
298 category: the file's category ("demographics", "service",
299 "other_service", "education", "career", or "other_nominations").
300 folder: the data folder. Defaults to None, in which case
301 `downloader.data_folder()` is used.
302 update: whether to download the file again even if it is there, to
303 get the judges and corrections since it was downloaded. Defaults
304 to False.
306 Returns:
307 The table (see `tidy`).
309 """
310 paths = downloader.download(category, folder, overwrite = update)
311 return read(paths[category])
314def load_all(
315 folder: pathlib.Path | str | None = None,
316 *,
317 update: bool = False) -> dict[str, pd.DataFrame]:
318 """Returns the tidy tables of all of the FJC's files.
320 Args:
321 folder: the data folder. Defaults to None, in which case
322 `downloader.data_folder()` is used.
323 update: whether to download the files again even if they are there.
324 Defaults to False.
326 Returns:
327 The table of each file (see `tidy`), by its category.
329 """
330 paths = downloader.download(folder = folder, overwrite = update)
331 return {category: read(path) for category, path in paths.items()}
334def read(path: pathlib.Path | str, **kwargs: Any) -> pd.DataFrame:
335 """Returns the tidy table of a CSV file of the directory.
337 The FJC's files are in UTF-8, but a file that was saved again by a
338 spreadsheet may be in Windows-1252, which is read too.
340 Args:
341 path: the path of the file, which may begin with "~" for the user's
342 home folder.
343 **kwargs: arguments for `pandas.read_csv`.
345 Returns:
346 The table (see `tidy`).
348 """
349 path = pathlib.Path(path).expanduser()
350 settings: dict[str, Any] = {
351 'dtype': str, 'keep_default_na': False, **kwargs}
352 try:
353 table = pd.read_csv(path, encoding = 'utf-8-sig', **settings)
354 except UnicodeDecodeError:
355 table = pd.read_csv(path, encoding = 'cp1252', **settings)
356 return tidy(table)
359def roster_by_judge(roster: pd.DataFrame) -> pd.DataFrame:
360 """Returns a roster with one row for each judge.
362 A roster made by `build_roster` has a row for each appointment: each
363 court that each judge has served on. This is the same information
364 organized by judge, as the FJC's own "organized by judge" export is. The
365 columns that describe the judge (such as the judge's name, "birth_year",
366 "woman", and the fields of the judge's career) are given once.
367 "appointments" is how many courts the judge has served on. Then the
368 columns that describe an appointment (such as "court", "start_date", and
369 "party") are given for each, numbered in the order of the judge's
370 service: "court_1", "start_date_1", ..., "court_2", and so on. They are
371 missing for a judge with fewer appointments than the most that any
372 judge has.
374 Args:
375 roster: a roster with a row for each appointment, as `build_roster`
376 makes it, in the order of each judge's service.
378 Returns:
379 The roster with a row for each judge, in the order that the judges
380 first appear in `roster`.
382 Raises:
383 KeyError: if `roster` lacks "nid" or "court".
385 """
386 _require(roster, ['nid', 'court'], 'a roster')
387 roster = roster.dropna(subset = ['nid']).reset_index(drop = True)
388 fields = [c for c in _BY_APPOINTMENT if c in roster.columns]
389 personal = [c for c in roster.columns if c not in _BY_APPOINTMENT]
390 judges = roster[personal].drop_duplicates('nid').set_index('nid')
391 judges.insert(
392 len(judges.columns), 'appointments',
393 roster.groupby('nid').size().reindex(judges.index).astype('int64'))
394 place = roster.groupby('nid').cumcount() + 1
395 # A boolean column gets missing values for the judges with fewer
396 # appointments, so it is made one that can have them.
397 appointed = roster[fields].astype({
398 c: 'boolean' for c in fields if roster[c].dtype == bool})
399 # `pivot_table` would aggregate each cell, which loses text and types, so
400 # each judge's appointments are unstacked instead.
401 indexed = appointed.set_index([roster['nid'], place.rename('place')])
402 wide = pd.DataFrame(indexed.unstack('place')) # noqa: PD010
403 numbered = [
404 (field, number) for number in range(1, int(place.max() or 0) + 1)
405 for field in fields]
406 wide = wide.reindex(columns = pd.MultiIndex.from_tuples(numbered))
407 wide.columns = [f'{field}_{number}' for field, number in numbered]
408 return pd.concat(
409 [judges, wide.reindex(judges.index)], axis = 1).reset_index()
412def tidy(table: pd.DataFrame) -> pd.DataFrame:
413 """Returns a table of the directory that is easy to use.
415 A tidy table is not changed, so a table can be tidied more than once.
417 Args:
418 table: a table of one of the FJC's files, as it was read.
420 Returns:
421 A new table whose columns have lower case names, with underscores in
422 place of spaces and punctuation (and some shorter names, such as
423 "position" for "Professional Career", which are in
424 `options._COLUMN_NAMES`). Text has no spaces at its ends, and a
425 blank cell is missing. "nid", "jid", "sequence", and the months
426 and days of births and deaths are whole numbers, as are the
427 years, and the columns whose names end in "_date" are dates.
428 The empty, unnamed columns at the end of the file of other
429 federal judicial service are left out.
431 """
432 tidied: dict[str, pd.Series] = {}
433 for column in table.columns:
434 name = _column_name(column)
435 values = table[column]
436 if name in _NUMBERS:
437 values = _number(values, r'(\d+)')
438 elif name in _YEARS:
439 values = _number(values, r'(\d{4})')
440 elif pd.api.types.is_datetime64_any_dtype(values.dtype):
441 pass
442 elif name.endswith('_date'):
443 values = pd.to_datetime(
444 _text(values), format = 'mixed', errors = 'coerce')
445 elif not pd.api.types.is_numeric_dtype(values.dtype): 445 ↛ 447line 445 didn't jump to line 447 because the condition on line 445 was always true
446 values = _text(values)
447 unnamed = not name or name.startswith('unnamed_')
448 if unnamed and values.isna().all():
449 continue
450 tidied[name] = values
451 return pd.DataFrame(tidied, index = table.index)
454""" Private Functions """
457def _circuit(kind: Any, court: Any) -> int | None:
458 """Returns the number of the circuit of a court.
460 Args:
461 kind: the type of the court, such as "U.S. District Court".
462 court: the name of the court, such as "U.S. District Court for the
463 Middle District of Georgia".
465 Returns:
466 The number of the circuit of a court of appeals, or that of the state
467 or territory of a district court now (12 for the District of
468 Columbia), or None for any other court or a name that
469 `options._APPEALS_CIRCUITS` or `options._DISTRICT_CIRCUITS` does
470 not know.
472 """
473 if not isinstance(court, str): 473 ↛ 474line 473 didn't jump to line 474 because the condition on line 473 was never true
474 return None
475 if kind == _APPEALS:
476 match = _CIRCUIT.search(court)
477 return options._APPEALS_CIRCUITS.get(match.group(1)) if match else None
478 if kind == _DISTRICT_COURT:
479 match = _DISTRICT.search(court)
480 return options._DISTRICT_CIRCUITS.get(match.group(1)) if match else None
481 return None
484def _column_name(column: Any) -> str:
485 """Returns the tidy name of a column.
487 Args:
488 column: the name of a column of one of the FJC's files.
490 Returns:
491 Its name in `options._COLUMN_NAMES`, or else the name in lower case,
492 with an underscore in place of each run of spaces and
493 punctuation.
495 """
496 name = str(column).strip().lower()
497 if name in options._COLUMN_NAMES:
498 return options._COLUMN_NAMES[name]
499 return re.sub(r'[^a-z0-9]+', '_', name).strip('_')
502def _histories(positions: pd.DataFrame) -> pd.Series:
503 """Returns each judge's professional career as one text.
505 The positions are joined with line breaks, rather than the semicolons of
506 the FJC's export organized by judge, because some positions have
507 semicolons in them.
509 Args:
510 positions: the tidy table of the file of professional careers.
512 Returns:
513 The judge's positions in the order of their sequence numbers, one on
514 each line, labeled by the judge's number ("nid").
516 """
517 order = ['nid', 'sequence'] if 'sequence' in positions.columns else ['nid']
518 histories: pd.Series = (
519 positions.dropna(subset = ['nid', 'position'])
520 .sort_values(order, kind = 'stable')
521 .groupby('nid')['position'].agg('\n'.join))
522 return histories
525def _number(values: pd.Series, pattern: str) -> pd.Series:
526 """Returns the whole numbers in a column.
528 Args:
529 values: numbers, or text with numbers.
530 pattern: a pattern whose first group is the number in text.
532 Returns:
533 The first number in each cell, which is missing if there is none.
535 """
536 if pd.api.types.is_numeric_dtype(values.dtype):
537 return values.astype('Int64')
538 found = _text(values).str.extract(pattern, expand = False)
539 return pd.to_numeric(found, errors = 'coerce').astype('Int64')
542def _require(data: pd.DataFrame, columns: Iterable[str], name: str) -> None:
543 """Raises an error if a table lacks columns.
545 Args:
546 data: the table.
547 columns: names of the columns that are needed.
548 name: what the table is, for the message.
550 Raises:
551 KeyError: if a column is missing.
553 """
554 missing = [c for c in columns if c not in data.columns]
555 if missing:
556 message = f'{name} needs the columns {missing}, which it does not have'
557 raise KeyError(message)
560def _table(source: pd.DataFrame | pathlib.Path | str) -> pd.DataFrame:
561 """Returns the tidy table of a file.
563 Args:
564 source: the path of the file, the table that was read from it, or its
565 tidy table.
567 Returns:
568 The tidy table, which is a new table.
570 """
571 if isinstance(source, pd.DataFrame):
572 return tidy(source)
573 return read(source)
576def _text(values: pd.Series) -> pd.Series:
577 """Returns a column as text without spaces at its ends.
579 Args:
580 values: the column.
582 Returns:
583 The text of each cell, which is missing if the cell is missing or
584 blank.
586 """
587 cleaned = [
588 None if pd.isna(value) or not str(value).strip() else str(value).strip()
589 for value in values]
590 return pd.Series(cleaned, index = values.index, dtype = 'str')
593def _vote_share(kind: Any, votes: Any) -> float | None:
594 """Returns the share of senators who voted to confirm a judge.
596 Args:
597 kind: the type of vote: "Voice", "Roll Call", or anything else.
598 votes: the ayes and nays of a roll call, such as "96/2".
600 Returns:
601 1 for a voice vote, the share of ayes for a roll call, or None if
602 there was neither (or no votes are recorded).
604 """
605 if kind == 'Voice':
606 return 1.0
607 match = _VOTES.search(votes) if isinstance(votes, str) else None
608 if match is None:
609 return None
610 ayes, nays = int(match.group(1)), int(match.group(2))
611 return ayes / (ayes + nays) if ayes + nays else None
614def _year(dates: pd.Series) -> pd.Series:
615 """Returns the years of a column of dates.
617 Args:
618 dates: dates, which may be missing.
620 Returns:
621 The year of each date as a whole number, which is missing if the
622 date is.
624 """
625 years: pd.Series = dates.dt.year.astype('Int64')
626 return years