Coverage for tests/test_parser.py: 100%
247 statements
« prev ^ index » next coverage.py v7.17.0, created at 2026-10-22 22:47 +0000
1"""Tests for parsing the FJC's files and building a roster."""
3from __future__ import annotations
5import pathlib
7import pandas as pd
8import pytest
9from conftest import APPEALS, DISTRICT, FIRST, NINTH, SERVICE, tables
11import fjc_biographies
12from fjc_biographies import parser
14FLAGS = [
15 'prosecutor', 'public_defender', 'law_clerk', 'supreme_court_clerk',
16 'solicitor_general', 'law_professor']
19@pytest.fixture
20def table() -> pd.DataFrame:
21 """Returns the roster of the invented judges, labeled by court."""
22 files = tables()
23 return fjc_biographies.build_roster(
24 files['service'], files['demographics'], files['career']).set_index(
25 'court', drop = False)
28def test_tidy_names_and_types() -> None:
29 service = parser.tidy(tables()['service'])
30 assert list(service.columns[:5]) == [
31 'nid', 'sequence', 'judge_name', 'court_type', 'court_name']
32 assert 'ayes_nays' in service.columns
33 assert service['nid'].dtype == 'Int64'
34 assert pd.api.types.is_datetime64_any_dtype(service['commission_date'])
35 assert service['commission_date'].iloc[0] == pd.Timestamp('1995-03-17')
36 # Blank text and text of spaces are missing, and text is trimmed.
37 assert pd.isna(service['senior_status_date'].iloc[0])
38 assert pd.isna(service['recess_appointment_date'].iloc[0])
39 assert service['ayes_nays'].iloc[1] == '99/1'
40 people = parser.tidy(tables()['demographics'])
41 assert people['birth_year'].dtype == 'Int64'
42 assert pd.isna(people.set_index('nid').loc[112, 'middle_name'])
43 # Tidying a tidy table changes nothing.
44 pd.testing.assert_frame_equal(parser.tidy(service), service)
47def test_tidy_other_tables() -> None:
48 files = tables()
49 other = parser.tidy(files['other_service'])
50 # The empty, unnamed columns are left out.
51 assert list(other.columns) == [
52 'nid', 'sequence', 'judge_name', 'type', 'description']
53 education = parser.tidy(files['education'])
54 assert list(education['degree_year']) == [1971, pd.NA]
55 assert pd.isna(education['degree'].iloc[1])
56 assert list(parser.tidy(files['career']).columns) == [
57 'nid', 'sequence', 'position']
58 assert list(parser.tidy(files['other_nominations']).columns) == [
59 'nid', 'sequence', 'judge_name', 'description']
62def test_tidy_dates_in_either_format() -> None:
63 # The FJC's page says that its CSV files write dates as mm/dd/yyyy,
64 # although they write them as yyyy-mm-dd.
65 dates = parser.tidy(pd.DataFrame({
66 'Commission Date': ['11/20/2014', '2014-11-20', '', 'unknown']}))
67 assert list(dates['commission_date'][:2]) == [pd.Timestamp('2014-11-20')] * 2
68 assert dates['commission_date'][2:].isna().all()
71def test_read(tmp_path: pathlib.Path) -> None:
72 files = tables()
73 path = tmp_path / 'demographics.csv'
74 files['demographics'].to_csv(path, index = False)
75 assert parser.read(path).loc[0, 'last_name'] == 'Lynch'
76 # A file that is not in UTF-8 is read as Windows-1252.
77 files['demographics'].loc[0, 'Last Name'] = 'Peña'
78 files['demographics'].to_csv(path, index = False, encoding = 'cp1252')
79 assert parser.read(path).loc[0, 'last_name'] == 'Peña'
82def test_build_roster(table: pd.DataFrame) -> None:
83 assert len(table) == len(SERVICE)
84 # Rows are sorted by judge and then by the order of the judge's service.
85 assert list(table.loc[table['nid'] == 107, 'court_type']) == [DISTRICT, APPEALS]
86 lynch = table.loc[FIRST].iloc[0]
87 assert lynch['judge'] == 'Lynch, Mary Ann'
88 assert (lynch['last_name'], lynch['first_name'], lynch['middle_name']) == ('Lynch', 'Mary', 'Ann')
89 assert lynch['circuit_num'] == 1
90 assert lynch['start_date'] == pd.Timestamp('1995-03-17')
91 assert (lynch['start_year'], lynch['birth_year']) == (1995, 1946)
92 assert pd.isna(lynch['end_year']) and pd.isna(lynch['senior_year'])
93 assert (lynch['party'], lynch['aba_rating'], lynch['senate_vote']) == (-1, 3, 1.0)
94 assert lynch['woman'] and not lynch['minority'] and not lynch['recess']
95 by_nid = table.drop_duplicates('nid', keep = 'last').set_index('nid')
96 assert list(by_nid.loc[[102, 103, 106], 'aba_rating']) == [2, 4, 1]
97 assert pd.isna(by_nid.loc[105, 'aba_rating'])
98 assert by_nid.loc[102, 'senate_vote'] == pytest.approx(0.99)
99 assert by_nid.loc[103, 'minority'] and by_nid.loc[105, 'minority']
100 assert by_nid.loc[110, 'end_year'] == 2017
101 assert by_nid.loc[106, 'senior_year'] == 2015
102 # A gender that is not reported is missing.
103 assert pd.isna(by_nid.loc[113, 'woman'])
106def test_build_roster_circuits(table: pd.DataFrame) -> None:
107 circuits = table.drop_duplicates('court')['circuit_num']
108 assert circuits[NINTH] == 9
109 assert circuits['U.S. Court of Appeals for the Eleventh Circuit'] == 11
110 assert circuits['U.S. Court of Appeals for the District of Columbia Circuit'] == 12
111 assert circuits['U.S. Court of Appeals for the Federal Circuit'] == 13
112 assert pd.isna(circuits['Supreme Court of the United States'])
113 # A district court's circuit is its state's.
114 assert circuits['U.S. District Court for the District of Massachusetts'] == 1
115 assert circuits['U.S. District Court for the Southern District of New York'] == 2
116 assert parser._circuit(DISTRICT, 'U.S. District Court for the District of Columbia (Supreme Court)') == 12
117 assert parser._circuit(DISTRICT, 'U.S. District Court for the Edenton & Wilmington Districts of North Carolina') == 4
118 assert parser._circuit(DISTRICT, 'U.S. District Court for the District of Orleans') is None
119 assert parser._circuit('Other', 'U.S. Court of International Trade') is None
122def test_build_roster_appointments(table: pd.DataFrame) -> None:
123 fifth = table.loc['U.S. Court of Appeals for the Fifth Circuit']
124 eleventh = table.loc['U.S. Court of Appeals for the Eleventh Circuit']
125 # A reassigned judge keeps the appointment of the judge's earlier service.
126 assert (fifth['start_year'], fifth['end_year'], eleventh['start_year']) == (1979, 1981, 1981)
127 for column in ('president', 'party', 'aba_rating', 'senate_vote'):
128 assert eleventh[column] == fifth[column]
129 assert eleventh['president'] == 'Jimmy Carter'
130 assert eleventh['senate_vote'] == pytest.approx(60 / 90)
131 federal = table.loc['U.S. Court of Appeals for the Federal Circuit']
132 assert (federal['president'], federal['party']) == ('Barack Obama', -1)
133 # A recess appointment starts a judge's service, and a party other than
134 # the two of today, an unrecorded vote, and an unknown gender are missing.
135 fourth = table.loc['U.S. Court of Appeals for the Fourth Circuit']
136 assert fourth['recess'] and fourth['start_year'] == 2000
137 assert pd.isna(fourth['senate_vote'])
138 supreme = table.loc['Supreme Court of the United States']
139 assert pd.isna(supreme['party']) and pd.isna(supreme['woman']) and pd.isna(supreme['minority'])
142def test_build_roster_reassignment_takes_whole_appointment() -> None:
143 # A reassigned judge takes every column of the appointment from the row
144 # above, even one that is missing there.
145 files = tables()
146 service = files['service']
147 service.loc[service['nid'] == 108, 'ABA Rating'] = ''
148 service.loc[service['nid'] == 107, 'Appointing President'] = [
149 'None (reassignment)', 'George H.W. Bush']
150 table = fjc_biographies.build_roster(service, files['demographics'])
151 assert table.loc[table['nid'] == 108, 'aba_rating'].isna().all()
152 stone = table.loc[table['nid'] == 107]
153 assert list(stone['aba_rating']) == [2, 2]
156def test_build_roster_careers(table: pd.DataFrame) -> None:
157 careers = table.drop_duplicates('nid').set_index('nid')
158 assert list(careers.loc[101, FLAGS]) == [True, False, True, False, False, False]
159 assert list(careers.loc[102, FLAGS]) == [True, False, False, False, False, False]
160 assert list(careers.loc[103, FLAGS]) == [False, True, False, False, False, False]
161 assert list(careers.loc[104, FLAGS]) == [False, False, True, True, False, True]
162 # A clerkship and the Supreme Court in different positions are not a
163 # clerkship at the Supreme Court, and "District" alone is not a prosecutor.
164 assert list(careers.loc[105, FLAGS]) == [False, False, True, False, False, False]
165 assert careers.loc[113, 'solicitor_general']
166 # A judge with no career in the file has no flags.
167 assert not careers.loc[106, FLAGS].any()
168 assert careers['prosecutor'].dtype == bool
171def test_build_roster_keeps_career_text(table: pd.DataFrame) -> None:
172 # The career that the rules searched comes before the fields made from
173 # it, with each position on its own line, in order.
174 columns = list(table.columns)
175 assert columns.index('career') == columns.index(FLAGS[0]) - 1
176 careers = table.drop_duplicates('nid').set_index('nid')['career']
177 assert careers[101] == (
178 'Law clerk, Hon. A. Judge, U.S. District Court, District of Rhode Island, 1971-1973\n'
179 'Assistant attorney general, State of Massachusetts, 1973-1974')
180 assert careers[102] == 'Assistant U.S. attorney, District of New Hampshire, 1981-1987'
181 # Every row of a judge has it, and a judge with no career has none.
182 assert table.groupby('nid')['career'].nunique(dropna = False).max() == 1
183 assert pd.isna(careers[106])
184 # The positions are ordered by their sequence numbers, not their rows.
185 files = tables()
186 shuffled = fjc_biographies.build_roster(
187 files['service'], files['demographics'], files['career'].iloc[::-1])
188 assert shuffled.loc[shuffled['nid'] == 101, 'career'].iloc[0] == careers[101]
189 # Without the file of careers, there is no career.
190 assert 'career' not in fjc_biographies.build_roster(files['service'], files['demographics']).columns
193def test_build_roster_from_paths(tmp_path: pathlib.Path) -> None:
194 paths = {}
195 for kind in ('service', 'demographics'):
196 paths[kind] = tmp_path / f'{kind}.csv'
197 tables()[kind].to_csv(paths[kind], index = False)
198 table = fjc_biographies.build_roster(**paths)
199 assert 'prosecutor' not in table.columns
200 assert len(table) == len(SERVICE)
203def test_build_roster_merges_by_judge() -> None:
204 files = tables()
205 # A judge who is not in the file of demographics has missing ones.
206 table = fjc_biographies.build_roster(files['service'], files['demographics'].iloc[1:])
207 lynch = table.loc[table['nid'] == 101].iloc[0]
208 assert pd.isna(lynch['last_name']) and pd.isna(lynch['woman'])
209 assert table.loc[table['nid'] == 102, 'last_name'].iloc[0] == 'Howard'
210 # A judge can only be one row of the demographics.
211 twice = pd.concat([files['demographics'], files['demographics'].iloc[:1]])
212 with pytest.raises(ValueError, match = 'many-to-one'):
213 fjc_biographies.build_roster(files['service'], twice)
216def test_build_roster_missing_columns() -> None:
217 files = tables()
218 with pytest.raises(KeyError, match = 'federal judicial service'):
219 fjc_biographies.build_roster(files['service'].drop(columns = 'Court Name'), files['demographics'])
220 with pytest.raises(KeyError, match = 'demographics'):
221 fjc_biographies.build_roster(files['service'], files['demographics'].drop(columns = 'Gender'))
222 with pytest.raises(KeyError, match = 'careers'):
223 fjc_biographies.build_roster(files['service'], files['demographics'], files['career'].drop(columns = 'nid'))
226def test_code_careers() -> None:
227 career = tables()['career']
228 flags = fjc_biographies.code_careers(career)
229 assert flags.index.name == 'nid'
230 assert list(flags.index) == [101, 102, 103, 104, 105, 113]
231 assert list(flags.columns) == FLAGS
232 assert (flags.dtypes == bool).all()
233 # Judges can be chosen, and a judge with no career has no flags.
234 chosen = fjc_biographies.code_careers(career, [104, 999, 104])
235 assert list(chosen.index) == [104, 999]
236 assert not chosen.loc[999].any()
239def test_code_careers_with_own_rulebook(tmp_path: pathlib.Path) -> None:
240 rules = tmp_path / 'careers.csv'
241 rules.write_text(
242 'target,variable,kind,pattern,value,ignorecase,dotall,note\n'
243 'career,positions,count,\\d{4}-\\d{4},,FALSE,FALSE,\n'
244 'career,first_job,section,^[^\\n]*,,FALSE,FALSE,The first position\n'
245 'career,kind,label,clerk,clerk,TRUE,FALSE,\n'
246 'career,kind,label,attorney,lawyer,TRUE,FALSE,\n'
247 'career,years,matches,\\d{4}-\\d{4},,FALSE,FALSE,\n', encoding = 'utf-8')
248 coded = fjc_biographies.code_careers(tables()['career'], [101, 102, 103, 106], rulebook = rules)
249 assert list(coded.columns) == ['positions', 'first_job', 'kind', 'years']
250 assert list(coded['positions']) == [2, 1, 1, 0]
251 # The positions are in the order of their sequence numbers.
252 assert coded.loc[101, 'first_job'].startswith('Law clerk')
253 assert list(coded['kind'][:2]) == ['clerk', 'lawyer']
254 assert coded['kind'][2:].isna().all()
255 assert coded.loc[101, 'years'] == ['1971-1973', '1973-1974']
256 # A roster gets the rulebook's fields in place of the built-in ones.
257 files = tables()
258 table = fjc_biographies.build_roster(
259 files['service'], files['demographics'], files['career'], rulebook = rules)
260 assert 'prosecutor' not in table.columns
261 assert list(table.loc[table['nid'].isin([101, 106]), 'positions']) == [2, 0]
264def test_code_careers_sorts_positions() -> None:
265 career = tables()['career'].iloc[::-1]
266 book = fjc_biographies.Rulebook.create('judges')
267 coded = fjc_biographies.code_careers(career, rulebook = book)
268 assert list(coded.index) == [101, 102, 103, 104, 105, 113]
271def test_build_roster_rulebook_clash(tmp_path: pathlib.Path) -> None:
272 rules = tmp_path / 'clash.csv'
273 rules.write_text(
274 'target,variable,kind,pattern,ignorecase,dotall\n'
275 'career,party,flag,x,TRUE,FALSE\n', encoding = 'utf-8')
276 files = tables()
277 with pytest.raises(ValueError, match = 'party'):
278 fjc_biographies.build_roster(
279 files['service'], files['demographics'], files['career'], rulebook = rules)
282def test_load(requests: list[str], tmp_path: pathlib.Path) -> None:
283 service = fjc_biographies.load('service', tmp_path)
284 assert len(requests) == 1
285 assert service['court_name'].iloc[0] == FIRST
286 assert (tmp_path / 'raw' / 'federal-judicial-service.csv').is_file()
287 # A file that was downloaded is read again without downloading it.
288 fjc_biographies.load('service', tmp_path)
289 assert len(requests) == 1
290 fjc_biographies.load('service', tmp_path, update = True)
291 assert len(requests) == 2
294def test_load_all(requests: list[str]) -> None:
295 loaded = fjc_biographies.load_all()
296 assert list(loaded) == [
297 'demographics', 'service', 'other_service', 'education', 'career',
298 'other_nominations']
299 assert len(requests) == 6
300 assert list(loaded['other_service'].columns) == [
301 'nid', 'sequence', 'judge_name', 'type', 'description']
302 roster = fjc_biographies.build_roster(
303 loaded['service'], loaded['demographics'], loaded['career'])
304 assert len(roster) == len(SERVICE)
305 assert roster.loc[roster['nid'] == 104, 'supreme_court_clerk'].all()
308def test_roster_by_judge(table: pd.DataFrame) -> None:
309 roster = table.reset_index(drop = True)
310 judges = fjc_biographies.roster_by_judge(roster)
311 # A row for each judge, in the order of the roster.
312 assert list(judges['nid']) == list(dict.fromkeys(roster['nid']))
313 assert judges['nid'].is_unique
314 by_nid = judges.set_index('nid')
315 # The judge's own columns are given once, then the number of
316 # appointments, then each appointment's columns, numbered.
317 columns = list(judges.columns)
318 assert columns[:6] == ['nid', 'judge', 'last_name', 'first_name', 'middle_name', 'suffix']
319 assert columns.index('appointments') < columns.index('sequence_1') < columns.index('sequence_2')
320 assert columns[columns.index('sequence_1'):columns.index('sequence_2')] == [
321 'sequence_1', 'court_1', 'court_type_1', 'circuit_num_1', 'start_date_1',
322 'end_date_1', 'senior_date_1', 'start_year_1', 'end_year_1',
323 'senior_year_1', 'president_1', 'party_1', 'aba_rating_1', 'recess_1',
324 'senate_vote_1']
325 assert 'court_3' not in columns
326 for column in ('birth_year', 'woman', 'minority', 'career', *FLAGS):
327 assert column in columns and f'{column}_1' not in columns
328 # Appointments are numbered in the order of the judge's service.
329 stone = by_nid.loc[107]
330 assert stone['appointments'] == 2
331 assert stone['court_type_1'] == DISTRICT and stone['court_2'] == FIRST
332 assert (stone['start_year_1'], stone['end_year_1'], stone['start_year_2']) == (1990, 1992, 1992)
333 assert (stone['aba_rating_1'], stone['aba_rating_2']) == (2, 3)
334 # A judge with one appointment has missing values for a second.
335 lynch = by_nid.loc[101]
336 assert lynch['appointments'] == 1 and lynch['court_1'] == FIRST
337 assert pd.isna(lynch['court_2']) and pd.isna(lynch['recess_2']) and pd.isna(lynch['start_date_2'])
338 assert lynch['prosecutor'] and lynch['woman']
339 assert lynch['career'].startswith('Law clerk') and lynch['career'].count('\n') == 1
340 # The types of the roster are kept, with booleans that can be missing.
341 assert judges['start_date_2'].dtype == roster['start_date'].dtype
342 assert judges['party_2'].dtype == 'Int64'
343 assert judges['recess_1'].dtype == 'boolean'
344 assert judges['appointments'].sum() == len(roster)
347def test_roster_by_judge_has_every_appointment(table: pd.DataFrame) -> None:
348 # Making a row for each appointment again gives back the roster.
349 roster = table.reset_index(drop = True)
350 judges = fjc_biographies.roster_by_judge(roster)
351 fields = [c.removesuffix('_1') for c in judges.columns if c.endswith('_1')]
352 rows = []
353 for number in (1, 2):
354 part = judges[[c for c in judges.columns if not c[-2:] in ('_1', '_2')]].copy()
355 for field in fields:
356 part[field] = judges[f'{field}_{number}']
357 part['place'] = number
358 rows.append(part[part['court'].notna()])
359 back = pd.concat(rows).sort_values(['nid', 'place'], kind = 'stable')
360 back = back[list(roster.columns)].reset_index(drop = True)
361 pd.testing.assert_frame_equal(
362 back.astype(str), roster.astype({'recess': 'boolean'}).astype(str))
365def test_roster_by_judge_needs_a_roster() -> None:
366 with pytest.raises(KeyError, match = 'a roster'):
367 fjc_biographies.roster_by_judge(pd.DataFrame({'nid': [1]}))