Coverage for tests/test_nibrs_reader.py: 100%
160 statements
« prev ^ index » next coverage.py v7.17.0, created at 2026-10-30 18:01 +0000
1"""Tests of reading and parsing NIBRS master files.
3Contents:
4 test_field_rejects_bad_kinds: checks the kinds and widths of fields.
5 test_layout_widths_match_the_documentation: checks layouts' widths.
6 test_layouts_have_no_overlapping_fields: checks layouts' positions.
7 test_layouts_have_unique_names_and_known_codebooks: checks layouts' names.
8 test_open_master_reads_plain_text: checks master files outside archives.
9 test_parse_converts_each_kind: checks parsing each kind of field.
10 test_parse_of_no_records_has_the_layouts_columns: checks empty tables.
11 test_read_decodes_fields: checks that reading decodes codes.
12 test_read_joins_split_batch_headers: checks pre-2013 batch headers.
13 test_read_keeps_states: checks keeping only some states.
14 test_read_makes_every_table: checks that reading makes every table.
15 test_read_of_a_recent_year: checks a recent master file.
16 test_read_rejects_unknown_tables: checks reading unknown tables.
17 test_read_stops_at_the_limit: checks the limit of records.
18 test_records_reads_both_compressions: checks Deflate and Deflate64.
19 test_tables_parses_in_chunks: checks parsing in chunks.
20 test_year_of_finds_years_in_names: checks finding years in names.
22"""
24from __future__ import annotations
26import datetime
27import pathlib
29from typing import Any
31import amos
32import pandas as pd
33import pytest
35from crimepy.nibrs import codes, layouts, reader
37""" Private Functions """
40def _read(*args: Any, **kwargs: Any) -> dict[str, pd.DataFrame]:
41 """Reads tables with `reader.read`, checking that they are datasets.
43 Args:
44 *args: arguments for `reader.read`.
45 **kwargs: arguments for `reader.read`.
47 Returns:
48 The data frame of each dataset, by name.
50 """
51 datasets = reader.read(*args, **kwargs)
52 assert all(isinstance(d, amos.Dataset) for d in datasets.values())
53 return {name: dataset.data for name, dataset in datasets.items()}
56""" Tests """
59def test_layouts_have_no_overlapping_fields() -> None:
60 """Checks that no two fields of a layout share a position."""
61 for layout in layouts.LAYOUTS.values():
62 fields = sorted(layout, key = lambda f: f.start)
63 for first, second in zip(fields, fields[1:], strict = False):
64 assert first.end <= second.start, (layout.level, first, second)
67def test_layouts_have_unique_names_and_known_codebooks() -> None:
68 """Checks that each layout's names are unique and its codebooks exist."""
69 for layout in layouts.LAYOUTS.values():
70 assert len(set(layout.names)) == len(layout.names)
71 for field in layout:
72 if field.codes is not None:
73 assert field.codes in codes.CODEBOOKS
76def test_layout_widths_match_the_documentation() -> None:
77 """Checks the widths of layouts against the record descriptions."""
78 assert layouts.LAYOUTS['BH'].width == 292
79 assert layouts.LAYOUTS['02'].width == 67
80 assert layouts.LAYOUTS['03'].width == 102
81 assert layouts.LAYOUTS['04'].width == 135
82 assert layouts.LAYOUTS['05'].width == 40
83 assert layouts.LAYOUTS['06'].width == 73
84 assert layouts.LAYOUTS['07'].width == 56
85 assert layouts.LAYOUTS['W6'].width == 104
88def test_field_rejects_bad_kinds() -> None:
89 """Checks that a field must have a known kind and a width."""
90 with pytest.raises(ValueError, match = 'kind'):
91 layouts.Field('x', 0, 1, 'number')
92 with pytest.raises(ValueError, match = 'width'):
93 layouts.Field('x', 3, 3)
96def test_year_of_finds_years_in_names(tmp_path: pathlib.Path) -> None:
97 """Checks that years are found in new and old names of master files.
99 Args:
100 tmp_path: the test's temporary folder.
102 """
103 assert reader.year_of('nibrs-2023.zip') == 2023
104 assert reader.year_of(tmp_path / 'NIBRS04.ENC') == 2004
105 assert reader.year_of(tmp_path / 'KCNIBR91.DAT') == 1991
106 assert reader.year_of(tmp_path / 'master.txt') is None
109def test_records_reads_both_compressions(
110 master_2004: pathlib.Path,
111 master_2025: pathlib.Path) -> None:
112 """Checks that records are read from Deflate and Deflate64 archives.
114 Args:
115 master_2004: the path of the 2004 master file.
116 master_2025: the path of the 2025 master file, compressed with
117 Deflate64.
119 """
120 old = list(reader.records(master_2004))
121 new = list(reader.records(master_2025))
122 assert len(old) == 20
123 assert all(len(r) == 132 for r in old)
124 assert new[0].startswith(b'BH50AK0010200')
125 assert new[-1].endswith(b'30MINN')
128def test_open_master_reads_plain_text(tmp_path: pathlib.Path) -> None:
129 """Checks that a master file that is not in an archive is read.
131 Args:
132 tmp_path: the test's temporary folder.
134 """
135 path = tmp_path / 'nibrs-2025.txt'
136 path.write_bytes(b'0550AK0010200C30BRCZ9728N202503010125FIN\n')
137 assert list(reader.records(path)) == [
138 b'0550AK0010200C30BRCZ9728N202503010125FIN']
139 with pytest.raises(FileNotFoundError):
140 list(reader.records(tmp_path / 'missing.zip'))
143def test_parse_converts_each_kind() -> None:
144 """Checks that parsing converts each kind of field."""
145 table = reader.parse(
146 [b'0350AK0010200C30BREBQ728N20250601738000000050',
147 b'0311IDIBN01006B-4RHP 728N19950118610000000000 '
148 b'L000000001250GM'],
149 '03', data_year = 2025)
150 rows = table.to_pylist()
151 assert rows[0]['data_year'] == 2025
152 assert rows[0]['incident_date'] == datetime.date(2025, 6, 1)
153 assert rows[0]['loss_type'] == '7'
154 assert rows[0]['property_description'] == '38'
155 assert rows[0]['property_value'] == 50
156 assert rows[0]['drug_quantity_1'] is None
157 assert rows[1]['drug_type_1'] == 'L'
158 assert rows[1]['drug_quantity_1'] == pytest.approx(1.25)
159 assert rows[1]['drug_measure_1'] == 'GM'
162def test_parse_of_no_records_has_the_layouts_columns() -> None:
163 """Checks that a table of no records has the layout's columns."""
164 table = reader.parse([], '05', data_year = 2025)
165 assert table.num_rows == 0
166 assert table.column_names == ['data_year', *layouts.LAYOUTS['05'].names]
169def test_read_makes_every_table(master_2004: pathlib.Path) -> None:
170 """Checks that reading a master file makes every table.
172 Args:
173 master_2004: the path of the 2004 master file.
175 """
176 frames = _read(master_2004)
177 assert set(frames) == set(layouts.SEGMENTS)
178 assert len(frames['administrative']) == 2
179 assert len(frames['offenses']) == 2
180 assert len(frames['group_b_arrests']) == 1
181 assert len(frames['window_clearances']) == 1
182 assert len(frames['window_properties']) == 1
183 assert len(frames['window_arrestees']) == 1
184 for name, frame in frames.items():
185 assert set(reader.schema(name).names) <= set(frame.columns)
188def test_read_decodes_fields(master_2004: pathlib.Path) -> None:
189 """Checks that reading replaces codes with their labels.
191 Args:
192 master_2004: the path of the 2004 master file.
194 """
195 frames = _read(master_2004)
196 offenses = frames['offenses'].set_index('incident_number')
197 burglary = offenses.loc['E0DROSC-2 N']
198 assert burglary['offense_code'] == 'Burglary/Breaking & Entering'
199 assert burglary['location'] == 'Specialty Store'
200 assert burglary['entry_method'] == 'Force'
201 assert burglary['crime_against'] == 'Property'
202 assert burglary['bias_motivation_1'] == 'None (No Bias)'
203 assault = offenses.loc['E0DROSCQ2 N']
204 assert assault['weapon_1'] == 'Personal Weapons'
205 victims = frames['victims'].set_index('incident_number')
206 person = victims.loc['E0DROSCQ2 N']
207 assert person['victim_age'] == 29
208 assert person['victim_sex'] == 'Male'
209 assert person['injury_1'] == 'Apparent Minor Injury'
210 assert person['offender_1'] == 1
211 assert person['relationship_1'] == 'Spouse'
212 administrative = frames['administrative'].set_index('incident_number')
213 assert bool(administrative.loc['E0DROSC-2 N', 'report_date'])
214 assert pd.isna(administrative.loc['E0DROSC-2 N', 'incident_hour'])
215 assert administrative.loc['E0DROSCQ2 N', 'incident_hour'] == 11
216 assert (frames['administrative']['agency_ori'] == 'AZ0111300').all()
217 window = frames['window_properties'].iloc[0]
218 assert window['offense_1'] == 'Burglary/Breaking & Entering'
219 assert window['property_description'] == 'Firearms'
222def test_read_joins_split_batch_headers(master_2004: pathlib.Path) -> None:
223 """Checks that the three batch header records of 2004 become one row.
225 Args:
226 master_2004: the path of the 2004 master file.
228 """
229 agencies = _read(master_2004, 'agencies')['agencies']
230 assert list(agencies['ori']) == ['ALAST0000', 'AZ0111300']
231 tempe = agencies.set_index('ori').loc['AZ0111300']
232 assert tempe['city'] == 'TEMPE'
233 assert tempe['population'] == 158880
234 assert tempe['population_group'] == 'Cities 100,000 to 249,999'
235 assert tempe['agency_type'] == 'City'
236 assert bool(tempe['core_city'])
237 assert tempe['months_reported'] == 12
238 assert tempe['months_active'] == 12
239 assert tempe['master_file_year'] == 2004
240 assert tempe['date_ori_went_nibrs'] == pd.Timestamp('1996-01-01')
241 patrol = agencies.set_index('ori').loc['ALAST0000']
242 assert patrol['agency_type'] == 'State Police'
243 assert patrol['months_active'] == 0
246def test_read_of_a_recent_year(master_2025: pathlib.Path) -> None:
247 """Checks the fields of a recent master file, without decoding.
249 Args:
250 master_2025: the path of the 2025 master file, compressed with
251 Deflate64.
253 """
254 frames = _read(master_2025, decode = False)
255 offender = frames['offenders'].iloc[0]
256 assert offender['offender_age'] == 25
257 assert offender['offender_sex'] == 'F'
258 assert offender['offender_race'] == 'I'
259 assert offender['offender_ethnicity'] == 'N'
260 assert not frames['administrative'].iloc[0]['cargo_theft']
261 agency = frames['agencies'].iloc[0]
262 assert agency['population'] == 31961
263 assert agency['state'] == 'AK'
264 arrest = frames['group_b_arrests'].iloc[0]
265 assert arrest['arrest_offense'] == '90C'
266 assert arrest['arrest_type'] == 'O'
269def test_read_keeps_states(master_2004: pathlib.Path) -> None:
270 """Checks that reading keeps only the states asked for.
272 Args:
273 master_2004: the path of the 2004 master file.
275 """
276 frames = _read(master_2004, ['agencies', 'victims'], states = 'AL')
277 assert list(frames['agencies']['ori']) == ['ALAST0000']
278 assert frames['victims'].empty
279 frames = _read(master_2004, 'victims', states = ['az'])
280 assert len(frames['victims']) == 2
281 with pytest.raises(ValueError, match = 'state'):
282 reader.read(master_2004, states = 'Arizona')
285def test_tables_parses_in_chunks(master_2004: pathlib.Path) -> None:
286 """Checks that tables are parsed in chunks and report progress.
288 Args:
289 master_2004: the path of the 2004 master file.
291 """
292 seen = []
293 chunks = list(reader.tables(
294 master_2004, 'offenses', chunk_size = 1, progress = seen.append))
295 assert sum(c['offenses'].num_rows for c in chunks if 'offenses' in c) == 2
296 assert seen[-1] == 20
299def test_read_stops_at_the_limit(master_2004: pathlib.Path) -> None:
300 """Checks that reading stops after the limit of records.
302 Args:
303 master_2004: the path of the 2004 master file.
305 """
306 frames = _read(master_2004, limit = 8)
307 assert len(frames['administrative']) == 1
308 assert frames['victims'].empty
311def test_read_rejects_unknown_tables(master_2004: pathlib.Path) -> None:
312 """Checks that reading a table that does not exist fails.
314 Args:
315 master_2004: the path of the 2004 master file.
317 """
318 with pytest.raises(KeyError):
319 reader.read(master_2004, 'suspects')