Coverage for tests/conftest.py: 100%
51 statements
« prev ^ index » next coverage.py v7.17.0, created at 2026-10-30 18:01 +0000
1"""Shared fixtures: small NIBRS master files.
3Most records are copied from the FBI's master files for 2004 (fixed-length
4records, with three batch header records for each agency) and 2025
5(variable-length records, with one batch header record). A few are made with
6`_record`, from the positions in the FBI's record descriptions, so that the
7agencies have populations and monthly activity and an incident has a known
8offender and an arrest. The 2004 file is a zip archive compressed with
9Deflate, and the 2025 file is compressed with Deflate64, as the CDE
10compressed some years.
12Contents:
13 RECORDS_2004: the records of the 2004 master file.
14 RECORDS_2025: the records of the 2025 master file.
15 data_folder: a data folder with the master files of 2004 and 2025.
16 master_2004: the path of the 2004 master file.
17 master_2025: the path of the 2025 master file.
18 workspace: keeps each test's data and results in a temporary folder.
19 write_deflate64_zip: writes a zip archive compressed with Deflate64.
21"""
23from __future__ import annotations
25import pathlib
26import struct
27import zipfile
28import zlib
30import inflate64
31import pytest
33from crimepy import options
35""" Records """
38def _record(start: str, fields: dict[int, str]) -> str:
39 """Returns a record from its first characters and fields.
41 Args:
42 start: the first characters of the record (its segment level and
43 more).
44 fields: the text of each field, by its first position in the FBI's
45 record descriptions (counted from 1).
47 Returns:
48 The record.
50 """
51 width = max([len(start), *(p - 1 + len(t) for p, t in fields.items())])
52 characters = list(start.ljust(width))
53 for position, text in fields.items():
54 characters[position - 1:position - 1 + len(text)] = list(text)
55 return ''.join(characters)
58RECORDS_2004: list[str] = [
59 (
60 'B101ALAST0000 20050822 ALABAMA HIGHWAY '
61 'PATROL AL8D634N 3490005M'),
62 (
63 'B201ALAST0000 000000000 000000000000000000 '
64 '000000000000000000 000000000000000000 000000000'),
65 'B301ALAST0000 000000000 000000000 002004',
66 _record('B102AZ0111300', {
67 26: '20060302', 34: '19960101', 42: 'TEMPE', 72: 'AZ', 74: '2',
68 76: '8', 77: '4', 78: '1', 79: 'Y', 89: '3060', 93: '015A',
69 97: 'A'}),
70 _record('B202AZ0111300', {
71 26: '000158880', 35: '013', 38: '090', 41: '000141865'}),
72 _record('B302AZ0111300', {
73 50: '12', 52: '12', 54: '2004', 58: ' Y ' * 12, 94: '013'}),
74 '0102AZ0111300 E0DROSC-2 N20040701R 010010100 N',
75 '0202AZ0111300 E0DROSC-2 N20040701220CN 2400F 88',
76 '0302AZ0111300 E0DROSC-2 N20040701720000001500',
77 '0402AZ0111300 E0DROSC-2 N20040701001220 B',
78 '0502AZ0111300 E0DROSC-2 N2004070100',
79 '0102AZ0111300 E0DROSCQ2 N20040701 11010010101 N',
80 '0202AZ0111300 E0DROSCQ2 N2004070113BCN 20 N 40 88',
81 _record('0402AZ0111300 E0DROSCQ2 N20040701', {
82 34: '001', 37: '13B', 67: 'I', 68: '29', 70: 'M', 71: 'W', 72: 'N',
83 73: 'R', 79: 'M', 84: '01', 86: 'SE'}),
84 _record('0502AZ0111300 E0DROSCQ2 N20040701', {
85 34: '01', 36: '31', 38: 'F', 39: 'W'}),
86 _record('0602AZ0111300 E0DROSCQ2 N20040701', {
87 34: '01', 36: 'E0DROSLT2 N', 48: '20040703', 56: 'T', 57: 'N',
88 58: '13B', 61: '01', 67: '31', 69: 'F', 70: 'W', 71: 'N', 72: 'R'}),
89 '0702AZ01113001T1M1VF3TX I2004072601 S90D01 18MWNN',
90 'W102AZ0111300 T0IRESC12 N20020902 16000000000 B2004120811A',
91 (
92 'W302AZ0111300 60DROSL62 N '
93 '51300000013520040923 '
94 '220'),
95 (
96 'W602AZ0111300 604AVSC-2 N 01 604AVSC-2 N20041228ON13B01 '
97 '32MANR N13B')]
99RECORDS_2025: list[str] = [
100 _record('BH50AK0010200000000000000', {
101 34: '20210101', 42: 'FAIRBANKS', 72: 'AK', 74: '4', 76: '9', 77: '4',
102 78: '1', 79: 'Y', 89: '3030', 93: '020A', 97: 'A', 106: '000031961',
103 115: '00', 118: '258', 121: '000000000', 226: '12', 228: '12',
104 230: '2025', 234: ' Y ' * 12, 270: '090'}),
105 (
106 '0150AK0010200C30BRCZ9728N20250301 16010010100 '
107 'N N'),
108 '0250AK0010200C30BRCZ9728N2025030113CCN 08 N 88',
109 (
110 '0450AK0010200C30BRCZ9728N2025030100113C '
111 'I25FWNR 01OK'),
112 '0550AK0010200C30BRCZ9728N202503010125FIN',
113 '0750AK00102000L1G0BRVSCTD2025020301 O90C01 30MINN']
116""" Public Functions """
119def write_deflate64_zip(path: pathlib.Path, name: str, data: bytes) -> None:
120 """Writes a zip archive with one member compressed with Deflate64.
122 `zipfile` cannot write Deflate64, so the archive's headers are written
123 here.
125 Args:
126 path: where to write the archive.
127 name: the name of the member.
128 data: the member's contents.
130 """
131 deflater = inflate64.Deflater()
132 compressed = deflater.deflate(data) + deflater.flush()
133 crc = zlib.crc32(data)
134 encoded = name.encode()
135 local = struct.pack(
136 '<IHHHHHIIIHH', 0x04034B50, 21, 0, 9, 0, 0, crc, len(compressed),
137 len(data), len(encoded), 0) + encoded
138 central = struct.pack(
139 '<IHHHHHHIIIHHHHHII', 0x02014B50, 21, 21, 0, 9, 0, 0, crc,
140 len(compressed), len(data), len(encoded), 0, 0, 0, 0, 0, 0) + encoded
141 end = struct.pack(
142 '<IHHHHIIH', 0x06054B50, 0, 0, 1, 1, len(central),
143 len(local) + len(compressed), 0)
144 path.write_bytes(local + compressed + central + end)
147""" Fixtures """
150@pytest.fixture(autouse = True)
151def workspace(
152 tmp_path: pathlib.Path,
153 monkeypatch: pytest.MonkeyPatch) -> pathlib.Path:
154 """Keeps each test's data and results in its own temporary folder.
156 crimepy's data folder (and an amos project's results) would otherwise be
157 in amos's workspace, the folder that holds the "code" folder of this
158 repository.
160 Args:
161 tmp_path: the test's temporary folder.
162 monkeypatch: sets the workspace and options for the test.
164 Returns:
165 The temporary folder, which is amos's workspace in the test.
167 """
168 monkeypatch.setenv('AMOS_WORKSPACE', str(tmp_path))
169 monkeypatch.setattr(options, 'DATA_FOLDER', None)
170 return tmp_path
173@pytest.fixture
174def data_folder(tmp_path: pathlib.Path) -> pathlib.Path:
175 """Returns a data folder with the master files of 2004 and 2025.
177 Args:
178 tmp_path: the test's temporary folder, which becomes the data folder.
180 Returns:
181 The data folder.
183 """
184 raw = tmp_path / 'nibrs' / 'raw'
185 raw.mkdir(parents = True)
186 with zipfile.ZipFile(
187 raw / 'nibrs-2004.zip', 'w', zipfile.ZIP_DEFLATED) as archive:
188 archive.writestr('NIBRS04.ENC', _pad(RECORDS_2004, 132))
189 write_deflate64_zip(
190 raw / 'nibrs-2025.zip', 'nibrs-2025.txt', _pad(RECORDS_2025, None))
191 return tmp_path
194@pytest.fixture
195def master_2004(data_folder: pathlib.Path) -> pathlib.Path:
196 """Returns the path of the 2004 master file.
198 Args:
199 data_folder: the data folder with the master files.
201 Returns:
202 The path of the zip archive.
204 """
205 return data_folder / 'nibrs' / 'raw' / 'nibrs-2004.zip'
208@pytest.fixture
209def master_2025(data_folder: pathlib.Path) -> pathlib.Path:
210 """Returns the path of the 2025 master file.
212 Args:
213 data_folder: the data folder with the master files.
215 Returns:
216 The path of the zip archive, which is compressed with Deflate64.
218 """
219 return data_folder / 'nibrs' / 'raw' / 'nibrs-2025.zip'
222""" Private Functions """
225def _pad(records: list[str], width: int | None) -> bytes:
226 """Returns records as the text of a master file.
228 Args:
229 records: the records.
230 width: the number of characters that every record is padded to (as
231 in fixed-length files), or `None` to leave them as they are.
233 Returns:
234 The records, each followed by a carriage return and a new line, as
235 Latin-1 bytes.
237 """
238 lines = [r.ljust(width) if width else r for r in records]
239 return ('\r\n'.join(lines) + '\r\n').encode('latin-1')