Coverage for src/crimepy/nibrs/cde.py: 89%
116 statements
« prev ^ index » next coverage.py v7.17.0, created at 2026-10-30 18:01 +0000
1"""Downloads NIBRS master files from the FBI's Crime Data Explorer.
3The Crime Data Explorer (CDE) has a master file of NIBRS data for each year
4from 1991. Each is a zip archive (from about 25 MB for 1991 to about 600 MB
5for recent years) of one fixed-width text file. The CDE does not link to the
6files directly: it asks its own web service for a link that works for 15
7minutes, and so does `crimepy`.
9Downloads are saved as "nibrs-{year}.zip" in the "nibrs/raw" folder of the
10data folder (`crimepy.utilities.data_folder()`, unless another is given). A
11file that is already there is not downloaded again unless `overwrite` is
12true.
14Contents:
15 available_years: returns the years of master files that the CDE has.
16 download: downloads the master files of years.
17 download_documentation: downloads the FBI's documentation of the files.
18 master_path: returns where the master file of a year is saved.
20"""
22from __future__ import annotations
24import functools
25import logging
26import pathlib
27import sys
28from collections.abc import Sequence
29from typing import Any
31import requests
33from .. import options, utilities
35logger = logging.getLogger(__name__)
37# The CDE's list of master files, with the first and last year of each.
38_MASTERS_URL: str = (
39 'https://cde.ucr.cjis.gov/LATEST/webapp/assets/JSON/downloads/masters.json')
40# The CDE's service that returns a temporary link to a file in its bucket.
41_SIGNED_URL: str = 'https://cde.ucr.cjis.gov/LATEST/s3/signedurl'
42# The key of a year's master file in the CDE's bucket.
43_KEY: str = 'master_files/nibrs/nibrs-{year}.zip'
44# The key of the CDE's documentation of the master files.
45_HELP_KEY: str = 'master_files/nibrs/nibrs-help.zip'
46# The first year of NIBRS master files, used if the CDE's list cannot be read.
47_FIRST_YEAR: int = 1991
48# Bytes read from the server at a time.
49_BLOCK: int = 1 << 20
50# Tries in a row that download nothing before a download gives up.
51_ATTEMPTS: int = 5
52# Errors of a connection that broke, after which a download resumes.
53_BROKEN: tuple[type[Exception], ...] = (
54 requests.ConnectionError, requests.exceptions.ChunkedEncodingError,
55 requests.Timeout)
58""" Public Functions """
61def available_years(*, timeout: float | None = None) -> list[int]:
62 """Returns the years of NIBRS master files that the CDE has.
64 Args:
65 timeout: seconds to wait for the CDE. Defaults to `None`, in which
66 case it is `crimepy.options.TIMEOUT`.
68 Raises:
69 requests.HTTPError: if the CDE refuses the request.
71 Returns:
72 Every year from the first to the last master file.
74 """
75 first, last = _year_range(timeout or options.TIMEOUT)
76 return list(range(first, last + 1))
79def master_path(
80 year: int,
81 folder: pathlib.Path | str | None = None) -> pathlib.Path:
82 """Returns where the NIBRS master file of `year` is saved.
84 Args:
85 year: the year of the master file.
86 folder: the data folder. Defaults to `None`, in which case it is
87 `crimepy.utilities.data_folder()`.
89 Returns:
90 The path of the zip archive, which may not exist yet.
92 """
93 raw = utilities.data_folder(folder) / 'nibrs' / 'raw'
94 return raw / f'nibrs-{int(year)}.zip'
97def download(
98 years: Any,
99 folder: pathlib.Path | str | None = None,
100 *,
101 overwrite: bool = False,
102 progress: bool = True,
103 timeout: float | None = None) -> list[pathlib.Path]:
104 """Downloads the NIBRS master files of `years` from the CDE.
106 Args:
107 years: a year, years, or a range of years (see
108 `crimepy.utilities.to_years`), such as 2022, [2019, 2020], or
109 "2015-2023".
110 folder: the data folder. Defaults to `None`, in which case it is
111 `crimepy.utilities.data_folder()`.
112 overwrite: whether to download a file that was downloaded before.
113 Defaults to `False`.
114 progress: whether to show how much of each file has been downloaded.
115 Defaults to `True`.
116 timeout: seconds to wait for the CDE. Defaults to `None`, in which
117 case it is `crimepy.options.TIMEOUT`.
119 Raises:
120 ValueError: if a year has no master file.
121 requests.HTTPError: if the CDE refuses a request.
123 Returns:
124 The paths of the zip archives, in the order of the years.
126 """
127 wanted = utilities.to_years(years)
128 if not wanted:
129 message = 'there are no years to download'
130 raise ValueError(message)
131 timeout = timeout or options.TIMEOUT
132 first, last = _year_range(timeout)
133 missing = [y for y in wanted if not first <= y <= last]
134 if missing:
135 message = (
136 f'the CDE has NIBRS master files for {first} to {last}, so there '
137 f'are none for {missing}')
138 raise ValueError(message)
139 paths = []
140 for year in wanted:
141 path = master_path(year, folder)
142 if path.is_file() and not overwrite: 142 ↛ 143line 142 didn't jump to line 143 because the condition on line 142 was never true
143 logger.info(f'{path} was downloaded before')
144 else:
145 _fetch(_KEY.format(year = year), path, progress, timeout)
146 paths.append(path)
147 return paths
150def download_documentation(
151 folder: pathlib.Path | str | None = None,
152 *,
153 overwrite: bool = False,
154 timeout: float | None = None) -> pathlib.Path:
155 """Downloads the FBI's documentation of the NIBRS master files.
157 The CDE's documentation is a zip archive with one PDF, a scan of the
158 "NIBRS Flatfile Yearly Master Record Descriptions" (12/18/95), which
159 `crimepy.nibrs.layouts` follows.
161 Args:
162 folder: the data folder. Defaults to `None`, in which case it is
163 `crimepy.utilities.data_folder()`.
164 overwrite: whether to download it again if it was downloaded before.
165 Defaults to `False`.
166 timeout: seconds to wait for the CDE. Defaults to `None`, in which
167 case it is `crimepy.options.TIMEOUT`.
169 Returns:
170 The path of the zip archive.
172 """
173 path = master_path(0, folder).with_name('nibrs-help.zip')
174 if overwrite or not path.is_file(): 174 ↛ 176line 174 didn't jump to line 176 because the condition on line 174 was always true
175 _fetch(_HELP_KEY, path, False, timeout or options.TIMEOUT)
176 return path
179""" Private Functions """
182def _fetch(
183 key: str,
184 path: pathlib.Path,
185 progress: bool,
186 timeout: float) -> None:
187 """Downloads the file at `key` in the CDE's bucket to `path`.
189 The file is downloaded to a temporary file that replaces `path` only when
190 the download is complete, so an interrupted download leaves nothing that
191 looks finished. The CDE's links work for about 15 minutes, which is less
192 than a slow connection needs for a recent master file, so when the
193 connection breaks, the download asks for a new link and resumes where it
194 stopped. It gives up after `_ATTEMPTS` tries in a row that add nothing.
196 Args:
197 key: the key of the file in the CDE's bucket.
198 path: where to save the file.
199 progress: whether to show how much has been downloaded.
200 timeout: seconds to wait for a server.
202 Raises:
203 requests.HTTPError: if a server refuses a request.
204 requests.ConnectionError: if the connection breaks `_ATTEMPTS` times
205 in a row without downloading anything.
207 """
208 path.parent.mkdir(parents = True, exist_ok = True)
209 partial = path.with_name(f'{path.name}.part')
210 partial.unlink(missing_ok = True)
211 done, size, failures = 0, 0, 0
212 while True:
213 before = done
214 try:
215 done, size = _resume(
216 _signed_url(key, timeout), partial, done, size,
217 path.name if progress else None, timeout)
218 except _BROKEN as error:
219 done = partial.stat().st_size if partial.is_file() else 0
220 failures = 0 if done > before else failures + 1
221 if failures >= _ATTEMPTS:
222 raise
223 logger.info(f'resuming {path.name} at {done:,} bytes after {error}')
224 continue
225 if not size or done >= size: 225 ↛ 212line 225 didn't jump to line 212 because the condition on line 225 was always true
226 break
227 if progress:
228 print(file = sys.stderr)
229 partial.replace(path)
230 logger.info(f'downloaded {path}')
233def _resume(
234 url: str,
235 partial: pathlib.Path,
236 done: int,
237 size: int,
238 name: str | None,
239 timeout: float) -> tuple[int, int]:
240 """Downloads the rest of a file, after the bytes already in `partial`.
242 Args:
243 url: a link to the file.
244 partial: the temporary file, with the first `done` bytes.
245 done: the bytes already downloaded.
246 size: the bytes in the whole file, or 0 if not known yet.
247 name: the name of the file to show progress for, or `None` to not
248 show progress.
249 timeout: seconds to wait for a server.
251 Raises:
252 requests.HTTPError: if the server refuses the request.
253 requests.ConnectionError: if the connection breaks before the file
254 is complete.
256 Returns:
257 The bytes downloaded and the bytes in the whole file.
259 """
260 headers = {'User-Agent': options.USER_AGENT}
261 if done:
262 headers['Range'] = f'bytes={done}-'
263 with requests.get(
264 url, stream = True, timeout = timeout, headers = headers) as response:
265 response.raise_for_status()
266 # A server that ignores the range sends the whole file again.
267 if done and response.status_code != 206:
268 done = 0
269 length = int(response.headers.get('Content-Length', 0))
270 size = size or (done + length if length else 0)
271 with open(partial, 'ab' if done else 'wb') as stream:
272 for block in response.iter_content(chunk_size = _BLOCK):
273 stream.write(block)
274 done += len(block)
275 if name is not None:
276 _show(name, done, size)
277 if size and done < size: 277 ↛ 278line 277 didn't jump to line 278 because the condition on line 277 was never true
278 message = f'the connection closed after {done:,} of {size:,} bytes'
279 raise requests.ConnectionError(message)
280 return done, size
283def _show(name: str, done: int, size: int) -> None:
284 """Shows how much of a file has been downloaded.
286 Args:
287 name: name of the file.
288 done: bytes downloaded so far.
289 size: bytes in the file, or 0 if not known.
291 """
292 megabytes = done / 1e6
293 if size: 293 ↛ 296line 293 didn't jump to line 296 because the condition on line 293 was always true
294 text = f'\r{name}: {megabytes:,.0f} of {size / 1e6:,.0f} MB'
295 else:
296 text = f'\r{name}: {megabytes:,.0f} MB'
297 print(text, end = '', file = sys.stderr, flush = True)
300def _signed_url(key: str, timeout: float) -> str:
301 """Returns a temporary link to the file at `key` in the CDE's bucket.
303 Args:
304 key: the key of the file in the CDE's bucket.
305 timeout: seconds to wait for the CDE.
307 Raises:
308 requests.HTTPError: if the CDE refuses the request.
309 ValueError: if the CDE does not return a link.
311 Returns:
312 A link that works for about 15 minutes.
314 """
315 response = requests.get(
316 _SIGNED_URL, params = {'key': key}, timeout = timeout,
317 headers = {'User-Agent': options.USER_AGENT})
318 response.raise_for_status()
319 links = response.json()
320 if not isinstance(links, dict) or not links.get(key): 320 ↛ 321line 320 didn't jump to line 321 because the condition on line 320 was never true
321 message = f'the CDE did not return a link to {key!r}: {links!r}'
322 raise ValueError(message)
323 return str(links[key])
326@functools.cache
327def _year_range(timeout: float) -> tuple[int, int]:
328 """Returns the first and last years of the CDE's NIBRS master files.
330 Args:
331 timeout: seconds to wait for the CDE.
333 Raises:
334 requests.HTTPError: if the CDE refuses the request.
335 ValueError: if the CDE does not list a NIBRS master file.
337 Returns:
338 The first and last years.
340 """
341 response = requests.get(
342 _MASTERS_URL, timeout = timeout,
343 headers = {'User-Agent': options.USER_AGENT})
344 response.raise_for_status()
345 masters: Sequence[dict[str, Any]] = response.json()
346 for master in masters: 346 ↛ 351line 346 didn't jump to line 351 because the loop on line 346 didn't complete
347 if master.get('id') == 'nibrs': 347 ↛ 346line 347 didn't jump to line 346 because the condition on line 347 was always true
348 return (
349 int(master.get('minYear', _FIRST_YEAR)),
350 int(master['maxYear']))
351 message = 'the CDE does not list a NIBRS master file'
352 raise ValueError(message)