Coverage for src/crimepy/nibrs/layouts.py: 100%
90 statements
« prev ^ index » next coverage.py v7.17.0, created at 2026-10-30 18:01 +0000
1"""Where each field is in the records of the NIBRS master files.
3A NIBRS master file has one record (line) for each segment of each report.
4The first two characters of a record name its segment (its "level"), and the
5rest are fixed-width fields. The positions are those of the FBI's "NIBRS
6Flatfile Yearly Master Record Descriptions" (12/18/95), which is still the
7documentation that the Crime Data Explorer gives with the files, and of the
8fields that the FBI added later and that are in the files but not in that
9document:
11* Administrative segment (01), position 89: cargo theft (Y or N), in 2013
12 and later.
13* Offense segment (02), positions 60-67: bias motivations #2 to #5, in 2013
14 to 2016.
15* Victim segment (04), positions 124-135: the officer's activity and
16 assignment, and the ORI of another jurisdiction (handwritten in the
17 document), in every year.
18* Offender segment (05), position 40: the ethnicity of the offender, in 2013
19 and later.
20* Batch header (BH), positions 285-292: a date that is the same for every
21 agency in a state, in 2013 to 2016.
23Files before 2013 split each agency's batch header into three records ("B1",
24"B2", and "B3"), and files from 2013 on have one ("BH"). Their fields have the
25same names here, so `crimepy` makes one table of agencies from either.
27Positions in `Field` are those of Python slices (counted from 0, with the end
28excluded), so the document's positions "34-36" are `start = 33, end = 36`.
29Records shorter than a layout (most records after 2012 drop trailing blanks)
30are padded with blanks, so fields past their end are missing.
32Contents:
33 Field: a field of a record: its name, position, and kind.
34 Layout: the fields of one kind of record.
35 KINDS: the kinds of fields.
36 LAYOUTS: the layout of each segment level, by its two-character code.
37 TABLES: the name of the table made from each segment level.
38 SEGMENTS: the segment levels of each table.
40"""
42from __future__ import annotations
44import dataclasses
45from collections.abc import Iterator
47# The kinds of fields, and what each becomes:
48# 'code': a code (such as an offense code), as text, which
49# `crimepy.nibrs.munge.decode` can replace with its label.
50# 'text': text, without leading and trailing blanks.
51# 'int': a whole number.
52# 'date': a date written as YYYYMMDD.
53# 'bool': true for "Y", false for "N", and missing otherwise.
54# 'yes': true if the field is `Field.true`, and false otherwise.
55# 'quantity': a 12-digit amount of a drug: 9 digits and 3 more digits of
56# thousandths.
57KINDS: frozenset[str] = frozenset({
58 'bool', 'code', 'date', 'int', 'quantity', 'text', 'yes'})
61""" Classes """
64@dataclasses.dataclass(frozen = True)
65class Field:
66 """A field of a record in a NIBRS master file.
68 Args:
69 name: name of the column made from the field.
70 start: position of the first character of the field, counted from 0.
71 end: position after the last character of the field.
72 kind: what the field becomes (see `KINDS`). Defaults to 'code'.
73 codes: name of the codebook in `crimepy.nibrs.codes.CODEBOOKS` that
74 gives the labels of a 'code' field. Defaults to `None`, in which
75 case the field has no labels.
76 true: the character that means true in a 'yes' field. Defaults to
77 'Y'.
79 """
81 name: str
82 start: int
83 end: int
84 kind: str = 'code'
85 codes: str | None = None
86 true: str = 'Y'
88 """ Initialization Methods """
90 def __post_init__(self) -> None:
91 """Checks the field.
93 Raises:
94 ValueError: if the kind is not one of `KINDS` or the field has no
95 width.
97 """
98 if self.kind not in KINDS:
99 message = (
100 f'{self.kind!r} is not a kind of field: use one of '
101 f'{sorted(KINDS)}')
102 raise ValueError(message)
103 if self.end <= self.start:
104 message = f'the field {self.name!r} has no width'
105 raise ValueError(message)
107 """ Properties """
109 @property
110 def width(self) -> int:
111 """Returns the number of characters in the field.
113 Returns:
114 The number of characters from `start` to `end`.
116 """
117 return self.end - self.start
120@dataclasses.dataclass(frozen = True)
121class Layout:
122 """The fields of one kind of record in a NIBRS master file.
124 Args:
125 level: the two-character segment level that starts the records.
126 name: what the records are.
127 fields: the fields of the records, in order.
129 """
131 level: str
132 name: str
133 fields: tuple[Field, ...]
135 """ Properties """
137 @property
138 def names(self) -> list[str]:
139 """Returns the names of the fields.
141 Returns:
142 The names, in the order of the fields.
144 """
145 return [f.name for f in self.fields]
147 @property
148 def width(self) -> int:
149 """Returns the number of characters up to the end of the last field.
151 Returns:
152 The position after the last character of the last field.
154 """
155 return max(f.end for f in self.fields)
157 """ Dunder Methods """
159 def __iter__(self) -> Iterator[Field]:
160 """Returns an iterator of the fields.
162 Returns:
163 An iterator of the fields, in order.
165 """
166 return iter(self.fields)
169""" Private Functions """
172def _field(
173 name: str,
174 first: int,
175 last: int,
176 kind: str = 'code',
177 codes: str | None = None,
178 true: str = 'Y') -> Field:
179 """Returns a field from the positions in the FBI's documentation.
181 Args:
182 name: name of the column made from the field.
183 first: position of the first character, counted from 1.
184 last: position of the last character, counted from 1.
185 kind: what the field becomes. Defaults to 'code'.
186 codes: name of the field's codebook. Defaults to `None`.
187 true: the character that means true in a 'yes' field. Defaults to
188 'Y'.
190 Returns:
191 The field, with the positions of Python slices.
193 """
194 return Field(name, first - 1, last, kind, codes, true)
197def _repeated(
198 name: str,
199 first: int,
200 width: int,
201 times: int,
202 kind: str = 'code',
203 codes: str | None = None) -> list[Field]:
204 """Returns fields that repeat one after another, numbered from 1.
206 Args:
207 name: name of the columns, which are numbered ("offense_1").
208 first: position of the first character of the first field, counted
209 from 1.
210 width: number of characters in each field.
211 times: number of fields.
212 kind: what the fields become. Defaults to 'code'.
213 codes: name of the fields' codebook. Defaults to `None`.
215 Returns:
216 The fields.
218 """
219 return [
220 _field(
221 f'{name}_{n + 1}', first + n * width, first + (n + 1) * width - 1,
222 kind, codes)
223 for n in range(times)]
226def _arrestee(first: int) -> list[Field]:
227 """Returns the fields that describe an arrestee and the arrest.
229 The fields are the same in the arrestee segment and in the Group "B"
230 Arrest Report, but start at different positions.
232 Args:
233 first: position of the type of arrest, counted from 1.
235 Returns:
236 The fields from the type of arrest to the disposition of an arrestee
237 under 18.
239 """
240 offense = first + 1 if first == 40 else first + 2
241 fields = [_field('arrest_type', first, first, 'code', 'arrest_types')]
242 if first != 40:
243 fields.append(_field(
244 'multiple_arrestee', first + 1, first + 1, 'code',
245 'multiple_arrestees'))
246 weapon = offense + 3
247 fields.extend([
248 _field('arrest_offense', offense, offense + 2, 'code', 'offenses'),
249 _field('arrestee_weapon_1', weapon, weapon + 1, 'code', 'weapons'),
250 _field(
251 'arrestee_weapon_1_automatic', weapon + 2, weapon + 2, 'yes',
252 true = 'A'),
253 _field(
254 'arrestee_weapon_2', weapon + 3, weapon + 4, 'code', 'weapons'),
255 _field(
256 'arrestee_weapon_2_automatic', weapon + 5, weapon + 5, 'yes',
257 true = 'A'),
258 _field('arrestee_age_code', weapon + 6, weapon + 7, 'code', 'ages'),
259 _field('arrestee_sex', weapon + 8, weapon + 8, 'code', 'sexes'),
260 _field('arrestee_race', weapon + 9, weapon + 9, 'code', 'races'),
261 _field(
262 'arrestee_ethnicity', weapon + 10, weapon + 10, 'code',
263 'ethnicities'),
264 _field(
265 'arrestee_resident', weapon + 11, weapon + 11, 'code',
266 'residents'),
267 _field(
268 'juvenile_disposition', weapon + 12, weapon + 12, 'code',
269 'dispositions')])
270 return fields
273def _county(number: int, first: int) -> list[Field]:
274 """Returns the fields of one of an agency's (up to five) counties.
276 Args:
277 number: which county (1 to 5).
278 first: position of the county's current population, counted from 1.
280 Returns:
281 The county's current population, UCR county code, MSA code, and
282 population in the previous census.
284 """
285 return [
286 _field(f'population_{number}', first, first + 8, 'int'),
287 _field(f'ucr_county_{number}', first + 9, first + 11, 'text'),
288 _field(f'msa_{number}', first + 12, first + 14, 'text'),
289 _field(f'last_population_{number}', first + 15, first + 23, 'int')]
292def _activity(first: int) -> list[Field]:
293 """Returns the 36 flags of an agency's submissions in each month.
295 Args:
296 first: position of January's first flag, counted from 1.
298 Returns:
299 For each month (named "jan" to "dec"), whether a zero report, a Group
300 "A" or "B" report, and a window record was submitted.
302 """
303 months = (
304 'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct',
305 'nov', 'dec')
306 fields = []
307 for n, month in enumerate(months):
308 position = first + 3 * n
309 fields.extend([
310 _field(f'zero_report_{month}', position, position, 'yes'),
311 _field(
312 f'incident_report_{month}', position + 1, position + 1,
313 'yes'),
314 _field(
315 f'window_report_{month}', position + 2, position + 2,
316 'yes')])
317 return fields
320def _tail(first: int) -> list[Field]:
321 """Returns the fields of an agency's batch header after its counties.
323 Args:
324 first: position of the "01-06-12 indicator", counted from 1.
326 Returns:
327 The reporting indicator, number of months reported, master file year,
328 monthly activity flags, and FIPS county codes.
330 """
331 return [
332 _field('reporting_indicator', first, first + 1, 'text'),
333 _field('months_reported', first + 2, first + 3, 'int'),
334 _field('master_file_year', first + 4, first + 7, 'int'),
335 *_activity(first + 8),
336 *_repeated('fips_county', first + 44, 3, 5, 'text')]
339""" Layouts """
342# Fields at the start of every record of a Group "A" Incident Report. The
343# numeric state code is kept as `state_code`, and `crimepy` adds the postal
344# abbreviation as `state`.
345_INCIDENT: list[Field] = [
346 _field('state_code', 3, 4, 'code', 'states'),
347 _field('ori', 5, 13, 'text'),
348 _field('incident_number', 14, 25, 'text'),
349 _field('incident_date', 26, 33, 'date')]
350# The same fields of a window property or arrestee segment, which has no
351# incident date. (A window exceptional clearance segment has one.)
352_WINDOW: list[Field] = _INCIDENT[:3]
354_ADMINISTRATIVE: list[Field] = [
355 _field('report_date', 34, 34, 'yes', true = 'R'),
356 _field('incident_hour', 35, 36, 'int'),
357 _field('total_offenses', 37, 38, 'int'),
358 _field('total_victims', 39, 41, 'int'),
359 _field('total_offenders', 42, 43, 'int'),
360 _field('total_arrestees', 44, 45, 'int'),
361 _field('city_submission', 46, 49, 'text'),
362 _field('cleared_exceptionally', 50, 50, 'code', 'clearances'),
363 _field('exceptional_clearance_date', 51, 58, 'date'),
364 _field('cargo_theft', 89, 89, 'bool')]
366_OFFENSE: list[Field] = [
367 _field('offense_code', 34, 36, 'code', 'offenses'),
368 _field('completion', 37, 37, 'code', 'completions'),
369 *_repeated('suspected_using', 38, 1, 3, 'code', 'using'),
370 _field('location', 41, 42, 'code', 'locations'),
371 _field('premises_entered', 43, 44, 'int'),
372 _field('entry_method', 45, 45, 'code', 'entries'),
373 *_repeated('criminal_activity', 46, 1, 3, 'code', 'activities'),
374 _field('weapon_1', 49, 50, 'code', 'weapons'),
375 _field('weapon_1_automatic', 51, 51, 'yes', true = 'A'),
376 _field('weapon_2', 52, 53, 'code', 'weapons'),
377 _field('weapon_2_automatic', 54, 54, 'yes', true = 'A'),
378 _field('weapon_3', 55, 56, 'code', 'weapons'),
379 _field('weapon_3_automatic', 57, 57, 'yes', true = 'A'),
380 *_repeated('bias_motivation', 58, 2, 5, 'code', 'biases')]
382# Fields of the property segment after the common fields.
383_PROPERTY: list[Field] = [
384 _field('loss_type', 34, 34, 'code', 'losses'),
385 _field('property_description', 35, 36, 'code', 'properties'),
386 _field('property_value', 37, 45, 'int'),
387 _field('date_recovered', 46, 53, 'date'),
388 _field('stolen_vehicles', 54, 55, 'int'),
389 _field('recovered_vehicles', 56, 57, 'int')]
390for _n, _first in enumerate((58, 73, 88), start = 1):
391 _PROPERTY.extend([
392 _field(f'drug_type_{_n}', _first, _first, 'code', 'drugs'),
393 _field(f'drug_quantity_{_n}', _first + 1, _first + 12, 'quantity'),
394 _field(
395 f'drug_measure_{_n}', _first + 13, _first + 14, 'code',
396 'measures')])
398_VICTIM: list[Field] = [
399 _field('victim_sequence', 34, 36, 'int'),
400 *_repeated('offense', 37, 3, 10, 'code', 'offenses'),
401 _field('victim_type', 67, 67, 'code', 'victim_types'),
402 _field('victim_age_code', 68, 69, 'code', 'ages'),
403 _field('victim_sex', 70, 70, 'code', 'sexes'),
404 _field('victim_race', 71, 71, 'code', 'races'),
405 _field('victim_ethnicity', 72, 72, 'code', 'ethnicities'),
406 _field('victim_resident', 73, 73, 'code', 'residents'),
407 *_repeated('circumstance', 74, 2, 2, 'code', 'circumstances'),
408 _field('justifiable_homicide', 78, 78, 'code', 'justifications'),
409 *_repeated('injury', 79, 1, 5, 'code', 'injuries')]
410for _n in range(1, 11):
411 _first = 84 + 4 * (_n - 1)
412 _VICTIM.extend([
413 _field(f'offender_{_n}', _first, _first + 1, 'int'),
414 _field(
415 f'relationship_{_n}', _first + 2, _first + 3, 'code',
416 'relationships')])
417_VICTIM.extend([
418 _field('officer_activity', 124, 125, 'code', 'officer_activities'),
419 _field('officer_assignment', 126, 126, 'code', 'officer_assignments'),
420 _field('officer_other_ori', 127, 135, 'text')])
422_OFFENDER: list[Field] = [
423 _field('offender_sequence', 34, 35, 'int'),
424 _field('offender_age_code', 36, 37, 'code', 'ages'),
425 _field('offender_sex', 38, 38, 'code', 'sexes'),
426 _field('offender_race', 39, 39, 'code', 'races'),
427 _field('offender_ethnicity', 40, 40, 'code', 'ethnicities')]
430_ARRESTEE: list[Field] = [
431 _field('arrestee_sequence', 34, 35, 'int'),
432 _field('arrest_transaction_number', 36, 47, 'text'),
433 _field('arrest_date', 48, 55, 'date'),
434 *_arrestee(56)]
436_GROUP_B: list[Field] = [
437 _field('state_code', 3, 4, 'code', 'states'),
438 _field('ori', 5, 13, 'text'),
439 _field('arrest_transaction_number', 14, 25, 'text'),
440 _field('arrest_date', 26, 33, 'date'),
441 _field('arrestee_sequence', 34, 35, 'int'),
442 _field('city_submission', 36, 39, 'text'),
443 *_arrestee(40)]
445# Fields of an agency's batch header that are the same in "BH" and "B1".
446_AGENCY: list[Field] = [
447 _field('state_code', 3, 4, 'code', 'states'),
448 _field('ori', 5, 13, 'text'),
449 _field('date_ori_added', 26, 33, 'date'),
450 _field('date_ori_went_nibrs', 34, 41, 'date'),
451 _field('city', 42, 71, 'text'),
452 _field('state_abbreviation', 72, 73, 'text'),
453 _field('population_group', 74, 75, 'code', 'population_groups'),
454 _field('division', 76, 76, 'code', 'divisions'),
455 _field('region', 77, 77, 'code', 'regions'),
456 _field('agency_type', 78, 78, 'code', 'agency_types'),
457 _field('core_city', 79, 79, 'bool'),
458 _field('covered_by_ori', 80, 88, 'text'),
459 _field('field_office', 89, 92, 'text'),
460 _field('judicial_district', 93, 96, 'text'),
461 _field('nibrs_active', 97, 97, 'yes', true = 'A'),
462 _field('inactive_date', 98, 105, 'date')]
465_BH: list[Field] = [
466 *_AGENCY,
467 *[f for n in range(5) for f in _county(n + 1, 106 + 24 * n)],
468 *_tail(226),
469 _field('state_nibrs_date', 285, 292, 'date')]
470_B1: list[Field] = list(_AGENCY)
471_B2: list[Field] = [
472 _AGENCY[1],
473 *[f for n in range(4) for f in _county(n + 1, 26 + 24 * n)]]
474_B3: list[Field] = [_AGENCY[1], *_county(5, 26), *_tail(50)]
476LAYOUTS: dict[str, Layout] = {
477 'BH': Layout('BH', 'batch header', tuple(_BH)),
478 'B1': Layout('B1', 'batch header (1 of 3)', tuple(_B1)),
479 'B2': Layout('B2', 'batch header (2 of 3)', tuple(_B2)),
480 'B3': Layout('B3', 'batch header (3 of 3)', tuple(_B3)),
481 '01': Layout(
482 '01', 'administrative segment', tuple(_INCIDENT + _ADMINISTRATIVE)),
483 '02': Layout('02', 'offense segment', tuple(_INCIDENT + _OFFENSE)),
484 '03': Layout('03', 'property segment', tuple(_INCIDENT + _PROPERTY)),
485 '04': Layout('04', 'victim segment', tuple(_INCIDENT + _VICTIM)),
486 '05': Layout('05', 'offender segment', tuple(_INCIDENT + _OFFENDER)),
487 '06': Layout('06', 'arrestee segment', tuple(_INCIDENT + _ARRESTEE)),
488 '07': Layout('07', 'Group "B" Arrest Report', tuple(_GROUP_B)),
489 'W1': Layout(
490 'W1', 'window exceptional clearance segment',
491 tuple(_INCIDENT + _ADMINISTRATIVE[:-1]
492 + _repeated('offense', 59, 3, 10, 'code', 'offenses'))),
493 'W3': Layout(
494 'W3', 'window recovered property segment',
495 tuple(_WINDOW + _PROPERTY
496 + _repeated('offense', 103, 3, 10, 'code', 'offenses'))),
497 'W6': Layout(
498 'W6', 'window arrestee segment',
499 tuple(_WINDOW + _ARRESTEE
500 + [_field('window_clearance', 74, 74, 'bool')]
501 + _repeated('offense', 75, 3, 10, 'code', 'offenses')))}
503TABLES: dict[str, str] = {
504 'BH': 'agencies',
505 'B1': 'agencies',
506 'B2': 'agencies',
507 'B3': 'agencies',
508 '01': 'administrative',
509 '02': 'offenses',
510 '03': 'properties',
511 '04': 'victims',
512 '05': 'offenders',
513 '06': 'arrestees',
514 '07': 'group_b_arrests',
515 'W1': 'window_clearances',
516 'W3': 'window_properties',
517 'W6': 'window_arrestees'}
519SEGMENTS: dict[str, tuple[str, ...]] = {}
520for _level, _table in TABLES.items():
521 SEGMENTS[_table] = (*SEGMENTS.get(_table, ()), _level)