Coverage for src/crimepy/techniques.py: 96%

1"""Techniques that analyze crime data, on `amos` datasets. 

2 

3Each technique is added to the `amos` library when crimepy is imported, so 

4it can be named in the settings of any `amos` worker, and each can be applied 

5to an `amos.Dataset` (or a data frame) directly. A describer stores its table 

6in the dataset's `tables` under its name, and a shaper replaces the 

7dataset's rows: 

8 

9* `count_crimes` (an `amos.Shaper`) turns one row for each event into one 

10 row for each group (and period), with the number of events, so that 

11 counts can be modeled (for example, with `amos`'s `poisson` or 

12 `negative_binomial` models). 

13* `crime_trend` (an `amos.Describer`) makes a table of counts over time. 

14* `population_rates` (an `amos.Describer`) makes counts per 100,000 people. 

15* `crime_rates` (an `amos.Describer`) makes the rate of crime in each group, 

16 using only agencies that reported all year. 

17* `clearance_rates` (an `amos.Describer`) makes a table of the share of 

18 incidents cleared in each group. 

19* `percent_change` (an `amos.Describer`) makes the change from one period to 

20 the next. 

21* `crosstab` (an `amos.Describer`) makes counts (or shares) of two variables 

22 against each other. 

23* `coverage` (an `amos.Describer`) makes a table of the agencies that 

24 reported and the people they serve, from a dataset of agencies. 

25 

26The functions of `crimepy.analyze` apply these techniques. The figures of 

27`crimepy.plot` (`trend_plot`, `top_values`, `time_heatmap`, and 

28`age_pyramid`) are techniques too. 

29 

30Contents: 

31 ClearanceRates: the share of incidents cleared in each group. 

32 CountCrimes: counts events by group and period. 

33 Coverage: the agencies that reported and the people they serve. 

34 CrimeRates: crimes per 100,000 people, by group. 

35 CrimeTrend: counts of events over time. 

36 Crosstab: counts or shares of two variables against each other. 

37 PercentChange: the change from one period to the next. 

38 PopulationRates: counts per 100,000 people. 

39 

40""" 

41 

42from __future__ import annotations 

43 

44import dataclasses 

45from typing import Any, Literal 

46 

47import amos 

48import pandas as pd 

49 

50from . import utilities 

51 

52""" Techniques """ 

53 

54 

55@dataclasses.dataclass 

56class CountCrimes(amos.Shaper): 

57 """Counts events by group (and period). 

58 

59 Parameters (set in the "count_crimes_parameters" section of the 

60 settings): 

61 

62 * `by`: the columns to group by, such as "state, offense_category". 

63 * `date`: the column of dates. Defaults to "incident_date" if `freq` is 

64 set. 

65 * `freq`: the length of each period ("YS", "QS", "MS", "W", or "D"). 

66 Defaults to none, which does not group by period. 

67 * `weight`: a column whose values are summed instead of counting rows. 

68 Defaults to none. 

69 * `name`: the name of the column of counts. Defaults to "count". 

70 * `label`: the label of the counted data. Defaults to the dataset's 

71 label, or, if it has none, the column of counts. 

72 

73 """ 

74 

75 """ Required Methods """ 

76 

77 def shape( 

78 self, 

79 data: pd.DataFrame, 

80 by: Any = None, 

81 date: str | None = None, 

82 freq: str | None = None, 

83 weight: str | None = None, 

84 name: str = 'count', 

85 **kwargs: Any) -> pd.DataFrame: 

86 """Returns the number of events in each group (and period). 

87 

88 Args: 

89 data: one row for each event. 

90 by: the columns to group by. Defaults to `None`, which counts all 

91 rows (in each period, if `freq` is set). 

92 date: the column of dates that `freq` groups by. Defaults to 

93 `None`, in which case it is "incident_date" if `freq` is set. 

94 freq: the length of each period, as a `pandas` frequency ("YS" 

95 for years, "QS" for quarters, "MS" for months, "W" for weeks, 

96 "D" for days). Defaults to `None`, which does not group by 

97 period. 

98 weight: a column whose values are summed instead of counting 

99 rows. Defaults to `None`. 

100 name: the name of the column of counts. Defaults to "count". 

101 **kwargs: not used. 

102 

103 Returns: 

104 A table with a column for each group (and "period"), and the 

105 counts. Groups with no rows are left out. 

106 

107 """ 

108 keys = utilities.to_names(by) 

109 freq = freq or None 

110 groupers: list[Any] = list(keys) 

111 if freq is not None: 

112 date = date or 'incident_date' 

113 groupers.append(pd.Grouper(key = date, freq = freq)) 

114 if not groupers: 

115 total = data[weight].sum() if weight else len(data) 

116 return pd.DataFrame({name: [total]}) 

117 grouped = data.groupby(groupers, observed = True, dropna = False) 

118 counts = grouped[weight].sum() if weight else grouped.size() 

119 table = counts.rename(name).reset_index() 

120 if freq is not None: 

121 table = table.rename(columns = {date: 'period'}) 

122 return table[table[name] > 0].reset_index(drop = True) 

123 

124 """ Public Methods """ 

125 

126 def apply( 

127 self, 

128 item: amos.Dataset, 

129 *, 

130 label: str | None = None, 

131 task: str | None = None, 

132 groups: Any = None, 

133 **kwargs: Any) -> amos.Dataset: 

134 """Counts the events of `item`. 

135 

136 Args: 

137 item: the dataset, with one row for each event. 

138 label: the label of the counted data. Defaults to `None`, which 

139 keeps the label of `item` or, if it has none, makes the 

140 column of counts the label. 

141 task: "classify" or "regress". Defaults to `None`. 

142 groups: the groups of the counted data. Defaults to `None`, which 

143 keeps the groups of `item`. 

144 **kwargs: the parameters of `shape`. 

145 

146 Returns: 

147 The counted dataset. 

148 

149 """ 

150 if label is None and item.label is None: 

151 label = kwargs.get('name', 'count') 

152 return super().apply( 

153 item, label = label, task = task, groups = groups, **kwargs) 

154 

155 

156@dataclasses.dataclass 

157class CrimeTrend(amos.Describer): 

158 """A table of counts of events over time, with a column for each group. 

159 

160 Parameters (set in the "crime_trend_parameters" section of the 

161 settings): `by`, `date`, `freq`, and `top` (see `describe`). 

162 

163 """ 

164 

165 """ Required Methods """ 

166 

167 def describe( 

168 self, 

169 item: amos.Dataset, 

170 by: str | None = None, 

171 date: str = 'incident_date', 

172 freq: str = 'MS', 

173 top: Any = None, 

174 **kwargs: Any) -> pd.DataFrame: 

175 """Returns counts over time. 

176 

177 Args: 

178 item: the dataset, with one row for each event. 

179 by: the column whose values become columns. Defaults to `None`, 

180 which makes one column, "count". 

181 date: the column of dates. Defaults to "incident_date". 

182 freq: the length of each period, as a `pandas` frequency. 

183 Defaults to "MS" (months). 

184 top: keep only the groups with the most rows. Defaults to `None`, 

185 which keeps them all. 

186 **kwargs: not used. 

187 

188 Returns: 

189 A table indexed by the start of each period, with a column of 

190 counts for each group (0 in periods with none). 

191 

192 """ 

193 counted = CountCrimes().shape(item.data, by, date = date, freq = freq) 

194 if by is None: 

195 return counted.set_index('period') 

196 wide = counted.pivot_table( 

197 index = 'period', columns = by, values = 'count', aggfunc = 'sum', 

198 fill_value = 0, observed = True) 

199 if top is not None: 

200 largest = wide.sum().nlargest(int(top)).index 

201 wide = wide[largest] 

202 wide.columns = [str(c) for c in wide.columns] 

203 return wide 

204 

205 

206@dataclasses.dataclass 

207class PopulationRates(amos.Describer): 

208 """Counts per 100,000 people (or another base), from a table of counts. 

209 

210 Parameters (set in the "population_rates_parameters" section of the 

211 settings): `population`, `on`, `count`, `per`, and `name` (see 

212 `describe`). 

213 

214 """ 

215 

216 """ Required Methods """ 

217 

218 def describe( 

219 self, 

220 item: amos.Dataset, 

221 population: Any = 'population', 

222 on: Any = None, 

223 count: str = 'count', 

224 per: float = 100_000, 

225 name: str = 'rate', 

226 **kwargs: Any) -> pd.DataFrame: 

227 """Returns counts per `per` people. 

228 

229 Args: 

230 item: a dataset of counts (such as one from `count_crimes`). 

231 population: a column of the counts with the population, or a 

232 table (a dataset or a data frame) of populations to join to 

233 the counts on `on`, with a "population" column. Defaults to 

234 "population". 

235 on: the columns that join `population` to the counts. Defaults to 

236 `None`, in which case they are the columns that both have 

237 (other than "population"). 

238 count: the column of counts. Defaults to "count". 

239 per: the number of people that rates are per. Defaults to 

240 100,000. 

241 name: name of the column of rates. Defaults to "rate". 

242 **kwargs: not used. 

243 

244 Returns: 

245 The counts, with the population (if it was joined) and the rates 

246 (missing where the population is 0 or missing). 

247 

248 """ 

249 table = item.data.copy() 

250 column = 'population' 

251 if isinstance(population, str): 

252 column = population 

253 else: 

254 people = utilities.to_frame(population) 

255 keys = utilities.to_names(on) or [ 

256 c for c in people.columns 

257 if c in table.columns and c != 'population'] 

258 table = table.merge(people, on = keys, how = 'left') 

259 values = pd.to_numeric( 

260 table[column], errors = 'coerce').astype('Float64') 

261 values = values.where(values > 0) 

262 table[name] = table[count] / values * float(per) 

263 return table 

264 

265 

266@dataclasses.dataclass 

267class CrimeRates(amos.Describer): 

268 """Crimes per 100,000 people in each group, from agencies that reported. 

269 

270 Only agencies that reported in at least `months` months of the year (see 

271 "months_active" in `crimepy.nibrs.munge`) count, both for the events and 

272 for the population, so that the rate is not lowered by agencies that did 

273 not report. 

274 

275 Parameters (set in the "crime_rates_parameters" section of the settings): 

276 `by`, `months`, `per`, and `folder` (see `describe`). In settings, the 

277 agencies are loaded for the years and states of the data. 

278 

279 """ 

280 

281 """ Required Methods """ 

282 

283 def describe( 

284 self, 

285 item: amos.Dataset, 

286 agencies: Any = None, 

287 by: Any = ('data_year', 'state'), 

288 months: Any = 12, 

289 per: float = 100_000, 

290 folder: Any = None, 

291 **kwargs: Any) -> pd.DataFrame: 

292 """Returns the number of events per `per` people in each group. 

293 

294 Args: 

295 item: a dataset with one row for each event (such as an 

296 incident), with "data_year" and "agency_ori" (or "ori"). 

297 agencies: the agencies (a dataset or a data frame), with 

298 "data_year", "ori", "population", and "months_active". 

299 Defaults to `None`, in which case the agencies of the years 

300 (and states, if the data has a "state" column) of the data 

301 are loaded. 

302 by: columns of the agencies to group by. Defaults to 

303 ("data_year", "state"). 

304 months: the fewest months an agency must have reported in. 

305 Defaults to 12. 

306 per: the number of people that rates are per. Defaults to 

307 100,000. 

308 folder: the data folder to load agencies from. Defaults to 

309 `None`, in which case it is `crimepy.utilities.data_folder()`. 

310 **kwargs: not used. 

311 

312 Returns: 

313 A table with a column for each group, "count", "population", 

314 "agencies", and "rate". 

315 

316 """ 

317 events = item.data 

318 if agencies is None: 

319 agencies = _agencies_of(events, folder) 

320 agencies = utilities.to_frame(agencies) 

321 keys = utilities.to_names(by) 

322 active = agencies[agencies['months_active'].fillna(0) >= int(months)] 

323 active = active.drop_duplicates(['data_year', 'ori']) 

324 key = 'agency_ori' if 'agency_ori' in events.columns else 'ori' 

325 columns = [k for k in keys if k not in ('data_year',)] 

326 joined = [c for c in columns if c in active.columns] 

327 kept = events[['data_year', key]].merge( 

328 active[['data_year', 'ori', *joined]], 

329 left_on = ['data_year', key], right_on = ['data_year', 'ori'], 

330 how = 'inner', suffixes = ('_event', '')) 

331 counts = kept.groupby(keys, observed = True).size().rename('count') 

332 people = active.groupby(keys, observed = True).agg( 

333 population = ('population', 'sum'), agencies = ('ori', 'nunique')) 

334 table = people.join(counts, how = 'left').reset_index() 

335 table['count'] = table['count'].fillna(0).astype('Int64') 

336 table = PopulationRates().describe( 

337 amos.Dataset(table), 'population', per = per) 

338 return table[[*keys, 'count', 'population', 'agencies', 'rate']] 

339 

340 

341@dataclasses.dataclass 

342class ClearanceRates(amos.Describer): 

343 """A table of the share of incidents cleared in each group. 

344 

345 Parameters (set in the "clearance_rates_parameters" section of the 

346 settings): `by` (the columns to group by) and `cleared` (the column 

347 saying whether each incident was cleared, "cleared" by default). 

348 

349 """ 

350 

351 """ Required Methods """ 

352 

353 def describe( 

354 self, 

355 item: amos.Dataset, 

356 by: Any = None, 

357 cleared: str = 'cleared', 

358 **kwargs: Any) -> pd.DataFrame: 

359 """Returns the share of incidents that were cleared in each group. 

360 

361 Args: 

362 item: the dataset, with one row for each incident and a column 

363 saying whether it was cleared (such as from 

364 `crimepy.nibrs.tables.incidents`). 

365 by: the columns to group by. Defaults to `None`, which uses all 

366 rows. 

367 cleared: the column saying whether each incident was cleared. 

368 Defaults to "cleared". 

369 **kwargs: not used. 

370 

371 Returns: 

372 A table with a column for each group, "incidents", "cleared", and 

373 "clearance_rate" (from 0 to 1). 

374 

375 """ 

376 incidents = item.data 

377 keys = utilities.to_names(by) 

378 flags = incidents[cleared].astype('boolean').fillna(False).astype(int) 

379 if keys: 

380 grouped = flags.groupby( 

381 [incidents[k] for k in keys], observed = True, dropna = False) 

382 table = grouped.agg(['size', 'sum']).reset_index() 

383 else: 

384 table = pd.DataFrame({'size': [len(flags)], 'sum': [flags.sum()]}) 

385 table = table.rename(columns = {'size': 'incidents', 'sum': 'cleared'}) 

386 table['clearance_rate'] = table['cleared'] / table['incidents'] 

387 return table 

388 

389 

390@dataclasses.dataclass 

391class PercentChange(amos.Describer): 

392 """The percent change from one period to the next. 

393 

394 Applied to a dataset of events, it counts them over time first (as 

395 `crime_trend` does, if `freq` is set). Applied to a table of counts or 

396 rates in each period (in order), it changes those. 

397 

398 Parameters (set in the "percent_change_parameters" section of the 

399 settings): `periods`, `by`, `date`, and `freq` (see `describe`). 

400 

401 """ 

402 

403 """ Required Methods """ 

404 

405 def describe( 

406 self, 

407 item: amos.Dataset, 

408 periods: Any = 1, 

409 by: str | None = None, 

410 date: str = 'incident_date', 

411 freq: str | None = None, 

412 **kwargs: Any) -> pd.DataFrame: 

413 """Returns the percent change from one period to the next. 

414 

415 Args: 

416 item: a dataset of events (if `freq` is set) or of counts or 

417 rates in each period, in order. 

418 periods: the number of periods to compare across (12 compares 

419 each month with the same month of the year before). Defaults 

420 to 1. 

421 by: the column of groups whose counts are changed, for events. 

422 Defaults to `None`. 

423 date: the column of dates of events. Defaults to "incident_date". 

424 freq: the length of each period, for events. Defaults to `None`, 

425 in which case the data is already counts in each period. 

426 **kwargs: not used. 

427 

428 Returns: 

429 The change, in percent, with the index of the counts. 

430 

431 """ 

432 if freq is not None: 

433 data = CrimeTrend().describe(item, by, date = date, freq = freq) 

434 else: 

435 data = item.data 

436 return data.pct_change(periods = int(periods), fill_method = None) * 100 

437 

438 

439@dataclasses.dataclass 

440class Crosstab(amos.Describer): 

441 """Counts (or shares) of two variables against each other. 

442 

443 Parameters (set in the "crosstab_parameters" section of the settings): 

444 `rows`, `columns`, `normalize`, and `margins` (see `describe`). 

445 

446 """ 

447 

448 """ Required Methods """ 

449 

450 def describe( 

451 self, 

452 item: amos.Dataset, 

453 rows: str | None = None, 

454 columns: str | None = None, 

455 normalize: bool | Literal['all', 'index', 'columns'] = False, 

456 margins: Any = False, 

457 **kwargs: Any) -> pd.DataFrame: 

458 """Returns counts (or shares) of two variables against each other. 

459 

460 Args: 

461 item: the dataset, with one row for each event. 

462 rows: the column whose values become rows. 

463 columns: the column whose values become columns. 

464 normalize: False for counts, True for shares of all rows, "index" 

465 for shares of each row, or "columns" for shares of each 

466 column. Defaults to `False`. 

467 margins: whether to add totals. Defaults to `False`. 

468 **kwargs: not used. 

469 

470 Raises: 

471 ValueError: if `rows` or `columns` is not set. 

472 

473 Returns: 

474 The table. 

475 

476 """ 

477 if rows is None or columns is None: 477 ↛ 478line 477 didn't jump to line 478 because the condition on line 477 was never true

478 message = f'set the "rows" and "columns" of {self.name!r}' 

479 raise ValueError(message) 

480 if (isinstance(normalize, str) 480 ↛ 482line 480 didn't jump to line 482 because the condition on line 480 was never true

481 and normalize not in ('all', 'index', 'columns')): 

482 normalize = utilities.to_bool(normalize) 

483 data = item.data 

484 return pd.crosstab( 

485 data[rows], data[columns], normalize = normalize, 

486 margins = utilities.to_bool(margins), dropna = True) 

487 

488 

489@dataclasses.dataclass 

490class Coverage(amos.Describer): 

491 """The agencies that reported and the people they serve. 

492 

493 Agencies covered by another agency (whose crimes the other agency 

494 reports) are left out, so that people are not counted twice. 

495 

496 Parameters (set in the "coverage_parameters" section of the settings): 

497 `by` and `months` (see `describe`). 

498 

499 """ 

500 

501 """ Required Methods """ 

502 

503 def describe( 

504 self, 

505 item: amos.Dataset, 

506 by: Any = ('data_year', 'state'), 

507 months: Any = 12, 

508 **kwargs: Any) -> pd.DataFrame: 

509 """Returns how many agencies reported and how many people they serve. 

510 

511 Args: 

512 item: a dataset of agencies, with "population" and 

513 "months_active". 

514 by: columns to group by. Defaults to ("data_year", "state"). 

515 months: the months that an agency must have reported in to count 

516 as reporting for the year. Defaults to 12. 

517 **kwargs: not used. 

518 

519 Returns: 

520 A table with a column for each group, "agencies", "reporting" 

521 (the agencies that reported in at least `months` months), 

522 "population", "population_reporting", and "share_reporting" 

523 (the share of the population served by agencies that 

524 reported). 

525 

526 """ 

527 keys = utilities.to_names(by) 

528 table = item.data.copy() 

529 if 'covered_by_ori' in table.columns: 529 ↛ 531line 529 didn't jump to line 531 because the condition on line 529 was always true

530 table = table[table['covered_by_ori'].isna()] 

531 reporting = table['months_active'].fillna(0) >= int(months) 

532 table['_reporting'] = reporting.astype(int) 

533 table['_population'] = table['population'].fillna(0) 

534 table['_population_reporting'] = table['_population'].where( 

535 reporting, 0) 

536 summary = table.groupby(keys, observed = True).agg( 

537 agencies = ('ori', 'nunique'), 

538 reporting = ('_reporting', 'sum'), 

539 population = ('_population', 'sum'), 

540 population_reporting = ( 

541 '_population_reporting', 'sum')).reset_index() 

542 share = summary['population_reporting'] / summary['population'].where( 

543 summary['population'] > 0) 

544 summary['share_reporting'] = share 

545 return summary 

546 

547 

548""" Private Functions """ 

549 

550 

551def _agencies_of(events: pd.DataFrame, folder: Any) -> pd.DataFrame: 

552 """Loads the agencies of the years (and states) of `events`. 

553 

554 Args: 

555 events: one row for each event, with "data_year" (and "state"). 

556 folder: the data folder. 

557 

558 Returns: 

559 The agencies. 

560 

561 """ 

562 # `nibrs` is imported here, when agencies are needed, so that importing 

563 # this module does not import the NIBRS readers. 

564 from .nibrs import store # noqa: PLC0415 

565 years = sorted(int(y) for y in events['data_year'].dropna().unique()) 

566 states = ( 

567 sorted(str(s) for s in events['state'].dropna().unique()) 

568 if 'state' in events.columns else None) 

569 return store.load_frame( 

570 'agencies', years, states = states, folder = folder, progress = False)