Downloads FJC IDB files from the FJC website.

The FJC publishes each yearly file of the Integrated Database on a dataset page for its case type (see options._DATASET_PAGES). The functions in this module read those pages and save the yearly SAS files, and the codebook for each case type, in the layout that run expects, creating any folders that do not exist yet:

data_folder/
├── appellate/          ap08.sas7bdat, ...
├── district/
│   ├── civil/          cv08_0.sas7bdat, ...
│   └── criminal/       cr08.sas7bdat, ...
└── docs/               the codebooks

create_folders(data_folder=options._DATA_FOLDER)

Creates the folders of the data folder that do not exist yet.

The data folder gets a folder for the yearly files of each case type and a 'docs' folder for the codebooks. Folders that already exist, and the files in them, are left as they are.

Parameters:

Name	Type	Description	Default
data_folder	str | Path	The path to the folder containing the FJC IDB dataset. Defaults to options._DATA_FOLDER.	_DATA_FOLDER

Returns:

Type	Description
list[Path]	list[pathlib.Path]: The folders of the data folder.

Examples:

>>> import fjc_idb
>>> fjc_idb.create_folders('path/to/fjc_idb')

Source code in src/fjc_idb/downloader.py

def create_folders(
    data_folder: str | pathlib.Path = options._DATA_FOLDER,
) -> list[pathlib.Path]:
    """Creates the folders of the data folder that do not exist yet.

    The data folder gets a folder for the yearly files of each case type and a
    'docs' folder for the codebooks. Folders that already exist, and the files
    in them, are left as they are.

    Args:
        data_folder: The path to the folder containing the FJC IDB dataset.
            Defaults to options._DATA_FOLDER.

    Returns:
        list[pathlib.Path]: The folders of the data folder.

    Examples:
        >>> import fjc_idb
        >>> fjc_idb.create_folders('path/to/fjc_idb')  # doctest: +SKIP

    """
    root = camina.pathlibify(data_folder)
    folders = [
        *(main._get_data_path(case_type, root)
          for case_type in options._DATASET_PAGES),
        root / 'docs']
    for folder in folders:
        folder.mkdir(parents = True, exist_ok = True)
    return folders

download(case_type, start_year, stop_year=None, data_folder=options._DATA_FOLDER, *, overwrite=False, codebook=True)

Downloads the yearly FJC IDB files for a range of years.

Files are saved under their FJC names (such as 'ap12_0.sas7bdat') in the folder that run reads for case_type. A year that already has a file is skipped unless overwrite is True, in which case its file is replaced. The FJC updates the file for the current year during the year, so overwrite it to get the latest data. Years the FJC has not published are skipped with a warning. Any folders of the data folder that do not exist yet are created (see create_folders).

Parameters:

Name	Type	Description	Default
case_type	str	The source type ('appellate', 'civil', or 'criminal').	required
start_year	int	The first statistical year to download.	required
stop_year	int | None	The last statistical year to download. Defaults to None, in which case the current year is used.	None
data_folder	str | Path	The path to the folder containing the FJC IDB dataset. Defaults to options._DATA_FOLDER.	_DATA_FOLDER
overwrite	bool	Whether to replace files that already exist. Defaults to False.	False
codebook	bool	Whether to also download the codebook for case_type into the 'docs' folder of data_folder. Defaults to True.	True

Returns:

Type	Description
list[Path]	list[pathlib.Path]: The files that were downloaded.

Raises:

Type	Description
ValueError	If case_type is not a recognized source type.

Examples:

>>> import fjc_idb
>>> fjc_idb.download('appellate', 2008, 2026)

Source code in src/fjc_idb/downloader.py

def download(
    case_type: str,
    start_year: int,
    stop_year: int | None = None,
    data_folder: str | pathlib.Path = options._DATA_FOLDER,
    *,
    overwrite: bool = False,
    codebook: bool = True) -> list[pathlib.Path]:
    """Downloads the yearly FJC IDB files for a range of years.

    Files are saved under their FJC names (such as 'ap12_0.sas7bdat') in the
    folder that `run` reads for `case_type`. A year that already has a file is
    skipped unless `overwrite` is True, in which case its file is replaced. The
    FJC updates the file for the current year during the year, so overwrite it
    to get the latest data. Years the FJC has not published are skipped with a
    warning. Any folders of the data folder that do not exist yet are created
    (see `create_folders`).

    Args:
        case_type: The source type ('appellate', 'civil', or 'criminal').
        start_year: The first statistical year to download.
        stop_year: The last statistical year to download. Defaults to None, in
            which case the current year is used.
        data_folder: The path to the folder containing the FJC IDB dataset.
            Defaults to options._DATA_FOLDER.
        overwrite: Whether to replace files that already exist. Defaults to
            False.
        codebook: Whether to also download the codebook for `case_type` into
            the 'docs' folder of `data_folder`. Defaults to True.

    Returns:
        list[pathlib.Path]: The files that were downloaded.

    Raises:
        ValueError: If `case_type` is not a recognized source type.

    Examples:
        >>> import fjc_idb
        >>> fjc_idb.download('appellate', 2008, 2026)  # doctest: +SKIP

    """
    if case_type not in options._DATASET_PAGES:
        message = (
            f'Invalid source type: {case_type}. '
            f'Must be one of {list(options._DATASET_PAGES)}.')
        raise ValueError(message)
    stop_year = stop_year or dt.datetime.now(tz = dt.UTC).year
    root = camina.pathlibify(data_folder)
    create_folders(root)
    folder = main._get_data_path(case_type, root)
    links = _get_links(options._DATASET_PAGES[case_type])
    files = _get_yearly_files(links)
    downloaded = []
    for year in range(start_year, stop_year + 1):
        if year not in files:
            logger.warning(
                'The FJC has not published a %s file for %s.', case_type, year)
            continue
        prefix = options._SOURCE_PREFIXES[case_type]
        existing = sorted(folder.glob(f'{prefix}{year % 100:02d}*.sas7bdat'))
        if existing and not overwrite:
            logger.info('Skipped %s, which already exists.', existing[0].name)
            continue
        path = _save(files[year], folder)
        # Remove an older file for the year saved under a different name.
        for file in existing:
            if file != path:
                file.unlink()
        downloaded.append(path)
    if codebook:
        codebooks = [url for url in links if '/idb/codebooks/' in url]
        for url in codebooks:
            path = root / 'docs' / _file_name(url)
            if overwrite or not path.exists():
                downloaded.append(_save(url, path.parent))
    return downloaded

list_files(case_type)

Returns the yearly FJC IDB files the FJC publishes for a case type.

Only the files for single years are included, not the cumulative, multi-year, or twelve-months-ending-June files on the same page.

Parameters:

Name	Type	Description	Default
case_type	str	The source type ('appellate', 'civil', or 'criminal').	required

Returns:

Type	Description
dict[int, str]	dict[int, str]: The address of each yearly file, keyed by its year.

Source code in src/fjc_idb/downloader.py

def list_files(case_type: str) -> dict[int, str]:
    """Returns the yearly FJC IDB files the FJC publishes for a case type.

    Only the files for single years are included, not the cumulative,
    multi-year, or twelve-months-ending-June files on the same page.

    Args:
        case_type: The source type ('appellate', 'civil', or 'criminal').

    Returns:
        dict[int, str]: The address of each yearly file, keyed by its year.

    """
    return _get_yearly_files(_get_links(options._DATASET_PAGES[case_type]))