<commit_before>"""
Utilities to download NeuroImaging datasets
Author: <NAME>
"""

import os
import re
import glob
import json
import nibabel
from sklearn.datasets.base import Bunch

from nilearn.datasets.utils import (
    _get_dataset_dir, _fetch_files, _fetch_file, _uncompress_file)

SPM_AUDITORY_DATA_FILES = ["fM00223/fM00223_%03i.img" % index
                           for index in range(4, 100)]
SPM_AUDITORY_DATA_FILES.append("sM00223/sM00223_002.img")


def fetch_bids_langloc_dataset(data_dir=None, verbose=1):
    """Download language localizer example bids dataset.

    Parameters
    ----------
    data_dir: string, optional
        Path to store the downloaded dataset. if None employ nilearn
        datasets default download directory.

    verbose: int, optional
        verbosity level (0 means no message).

    Returns
    -------
    data_dir: string
        Path to downloaded dataset

    downloaded_files: list of string
        Absolute paths of downloaded files on disk
    """
    url = 'https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/5888d9a76c613b01fc6acc4e'
    dataset_name = 'bids_langloc_example'
    main_folder = 'bids_langloc_dataset'
    data_dir = _get_dataset_dir(dataset_name, data_dir=data_dir,
                                verbose=verbose)
    # The files_spec needed for _fetch_files
    files_spec = [(main_folder + '.zip', url, {'move': main_folder + '.zip'})]
    if not os.path.exists(os.path.join(data_dir, main_folder)):
        downloaded_files = _fetch_files(data_dir, files_spec, resume=True,
                                        verbose=verbose)
        _uncompress_file(downloaded_files[0])
    main_path = os.path.join(data_dir, main_folder)
    file_list = [os.path.join(path, f) for
                 path, dirs, files in os.walk(main_path) for f in files]
    return os.path.join(data_dir, main_folder), sorted(file_list)


def fetch_openfmri_dataset(dataset_name='ds000001', dataset_revision=None,
                           data_dir=None, verbose=1):
    """Download latest revision of specified bids dataset.

    Compressed files will not be uncompressed automatically due to the expected
    great size of downloaded dataset.

    Only datasets that contain preprocessed files following the official
    conventions of the future BIDS derivatives specification can be used out
    of the box with Nistats. Otherwise custom preprocessing would need to be
    performed, optionally following the BIDS derivatives specification for the
    preprocessing output files.

    Parameters
    ----------
    dataset_name: string, optional
        Accesion number as published in https://openfmri.org/dataset/.
        Downloads by default dataset ds000001.

    dataset_revision: string, optional
        Revision as presented in the specific dataset link accesible
        from https://openfmri.org/dataset/. Looks for the latest by default.

    data_dir: string, optional
        Path to store the downloaded dataset. if None employ nilearn
        datasets default download directory.<commit_msg>stated function<commit_after>"""
Utilities to download NeuroImaging datasets
Author: <NAME>
"""

import os
import re
import glob
import json
import nibabel
from sklearn.datasets.base import Bunch

from nilearn.datasets.utils import (
    _get_dataset_dir, _fetch_files, _fetch_file, _uncompress_file)

SPM_AUDITORY_DATA_FILES = ["fM00223/fM00223_%03i.img" % index
                           for index in range(4, 100)]
SPM_AUDITORY_DATA_FILES.append("sM00223/sM00223_002.img")


def fetch_bids_langloc_dataset(data_dir=None, verbose=1):
    """Download language localizer example bids dataset.

    Parameters
    ----------
    data_dir: string, optional
        Path to store the downloaded dataset. if None employ nilearn
        datasets default download directory.

    verbose: int, optional
        verbosity level (0 means no message).

    Returns
    -------
    data_dir: string
        Path to downloaded dataset

    downloaded_files: list of string
        Absolute paths of downloaded files on disk
    """
    url = 'https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/5888d9a76c613b01fc6acc4e'
    dataset_name = 'bids_langloc_example'
    main_folder = 'bids_langloc_dataset'
    data_dir = _get_dataset_dir(dataset_name, data_dir=data_dir,
                                verbose=verbose)
    # The files_spec needed for _fetch_files
    files_spec = [(main_folder + '.zip', url, {'move': main_folder + '.zip'})]
    if not os.path.exists(os.path.join(data_dir, main_folder)):
        downloaded_files = _fetch_files(data_dir, files_spec, resume=True,
                                        verbose=verbose)
        _uncompress_file(downloaded_files[0])
    main_path = os.path.join(data_dir, main_folder)
    file_list = [os.path.join(path, f) for
                 path, dirs, files in os.walk(main_path) for f in files]
    return os.path.join(data_dir, main_folder), sorted(file_list)


def fetch_openneuro_dataset(data_dir=None, dataset='ds000030', subjects=[''],
                            verbose=1):
    pass


def fetch_openfmri_dataset(dataset_name='ds000001', dataset_revision=None,
                           data_dir=None, verbose=1):
    """Download latest revision of specified bids dataset.

    Compressed files will not be uncompressed automatically due to the expected
    great size of downloaded dataset.

    Only datasets that contain preprocessed files following the official
    conventions of the future BIDS derivatives specification can be used out
    of the box with Nistats. Otherwise custom preprocessing would need to be
    performed, optionally following the BIDS derivatives specification for the
    preprocessing output files.

    Parameters
    ----------
    dataset_name: string, optional
        Accesion number as published in https://openfmri.org/dataset/.
        Downloads by default dataset ds000001.

    dataset_revision: string, optional
        Revision as presented in the specific dataset link accesible
        from https://openfmri.org/dataset/. Looks for the latest by default.

    data_dir: string, optional
        Path to store the downloaded dataset. if None employ nilearn
        datasets default download directory.
{'fraction_non_alphanumeric': 0.07634801972323843, 'fraction_numerical': 0.020836647049467156, 'mean_word_length': 2.9847908745247147, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6675988', 'n_tokens_mistral': 1893, 'n_tokens_neox': 1745, 'n_words': 542}