# Path: cogspaces/datasets/contrast.py
# def fetch_contrasts(studies: str or List[str] = 'all', data_dir=None):
#     dfs = []
#     if studies == 'all':
#         studies = nv_ids.keys()
#     for study in studies:
#         if study not in nv_ids:
#             return ValueError('Wrong dataset.')
#         data = fetch_neurovault_ids([nv_ids[study]], data_dir=data_dir, verbose=10,
#                                     mode='download_new')
#         dfs.append(_assemble(data['images'], data['images_meta'], study))
#     return pd.concat(dfs)
# 
# Path: cogspaces/datasets/derivative.py
# def fetch_atlas_modl(data_dir=None,
#                      url=None,
#                      resume=True, verbose=1):
#     """Download and load a multi-scale atlas computed using MODL over HCP900.
# 
#     Parameters
#     ----------
#     data_dir: string, optional
#         Path of the data directory. Used to force data storage in a non-
#         standard location. Default: None (meaning: default)
#     url: string, optional
#         Download URL of the dataset. Overwrite the default URL.
#     """
# 
#     if url is None:
#         url = 'http://cogspaces.github.io/assets/data/modl/'
# 
#     data_dir = get_data_dir(data_dir)
#     dataset_name = 'modl'
#     data_dir = _get_dataset_dir(dataset_name, data_dir=data_dir,
#                                 verbose=verbose)
# 
#     keys = ['components_64',
#             'components_128',
#             'components_453_gm',
#             'loadings_128_gm'
#             ]
# 
#     paths = [
#         'components_64.nii.gz',
#         'components_128.nii.gz',
#         'components_453_gm.nii.gz',
#         'loadings_128_gm.npy',
#     ]
#     urls = [url + path for path in paths]
#     files = [(path, url, {}) for path, url in zip(paths, urls)]
# 
#     files = _fetch_files(data_dir, files, resume=resume,
#                          verbose=verbose)
# 
#     params = {key: file for key, file in zip(keys, files)}
# 
#     fdescr = 'Components computed using the MODL package, at various scale,' \
#              'from HCP900 data'
# 
#     params['description'] = fdescr
#     params['data_dir'] = data_dir
# 
#     return Bunch(**params)
# 
# def fetch_mask(data_dir=None, url=None, resume=True, verbose=1):
#     if url is None:
#         url = 'http://cogspaces.github.io/assets/data/hcp_mask.nii.gz'
#     files = [('hcp_mask.nii.gz', url, {})]
# 
#     dataset_name = 'mask'
#     data_dir = get_data_dir(data_dir)
#     dataset_dir = _get_dataset_dir(dataset_name, data_dir=data_dir,
#                                    verbose=verbose)
#     files = _fetch_files(dataset_dir, files, resume=resume,
#                          verbose=verbose)
#     return files[0]
# 
# STUDY_LIST = ['knops2009recruitment', 'ds009', 'gauthier2010resonance',
#               'ds017B', 'ds110', 'vagharchakian2012temporal', 'ds001',
#               'devauchelle2009sentence', 'camcan', 'archi',
#               'henson2010faces', 'ds052', 'ds006A', 'ds109', 'ds108', 'la5c',
#               'gauthier2009resonance', 'ds011', 'ds107', 'ds116', 'ds101',
#               'ds002', 'ds003', 'ds051', 'ds008', 'pinel2009twins', 'ds017A',
#               'ds105', 'ds007', 'ds005', 'amalric2012mathematicians', 'ds114',
#               'brainomics', 'cauvet2009muslang', 'hcp']
# 
# Path: cogspaces/datasets/utils.py
# def get_data_dir(data_dir=None):
#     """ Returns the directories in which to look for data.
# 
#     Parameters
#     ----------
#     data_dir: string, optional
#         Path of the utils directory. Used to force utils storage in a specified
#         location. Default: None
# 
#     Returns
#     -------
#     path: string
#         Path of the data directory.
#     """
# 
#     # Check data_dir which force storage in a specific location
#     if data_dir is not None:
#         assert (isinstance(data_dir, str))
#         return data_dir
#     elif 'COGSPACES_DATA' in os.environ:
#         return os.environ['COGSPACES_DATA']
#     else:
#         return os.path.expanduser('~/cogspaces_data')
# 
# Path: cogspaces/raw_datasets/contrast.py
# def fetch_all(data_dir=None):
#     dfs = []
#     for study in ['archi', 'brainomics', 'camcan', 'hcp',
#                   'la5c', 'brainpedia']:
#         df = fetch_contrasts(study, data_dir=data_dir)
#         dfs.append(df)
#     df = pd.concat(dfs)
#     df = clean_targets(df)
#     return df


import os
import numpy as np
import pandas as pd
from os.path import join
from typing import List, Union
from joblib import Parallel, delayed, dump, load
from nilearn.input_data import NiftiMasker
from sklearn.utils import gen_batches
from cogspaces.datasets import fetch_mask, fetch_atlas_modl, \
    fetch_contrasts, STUDY_LIST
from cogspaces.datasets.utils import get_data_dir
from cogspaces.raw_datasets.contrast import fetch_all



idx = pd.IndexSlice


def single_mask(masker, imgs):
    return masker.transform(imgs)


def single_reduce(components, data, lstsq=False):
    if not lstsq:
        return data.dot(components.T)
    else:
        X, _, _, _ = np.linalg.lstsq(components.T, data.T)
        return X.T


def mask_contrasts(studies: Union[str, List[str]] = 'all',
                   output_dir: str = 'masked',
                   use_raw=False,
                   n_jobs: int = 1):
    batch_size = 10

    if not os.path.exists(output_dir):
        os.makedirs(output_dir)
    if use_raw and studies == 'all':
        data = fetch_all()
    else:

# Path: cogspaces/datasets/contrast.py
# def fetch_contrasts(studies: str or List[str] = 'all', data_dir=None):
#     dfs = []
#     if studies == 'all':
#         studies = nv_ids.keys()
#     for study in studies:
#         if study not in nv_ids:
#             return ValueError('Wrong dataset.')
#         data = fetch_neurovault_ids([nv_ids[study]], data_dir=data_dir, verbose=10,
#                                     mode='download_new')
#         dfs.append(_assemble(data['images'], data['images_meta'], study))
#     return pd.concat(dfs)
# 
# Path: cogspaces/datasets/derivative.py
# def fetch_atlas_modl(data_dir=None,
#                      url=None,
#                      resume=True, verbose=1):
#     """Download and load a multi-scale atlas computed using MODL over HCP900.
# 
#     Parameters
#     ----------
#     data_dir: string, optional
#         Path of the data directory. Used to force data storage in a non-
#         standard location. Default: None (meaning: default)
#     url: string, optional
#         Download URL of the dataset. Overwrite the default URL.
#     """
# 
#     if url is None:
#         url = 'http://cogspaces.github.io/assets/data/modl/'
# 
#     data_dir = get_data_dir(data_dir)
#     dataset_name = 'modl'
#     data_dir = _get_dataset_dir(dataset_name, data_dir=data_dir,
#                                 verbose=verbose)
# 
#     keys = ['components_64',
#             'components_128',
#             'components_453_gm',
#             'loadings_128_gm'
#             ]
# 
#     paths = [
#         'components_64.nii.gz',
#         'components_128.nii.gz',
#         'components_453_gm.nii.gz',
#         'loadings_128_gm.npy',
#     ]
#     urls = [url + path for path in paths]
#     files = [(path, url, {}) for path, url in zip(paths, urls)]
# 
#     files = _fetch_files(data_dir, files, resume=resume,
#                          verbose=verbose)
# 
#     params = {key: file for key, file in zip(keys, files)}
# 
#     fdescr = 'Components computed using the MODL package, at various scale,' \
#              'from HCP900 data'
# 
#     params['description'] = fdescr
#     params['data_dir'] = data_dir
# 
#     return Bunch(**params)
# 
# def fetch_mask(data_dir=None, url=None, resume=True, verbose=1):
#     if url is None:
#         url = 'http://cogspaces.github.io/assets/data/hcp_mask.nii.gz'
#     files = [('hcp_mask.nii.gz', url, {})]
# 
#     dataset_name = 'mask'
#     data_dir = get_data_dir(data_dir)
#     dataset_dir = _get_dataset_dir(dataset_name, data_dir=data_dir,
#                                    verbose=verbose)
#     files = _fetch_files(dataset_dir, files, resume=resume,
#                          verbose=verbose)
#     return files[0]
# 
# STUDY_LIST = ['knops2009recruitment', 'ds009', 'gauthier2010resonance',
#               'ds017B', 'ds110', 'vagharchakian2012temporal', 'ds001',
#               'devauchelle2009sentence', 'camcan', 'archi',
#               'henson2010faces', 'ds052', 'ds006A', 'ds109', 'ds108', 'la5c',
#               'gauthier2009resonance', 'ds011', 'ds107', 'ds116', 'ds101',
#               'ds002', 'ds003', 'ds051', 'ds008', 'pinel2009twins', 'ds017A',
#               'ds105', 'ds007', 'ds005', 'amalric2012mathematicians', 'ds114',
#               'brainomics', 'cauvet2009muslang', 'hcp']
# 
# Path: cogspaces/datasets/utils.py
# def get_data_dir(data_dir=None):
#     """ Returns the directories in which to look for data.
# 
#     Parameters
#     ----------
#     data_dir: string, optional
#         Path of the utils directory. Used to force utils storage in a specified
#         location. Default: None
# 
#     Returns
#     -------
#     path: string
#         Path of the data directory.
#     """
# 
#     # Check data_dir which force storage in a specific location
#     if data_dir is not None:
#         assert (isinstance(data_dir, str))
#         return data_dir
#     elif 'COGSPACES_DATA' in os.environ:
#         return os.environ['COGSPACES_DATA']
#     else:
#         return os.path.expanduser('~/cogspaces_data')
# 
# Path: cogspaces/raw_datasets/contrast.py
# def fetch_all(data_dir=None):
#     dfs = []
#     for study in ['archi', 'brainomics', 'camcan', 'hcp',
#                   'la5c', 'brainpedia']:
#         df = fetch_contrasts(study, data_dir=data_dir)
#         dfs.append(df)
#     df = pd.concat(dfs)
#     df = clean_targets(df)
#     return df


# Path: exps/reduce.py
import os
import numpy as np
import pandas as pd
from os.path import join
from typing import List, Union
from joblib import Parallel, delayed, dump, load
from nilearn.input_data import NiftiMasker
from sklearn.utils import gen_batches
from cogspaces.datasets import fetch_mask, fetch_atlas_modl, \
    fetch_contrasts, STUDY_LIST
from cogspaces.datasets.utils import get_data_dir
from cogspaces.raw_datasets.contrast import fetch_all




idx = pd.IndexSlice


def single_mask(masker, imgs):
    return masker.transform(imgs)


def single_reduce(components, data, lstsq=False):
    if not lstsq:
        return data.dot(components.T)
    else:
        X, _, _, _ = np.linalg.lstsq(components.T, data.T)
        return X.T


def mask_contrasts(studies: Union[str, List[str]] = 'all',
                   output_dir: str = 'masked',
                   use_raw=False,
                   n_jobs: int = 1):
    batch_size = 10

    if not os.path.exists(output_dir):
        os.makedirs(output_dir)
    if use_raw and studies == 'all':
        data = fetch_all()
    else:

        data = fetch_contrasts(studies)