<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function

import os
import re

import niftynet.io.misc_io as util


class KeywordsMatching(object):
    """
    This class is responsible for the search of the appropriate files to use
    as input based on the constraints given in the config file
    """

    def __init__(self, list_paths=(), list_contain=(), list_not_contain=()):
        self.path_to_search = list_paths
        self.filename_contains = list_contain
        self.filename_not_contains = list_not_contain

    @classmethod
    def from_tuple(cls, input_tuple):
        """
        In the config file, constraints for a given search can be of three
        types:
        path_to_search, filename_contains and filename_not_contains. Each
        associated value is a string. Multiple constraints are delimited by a ,
        This function creates the corresponding matching object with the list
        of constraints for each of these subtypes.
        :param input_tuple:
        :return:
        """
        path, contain, not_contain = [], (), ()
        for (name, value) in input_tuple:
            if not value:
                continue
            if name == "path_to_search":
                value = value.split(',')
                for path_i in value:
                    path_i = os.path.abspath(path_i.strip())
                    if os.path.exists(path_i):
                        path.append(path_i)
                    else:
                        raise ValueError('data input folder {} not found, did'
                                         ' you maybe forget to download data?'
                                         .format(value))
            elif name == "filename_contains":
                contain = tuple(set(value))
            elif name == "filename_not_contains":
                not_contain = tuple(set(value))
        path = tuple(set(path))
        new_matcher = cls(path, contain, not_contain)
        return new_matcher

    def matching_subjects_and_filenames(self):
        """
        This function perform the search of the relevant files (stored in
        list_final) and extract
        the corresponding possible list of subject names (stored in
        name_list_final).
        :returns list_final, name_list_final
        """
        path_file = [(p, filename)
                     for p in self.path_to_search
                     for filename in os.listdir(p)]
        matching_path_file = list(filter(self.__is_a_candidate, path_file))
        list_final = [os.path.join(p, filename)
                      for p, filename in matching_path_file]
        name_list_final = [self.__extract_subject_id_from(filename)
                           for p, filename in matching_path_file]
        if not list_final or not name_list_final:
            raise IOError('no file matched based on this matcher: {}'.format(
                self.__dict__))
        return list_final, name_list_final

    def __is_a_candidate(self, x):<commit_msg>`from_tuple()` now takes a default folder where data can be sourced from if they are not found in the current directory<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function

import os
import re

import niftynet.io.misc_io as util


class KeywordsMatching(object):
    """
    This class is responsible for the search of the appropriate files to use
    as input based on the constraints given in the config file
    """

    def __init__(self, list_paths=(), list_contain=(), list_not_contain=()):
        self.path_to_search = list_paths
        self.filename_contains = list_contain
        self.filename_not_contains = list_not_contain

    @classmethod
    def from_tuple(cls, input_tuple, default_folder=None):
        """
        In the config file, constraints for a given search can be of three
        types:
        path_to_search, filename_contains and filename_not_contains. Each
        associated value is a string. Multiple constraints are delimited by a ,
        This function creates the corresponding matching object with the list
        of constraints for each of these subtypes.
        :param default_folder: relative paths are first tested against the current folder, and then against this
               default folder
        :param input_tuple:
        :return:
        """
        path, contain, not_contain = [], (), ()
        for (name, value) in input_tuple:
            if not value:
                continue
            if name == "path_to_search":
                value = value.split(',')
                for path_i in value:
                    path_i = path_i.strip()
                    path_orig = os.path.abspath(path_i)
                    if os.path.exists(path_orig):
                        path.append(path_orig)
                    else:
                        if not default_folder:
                            raise ValueError('data input folder {} not found, did'
                                             ' you maybe forget to download data?'
                                             .format(path_i))
                        path_def = os.path.abspath(os.path.join(default_folder, path_i))
                        if os.path.exists(path_def):
                            path.append(path_def)
                        else:
                            raise ValueError('data input folder {} not found, did'
                                             ' you maybe forget to download data?'
                                             .format(path_i))
            elif name == "filename_contains":
                contain = tuple(set(value))
            elif name == "filename_not_contains":
                not_contain = tuple(set(value))
        path = tuple(set(path))
        new_matcher = cls(path, contain, not_contain)
        return new_matcher

    def matching_subjects_and_filenames(self):
        """
        This function perform the search of the relevant files (stored in
        list_final) and extract
        the corresponding possible list of subject names (stored in
        name_list_final).
        :returns list_final, name_list_final
        """
        path_file = [(p, filename)
                     for p in self.path_to_search
                     for filename in os.listdir(p)]
        matching_path_file = list(filter(self.__is_a_candidate, path_file))
        list_final = [os.path.join(p, filename)
                      for p, filename in matching_path_file]
        name_list_final = [self.__extract_subject_id_from(filename)
                           for p, filename in matching_path_file]
        if not list_final or not name_list_final:
            raise IOError('no file matched based on this matcher: {}'.format(
                self.__dict__))
        return list_final, name_list_final

    def __is_a_candidate(self, x):
{'fraction_non_alphanumeric': 0.0706308919506889, 'fraction_numerical': 0.000290065264684554, 'mean_word_length': 1.8216039279869067, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7543944', 'n_tokens_mistral': 1768, 'n_tokens_neox': 1717, 'n_words': 555}