<commit_before>import codecs
import json
import os

from collections import OrderedDict
from .utils import Normalizer

path = os.path.dirname(__file__)


class Parser:
    __names, __ratios = {}, {}
    __surnames = []

    def __init__(self):
        self._load_data()

    def _load_data(self):
        """
        Load all data files into memory.
        """
        self._load_names()
        self._load_name_surname_ratios()
        self._load_surnames()

    def _load_names(self):
        """
        Load names data file.

        This file contains a list of spanish given names with the probability for
        each one to be a male or female name.
        """
        for line in self.remove_file_comments('names_ine'):
                (name, prob_female, prob_male) = line.rstrip().split('\t')
                self.__names[name] = float(prob_male)

    def _load_name_surname_ratios(self):
        """
        Load name/surnames ratios data file.

        The file contains a list of names and surnames with the probability for each
        one to be a name (lower values) or a surname (higher values).
        """
        for line in self.remove_file_comments('name_surname_ratio'):
            (key, val) = line.rstrip().split('\t')
            self.__ratios[key] = float(val)

    def _load_surnames(self):
        """
        Load names data file.

        This file contains a list of spanish surnames.
        """
        for line in self.remove_file_comments('surnames_ine'):
            self.__surnames.append(line.rstrip())

    def remove_file_comments(self, relative_path):
        """
        Generator to remove comments from a file.

        Params:
            file: File to be processed.
        """
        with codecs.open(os.path.join(path, 'data', relative_path), 'r', 'UTF-8') as file:
            for line in file:
                line = line.strip()
                if not line.startswith('#'):
                    yield line

    def guess_gender(self, fullname):
        """
        Guess the gender of the given full name.

        Params:
            fullname: Full name from where we want to guess the gender.

        Returns:
            A JSON string with all the computed information.
        """
        if isinstance(fullname, str):
            fullname = Normalizer.normalize(fullname)
            names, surnames = self._classify(fullname)

            if names and surnames:
                real_name, ratio = self._get_gender_ratio(list(names.keys()))
                return self._create_answer(real_name, ratio, names, surnames)
            pass

    def _classify(self, fullname):
        """
        Classify fullname tokens into names and surnames based on datasets.

        Params:
            fullname: Full name to be classified.

        Returns:<commit_msg>Add an option to force name splitting<commit_after>import codecs
import json
import os

from collections import OrderedDict
from .utils import Normalizer

path = os.path.dirname(__file__)


class Parser:

    __names, __ratios = {}, {}
    __surnames = []

    def __init__(self, force_split=False):
        self.__force_split = force_split
        self._load_data()

    def _load_data(self):
        """
        Load all data files into memory.
        """
        self._load_names()
        self._load_name_surname_ratios()
        self._load_surnames()

    def _load_names(self):
        """
        Load names data file.

        This file contains a list of spanish given names with the probability for
        each one to be a male or female name.
        """
        for line in self.remove_file_comments('names_ine'):
                (name, prob_female, prob_male) = line.rstrip().split('\t')
                self.__names[name] = float(prob_male)

    def _load_name_surname_ratios(self):
        """
        Load name/surnames ratios data file.

        The file contains a list of names and surnames with the probability for each
        one to be a name (lower values) or a surname (higher values).
        """
        for line in self.remove_file_comments('name_surname_ratio'):
            (key, val) = line.rstrip().split('\t')
            self.__ratios[key] = float(val)

    def _load_surnames(self):
        """
        Load names data file.

        This file contains a list of spanish surnames.
        """
        for line in self.remove_file_comments('surnames_ine'):
            self.__surnames.append(line.rstrip())

    def remove_file_comments(self, relative_path):
        """
        Generator to remove comments from a file.

        Params:
            file: File to be processed.
        """
        with codecs.open(os.path.join(path, 'data', relative_path), 'r', 'UTF-8') as file:
            for line in file:
                line = line.strip()
                if not line.startswith('#'):
                    yield line

    def guess_gender(self, fullname):
        """
        Guess the gender of the given full name.

        Params:
            fullname: Full name from where we want to guess the gender.

        Returns:
            A JSON string with all the computed information.
        """
        if isinstance(fullname, str):
            fullname = Normalizer.normalize(fullname)
            names, surnames = self._classify(fullname)

            if names and (surnames or (self.__force_split and self._is_splittable(names))):
                real_name, ratio = self._get_gender_ratio(list(names.keys()))
                return self._create_answer(real_name, ratio, names, surnames)

    def _is_splittable(self, names):
        return names[next(reversed(names))] < 1


    def _classify(self, fullname):
        """
        Classify fullname tokens into names and surnames based on datasets.

        Params:
            fullname: Full name to be classified.

        Returns:
{'fraction_non_alphanumeric': 0.08946288060212111, 'fraction_numerical': 0.0005131713992473486, 'mean_word_length': 2.024831867563373, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1416802', 'n_tokens_mistral': 1663, 'n_tokens_neox': 1594, 'n_words': 503}