<commit_before>import io
import os
import collections
import xml.etree.ElementTree as ET

from ..fileio.datareaders import load_data

class BaseModel(object):
    """
    Base class from which all substitution model classes are descended.
    Implements generic functionality which is common to all substitution
    models, such as rate variation.
    """

    treewide_reconstruction = False
    """Should ASR be performed on the entire tree (if at all)?"""

    def __init__(self, model_config, global_config):
        """
        Parse configuration options, load data from file and pre-process data.
        """
        self.messages = []
        self.config = global_config

        self.name = model_config["name"]
        self.data_filename = model_config["data"]
        self.clock = model_config.get("clock", "")
        self.features = model_config.get("features",["*"])
        self.reconstruct = model_config.get("reconstruct", None)
        self.reconstruct_at = model_config.get("reconstruct_at", [])
        self.exclusions = model_config.get("exclusions",None)
        self.constant_feature = False
        self.constant_feature_removed = False
        self.frequencies = model_config.get("frequencies", "empirical")
        self.pruned = model_config.get("pruned", False)
        self.rate_variation = model_config.get("rate_variation", False)
        self.feature_rates = model_config.get("feature_rates", None)
        self.rate_partition = model_config.get("rate_partition", None)
        self.ascertained = model_config.get("ascertained", None)
        # Force removal of constant features here
        # This can be set by the user in BinaryModel only
        self.remove_constant_features = True
        self.minimum_data = float(model_config.get("minimum_data", 0))
        self.substitution_name = self.__class__.__name__
        self.data_separator = ","
        self.use_robust_eigensystem = model_config.get("use_robust_eigensystem", False)
        self.metadata = []
        self.treedata = []

        # Load the entire dataset from the file
        self.data, language_code_map = load_data(
            self.data_filename,
            file_format=model_config.get("file_format", None),
            lang_column=model_config.get("language_column", None),
            value_column=model_config.get("value_column", None),
            expect_multiple=True)

        # Augment the Glottolog classifications with human-friendly language
        # names which may have been read from a CLDF dataset.  Note that we
        # store both the actual language ID and its lowercase transformation.
        # This is kind of ugly, but we inconsistently convert things to
        # lowercase before doing Glottolog lookups all over the place, so
        # this is the easiest way to make this work everywhere.  We should
        # clean this up some day!
        for language_id, glottocode in language_code_map.items():
            print("Augmenting ", language_id, glottocode)
            if glottocode in global_config.classifications:
                global_config.classifications[language_id] = global_config.classifications[glottocode]
                global_config.classifications[language_id.lower()] = global_config.classifications[language_id]
                print(language_id, global_config.classifications[language_id])
            if glottocode in global_config.glotto_macroareas:
                global_config.glotto_macroareas[language_id] = global_config.glotto_macroareas[glottocode]
                global_config.glotto_macroareas[language_id.lower()] = global_config.glotto_macroareas[language_id]
            if glottocode in global_config.locations:
                global_config.locations[language_id] = global_config.locations[glottocode]
                global_config.locations[language_id.lower()] = global_config.locations[language_id]
                print(language_id, global_config.locations[language_id])

        # Remove features not wanted in this analysis
        self.build_feature_filter()
        self.apply_feature_filter()

        # Keep this around for later...
        self.global_config = global_config

    def build_feature_filter(self):
        """
        Create the self.feature_filter attribute, which is a set of feature
        names that functions analogously to Configuration.lang_filter
        attribute.
        """
        if self.features == ["*"]:
            random_iso = list(self.data.keys())[0]
            self.features = set()
            for lang_features in self.data.values():<commit_msg>Kill debugging prints.<commit_after>import io
import os
import collections
import xml.etree.ElementTree as ET

from ..fileio.datareaders import load_data

class BaseModel(object):
    """
    Base class from which all substitution model classes are descended.
    Implements generic functionality which is common to all substitution
    models, such as rate variation.
    """

    treewide_reconstruction = False
    """Should ASR be performed on the entire tree (if at all)?"""

    def __init__(self, model_config, global_config):
        """
        Parse configuration options, load data from file and pre-process data.
        """
        self.messages = []
        self.config = global_config

        self.name = model_config["name"]
        self.data_filename = model_config["data"]
        self.clock = model_config.get("clock", "")
        self.features = model_config.get("features",["*"])
        self.reconstruct = model_config.get("reconstruct", None)
        self.reconstruct_at = model_config.get("reconstruct_at", [])
        self.exclusions = model_config.get("exclusions",None)
        self.constant_feature = False
        self.constant_feature_removed = False
        self.frequencies = model_config.get("frequencies", "empirical")
        self.pruned = model_config.get("pruned", False)
        self.rate_variation = model_config.get("rate_variation", False)
        self.feature_rates = model_config.get("feature_rates", None)
        self.rate_partition = model_config.get("rate_partition", None)
        self.ascertained = model_config.get("ascertained", None)
        # Force removal of constant features here
        # This can be set by the user in BinaryModel only
        self.remove_constant_features = True
        self.minimum_data = float(model_config.get("minimum_data", 0))
        self.substitution_name = self.__class__.__name__
        self.data_separator = ","
        self.use_robust_eigensystem = model_config.get("use_robust_eigensystem", False)
        self.metadata = []
        self.treedata = []

        # Load the entire dataset from the file
        self.data, language_code_map = load_data(
            self.data_filename,
            file_format=model_config.get("file_format", None),
            lang_column=model_config.get("language_column", None),
            value_column=model_config.get("value_column", None),
            expect_multiple=True)

        # Augment the Glottolog classifications with human-friendly language
        # names which may have been read from a CLDF dataset.  Note that we
        # store both the actual language ID and its lowercase transformation.
        # This is kind of ugly, but we inconsistently convert things to
        # lowercase before doing Glottolog lookups all over the place, so
        # this is the easiest way to make this work everywhere.  We should
        # clean this up some day!
        for language_id, glottocode in language_code_map.items():
            if glottocode in global_config.classifications:
                global_config.classifications[language_id] = global_config.classifications[glottocode]
                global_config.classifications[language_id.lower()] = global_config.classifications[language_id]
            if glottocode in global_config.glotto_macroareas:
                global_config.glotto_macroareas[language_id] = global_config.glotto_macroareas[glottocode]
                global_config.glotto_macroareas[language_id.lower()] = global_config.glotto_macroareas[language_id]
            if glottocode in global_config.locations:
                global_config.locations[language_id] = global_config.locations[glottocode]
                global_config.locations[language_id.lower()] = global_config.locations[language_id]

        # Remove features not wanted in this analysis
        self.build_feature_filter()
        self.apply_feature_filter()

        # Keep this around for later...
        self.global_config = global_config

    def build_feature_filter(self):
        """
        Create the self.feature_filter attribute, which is a set of feature
        names that functions analogously to Configuration.lang_filter
        attribute.
        """
        if self.features == ["*"]:
            random_iso = list(self.data.keys())[0]
            self.features = set()
            for lang_features in self.data.values():
{'fraction_non_alphanumeric': 0.08397629430839763, 'fraction_numerical': 0.00044727720004472774, 'mean_word_length': 2.8819444444444446, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '899473', 'n_tokens_mistral': 2448, 'n_tokens_neox': 2360, 'n_words': 600}