<commit_before>from evolib.tools.GeneralMethods import create_ids

from evolib.stats.StatObjects import IOstats
from evolib.generic.AlignmentSite import Site

from evolib.data.DataObjects import SeqTable, IOtable
from evolib.tools.DNAmethods import booleanDNA, booleanIO
from evolib.generic.GeneticSequence import DNAsequence

from evolib.tools.GeneralMethods import loopByColumn

from evolib.tools.DNAmethods import dna_to_amino, synNonsynProbs

class DnaPopulationData(IOstats):
    """
    Class representation of DNA sequences from multiple 
    samples. 
    """
    def __init__(self, *args):

        if len(args) == 1:
            seqs, ids = self._from_sequence(args[0])
        elif len(args) == 2:
            seqs, ids = self._from_sequence(args[0], args[1])
        else:
            raise TypeError, "Wrong number of arguments"
        
        self._attach_data(seqs, ids)


    def _from_sequence(self, seqs, ids = None):
    
        if isinstance(seqs, list) is False:
            raise TypeError, 'List expected.'
        
        n = len(seqs)
        
        if isinstance(ids, list) is False:
            if ids is None:
                ids = create_ids(n, "seq")
            else:
                raise TypeError, 'List expected.'

        return seqs, ids
        

    def _attach_data(self, sequences, ids):
        
        self.DNAdata = SeqTable(sequences, ids)
        self.IOdata = self._get_IOdata(self.DNAdata)

    ######

    def __getitem__(self, item):
        
        if isinstance(item, str):
            seq = DNAequence(self.DNAdata[item], item)
        elif isinstance(item, int):
            seq = DNAequence(self.DNAdata[item], self.DNAdata.ids[item])
        else:
            raise TypeError, "String or integer required"
        
        return seq
    
    def __len__(self):
        return len(self.DNAdata)
        
    ######
    
    def _get_IOdata(self, seqs):

        self.validSites = 0
        io = []
        
        for site in seqs.iter_sites():

            SiteClass = Site(site)
            
            if SiteClass.has_missing_data():
                pass
            elif SiteClass.number_of_alleles() > 2:
                pass
            elif SiteClass.number_of_alleles() == 1:
                self.validSites += 1
            else:
                self.validSites += 1
                siteIO = booleanDNA(SiteClass.alleles())
                io.append(siteIO)

        IO = IOtable(io)
                
        return IO

    ######
    
    def iter_sites(self):
        for site in self.DNAdata.iter_sites():
            yield site
    
    ######
            
    def ids(self):
        return self.DNAdata.ids


    def nsamples(self):
        return self.__len__()


    def sequences(self):
        return self.DNAdata.sequences

    
    def length(self):
        return self.validSites

    ######

    def index(self, key):
        return self.DNAdata.index(key)

    def pop(self, index = None):
        seq, seqid = self.DNAdata.pop(index)
        self.IOdata = self._get_IOdata(self.DNAdata)
        
        return DNAsequence(seq, seqid)

    def sort(self):
        new_ids = sorted(self.ids())
        new_seqs = []
        for i in new_ids:
            new_seqs.append(self[i].sequence)
        self._attach_data(new_seqs, new_ids)
            
    <commit_msg>sort fsa alignments<commit_after>import random

from evolib.tools.GeneralMethods import create_ids

from evolib.stats.StatObjects import IOstats
from evolib.generic.AlignmentSite import Site

from evolib.data.DataObjects import SeqTable, IOtable
from evolib.tools.DNAmethods import booleanDNA, booleanIO
from evolib.generic.GeneticSequence import DNAsequence

from evolib.tools.GeneralMethods import loopByColumn

from evolib.tools.DNAmethods import dna_to_amino, synNonsynProbs

class DnaPopulationData(IOstats):
    """
    Class representation of DNA sequences from multiple 
    samples. 
    """
    def __init__(self, *args):

        if len(args) == 1:
            seqs, ids = self._from_sequence(args[0])
        elif len(args) == 2:
            seqs, ids = self._from_sequence(args[0], args[1])
        else:
            raise TypeError, "Wrong number of arguments"
        
        self._attach_data(seqs, ids)


    def _from_sequence(self, seqs, ids = None):
    
        if isinstance(seqs, list) is False:
            raise TypeError, 'List expected.'
        
        n = len(seqs)
        
        if isinstance(ids, list) is False:
            if ids is None:
                ids = create_ids(n, "seq")
            else:
                raise TypeError, 'List expected.'

        return seqs, ids
        

    def _attach_data(self, sequences, ids):
        
        self.DNAdata = SeqTable(sequences, ids)
        self.IOdata = self._get_IOdata(self.DNAdata)

    ######

    def __getitem__(self, item):
        
        if isinstance(item, str):
            seq = DNAequence(self.DNAdata[item], item)
        elif isinstance(item, int):
            seq = DNAequence(self.DNAdata[item], self.DNAdata.ids[item])
        else:
            raise TypeError, "String or integer required"
        
        return seq
    
    def __len__(self):
        return len(self.DNAdata)
        
    ######
    
    def _get_IOdata(self, seqs):

        self.validSites = 0
        io = []
        
        for site in seqs.iter_sites():

            SiteClass = Site(site)
            
            if SiteClass.has_missing_data():
                pass
            elif SiteClass.number_of_alleles() > 2:
                pass
            elif SiteClass.number_of_alleles() == 1:
                self.validSites += 1
            else:
                self.validSites += 1
                siteIO = booleanDNA(SiteClass.alleles())
                io.append(siteIO)

        IO = IOtable(io)
                
        return IO

    ######
    
    def iter_sites(self):
        for site in self.DNAdata.iter_sites():
            yield site
    
    ######
            
    def ids(self):
        return self.DNAdata.ids


    def nsamples(self):
        return self.__len__()


    def sequences(self):
        return self.DNAdata.sequences

    
    def length(self):
        return self.validSites

    ######

    def index(self, key):
        return self.DNAdata.index(key)

    def pop(self, index = None):
        seq, seqid = self.DNAdata.pop(index)
        self.IOdata = self._get_IOdata(self.DNAdata)
        
        return DNAsequence(seq, seqid)

    def sample(self, n, replace = False):
        rids = random.sample(self.ids(), n)
        rseqs = []
        seqs = self.sequences()
        for rid in rids:
            i = self.index(rid)
            rseqs.append(seqs[i])

        return type(self)(rseqs, rids)
    
    def sort(self):
        new_ids = sorted(self.ids())
        new_seqs = []
        for i in new_ids:
            new_seqs.append(self[i].sequence)
        self._attach_data(new_seqs, new_ids)
            
    
{'fraction_non_alphanumeric': 0.09688482423549585, 'fraction_numerical': 0.002857959416976279, 'mean_word_length': 1.7729793977812995, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5605795', 'n_tokens_mistral': 2166, 'n_tokens_neox': 2042, 'n_words': 485}