<commit_before>#!/usr/bin/python

import os
import itertools
import collections

from pypeline.node import Node, NodeError
from pypeline.fileutils import move_file, reroot_path

from pypeline.common.formats.msa import read_msa, join_msa, split_msa
from pypeline.common.formats.phylip import interleaved_phy, sequential_phy
from pypeline.common.utilities import grouper, safe_coerce_to_tuple



_VALID_KEYS = frozenset(["name", "partition_by"])




class FastaToPartitionsNode(Node):
    def __init__(self, infiles, out_partitions, partition_by = "123", dependencies = ()):
        assert len(partition_by) == 3
        if not isinstance(infiles, dict):
            raise TypeError("'infiles' must be a dictionary")
        elif not all(isinstance(dd, dict) for dd in infiles.values()):
            raise TypeError("'infiles' must be a dictionary of dictionaries")
        elif not any(("name" in dd) for dd in infiles.values()):
            raise ValueError("'name' must be specified for all input files")
        elif any((set(dd) - _VALID_KEYS) for dd in infiles.values()):
            raise ValueError("Invalid keys found: %s" % ", ".join(set(dd) - _VALID_KEYS))

        self._infiles   = infiles
        self._out_part  = out_partitions
        self._part_by   = partition_by

        description  = "<FastaToPartitions: %i file(s) -> '%s'>" % \
            (len(infiles), out_partitions)
            
        Node.__init__(self, 
                      description  = description,
                      input_files  = infiles,
                      output_files = out_partitions,
                      dependencies = dependencies)


    def _run(self, _config, temp):
        end = 0
        partitions = []
        for (filename, msa) in _read_sequences(self._infiles):
            length = len(msa.itervalues().next())
            start, end = end + 1, end + length

            for (group, offsets) in self._get_partition_by(filename):
                if len(offsets) != 3:
                    parts = [("%i-%i\\3" % (start + offset, end)) for offset in offsets]
                else:
                    parts = ["%i-%i" % (start, end)]

                partition = "DNA, %s_%s = %s\n" \<commit_msg>Improved input checking, and use of new fileutils location.<commit_after>#!/usr/bin/python

import os
import itertools
import collections

from pypeline.node import Node, NodeError
from pypeline.common.fileutils import move_file, reroot_path
from pypeline.common.formats.msa import read_msa, join_msa, split_msa
from pypeline.common.formats.phylip import interleaved_phy, sequential_phy
from pypeline.common.utilities import grouper, safe_coerce_to_tuple



_VALID_KEYS = frozenset(["name", "partition_by"])




class FastaToPartitionsNode(Node):
    def __init__(self, infiles, out_partitions, partition_by = "123", dependencies = ()):
        if (len(partition_by) != 3):
            raise ValueError("Default 'partition_by' must be 3 entires long!")
        elif not isinstance(infiles, dict):
            raise TypeError("'infiles' must be a dictionary")
        elif any(len(dd.get("partition_by", "123")) != 3 for dd in infiles.itervalues()):
            raise ValueError("'partition_by' must be 3 entires long!")
        elif not all(isinstance(dd, dict) for dd in infiles.values()):
            raise TypeError("'infiles' must be a dictionary of dictionaries")
        elif not any(("name" in dd) for dd in infiles.values()):
            raise ValueError("'name' must be specified for all input files")
        elif any((set(dd) - _VALID_KEYS) for dd in infiles.values()):
            raise ValueError("Invalid keys found: %s" % ", ".join(set(dd) - _VALID_KEYS))

        self._infiles   = infiles
        self._out_part  = out_partitions
        self._part_by   = partition_by

        description  = "<FastaToPartitions (default: %s): %i file(s) -> '%s'>" % \
            (partition_by, len(infiles), out_partitions)
            
        Node.__init__(self, 
                      description  = description,
                      input_files  = infiles,
                      output_files = out_partitions,
                      dependencies = dependencies)


    def _run(self, _config, temp):
        end = 0
        partitions = []
        for (filename, msa) in _read_sequences(self._infiles):
            length = len(msa.itervalues().next())
            start, end = end + 1, end + length

            for (group, offsets) in self._get_partition_by(filename):
                if len(offsets) != 3:
                    parts = [("%i-%i\\3" % (start + offset, end)) for offset in offsets]
                else:
                    parts = ["%i-%i" % (start, end)]

                partition = "DNA, %s_%s = %s\n" \
{'fraction_non_alphanumeric': 0.1112518471606502, 'fraction_numerical': 0.004644289634789952, 'mean_word_length': 2.377049180327869, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '182011', 'n_tokens_mistral': 1464, 'n_tokens_neox': 1379, 'n_words': 382}