<commit_before>import logging
import itertools
import re

logger = logging.getLogger(__name__)


def get_unique_age_records(age_records):
    unique_records = []

    for a1 in age_records:
        if a1.duplicate: continue
        unique_records.append(a1)

        for a2 in age_records:
            if a2.duplicate: continue
            if a1.contig.raw_name == a2.contig.raw_name: continue

            if a1.start1_end1s == a2.start1_end1s:
                a2.duplicate = True

    return unique_records


class AgeInput:
    def __init__(self, fname, length):
        self.fname = fname
        self.length = length

    def __str__(self):
        return "<%s(%s, %d)>" % (self.__class__.__name__, self.fname, self.length)

    def __repr__(self):
        return str(self)


class AgeFormatError(Exception):
    def __init__(self, context, line):
        self.context = context
        self.line = line
    def __str__(self):
        return "Error while reading AGE output, L%d (section %s)." % (self.line, self.context)


class AgeRecord:
    def __init__(self, age_out_file=None,tr_region_1=[]):
        # Index 0 corresponds to the reference sequence
        self.aligned_bases = 0
        self.inputs = []  # inputs[1] is expected to have an insertion w.r.t. inputs[0]
        self.percent = 100  # Percent of matching bases in alignment
        self.percents = [100, 100]  # Percent of matching bases in alignment for each flank
        self.start1_end1s = []  # Alignment intervals for the first sequence
        self.start1_end1s_orig = []  # Alignment intervals for the first sequence (without considering truncations)
        self.start2_end2s = []  # Alignment intervals for the second sequence
        self.polarities1 = []  # Alignment polarities of intervals for the second sequence
        self.polarities2 = []  # Alignment polarities of intervals for the second sequence
        self.excised_regions = []  # List of excised regions as intervals
        self.n_alt = 0  # Number of alternate regions
        self.alternate_regions = []

        self.flanking_regions = [0, 0]  # Length of each flank on the donor sequence
        self.ref_flanking_regions = [0, 0]  # Length of each flank on the ref requence
        self.hom = 0  # Length of sequence homology
        self.flank_percent = 0  # Percent of total length in the flanks
        self.nfrags = 1  # Number of fragments in the alignment

        # The following fields are set later on when processing the alignments
        self.very_bad_assembly = False  # Alignment failed some basic checks
        self.bad_assembly = False  # Only one fragment in the alignment

        self.start = -1
        self.end = -1
        self.excised_lengths = [0, 0]  # Lengths of each excised region

        self.assembly_contig = None
        self.used = False
        self.duplicate = False

        self.age_file = age_out_file
        self.score = 0
        self.tr_region_1=tr_region_1 # Truncation region in first sequence
        self.invalid_tr_intervals=[] #Indicies of invalid intervals in first sequence due to truncation
        
        if age_out_file is not None:
            self.read_from_age_file(age_out_file)

    class LineReader:
        """Behaves like file objects do but also counts lines for tracing the AGE parser."""
        def __init__(self, fd):
            self.fd = fd
            self.line_num = 0
        def readline(self):
            line = self.fd.readline()
            if line: self.line_num += 1
            return line

    rx_rng = re.compile(r"\[\s*(\d+),\s*(\d+)\]")
    rx_perc = re.compile(r"\(\s*(\d+)%\)")
    rx_input = re.compile(r"=>\s+(\d+) nucs '(.*?)'")
    rx_ex = re.compile(r"seq =>\s+(\d+) nucs( \[(\d+),(\d+)\])?")
    rx_ident = re.compile(r"seq =>\s+(\d+) nucs( \[(\d+),(\d+)\] to \[(\d+),(\d+)\])?")

    def read_alignment_ranges(self, age_fd, name):
        line = age_fd.readline().strip()
        if not line.startswith(name):
            raise AgeFormatError("ALIGNMENT RANGES", age_fd.line_num)
        start_end = []
        for m in self.rx_rng.finditer(line):
            start_end.append([int(m.group(1)), int(m.group(2))])
        return start_end

    def parse_input_descriptor(self, line):
        """Returns a tuple of filename and sequence length."""
        m = self.rx_input.search(line)
        if m is None:
            raise AgeFormatError("INPUT DESCRIPTOR", age_fd.line_num)
        return (m.group(2), int(m.group(1)))

    def read_excluded_range(self, age_fd, name, context):
        line = age_fd.readline().strip()
        if len(line) == 0:
            return None
        if not line.startswith(name):
            raise AgeFormatError(context, age_fd.line_num)
        m = self.rx_ex.search(line)
        seq_len = int(m.group(1))
        # ATTN: Do you intend to ever use start or stop of excluded regions? So far those are never read anywhere. 
        return [0] if seq_len == 0 else [seq_len, int(m.group(3)), int(m.group(4))]

    def read_excluded_regions(self, age_fd, context):
        excluded_regions = [];
        while True:
            seq_rng = self.read_excluded_range(age_fd, "first", context)
            if seq_rng is None:
                break
            excluded_regions.append(seq_rng)
            seq_rng = self.read_excluded_range(age_fd, "second", context)
            if seq_rng is None:
                raise AgeFormatError(context, age_fd.line_num)
            excluded_regions.append(seq_rng)
            # ATTN: Is it intentional that while AGE allows multiple excised regions, only the first is ever looked at?
        return excluded_regions

    def read_identities(self, age_fd, name, context):
        line = age_fd.readline().strip()
        if not line.startswith(name):
            raise AgeFormatError(context, age_fd.line_num)
        m = self.rx_ident.search(line)
        seq_len = int(m.group(1))
        # ATTN: Do you intend to ever use positions of identities? So far those are never read anywhere. 
        return [0] if seq_len == 0 else [seq_len, int(m.group(3)), int(m.group(4)), int(m.group(5)), int(m.group(6))]

    def read_from_age_file(self, age_out_file):
        with open(age_out_file) as raw_age_fd:
            age_fd = self.LineReader(raw_age_fd)
            while True:
                line = age_fd.readline()
                if not line: break
                line = line.strip()

                if line.startswith("Aligned:"):
                    self.aligned_bases = int(line.split()[1])
                    continue

                if line.startswith("First  seq"):
                    file1, len1 = self.parse_input_descriptor(line)
                    if self.tr_region_1:
                        len1+=self.tr_region_1[1]                        
                    self.inputs.append(AgeInput(file1, len1))
                    continue

                if line.startswith("Second seq"):
                    file2, len2 = self.parse_input_descriptor(line)
                    self.inputs.append(AgeInput(file2, len2))
                    continue

                if line.startswith("Identic:"):
                    nums = map(int, self.rx_perc.findall(line))
                    self.percent = nums[0]
                    if len(nums) > 1:
                        self.percents = nums[1:3]
                    continue

                if line.startswith("Score:"):
                    self.score = int(line.split()[1])
                    continue

                if line == "Alignment:":
                    self.start1_end1s_orig = self.read_alignment_ranges(age_fd, "first")
                    self.start2_end2s = self.read_alignment_ranges(age_fd, "second")
                    if self.tr_region_1:
                        self.update_pos_tr()
                        self.invalid_tr_intervals=[i for i,interval in enumerate(self.start1_end1s_orig) if <commit_msg>Fix bug whereby exception cannot be constructed when parsing input descriptor fails.<commit_after>import logging
import itertools
import re

logger = logging.getLogger(__name__)


def get_unique_age_records(age_records):
    unique_records = []

    for a1 in age_records:
        if a1.duplicate: continue
        unique_records.append(a1)

        for a2 in age_records:
            if a2.duplicate: continue
            if a1.contig.raw_name == a2.contig.raw_name: continue

            if a1.start1_end1s == a2.start1_end1s:
                a2.duplicate = True

    return unique_records


class AgeInput:
    def __init__(self, fname, length):
        self.fname = fname
        self.length = length

    def __str__(self):
        return "<%s(%s, %d)>" % (self.__class__.__name__, self.fname, self.length)

    def __repr__(self):
        return str(self)


class AgeFormatError(Exception):
    def __init__(self, context, line):
        self.context = context
        self.line = line
    def __str__(self):
        return "Error while reading AGE output, L%d (section %s)." % (self.line, self.context)


class AgeRecord:
    def __init__(self, age_out_file=None,tr_region_1=[]):
        # Index 0 corresponds to the reference sequence
        self.aligned_bases = 0
        self.inputs = []  # inputs[1] is expected to have an insertion w.r.t. inputs[0]
        self.percent = 100  # Percent of matching bases in alignment
        self.percents = [100, 100]  # Percent of matching bases in alignment for each flank
        self.start1_end1s = []  # Alignment intervals for the first sequence
        self.start1_end1s_orig = []  # Alignment intervals for the first sequence (without considering truncations)
        self.start2_end2s = []  # Alignment intervals for the second sequence
        self.polarities1 = []  # Alignment polarities of intervals for the second sequence
        self.polarities2 = []  # Alignment polarities of intervals for the second sequence
        self.excised_regions = []  # List of excised regions as intervals
        self.n_alt = 0  # Number of alternate regions
        self.alternate_regions = []

        self.flanking_regions = [0, 0]  # Length of each flank on the donor sequence
        self.ref_flanking_regions = [0, 0]  # Length of each flank on the ref requence
        self.hom = 0  # Length of sequence homology
        self.flank_percent = 0  # Percent of total length in the flanks
        self.nfrags = 1  # Number of fragments in the alignment

        # The following fields are set later on when processing the alignments
        self.very_bad_assembly = False  # Alignment failed some basic checks
        self.bad_assembly = False  # Only one fragment in the alignment

        self.start = -1
        self.end = -1
        self.excised_lengths = [0, 0]  # Lengths of each excised region

        self.assembly_contig = None
        self.used = False
        self.duplicate = False

        self.age_file = age_out_file
        self.score = 0
        self.tr_region_1=tr_region_1 # Truncation region in first sequence
        self.invalid_tr_intervals=[] #Indicies of invalid intervals in first sequence due to truncation
        
        if age_out_file is not None:
            self.read_from_age_file(age_out_file)

    class LineReader:
        """Behaves like file objects do but also counts lines for tracing the AGE parser."""
        def __init__(self, fd):
            self.fd = fd
            self.line_num = 0
        def readline(self):
            line = self.fd.readline()
            if line: self.line_num += 1
            return line

    rx_rng = re.compile(r"\[\s*(\d+),\s*(\d+)\]")
    rx_perc = re.compile(r"\(\s*(\d+)%\)")
    rx_input = re.compile(r"=>\s+(\d+) nucs '(.*?)'")
    rx_ex = re.compile(r"seq =>\s+(\d+) nucs( \[(\d+),(\d+)\])?")
    rx_ident = re.compile(r"seq =>\s+(\d+) nucs( \[(\d+),(\d+)\] to \[(\d+),(\d+)\])?")

    def read_alignment_ranges(self, age_fd, name):
        line = age_fd.readline().strip()
        if not line.startswith(name):
            raise AgeFormatError("ALIGNMENT RANGES", age_fd.line_num)
        start_end = []
        for m in self.rx_rng.finditer(line):
            start_end.append([int(m.group(1)), int(m.group(2))])
        return start_end

    def parse_input_descriptor(self, line, line_num):
        """Returns a tuple of filename and sequence length."""
        m = self.rx_input.search(line)
        if m is None:
            raise AgeFormatError("INPUT DESCRIPTOR", line_num)
        return (m.group(2), int(m.group(1)))

    def read_excluded_range(self, age_fd, name, context):
        line = age_fd.readline().strip()
        if len(line) == 0:
            return None
        if not line.startswith(name):
            raise AgeFormatError(context, age_fd.line_num)
        m = self.rx_ex.search(line)
        seq_len = int(m.group(1))
        # ATTN: Do you intend to ever use start or stop of excluded regions? So far those are never read anywhere. 
        return [0] if seq_len == 0 else [seq_len, int(m.group(3)), int(m.group(4))]

    def read_excluded_regions(self, age_fd, context):
        excluded_regions = [];
        while True:
            seq_rng = self.read_excluded_range(age_fd, "first", context)
            if seq_rng is None:
                break
            excluded_regions.append(seq_rng)
            seq_rng = self.read_excluded_range(age_fd, "second", context)
            if seq_rng is None:
                raise AgeFormatError(context, age_fd.line_num)
            excluded_regions.append(seq_rng)
            # ATTN: Is it intentional that while AGE allows multiple excised regions, only the first is ever looked at?
        return excluded_regions

    def read_identities(self, age_fd, name, context):
        line = age_fd.readline().strip()
        if not line.startswith(name):
            raise AgeFormatError(context, age_fd.line_num)
        m = self.rx_ident.search(line)
        seq_len = int(m.group(1))
        # ATTN: Do you intend to ever use positions of identities? So far those are never read anywhere. 
        return [0] if seq_len == 0 else [seq_len, int(m.group(3)), int(m.group(4)), int(m.group(5)), int(m.group(6))]

    def read_from_age_file(self, age_out_file):
        with open(age_out_file) as raw_age_fd:
            age_fd = self.LineReader(raw_age_fd)
            while True:
                line = age_fd.readline()
                if not line: break
                line = line.strip()

                if line.startswith("Aligned:"):
                    self.aligned_bases = int(line.split()[1])
                    continue

                if line.startswith("First  seq"):
                    file1, len1 = self.parse_input_descriptor(line, age_fd.line_num)
                    if self.tr_region_1:
                        len1+=self.tr_region_1[1]                        
                    self.inputs.append(AgeInput(file1, len1))
                    continue

                if line.startswith("Second seq"):
                    file2, len2 = self.parse_input_descriptor(line, age_fd.line_num)
                    self.inputs.append(AgeInput(file2, len2))
                    continue

                if line.startswith("Identic:"):
                    nums = map(int, self.rx_perc.findall(line))
                    self.percent = nums[0]
                    if len(nums) > 1:
                        self.percents = nums[1:3]
                    continue

                if line.startswith("Score:"):
                    self.score = int(line.split()[1])
                    continue

                if line == "Alignment:":
                    self.start1_end1s_orig = self.read_alignment_ranges(age_fd, "first")
                    self.start2_end2s = self.read_alignment_ranges(age_fd, "second")
                    if self.tr_region_1:
                        self.update_pos_tr()
                        self.invalid_tr_intervals=[i for i,interval in enumerate(self.start1_end1s_orig) if 
{'fraction_non_alphanumeric': 0.10130636854666498, 'fraction_numerical': 0.011933174224343675, 'mean_word_length': 2.0745317628885886, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5962805', 'n_tokens_mistral': 4973, 'n_tokens_neox': 4710, 'n_words': 1259}