<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

from __future__ import print_function

import os.path as op
import sys
import logging

import numpy as np
from collections import defaultdict

try:
    from collections.abc import Iterable
except ImportError:
    from collections import Iterable

from jcvi.algorithms.lis import heaviest_increasing_subsequence as his
from jcvi.formats.bed import Bed, BedLine
from jcvi.formats.blast import Blast
from jcvi.formats.base import BaseFile, SetFile, read_block, must_open
from jcvi.utils.grouper import Grouper
from jcvi.utils.cbook import gene_name, human_size
from jcvi.utils.range import Range, range_chain
from jcvi.apps.base import OptionParser, ActionDispatcher


class AnchorFile(BaseFile):
    def __init__(self, filename, minsize=0):
        super(AnchorFile, self).__init__(filename)
        self.blocks = list(self.iter_blocks(minsize=minsize))

    def iter_blocks(self, minsize=0):
        fp = open(self.filename)
        for header, lines in read_block(fp, "#"):
            lines = [x.split() for x in lines]
            if len(lines) >= minsize:
                yield lines

    def iter_pairs(self, minsize=0):
        block_id = -1
        for rows in self.iter_blocks(minsize=minsize):
            block_id += 1
            for row in rows:
                a, b = row[:2]
                yield a, b, block_id

    def make_ranges(self, order, clip=10):
        """ Prepare anchors information into a set of ranges for chaining
        """
        ranges = []
        block_pairs = defaultdict(dict)
        blocks = self.blocks
        for i, ib in enumerate(blocks):
            q, s, t = zip(*ib)
            if q[0] not in order:
                q, s = s, q

            r = make_range(q, s, t, i, order, block_pairs, clip=clip)
            ranges.append(r)

            assert q[0] in order
            if s[0] not in order:
                continue

            # is_self comparison
            q, s = s, q
            r = make_range(q, s, t, i, order, block_pairs, clip=clip)
            ranges.append(r)
        return ranges, block_pairs

    def print_to_file(self, filename="stdout", accepted=None):
        fw = must_open(filename, "w")
        blocks = self.blocks
        nremoved = 0
        ncorrected = 0
        for block in blocks:
            print("###", file=fw)
            for line in block:
                a, b, score = line
                pair = (a, b)
                if accepted:
                    if pair not in accepted:
                        nremoved += 1
                        continue
                    av = accepted[pair]
                    if score != av and score != av + "L":
                        score = av
                        ncorrected += 1
                print("\t".join((a, b, score)), file=fw)
        fw.close()

        logging.debug("Removed {0} existing anchors.".format(nremoved))
        logging.debug("Corrected scores for {0} anchors.".format(ncorrected))
        logging.debug("Anchors written to `{0}`.".format(filename))

    def blast(self, blastfile=None, outfile=None):
        """
        convert anchor file to 12 col blast file
        """
        from jcvi.formats.blast import BlastSlow, BlastLineByConversion

        if not outfile:
            outfile = self.filename + ".blast"

        if blastfile is not None:
            blasts = BlastSlow(blastfile).to_dict()
        else:
            blasts = None

        fw = must_open(outfile, "w", checkexists=True)
        nlines = 0
        for a, b, id in self.iter_pairs():
            if (a, b) in blasts:
                bline = blasts[(a, b)]
            elif (b, a) in blasts:
                bline = blasts[(b, a)]
            else:
                line = "\t".join((a, b))
                bline = BlastLineByConversion(line, mode="110000000000")

            print(bline, file=fw)
            nlines += 1
        fw.close()

        logging.debug(
            "A total of {0} BLAST lines written to `{1}`.".format(nlines, outfile)
        )

        return outfile


class BlockFile(BaseFile):
    def __init__(self, filename, defaultcolor="#fb8072", header=False):
        super(BlockFile, self).__init__(filename)
        fp = must_open(filename)
        hd = next(fp).rstrip().split("\t")
        ncols = len(hd)
        if header:
            self.header = hd
        else:
            fp.seek(0)
            self.header = range(ncols)

        data = []
        highlight = []
        for row in fp:
            hl = "*" in row
            # r* highlights the block in red color
            if hl:
                hl, row = row.split("*", 1)
                hl = hl or defaultcolor
            atoms = row.rstrip().split("\t")
            atoms = [x.strip() for x in atoms]
            atoms = ["." if x == "" else x for x in atoms]
            if len(atoms) > ncols:
                atoms = atoms[:ncols]
            elif len(atoms) < ncols:
                atoms = atoms + ["."] * (ncols - len(atoms))
            data.append(atoms)
            highlight.append(hl)

        self.data = data
        self.highlight = highlight
        self.columns = list(zip(*data))
        self.ncols = ncols

    def get_extent(self, i, order, debug=True):
        # Some blocks file, such as ones manually edited, will have garbled
        # order, which prompts the hack below
        acol = [order[x][0] for x in self.columns[0] if x in order]
        bcol = [order[x][0] for x in self.columns[i] if x in order]
        elen = min(len(acol), len(bcol))
        ia, ib = acol[:elen], bcol[:elen]
        slope = get_orientation(ia, ib)

        ocol = [order[x] for x in self.columns[i] if x in order]
        # orientation = '+' if ocol[0][0] <= ocol[-1][0] else '-'
        si, start = min(ocol)
        ei, end = max(ocol)
        same_chr = start.seqid == end.seqid
        chr = start.seqid if same_chr else None<commit_msg>[compara] Fix slope => orientation from recent change<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

from __future__ import print_function

import os.path as op
import sys
import logging

import numpy as np
from collections import defaultdict

try:
    from collections.abc import Iterable
except ImportError:
    from collections import Iterable

from jcvi.algorithms.lis import heaviest_increasing_subsequence as his
from jcvi.formats.bed import Bed, BedLine
from jcvi.formats.blast import Blast
from jcvi.formats.base import BaseFile, SetFile, read_block, must_open
from jcvi.utils.grouper import Grouper
from jcvi.utils.cbook import gene_name, human_size
from jcvi.utils.range import Range, range_chain
from jcvi.apps.base import OptionParser, ActionDispatcher


class AnchorFile(BaseFile):
    def __init__(self, filename, minsize=0):
        super(AnchorFile, self).__init__(filename)
        self.blocks = list(self.iter_blocks(minsize=minsize))

    def iter_blocks(self, minsize=0):
        fp = open(self.filename)
        for header, lines in read_block(fp, "#"):
            lines = [x.split() for x in lines]
            if len(lines) >= minsize:
                yield lines

    def iter_pairs(self, minsize=0):
        block_id = -1
        for rows in self.iter_blocks(minsize=minsize):
            block_id += 1
            for row in rows:
                a, b = row[:2]
                yield a, b, block_id

    def make_ranges(self, order, clip=10):
        """ Prepare anchors information into a set of ranges for chaining
        """
        ranges = []
        block_pairs = defaultdict(dict)
        blocks = self.blocks
        for i, ib in enumerate(blocks):
            q, s, t = zip(*ib)
            if q[0] not in order:
                q, s = s, q

            r = make_range(q, s, t, i, order, block_pairs, clip=clip)
            ranges.append(r)

            assert q[0] in order
            if s[0] not in order:
                continue

            # is_self comparison
            q, s = s, q
            r = make_range(q, s, t, i, order, block_pairs, clip=clip)
            ranges.append(r)
        return ranges, block_pairs

    def print_to_file(self, filename="stdout", accepted=None):
        fw = must_open(filename, "w")
        blocks = self.blocks
        nremoved = 0
        ncorrected = 0
        for block in blocks:
            print("###", file=fw)
            for line in block:
                a, b, score = line
                pair = (a, b)
                if accepted:
                    if pair not in accepted:
                        nremoved += 1
                        continue
                    av = accepted[pair]
                    if score != av and score != av + "L":
                        score = av
                        ncorrected += 1
                print("\t".join((a, b, score)), file=fw)
        fw.close()

        logging.debug("Removed {0} existing anchors.".format(nremoved))
        logging.debug("Corrected scores for {0} anchors.".format(ncorrected))
        logging.debug("Anchors written to `{0}`.".format(filename))

    def blast(self, blastfile=None, outfile=None):
        """
        convert anchor file to 12 col blast file
        """
        from jcvi.formats.blast import BlastSlow, BlastLineByConversion

        if not outfile:
            outfile = self.filename + ".blast"

        if blastfile is not None:
            blasts = BlastSlow(blastfile).to_dict()
        else:
            blasts = None

        fw = must_open(outfile, "w", checkexists=True)
        nlines = 0
        for a, b, id in self.iter_pairs():
            if (a, b) in blasts:
                bline = blasts[(a, b)]
            elif (b, a) in blasts:
                bline = blasts[(b, a)]
            else:
                line = "\t".join((a, b))
                bline = BlastLineByConversion(line, mode="110000000000")

            print(bline, file=fw)
            nlines += 1
        fw.close()

        logging.debug(
            "A total of {0} BLAST lines written to `{1}`.".format(nlines, outfile)
        )

        return outfile


class BlockFile(BaseFile):
    def __init__(self, filename, defaultcolor="#fb8072", header=False):
        super(BlockFile, self).__init__(filename)
        fp = must_open(filename)
        hd = next(fp).rstrip().split("\t")
        ncols = len(hd)
        if header:
            self.header = hd
        else:
            fp.seek(0)
            self.header = range(ncols)

        data = []
        highlight = []
        for row in fp:
            hl = "*" in row
            # r* highlights the block in red color
            if hl:
                hl, row = row.split("*", 1)
                hl = hl or defaultcolor
            atoms = row.rstrip().split("\t")
            atoms = [x.strip() for x in atoms]
            atoms = ["." if x == "" else x for x in atoms]
            if len(atoms) > ncols:
                atoms = atoms[:ncols]
            elif len(atoms) < ncols:
                atoms = atoms + ["."] * (ncols - len(atoms))
            data.append(atoms)
            highlight.append(hl)

        self.data = data
        self.highlight = highlight
        self.columns = list(zip(*data))
        self.ncols = ncols

    def get_extent(self, i, order, debug=True):
        # Some blocks file, such as ones manually edited, will have garbled
        # order, which prompts the hack below
        acol = [order[x][0] for x in self.columns[0] if x in order]
        bcol = [order[x][0] for x in self.columns[i] if x in order]
        elen = min(len(acol), len(bcol))
        ia, ib = acol[:elen], bcol[:elen]
        orientation = get_orientation(ia, ib)

        ocol = [order[x] for x in self.columns[i] if x in order]
        # orientation = '+' if ocol[0][0] <= ocol[-1][0] else '-'
        si, start = min(ocol)
        ei, end = max(ocol)
        same_chr = start.seqid == end.seqid
        chr = start.seqid if same_chr else None
{'fraction_non_alphanumeric': 0.10831656606304493, 'fraction_numerical': 0.008383635144198525, 'mean_word_length': 1.7473514509442654, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6851629', 'n_tokens_mistral': 3721, 'n_tokens_neox': 3532, 'n_words': 1065}