<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

"""
Scaffold Ordering with Weighted Maps.
"""

import os.path as op
import sys
import logging

import numpy as np
import networkx as nx

from itertools import combinations
from collections import defaultdict
from scipy.stats import spearmanr

from jcvi.algorithms.formula import reject_outliers
from jcvi.algorithms.lis import longest_monotonous_subseq_length
from jcvi.algorithms.tsp import hamiltonian
from jcvi.algorithms.matrix import determine_signs
from jcvi.formats.agp import AGP, order_to_agp, build as agp_build
from jcvi.formats.base import DictFile, FileMerger, must_open
from jcvi.formats.bed import Bed, BedLine, sort
from jcvi.formats.chain import fromagp
from jcvi.formats.sizes import Sizes
from jcvi.utils.cbook import human_size
from jcvi.utils.counter import Counter
from jcvi.utils.grouper import Grouper
from jcvi.utils.iter import flatten, pairwise
from jcvi.apps.base import OptionParser, ActionDispatcher, debug, sh, \
            need_update
debug()


START, END = "START", "END"
distance_choices = ("cM", "rank")
np.seterr(invalid="ignore")


class Scaffold (object):

    def __init__(self, seqid, mapc):
        self.markers = mapc.extract(seqid)
        self.seqid = seqid
        self.mapc = mapc

    @property
    def mlg_counts(self):
        return Counter([x.mlg for x in self.markers])

    def add_LG_pairs(self, G, mappair):
        # Computes co-occurrences of LG pairs
        cc = self.mlg_counts.items()
        mappair = sorted(mappair)
        for (ak, av), (bk, bv) in combinations(cc, 2):
            aks, bks = ak.split("-")[0], bk.split("-")[0]
            if sorted((aks, bks)) != mappair:
                continue
            weight = min(av, bv)
            G[ak, bk] += weight
            G[bk, ak] += weight


class LinkageGroup (object):

    def __init__(self, lg, length, markers, function=(lambda x: x.rank)):
        self.lg = lg
        self.length = length
        self.markers = markers

        self.mapname = lg.split("-")[0]
        self.series = {}
        self.nmarkers = {}
        self.oo = {}
        self.position = {}
        self.guide = {}
        for k, v in markers.items():  # keyed by scaffold ids
            self.series[k] = xs = [function(x) for x in v]
            self.nmarkers[k] = len(v)
            physical_to_cm = [(x.pos, function(x)) for x in v]
            self.oo[k] = get_rho(physical_to_cm)
            self.position[k] = np.median(xs)
            self.guide[k] = np.median([x.cm for x in v])

        path = sorted((v, self.guide[k], k) for k, v in self.position.items())
        vv, gg, path = zip(*path)
        self.path = path
        self.rho = 0


class ScaffoldOO (object):
    """
    This contains the routine to construct order and orientation for the
    scaffolds per partition.
    """
    def __init__(self, lgs, scaffolds, mapc, pivot, weights, sizes,
                 function=(lambda x: x.rank)):

        self.lgs = lgs
        self.lengths = mapc.lengths
        self.bins = mapc.bins
        self.function = function
        self.sizes = sizes
        self.scaffolds = scaffolds
        self.pivot = pivot
        self.weights = weights

        self.prepare_linkage_groups()  # populate all data
        signs = self.assign_orientation()
        assert len(signs) == len(scaffolds)
        scaffolds_oo = dict(zip(scaffolds, signs))

        tour = self.assign_order()
        links = 1
        iterations = 0
        solutions = []
        while links:
            score = self.evaluate(tour)
            logging.debug("Iteration {0} score: {1}".format(iterations, score))
            solutions.append((score, tour))
            tour, links = self.fix_order(tour)
            iterations += 1

        min_score, tour = min(solutions)
        logging.debug("Best solution score: {0}".format(min_score))
        tour = [(x, scaffolds_oo[x]) for x in tour]

        tour = self.fix_orientation(tour)
        recode = {0: '?', 1: '+', -1: '-'}
        tour = [(x, recode[o]) for x, o in tour]
        self.tour = tour

        for mlg in self.lgs:
            mapname, lg = mlg.rsplit("-", 1)
            if mapname == pivot:
                self.object = "chr{0}".format(lg)
                break

    def weighted_mean(self, a):
        a, w = zip(*a)
        w = [self.weights[x] for x in w]
        return np.average(a, weights=w)

    def get_markers(self, lg, scaffold, orientation=0):
        xs = self.bins.get((lg, scaffold), [])
        if orientation < 0:
            xs = xs[::-1]
        return xs

    def get_series(self, lg, scaffold, orientation=0):
        xs = self.get_markers(lg, scaffold, orientation=orientation)
        return [self.function(x) for x in xs]

    def prepare_linkage_groups(self):
        self.linkage_groups = []
        for lg in self.lgs:
            length = self.lengths[lg]
            markers = {}
            for s in self.scaffolds:
                xs = self.get_markers(lg, s)
                if xs:
                    markers[s] = xs
            LG = LinkageGroup(lg, length, markers, function=self.function)
            self.linkage_groups.append(LG)
            print LG.lg
            print sorted((v, k) for k, v in LG.position.items())

    def distances_to_tour(self):
        scaffolds = self.scaffolds
        distances = self.distances
        G = nx.DiGraph()
        for (a, b), v in distances.items():
            d = self.weighted_mean(v)
            G.add_edge(a, b, weight=d)
            if a == START or b == END:
                continue
            G.add_edge(b, a, weight=d)

        logging.debug("Graph size: |V|={0}, |E|={1}.".format(len(G), G.size()))

        L = nx.all_pairs_dijkstra_path_length(G)
        for a, b in combinations(scaffolds, 2):
            if G.has_edge(a, b):
                continue
            l = L[a][b]
            G.add_edge(a, b, weight=l)
            G.add_edge(b, a, weight=l)

        edges = []
        for a, b, d in G.edges(data=True):
            edges.append((a, b, d['weight']))

        tour = hamiltonian(edges, symmetric=False, precision=2)
        assert tour[0] == START and tour[-1] == END
        tour = tour[1:-1]
        return tour

    def assign_order(self):
        """
        The goal is to assign scaffold orders. To help order the scaffolds, two
        dummy node, START and END, mark the ends of the chromosome. We connect
        START to each scaffold (directed), and each scaffold to END.
        """
        linkage_groups = self.linkage_groups
        for mlg in linkage_groups:
            mapname = mlg.mapname
            if mapname == self.pivot:
                pivot_position = mlg.position

        for mlg in linkage_groups:
            position = mlg.position
            # Flip order if path goes in the opposite direction to the pivot
            common = []
            for a, ap in position.items():
                if a not in pivot_position:
                    continue
                pp = pivot_position[a]
                common.append((ap, pp))

            mlg.rho = get_rho(common)
            if mlg.rho < 0:
                mlg.path = mlg.path[::-1]

        # Preparation of TSP
        distances = defaultdict(list)
        for mlg in linkage_groups:
            mapname = mlg.mapname
            position = mlg.position
            length = mlg.length
            path = mlg.path
            rho = mlg.rho
            for a, b in combinations(path, 2):
                d = abs(position[a] - position[b])
                distances[a, b].append((d, mapname))
            for p in path:
                adist, bdist = position[p], length - position[p]
                if rho < 0:
                    adist, bdist = bdist, adist
                distances[START, p].append((adist, mapname))
                distances[p, END].append((bdist, mapname))
        self.distances = distances

        tour = self.distances_to_tour()
        return tour

    def fix_order(self, tour):
        """
        TSP will not produce the ideal order, but quite close. The reason for
        slight inaccuracies is due to scaffold connections (edges) that are
        unique to certain maps. In other words, edges like this will have no
        weight for certain maps. The following fix add those edges back by
        extrapolating distances based on the preliminary order and then
        iteratively perform updates after inserting those weights back.
        """
        distances = self.distances
        tour_index = dict((t, i) for i, t in enumerate(tour))
        links = 0
        for mlg in self.linkage_groups:
            mapname = mlg.mapname
            position = mlg.position
            for a, b in pairwise(tour):
                if not a in position:
                    continue
                if b in position:
                    continue
                # Find the closest scaffold in the same lg DOWNSTREAM
                ai = tour_index[a]
                flank = [x for x in tour[ai + 1:] if x in position]
                c = flank[0] if flank else None
                if not c:
                    continue
                ci = tour_index[c]
                distance = abs(position[c] - position[a])
                d = distance / 2.
                for i in xrange(ai + 1, ci):
                    t = tour[i]
                    for e in ((a, t), (t, a), (t, c), (c, t)):
                        if mapname in [x[-1] for x in distances[e]]:
                            continue
                        distances[e].append((d, mapname))
                        print e[0], "=>", e[1], d, mapname
                        links += 1

        logging.debug("A total of {0} new edges inserted.".format(links))
        if links:
            tour = self.distances_to_tour()
        return tour, links


    def get_orientation(self, si, sj):
        """
        si, sj are two number series. To compute whether these two series have
        same orientation or not. We combine them in the two orientation
        configurations and compute length of the longest monotonous series.
        """
        if not si or not sj:
            return 0
        # Same orientation configuration
        a = longest_monotonous_subseq_length(si + sj)
        b = longest_monotonous_subseq_length(sj + si)
        # Opposite orientation configuration
        c = longest_monotonous_subseq_length(si + sj[::-1])
        d = longest_monotonous_subseq_length(sj[::-1] + si)
        return max(a, b)[0] - max(c, d)[0]

    def assign_orientation(self):
        signs = defaultdict(list)
        scaffolds = self.scaffolds
        for mlg in self.linkage_groups:
            mapname = mlg.mapname
            series = mlg.series
            if mapname == self.pivot:
                pivot_oo = mlg.oo
                pivot_nmarkers = mlg.nmarkers

            for i, j in combinations(range(len(scaffolds)), 2):
                si, sj = scaffolds[i], scaffolds[j]
                si, sj = series.get(si, []), series.get(sj, [])
                d = self.get_orientation(si, sj)
                if not d:
                    continue
                signs[i, j].append((d, mapname))

        for e, v in signs.items():
            signs[e] = self.weighted_mean(v)

        signs_edges = sorted((a, b, w) for (a, b), w in signs.items())
        signs = determine_signs(scaffolds, signs_edges)

        # Finally flip this according to pivot map, then weight by #_markers
        pivot_oo = [pivot_oo.get(x, 0) for x in scaffolds]
        nmarkers = [pivot_nmarkers.get(x, 0) for x in scaffolds]
        flipr = signs * np.sign(np.array(pivot_oo)) * nmarkers
        if sum(flipr) < 0:
            signs = - signs
        return signs

    def fix_orientation(self, tour):
        """
        Test each scaffold if flipping will increass longest monotonous chain
        length.
        """
        orientations = dict(tour)  # old configuration here
        scaffold_oo = defaultdict(list)
        scaffolds, oos = zip(*tour)
        for mlg in self.linkage_groups:
            lg = mlg.lg
            mapname = mlg.mapname
            for s, o in tour:
                i = scaffolds.index(s)
                L = [self.get_series(lg, x, xo) for x, xo in tour[:i]]
                U = [self.get_series(lg, x, xo) for x, xo in tour[i + 1:]]
                L, U = list(flatten(L)), list(flatten(U))
                M = self.get_series(lg, s)
                plus = longest_monotonous_subseq_length(L + M + U)
                minus = longest_monotonous_subseq_length(L + M[::-1] + U)
                d = plus[0] - minus[0]
                if not d:
                    continue
                scaffold_oo[s].append((d, mapname))  # reset orientation

        for s, v in scaffold_oo.items():
            d = self.weighted_mean(v)
            if abs(d) > .5:  # only update when there is good evidence
                orientations[s] = np.sign(d)
            #print s, v, d

        tour = [(x, orientations[x]) for x in scaffolds]
        return tour

    def evaluate(self, tour):
        """
        Return score that correspond to the sum of all distances.
        """
        weights = self.weights
        score = 0
        for mlg in self.linkage_groups:
            position = mlg.position
            mapname = mlg.mapname
            rho = mlg.rho
            length = mlg.length
            lg_position = [position[x] for x in tour if x in position]
            if rho >= 0:
                lg_position = [0] + lg_position + [length]
            else:
                lg_position = [length] + lg_position + [0]
            s = sum(abs(a - b) for a, b in pairwise(lg_position))
            score += weights[mapname] * s

        return score


class CSVMapLine (object):

    def __init__(self, row, sep=",", mapname=None):
        # ScaffoldID,ScaffoldPosition,LinkageGroup,GeneticPosition
        args = row.strip().split(sep)
        self.seqid = args[0]
        self.pos = int(args[1])<commit_msg>implement single linkage distance function in assembly.allmaps<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

"""
Scaffold Ordering with Weighted Maps.
"""

import os.path as op
import sys
import logging

import numpy as np
import networkx as nx

from itertools import combinations, product
from collections import defaultdict
from scipy.stats import spearmanr

from jcvi.algorithms.formula import reject_outliers
from jcvi.algorithms.lis import longest_monotonous_subseq_length
from jcvi.algorithms.tsp import hamiltonian
from jcvi.algorithms.matrix import determine_signs
from jcvi.formats.agp import AGP, order_to_agp, build as agp_build
from jcvi.formats.base import DictFile, FileMerger, must_open
from jcvi.formats.bed import Bed, BedLine, sort
from jcvi.formats.chain import fromagp
from jcvi.formats.sizes import Sizes
from jcvi.utils.cbook import human_size
from jcvi.utils.counter import Counter
from jcvi.utils.grouper import Grouper
from jcvi.utils.iter import flatten, pairwise
from jcvi.apps.base import OptionParser, ActionDispatcher, debug, sh, \
            need_update
debug()


START, END = "START", "END"
distance_choices = ("cM", "rank")
np.seterr(invalid="ignore")


class Scaffold (object):

    def __init__(self, seqid, mapc):
        self.markers = mapc.extract(seqid)
        self.seqid = seqid
        self.mapc = mapc

    @property
    def mlg_counts(self):
        return Counter([x.mlg for x in self.markers])

    def add_LG_pairs(self, G, mappair):
        # Computes co-occurrences of LG pairs
        cc = self.mlg_counts.items()
        mappair = sorted(mappair)
        for (ak, av), (bk, bv) in combinations(cc, 2):
            aks, bks = ak.split("-")[0], bk.split("-")[0]
            if sorted((aks, bks)) != mappair:
                continue
            weight = min(av, bv)
            G[ak, bk] += weight
            G[bk, ak] += weight


class LinkageGroup (object):

    def __init__(self, lg, length, markers, function=(lambda x: x.rank)):
        self.lg = lg
        self.length = length
        self.markers = markers
        self.function = f = function

        self.mapname = lg.split("-")[0]
        self.series = {}
        self.nmarkers = {}
        self.oo = {}
        self.position = {}
        self.guide = {}
        for k, v in markers.items():  # keyed by scaffold ids
            self.series[k] = xs = [f(x) for x in v]
            self.nmarkers[k] = len(v)
            physical_to_cm = [(x.pos, f(x)) for x in v]
            self.oo[k] = get_rho(physical_to_cm)
            self.position[k] = np.median(xs)
            self.guide[k] = np.median([x.cm for x in v])

        path = sorted((v, self.guide[k], k) for k, v in self.position.items())
        vv, gg, path = zip(*path)
        self.path = path
        self.rho = 0
        self.distances = self.populate_pairwise_distance()

    def populate_pairwise_distance(self):
        distances = {}
        f = self.function
        markers = self.markers
        for a, b in combinations(self.path, 2):
            ma, mb = markers[a], markers[b]
            d = min(abs(f(x) - f(y)) for x, y in product(ma, mb))
            distances[a, b] = distances[b, a] = d
        return distances


class ScaffoldOO (object):
    """
    This contains the routine to construct order and orientation for the
    scaffolds per partition.
    """
    def __init__(self, lgs, scaffolds, mapc, pivot, weights, sizes,
                 function=(lambda x: x.rank)):

        self.lgs = lgs
        self.lengths = mapc.lengths
        self.bins = mapc.bins
        self.function = function
        self.sizes = sizes
        self.scaffolds = scaffolds
        self.pivot = pivot
        self.weights = weights

        self.prepare_linkage_groups()  # populate all data
        signs = self.assign_orientation()
        assert len(signs) == len(scaffolds)
        scaffolds_oo = dict(zip(scaffolds, signs))

        tour = self.assign_order()
        links = 1
        iterations = 0
        solutions = []
        while links:
            score = self.evaluate(tour)
            logging.debug("Iteration {0} score: {1}".format(iterations, score))
            solutions.append((score, tour))
            tour, links = self.fix_order(tour)
            iterations += 1

        min_score, tour = min(solutions)
        logging.debug("Best solution score: {0}".format(min_score))
        tour = [(x, scaffolds_oo[x]) for x in tour]

        tour = self.fix_orientation(tour)
        recode = {0: '?', 1: '+', -1: '-'}
        tour = [(x, recode[o]) for x, o in tour]
        self.tour = tour

        for mlg in self.lgs:
            mapname, lg = mlg.rsplit("-", 1)
            if mapname == pivot:
                self.object = "chr{0}".format(lg)
                break

    def weighted_mean(self, a):
        a, w = zip(*a)
        w = [self.weights[x] for x in w]
        return np.average(a, weights=w)

    def get_markers(self, lg, scaffold, orientation=0):
        xs = self.bins.get((lg, scaffold), [])
        if orientation < 0:
            xs = xs[::-1]
        return xs

    def get_series(self, lg, scaffold, orientation=0):
        xs = self.get_markers(lg, scaffold, orientation=orientation)
        return [self.function(x) for x in xs]

    def prepare_linkage_groups(self):
        self.linkage_groups = []
        for lg in self.lgs:
            length = self.lengths[lg]
            markers = {}
            for s in self.scaffolds:
                xs = self.get_markers(lg, s)
                if xs:
                    markers[s] = xs
            LG = LinkageGroup(lg, length, markers, function=self.function)
            self.linkage_groups.append(LG)
            print LG.lg
            print sorted((v, k) for k, v in LG.position.items())

    def distances_to_tour(self):
        scaffolds = self.scaffolds
        distances = self.distances
        G = nx.DiGraph()
        for (a, b), v in distances.items():
            d = self.weighted_mean(v)
            G.add_edge(a, b, weight=d)
            if a == START or b == END:
                continue
            G.add_edge(b, a, weight=d)

        logging.debug("Graph size: |V|={0}, |E|={1}.".format(len(G), G.size()))

        L = nx.all_pairs_dijkstra_path_length(G)
        for a, b in combinations(scaffolds, 2):
            if G.has_edge(a, b):
                continue
            l = L[a][b]
            G.add_edge(a, b, weight=l)
            G.add_edge(b, a, weight=l)

        edges = []
        for a, b, d in G.edges(data=True):
            edges.append((a, b, d['weight']))

        tour = hamiltonian(edges, symmetric=False, precision=2)
        assert tour[0] == START and tour[-1] == END
        tour = tour[1:-1]
        return tour

    def assign_order(self):
        """
        The goal is to assign scaffold orders. To help order the scaffolds, two
        dummy node, START and END, mark the ends of the chromosome. We connect
        START to each scaffold (directed), and each scaffold to END.
        """
        linkage_groups = self.linkage_groups
        for mlg in linkage_groups:
            mapname = mlg.mapname
            if mapname == self.pivot:
                pivot_position = mlg.position

        for mlg in linkage_groups:
            position = mlg.position
            # Flip order if path goes in the opposite direction to the pivot
            common = []
            for a, ap in position.items():
                if a not in pivot_position:
                    continue
                pp = pivot_position[a]
                common.append((ap, pp))

            mlg.rho = get_rho(common)
            if mlg.rho < 0:
                mlg.path = mlg.path[::-1]

        # Preparation of TSP
        distances = defaultdict(list)
        for mlg in linkage_groups:
            mapname = mlg.mapname
            position = mlg.position
            length = mlg.length
            path = mlg.path
            rho = mlg.rho
            dd = mlg.distances
            for a, b in combinations(path, 2):
                d = dd[a, b]
                distances[a, b].append((d, mapname))
            for p in path:
                adist, bdist = position[p], length - position[p]
                if rho < 0:
                    adist, bdist = bdist, adist
                distances[START, p].append((adist, mapname))
                distances[p, END].append((bdist, mapname))
        self.distances = distances

        tour = self.distances_to_tour()
        return tour

    def fix_order(self, tour):
        """
        TSP will not produce the ideal order, but quite close. The reason for
        slight inaccuracies is due to scaffold connections (edges) that are
        unique to certain maps. In other words, edges like this will have no
        weight for certain maps. The following fix add those edges back by
        extrapolating distances based on the preliminary order and then
        iteratively perform updates after inserting those weights back.
        """
        distances = self.distances
        tour_index = dict((t, i) for i, t in enumerate(tour))
        links = 0
        for mlg in self.linkage_groups:
            mapname = mlg.mapname
            position = mlg.position
            dd = mlg.distances
            for a, b in pairwise(tour):
                if not a in position:
                    continue
                if b in position:
                    continue
                # Find the closest scaffold in the same lg DOWNSTREAM
                ai = tour_index[a]
                flank = [x for x in tour[ai + 1:] if x in position]
                c = flank[0] if flank else None
                if not c:
                    continue
                ci = tour_index[c]
                distance = dd[c, a]
                d = distance / 2.
                for i in xrange(ai + 1, ci):
                    t = tour[i]
                    for e in ((a, t), (t, a), (t, c), (c, t)):
                        if mapname in [x[-1] for x in distances[e]]:
                            continue
                        distances[e].append((d, mapname))
                        print e[0], "=>", e[1], d, mapname
                        links += 1

        logging.debug("A total of {0} new edges inserted.".format(links))
        if links:
            tour = self.distances_to_tour()
        return tour, links


    def get_orientation(self, si, sj):
        """
        si, sj are two number series. To compute whether these two series have
        same orientation or not. We combine them in the two orientation
        configurations and compute length of the longest monotonous series.
        """
        if not si or not sj:
            return 0
        # Same orientation configuration
        a = longest_monotonous_subseq_length(si + sj)
        b = longest_monotonous_subseq_length(sj + si)
        # Opposite orientation configuration
        c = longest_monotonous_subseq_length(si + sj[::-1])
        d = longest_monotonous_subseq_length(sj[::-1] + si)
        return max(a, b)[0] - max(c, d)[0]

    def assign_orientation(self):
        signs = defaultdict(list)
        scaffolds = self.scaffolds
        for mlg in self.linkage_groups:
            mapname = mlg.mapname
            series = mlg.series
            if mapname == self.pivot:
                pivot_oo = mlg.oo
                pivot_nmarkers = mlg.nmarkers

            for i, j in combinations(range(len(scaffolds)), 2):
                si, sj = scaffolds[i], scaffolds[j]
                si, sj = series.get(si, []), series.get(sj, [])
                d = self.get_orientation(si, sj)
                if not d:
                    continue
                signs[i, j].append((d, mapname))

        for e, v in signs.items():
            signs[e] = self.weighted_mean(v)

        signs_edges = sorted((a, b, w) for (a, b), w in signs.items())
        signs = determine_signs(scaffolds, signs_edges)

        # Finally flip this according to pivot map, then weight by #_markers
        pivot_oo = [pivot_oo.get(x, 0) for x in scaffolds]
        nmarkers = [pivot_nmarkers.get(x, 0) for x in scaffolds]
        flipr = signs * np.sign(np.array(pivot_oo)) * nmarkers
        if sum(flipr) < 0:
            signs = - signs
        return signs

    def fix_orientation(self, tour):
        """
        Test each scaffold if flipping will increass longest monotonous chain
        length.
        """
        orientations = dict(tour)  # old configuration here
        scaffold_oo = defaultdict(list)
        scaffolds, oos = zip(*tour)
        for mlg in self.linkage_groups:
            lg = mlg.lg
            mapname = mlg.mapname
            for s, o in tour:
                i = scaffolds.index(s)
                L = [self.get_series(lg, x, xo) for x, xo in tour[:i]]
                U = [self.get_series(lg, x, xo) for x, xo in tour[i + 1:]]
                L, U = list(flatten(L)), list(flatten(U))
                M = self.get_series(lg, s)
                plus = longest_monotonous_subseq_length(L + M + U)
                minus = longest_monotonous_subseq_length(L + M[::-1] + U)
                d = plus[0] - minus[0]
                if not d:
                    continue
                scaffold_oo[s].append((d, mapname))  # reset orientation

        for s, v in scaffold_oo.items():
            d = self.weighted_mean(v)
            if abs(d) > .5:  # only update when there is good evidence
                orientations[s] = np.sign(d)
            #print s, v, d

        tour = [(x, orientations[x]) for x in scaffolds]
        return tour

    def evaluate(self, tour):
        """
        Return score that correspond to the sum of all distances.
        """
        weights = self.weights
        score = 0
        for mlg in self.linkage_groups:
            position = mlg.position
            mapname = mlg.mapname
            rho = mlg.rho
            length = mlg.length
            dd = mlg.distances
            lg_tour = [x for x in tour if x in position]
            s = sum(dd[a, b] for a, b in pairwise(lg_tour))
            start, end = lg_tour[0], lg_tour[-1]
            if rho < 0:
                start, end = end, start
            adist, bdist = position[start], length - position[end]
            s += adist + bdist
            score += weights[mapname] * s

        return score


class CSVMapLine (object):

    def __init__(self, row, sep=",", mapname=None):
        # ScaffoldID,ScaffoldPosition,LinkageGroup,GeneticPosition
        args = row.strip().split(sep)
        self.seqid = args[0]
        self.pos = int(args[1])
{'fraction_non_alphanumeric': 0.10352292168569534, 'fraction_numerical': 0.004434202716385601, 'mean_word_length': 1.7466436517069428, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7616437', 'n_tokens_mistral': 9079, 'n_tokens_neox': 8517, 'n_words': 2547}