<commit_before>import os
from stat import S_ISDIR
from math import ceil
from collections import defaultdict
from time import ctime, time
from math import log10
import matplotlib.pyplot as plt
import numpy as np
from matplotlib.lines import Line2D
from matplotlib.patches import Rectangle
from matplotlib import gridspec
from IPython.display import HTML

from dark import html
from dark.baseimage import BaseImage
from dark.dimension import dimensionalIterator
from dark.html import NCBISequenceLink
from dark.intervals import ReadIntervals
from dark import genbank
from dark import ncbidb
from dark import features

START_CODONS = set(['ATG'])
STOP_CODONS = set(['TAA', 'TAG', 'TGA'])

QUERY_COLORS = {
    'A': (1.0, 0.0, 0.0),  # Red.
    'C': (0.0, 0.0, 1.0),  # Blue.
    'G': (0.0, 1.0, 0.0),  # Green.
    'N': (1.0, 0.0, 1.0),  # Purple.
    'T': (1.0, 0.8, 0.0),  # Orange.
    'gap': (0.2, 0.2, 0.2),  # Almost black.
    'match': (0.9, 0.9, 0.9),  # Almost white.
}

# If we're making a plot that has a log-linear X axis, don't show
# background light grey rectangles for any gap whose (logged) width is less
# than SMALLEST_LOGGED_GAP_TO_DISPLAY.
SMALLEST_LOGGED_GAP_TO_DISPLAY = 20


def report(msg):
    print '%s: %s' % (ctime(time()), msg)


def _sortHTML(hits, by):
    """
    Return an C{IPython.display.HTML} object with the hits sorted by the
    given attribute.

    @param hits: An L{dark.blast.BlastHits} instance.
    @param by: A C{str}, one of 'eMean', 'eMedian', 'readCount', 'title'.
    @return: An HTML instance with sorted titles and information about
        hit read count, length, and e-values.
    """
    out = []
    for i, title in enumerate(hits.sortTitles(by), start=1):
        hitInfo = hits.titles[title]
        link = NCBISequenceLink(title, title)
        out.append(
            '%3d: count=%4d, len=%7d, median(e)=%20s mean(e)=%20s: %s' %
            (i, hitInfo['readCount'], hitInfo['length'],
             hitInfo['eMedian'], hitInfo['eMean'], link))
    return HTML('<pre><tt>' + '<br/>'.join(out) + '</tt></pre>')


def summarizeHitsByMeanEValue(hits):
    """
    Sort hit titles by mean e-value.

    @param hits: An L{dark.blast.BlastHits} instance.
    @return: An C{IPython.display.HTML} instance with hit titles sorted by
        mean e-value.
    """
    return _sortHTML(hits, 'eMean')


def summarizeHitsByMedianEValue(hits):
    """
    Sort hit titles by median e-value.

    @param hits: An L{dark.blast.BlastHits} instance.
    @return: An C{IPython.display.HTML} instance with hit titles sorted by
        median e-value.
    """
    return _sortHTML(hits, 'eMedian')


def summarizeHitsByCount(hits):
    """
    Sort hit titles by read count.

    @param hits: An L{dark.blast.BlastHits} instance.
    @return: An C{IPython.display.HTML} instance with hit titles sorted by
        read count.
    """
    return _sortHTML(hits, 'readCount')


def summarizeHitsByLength(hits):
    """
    Sort hit titles by sequence length.

    @param hits: An L{dark.blast.BlastHits} instance.
    @return: An C{IPython.display.HTML} instance with hit titles sorted by
        mean sequence length.
    """
    return _sortHTML(hits, 'length')


def alignmentGraph(blastHits, title, addQueryLines=True, showFeatures=True,
                   colorQueryBases=False, createFigure=True, showFigure=True,
                   readsAx=None, imageFile=None, quiet=False, idList=False,
                   xRange='subject'):
    """
    Align a set of matching reads against a BLAST hit.

    @param blastHits: A L{dark.blast.BlastHits} instance.
    @param title: A C{str} sequence title that was hit by BLAST. We plot the
        reads that hit this title.
    @param addQueryLines: if C{True}, draw query lines in full (these will then
        be partly overdrawn by the HSP match against the subject). These are
        the 'whiskers' that potentially protrude from each side of a query.
    @param showFeatures: if C{True}, look online for features of the subject
        sequence (given by hitId).
    @param colorQueryBases: if C{True}, color each base of a query string. If
        C{True}, then addQueryLines is meaningless since the whole query is
        shown colored.
    @param createFigure: If C{True}, create a figure and give it a title.
    @param showFigure: If C{True}, show the created figure. Set this to
        C{False} if you're creating a panel of figures or just want to save an
        image (with C{imageFile}).
    @param readsAx: If not None, use this as the subplot for displaying reads.
    @param imageFile: If not None, specifies a filename to write the image to.
    @param quiet: If C{True}, don't print progress / timing output.
    @param idList: a dictionary. The keys is a color and the values is a list
        of read identifiers that should be colored in the respective color.
    @param xRange: set to either 'subject' or 'reads' to indicate the range of
        the X axis.
    """

    assert xRange in ('subject', 'reads'), (
        'xRange must be either "subject" or "reads".')

    start = time()

    titleId = title.split(' ', 1)[0]
    sequence = ncbidb.getSequence(titleId, blastHits.records.blastDb)

    if createFigure:
        dpi = 80
        # width = max(float(len(sequence)) / float(dpi), 25)
        width = 20
        figure = plt.figure(figsize=(width, 20), dpi=dpi)

    createdReadsAx = readsAx is None

    if showFeatures:
        gbSeq = genbank.getSequence(titleId)
        gs = gridspec.GridSpec(4, 1, height_ratios=[2, 1, 1, 12])
        featureAx = plt.subplot(gs[0, 0])
        orfAx = plt.subplot(gs[1, 0])
        orfReversedAx = plt.subplot(gs[2, 0])
        readsAx = readsAx or plt.subplot(gs[3, 0])
    else:
        featureEndpoints = []
        readsAx = readsAx or plt.subplot(111)

    params = blastHits.plotParams
    logLinearXAxis = params['logLinearXAxis']
    rankEValues = params['rankEValues']

    plotInfo = blastHits.titles[title]['plotInfo']
    items = plotInfo['items']
    maxEIncludingRandoms = int(ceil(plotInfo['maxEIncludingRandoms']))
    maxE = int(ceil(plotInfo['maxE']))
    minE = int(plotInfo['minE'])
    maxX = plotInfo['maxX']
    minX = plotInfo['minX']

    # Add light grey vertical rectangles to show the logarithmic gaps. Add
    # these first so that reads will be plotted on top of them. Only draw
    # gaps that are more than 20 pixels wide as we could have millions of
    # tiny gaps for a bacteria and drawing them all will be slow and only
    # serves to make the entire background grey.
    if logLinearXAxis and len(plotInfo['offsetAdjuster'].adjustments()) < 100:
        offsetAdjuster = plotInfo['offsetAdjuster'].adjustOffset
        for (intervalType, interval) in plotInfo['readIntervals'].walk():
            if intervalType == ReadIntervals.EMPTY:
                adjustedStart = offsetAdjuster(interval[0])
                adjustedStop = offsetAdjuster(interval[1])
                width = adjustedStop - adjustedStart
                if width >= SMALLEST_LOGGED_GAP_TO_DISPLAY:
                    rect = Rectangle(
                        (adjustedStart, 0), width,
                        maxEIncludingRandoms + 1, color='#f4f4f4')
                    readsAx.add_patch(rect)

    if colorQueryBases:
        # Color each query by its bases.
        xScale = 3
        yScale = 2
        baseImage = BaseImage(
            maxX - minX,
            maxEIncludingRandoms - minE + (1 if rankEValues else 0),
            xScale, yScale)
        for item in items:
            hsp = item['hsp']
            e = item['convertedE'] - minE
            # If the product of the subject and query frame values is +ve,
            # then they're either both +ve or both -ve, so we just use the
            # query as is. Otherwise, we need to reverse complement it.
            if item['frame']['subject'] * item['frame']['query'] > 0:
                query = blastHits.fasta[item['readNum']].seq
            else:
                # One of the subject or query has negative sense.
                query = blastHits.fasta[
                    item['readNum']].reverse_complement().seq
            queryStart = hsp['queryStart']
            # There are 3 parts of the query string we need to display. 1)
            # the left part (if any) before the matched part of the
            # subject. 2) the matched part (which can include gaps in the
            # query and/or subject). 3) the right part (if any) after the
            # matched part.

            # 1. Left part.
            xOffset = queryStart - minX
            queryOffset = 0
            for queryIndex in xrange(hsp['subjectStart'] - queryStart):
                color = QUERY_COLORS[query[queryOffset + queryIndex]]
                baseImage.set(xOffset + queryIndex, e, color)

            # 2. Match part.
            xOffset = hsp['subjectStart'] - minX
            xIndex = 0
            queryOffset = hsp['subjectStart'] - hsp['queryStart']
            origSubject = item['origHsp'].sbjct
            origQuery = item['origHsp'].query
            for matchIndex in xrange(len(origSubject)):
                if origSubject[matchIndex] == '-':
                    # A gap in the subject was needed to match the query.
                    # In our graph we keep the subject the same even in the
                    # case where BLAST opened gaps in it, so we compensate
                    # for the gap in the subject by not showing this base
                    # of the query.
                    pass
                else:
                    if origSubject[matchIndex] == origQuery[matchIndex]:
                        # The query matched the subject at this location.
                        # Matching bases are all colored in the same
                        # 'match' color.
                        color = QUERY_COLORS['match']
                    else:
                        if origQuery[matchIndex] == '-':
                            # A gap in the query. All query gaps get the
                            # same 'gap' color.
                            color = QUERY_COLORS['gap']
                        else:
                            # Query doesn't match subject (and is not a gap).
                            color = QUERY_COLORS[origQuery[matchIndex]]
                    baseImage.set(xOffset + xIndex, e, color)
                    xIndex += 1

            # 3. Right part.
            xOffset = hsp['subjectEnd'] - minX
            queryOffset = hsp['subjectEnd'] - hsp['queryStart']
            for queryIndex in xrange(hsp['queryEnd'] - hsp['subjectEnd']):
                color = QUERY_COLORS[query[queryOffset + queryIndex]]
                baseImage.set(xOffset + queryIndex, e, color)

                # a gap in the subject was needed to match a different query
                # than the one we're matching against.
                # this throws off the indexing.
                # if this happen, show a gap when coloring in the base of the
                # query as gap.

        readsAx.imshow(baseImage.data, aspect='auto', origin='lower',
                       interpolation='nearest',
                       extent=[minX, maxX, minE, maxEIncludingRandoms])
    else:
        # Add horizontal lines for all the query sequences. These will be the
        # grey 'whiskers' in the plots once we (below) draw the matched part
        # on top of part of them.
        if addQueryLines:
            for item in items:
                e = item['convertedE']
                hsp = item['hsp']
                line = Line2D([hsp['queryStart'], hsp['queryEnd']], [e, e],
                              color='#aaaaaa')
                readsAx.add_line(line)

        # Add the horizontal BLAST alignment lines.
        for item in items:
            e = item['convertedE']
            hsp = item['hsp']
            line = Line2D([hsp['subjectStart'], hsp['subjectEnd']], [e, e],
                          color='blue')
            readsAx.add_line(line)

        # TODO: Barbara, add some kind of comment here please. I think this
        # could be done more efficiently.
        if idList:
            for item in items:
                for key in idList:
                    for ids in idList[key]:
                        if ids == item['query']:
                            e = item['convertedE']
                            hsp = item['hsp']
                            line = Line2D([hsp['subjectStart'],
                                           hsp['subjectEnd']], [e, e],
                                          color=key)
                            readsAx.add_line(line)

    # Add to ORF figures and add vertical lines for the sequence features.
    # The feature and ORF display are linked and they shouldn't be. This
    # code is a horrible mess.
    if showFeatures:
        if logLinearXAxis:
            offsetAdjuster = plotInfo['offsetAdjuster'].adjustOffset
        else:
            offsetAdjuster = lambda x: x

        featureEndpoints = features.addFeatures(featureAx, gbSeq, minX, maxX,
                                                offsetAdjuster)
        if len(featureEndpoints) < 20:
            for fe in featureEndpoints:
                readsAx.axvline(x=fe['start'], color=fe['color'])
                readsAx.axvline(x=fe['end'], color='#cccccc')
            features.addORFs(orfAx, sequence.seq, minX, maxX, featureEndpoints,
                             offsetAdjuster)
        else:
            features.addORFs(orfAx, sequence.seq, minX, maxX, [],
                             offsetAdjuster)

        features.addReversedORFs(orfReversedAx,
                                 sequence.reverse_complement().seq,
                                 minX, maxX, offsetAdjuster)

    # We'll return some information we've gathered.
    result = {
        'features': featureEndpoints,
    }

    # Add the horizontal divider between the highest e-value and the randomly
    # higher ones (if any).
    if plotInfo['zeroEValueFound']:
        readsAx.axhline(y=maxE + 1, color='#cccccc', linewidth=1)

    # Titles, axis, etc.
    if createFigure:
        readCount = blastHits.titles[title]['readCount']
        hspTotal = plotInfo['hspTotal']
        figure.suptitle(
            '%s\nLength %d, %d read%s hit, %d HSP%s in total (%d shown).' %
            (
                sequence.description, len(sequence),
                readCount, '' if readCount == 1 else 's',
                hspTotal, '' if hspTotal == 1 else 's',
                len(plotInfo['items'])<commit_msg>Draw line between max E value and first ranked zero e value in between the two reads, not on top of the first ranked one.<commit_after>import os
from stat import S_ISDIR
from math import ceil
from collections import defaultdict
from time import ctime, time
from math import log10
import matplotlib.pyplot as plt
import numpy as np
from matplotlib.lines import Line2D
from matplotlib.patches import Rectangle
from matplotlib import gridspec
from IPython.display import HTML

from dark import html
from dark.baseimage import BaseImage
from dark.dimension import dimensionalIterator
from dark.html import NCBISequenceLink
from dark.intervals import ReadIntervals
from dark import genbank
from dark import ncbidb
from dark import features

START_CODONS = set(['ATG'])
STOP_CODONS = set(['TAA', 'TAG', 'TGA'])

QUERY_COLORS = {
    'A': (1.0, 0.0, 0.0),  # Red.
    'C': (0.0, 0.0, 1.0),  # Blue.
    'G': (0.0, 1.0, 0.0),  # Green.
    'N': (1.0, 0.0, 1.0),  # Purple.
    'T': (1.0, 0.8, 0.0),  # Orange.
    'gap': (0.2, 0.2, 0.2),  # Almost black.
    'match': (0.9, 0.9, 0.9),  # Almost white.
}

# If we're making a plot that has a log-linear X axis, don't show
# background light grey rectangles for any gap whose (logged) width is less
# than SMALLEST_LOGGED_GAP_TO_DISPLAY.
SMALLEST_LOGGED_GAP_TO_DISPLAY = 20


def report(msg):
    print '%s: %s' % (ctime(time()), msg)


def _sortHTML(hits, by):
    """
    Return an C{IPython.display.HTML} object with the hits sorted by the
    given attribute.

    @param hits: An L{dark.blast.BlastHits} instance.
    @param by: A C{str}, one of 'eMean', 'eMedian', 'readCount', 'title'.
    @return: An HTML instance with sorted titles and information about
        hit read count, length, and e-values.
    """
    out = []
    for i, title in enumerate(hits.sortTitles(by), start=1):
        hitInfo = hits.titles[title]
        link = NCBISequenceLink(title, title)
        out.append(
            '%3d: count=%4d, len=%7d, median(e)=%20s mean(e)=%20s: %s' %
            (i, hitInfo['readCount'], hitInfo['length'],
             hitInfo['eMedian'], hitInfo['eMean'], link))
    return HTML('<pre><tt>' + '<br/>'.join(out) + '</tt></pre>')


def summarizeHitsByMeanEValue(hits):
    """
    Sort hit titles by mean e-value.

    @param hits: An L{dark.blast.BlastHits} instance.
    @return: An C{IPython.display.HTML} instance with hit titles sorted by
        mean e-value.
    """
    return _sortHTML(hits, 'eMean')


def summarizeHitsByMedianEValue(hits):
    """
    Sort hit titles by median e-value.

    @param hits: An L{dark.blast.BlastHits} instance.
    @return: An C{IPython.display.HTML} instance with hit titles sorted by
        median e-value.
    """
    return _sortHTML(hits, 'eMedian')


def summarizeHitsByCount(hits):
    """
    Sort hit titles by read count.

    @param hits: An L{dark.blast.BlastHits} instance.
    @return: An C{IPython.display.HTML} instance with hit titles sorted by
        read count.
    """
    return _sortHTML(hits, 'readCount')


def summarizeHitsByLength(hits):
    """
    Sort hit titles by sequence length.

    @param hits: An L{dark.blast.BlastHits} instance.
    @return: An C{IPython.display.HTML} instance with hit titles sorted by
        mean sequence length.
    """
    return _sortHTML(hits, 'length')


def alignmentGraph(blastHits, title, addQueryLines=True, showFeatures=True,
                   colorQueryBases=False, createFigure=True, showFigure=True,
                   readsAx=None, imageFile=None, quiet=False, idList=False,
                   xRange='subject'):
    """
    Align a set of matching reads against a BLAST hit.

    @param blastHits: A L{dark.blast.BlastHits} instance.
    @param title: A C{str} sequence title that was hit by BLAST. We plot the
        reads that hit this title.
    @param addQueryLines: if C{True}, draw query lines in full (these will then
        be partly overdrawn by the HSP match against the subject). These are
        the 'whiskers' that potentially protrude from each side of a query.
    @param showFeatures: if C{True}, look online for features of the subject
        sequence (given by hitId).
    @param colorQueryBases: if C{True}, color each base of a query string. If
        C{True}, then addQueryLines is meaningless since the whole query is
        shown colored.
    @param createFigure: If C{True}, create a figure and give it a title.
    @param showFigure: If C{True}, show the created figure. Set this to
        C{False} if you're creating a panel of figures or just want to save an
        image (with C{imageFile}).
    @param readsAx: If not None, use this as the subplot for displaying reads.
    @param imageFile: If not None, specifies a filename to write the image to.
    @param quiet: If C{True}, don't print progress / timing output.
    @param idList: a dictionary. The keys is a color and the values is a list
        of read identifiers that should be colored in the respective color.
    @param xRange: set to either 'subject' or 'reads' to indicate the range of
        the X axis.
    """

    assert xRange in ('subject', 'reads'), (
        'xRange must be either "subject" or "reads".')

    start = time()

    titleId = title.split(' ', 1)[0]
    sequence = ncbidb.getSequence(titleId, blastHits.records.blastDb)

    if createFigure:
        dpi = 80
        # width = max(float(len(sequence)) / float(dpi), 25)
        width = 20
        figure = plt.figure(figsize=(width, 20), dpi=dpi)

    createdReadsAx = readsAx is None

    if showFeatures:
        gbSeq = genbank.getSequence(titleId)
        gs = gridspec.GridSpec(4, 1, height_ratios=[2, 1, 1, 12])
        featureAx = plt.subplot(gs[0, 0])
        orfAx = plt.subplot(gs[1, 0])
        orfReversedAx = plt.subplot(gs[2, 0])
        readsAx = readsAx or plt.subplot(gs[3, 0])
    else:
        featureEndpoints = []
        readsAx = readsAx or plt.subplot(111)

    params = blastHits.plotParams
    logLinearXAxis = params['logLinearXAxis']
    rankEValues = params['rankEValues']

    plotInfo = blastHits.titles[title]['plotInfo']
    items = plotInfo['items']
    maxEIncludingRandoms = int(ceil(plotInfo['maxEIncludingRandoms']))
    maxE = int(ceil(plotInfo['maxE']))
    minE = int(plotInfo['minE'])
    maxX = plotInfo['maxX']
    minX = plotInfo['minX']

    # Add light grey vertical rectangles to show the logarithmic gaps. Add
    # these first so that reads will be plotted on top of them. Only draw
    # gaps that are more than 20 pixels wide as we could have millions of
    # tiny gaps for a bacteria and drawing them all will be slow and only
    # serves to make the entire background grey.
    if logLinearXAxis and len(plotInfo['offsetAdjuster'].adjustments()) < 100:
        offsetAdjuster = plotInfo['offsetAdjuster'].adjustOffset
        for (intervalType, interval) in plotInfo['readIntervals'].walk():
            if intervalType == ReadIntervals.EMPTY:
                adjustedStart = offsetAdjuster(interval[0])
                adjustedStop = offsetAdjuster(interval[1])
                width = adjustedStop - adjustedStart
                if width >= SMALLEST_LOGGED_GAP_TO_DISPLAY:
                    rect = Rectangle(
                        (adjustedStart, 0), width,
                        maxEIncludingRandoms + 1, color='#f4f4f4')
                    readsAx.add_patch(rect)

    if colorQueryBases:
        # Color each query by its bases.
        xScale = 3
        yScale = 2
        baseImage = BaseImage(
            maxX - minX,
            maxEIncludingRandoms - minE + (1 if rankEValues else 0),
            xScale, yScale)
        for item in items:
            hsp = item['hsp']
            e = item['convertedE'] - minE
            # If the product of the subject and query frame values is +ve,
            # then they're either both +ve or both -ve, so we just use the
            # query as is. Otherwise, we need to reverse complement it.
            if item['frame']['subject'] * item['frame']['query'] > 0:
                query = blastHits.fasta[item['readNum']].seq
            else:
                # One of the subject or query has negative sense.
                query = blastHits.fasta[
                    item['readNum']].reverse_complement().seq
            queryStart = hsp['queryStart']
            # There are 3 parts of the query string we need to display. 1)
            # the left part (if any) before the matched part of the
            # subject. 2) the matched part (which can include gaps in the
            # query and/or subject). 3) the right part (if any) after the
            # matched part.

            # 1. Left part.
            xOffset = queryStart - minX
            queryOffset = 0
            for queryIndex in xrange(hsp['subjectStart'] - queryStart):
                color = QUERY_COLORS[query[queryOffset + queryIndex]]
                baseImage.set(xOffset + queryIndex, e, color)

            # 2. Match part.
            xOffset = hsp['subjectStart'] - minX
            xIndex = 0
            queryOffset = hsp['subjectStart'] - hsp['queryStart']
            origSubject = item['origHsp'].sbjct
            origQuery = item['origHsp'].query
            for matchIndex in xrange(len(origSubject)):
                if origSubject[matchIndex] == '-':
                    # A gap in the subject was needed to match the query.
                    # In our graph we keep the subject the same even in the
                    # case where BLAST opened gaps in it, so we compensate
                    # for the gap in the subject by not showing this base
                    # of the query.
                    pass
                else:
                    if origSubject[matchIndex] == origQuery[matchIndex]:
                        # The query matched the subject at this location.
                        # Matching bases are all colored in the same
                        # 'match' color.
                        color = QUERY_COLORS['match']
                    else:
                        if origQuery[matchIndex] == '-':
                            # A gap in the query. All query gaps get the
                            # same 'gap' color.
                            color = QUERY_COLORS['gap']
                        else:
                            # Query doesn't match subject (and is not a gap).
                            color = QUERY_COLORS[origQuery[matchIndex]]
                    baseImage.set(xOffset + xIndex, e, color)
                    xIndex += 1

            # 3. Right part.
            xOffset = hsp['subjectEnd'] - minX
            queryOffset = hsp['subjectEnd'] - hsp['queryStart']
            for queryIndex in xrange(hsp['queryEnd'] - hsp['subjectEnd']):
                color = QUERY_COLORS[query[queryOffset + queryIndex]]
                baseImage.set(xOffset + queryIndex, e, color)

                # a gap in the subject was needed to match a different query
                # than the one we're matching against.
                # this throws off the indexing.
                # if this happen, show a gap when coloring in the base of the
                # query as gap.

        readsAx.imshow(baseImage.data, aspect='auto', origin='lower',
                       interpolation='nearest',
                       extent=[minX, maxX, minE, maxEIncludingRandoms])
    else:
        # Add horizontal lines for all the query sequences. These will be the
        # grey 'whiskers' in the plots once we (below) draw the matched part
        # on top of part of them.
        if addQueryLines:
            for item in items:
                e = item['convertedE']
                hsp = item['hsp']
                line = Line2D([hsp['queryStart'], hsp['queryEnd']], [e, e],
                              color='#aaaaaa')
                readsAx.add_line(line)

        # Add the horizontal BLAST alignment lines.
        for item in items:
            e = item['convertedE']
            hsp = item['hsp']
            line = Line2D([hsp['subjectStart'], hsp['subjectEnd']], [e, e],
                          color='blue')
            readsAx.add_line(line)

        # TODO: Barbara, add some kind of comment here please. I think this
        # could be done more efficiently.
        if idList:
            for item in items:
                for key in idList:
                    for ids in idList[key]:
                        if ids == item['query']:
                            e = item['convertedE']
                            hsp = item['hsp']
                            line = Line2D([hsp['subjectStart'],
                                           hsp['subjectEnd']], [e, e],
                                          color=key)
                            readsAx.add_line(line)

    # Add to ORF figures and add vertical lines for the sequence features.
    # The feature and ORF display are linked and they shouldn't be. This
    # code is a horrible mess.
    if showFeatures:
        if logLinearXAxis:
            offsetAdjuster = plotInfo['offsetAdjuster'].adjustOffset
        else:
            offsetAdjuster = lambda x: x

        featureEndpoints = features.addFeatures(featureAx, gbSeq, minX, maxX,
                                                offsetAdjuster)
        if len(featureEndpoints) < 20:
            for fe in featureEndpoints:
                readsAx.axvline(x=fe['start'], color=fe['color'])
                readsAx.axvline(x=fe['end'], color='#cccccc')
            features.addORFs(orfAx, sequence.seq, minX, maxX, featureEndpoints,
                             offsetAdjuster)
        else:
            features.addORFs(orfAx, sequence.seq, minX, maxX, [],
                             offsetAdjuster)

        features.addReversedORFs(orfReversedAx,
                                 sequence.reverse_complement().seq,
                                 minX, maxX, offsetAdjuster)

    # We'll return some information we've gathered.
    result = {
        'features': featureEndpoints,
    }

    # Add the horizontal divider between the highest e-value and the randomly
    # higher ones (if any).
    if plotInfo['zeroEValueFound']:
        readsAx.axhline(y=maxE + 0.5, color='#cccccc', linewidth=1)

    # Titles, axis, etc.
    if createFigure:
        readCount = blastHits.titles[title]['readCount']
        hspTotal = plotInfo['hspTotal']
        figure.suptitle(
            '%s\nLength %d, %d read%s hit, %d HSP%s in total (%d shown).' %
            (
                sequence.description, len(sequence),
                readCount, '' if readCount == 1 else 's',
                hspTotal, '' if hspTotal == 1 else 's',
                len(plotInfo['items'])
{'fraction_non_alphanumeric': 0.09750853242320819, 'fraction_numerical': 0.008156996587030717, 'mean_word_length': 1.9847203830090658, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4577702', 'n_tokens_mistral': 8696, 'n_tokens_neox': 8163, 'n_words': 2769}