<commit_before>#!/usr/bin/env python

#Copyright (C) 2013 by <NAME>
#
#Released under the MIT license, see LICENSE.txt
import sys
import os
import argparse
import copy
from pybedtools import BedTool, Interval

"""
Replace the ID column with L or R, depending whether its the first or second
instance.

scaffold_1	141	225	1+	43	+
scaffold_1	4479	4563	1+	43	+

becomes

scaffold_1	141	225	L_Term	43	+
scaffold_1	225	4479	R_Term	43	+

"""

def main(argv=None):
    if argv is None:
        argv = sys.argv

    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description="Change ID to L_term or R_term since the HMM can only use the "
        "two states.")
    parser.add_argument("inBed", help="bed with ltr results to process")
    parser.add_argument("outBed", help="bed to write forward strand output to.")
    parser.add_argument("--rev", help="bed to write reverse strand output to.",
                        default=None)
    
    args = parser.parse_args()
    assert os.path.exists(args.inBed)
    outFile = open(args.outBed, "w")
    outFile2 = None
    if args.rev is not None:
        outFile2 = open(args.rev, "w")

    prevInterval = None
    for interval in BedTool(args.inBed):
        
        origInterval = copy.deepcopy(interval)

        # Right termini
        if prevInterval is not None:
            if interval.name != prevInterval.name:
                raise RuntimeError("Consecutive intervals dont have same id"
                                   "\n%s%s" % (prevInterval, interval))

            interval.name = "R_Term"
            prevInterval = None
            
        # Left termini
        else:
            interval.name = "L_Term"
            prevInterval = origInterval
        
        if origInterval.name[-1] == "+":
            outFile.write(str(interval))
        elif args.rev is not None:
            assert origInterval.name[-1] == "-"
            outFile2.write(str(interval))
                    
    outFile.close()
    if outFile2 is not None:
        outFile2.close()
        
if __name__ == "__main__":
    sys.exit(main())<commit_msg>revamp yet again with new options for splitting output into diff files<commit_after>#!/usr/bin/env python

#Copyright (C) 2013 by <NAME>
#
#Released under the MIT license, see LICENSE.txt
import sys
import os
import argparse
import copy
from pybedtools import BedTool, Interval

"""
Replace the ID column with L or R, depending whether its the first or second
instance.

scaffold_1	141	225	1+	43	+
scaffold_1	4479	4563	1+	43	+

becomes

scaffold_1	141	225	L_Term	43	+
scaffold_1	225	4479	R_Term	43	+

Optionally split into up to for files depending on strand and side
"""

def main(argv=None):
    if argv is None:
        argv = sys.argv

    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description="Filter bed of lastz termini for use with HMM.  By default "
        "unique id in name column is changed to L_Term or R_Term")
    parser.add_argument("inBed", help="bed with ltr results to process")
    parser.add_argument("outBed", help="bed to write forward strand output to.")
    parser.add_argument("--splitStrand", help="write forwards strand to <outBed>_f and"
                        " reverse strand to <outBed>_b", action="store_true",
                        default=False)
    parser.add_argument("--splitSide", help="write left termini to <outBed>_l and"
                        " right side to <outBed>_r", action="store_true",
                        default=False)
    parser.add_argument("--leaveName", help="dont change the name column",
                        action="store_true", default=False)
    
    args = parser.parse_args()
    assert os.path.exists(args.inBed)

    # create the 4 file handles for the output. depending on
    # args some of them can actually be same. 
    name, ext = os.path.splitext(args.outBed)
    f, b, l, r = "","","",""
    if args.splitSide is True:
        l, r = "_l", "_r"
    if args.splitStrand is True:
        f, b = "_f", "_b"
    lfPath = "%s%s%s%s" % (name, l, f, ext)
    rfPath = "%s%s%s%s" % (name, r, f, ext)
    lbPath = "%s%s%s%s" % (name, l, b, ext)
    rbPath = "%s%s%s%s" % (name, r, b, ext)
    files = dict()
    def getFile(path):
        if path not in files:
            assert path != args.inBed
            file = open(path, "w")
            files[path] = file            
        return files[path]
    lfFile = getFile(lfPath)
    rfFile = getFile(rfPath)
    lbFile = getFile(lbPath)
    rbFile = getFile(rbPath)

    prevInterval = None
    for interval in BedTool(args.inBed):
        
        origInterval = copy.deepcopy(interval)
        # Right termini
        if prevInterval is not None:
            if interval.name != prevInterval.name:
                raise RuntimeError("Consecutive intervals dont have same id"
                                   "\n%s%s" % (prevInterval, interval))

            if args.leaveName is False:
                interval.name = "R_Term"
            prevInterval = None
            left = False
            
        # Left termini
        else:
            if args.leaveName is False:
                interval.name = "L_Term"
            prevInterval = origInterval
            left = True

        forward = origInterval.name[-1] == "+"

        file = None
        if left:
            if forward:
                file = lfFile
            else:
                file = lbFile
        else:
            if forward:
                file = rfFile
            else:
                file = rbFile
        file.write(str(interval))

    cset = set()
    for file in [lfFile, rfFile, lbFile, rbFile]:
        if file not in cset:
            file.close()
            cset.add(file)
        
if __name__ == "__main__":
    sys.exit(main())
{'fraction_non_alphanumeric': 0.09735720375106564, 'fraction_numerical': 0.01687979539641944, 'mean_word_length': 2.036231884057971, 'pattern_counts': {'":': 3, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3905073', 'n_tokens_mistral': 1864, 'n_tokens_neox': 1753, 'n_words': 549}