<commit_before>#!/usr/bin/env python

#Copyright (C) 2013 by <NAME>
#
#Released under the MIT license, see LICENSE.txt
import sys
import os
import argparse
import re

from pybedtools import BedTool, Interval

"""
Remove everything past the first | and / in the name column
"""

def main(argv=None):
    if argv is None:
        argv = sys.argv

    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description="Cut names off at first | and or /")
    parser.add_argument("inBed", help="bed with chaux results to process")
    parser.add_argument("--keepSlash", help="dont strip anything after slash "
                        "ex: DNA/HELITRONY1C -> DNA", action="store_true",
                        default=False)
    parser.add_argument("--leaveNumbers", help="by default, numbers as the end"
                        " of names are trimmed off.  ex: DNA/HELITRONY1C -> "
                        " DNA/HELITRONY. This option disables this behaviour",
                        default=False)
    parser.add_argument("--mapPrefix", help="Rename all strings with given "
                        "prefix to just the prefix. ex: --mapPrefix DNA/HELI"
                        " would cause any instance of DNA/HELITRONY1C or "
                        "HELITRON2 to be mapped to just DNA/HELI.  This option"
                        " overrides --keepSlash and --leaveNumbers for the"
                        " elements to which it applies.  This option can be"
                        " specified more than once. ex --mapPrefix DNA/HELI "
                        "--maxPrefix DNA/ASINE.", action="append")

    args = parser.parse_args()
    assert os.path.exists(args.inBed)

    for interval in BedTool(args.inBed).sort():
        prefix = findPrefix(interval.name, args.mapPrefix)
        if prefix is not None:
            # prefix was specified with --mapPrefix, that's what we use
            interval.name = prefix
        else:
            # otherwise, strip after |
            interval.name = interval.name[:interval.name.find("|")]
            # strip after "/" unless told not to
            if "/" in interval.name and args.keepSlash is False:
                interval.name = interval.name[:interval.name.find("/")]
            # strip trailing digits (and anything after) unless told not to
            if args.leaveNumbers is False:
                m = re.search("\d", interval.name)
                if m is not None:
                    interval.name = interval.name[:m.start()]
        
        sys.stdout.write(str(interval))


# should be a prefix tree...
def findPrefix(name, plist):
    if plist is None:
        return None
    for p in plist:
        if name[:len(p)] == p:
            return p
    return None

        
if __name__ == "__main__":<commit_msg>add score filter to cleanchaux<commit_after>#!/usr/bin/env python

#Copyright (C) 2013 by <NAME>
#
#Released under the MIT license, see LICENSE.txt
import sys
import os
import argparse
import re

from pybedtools import BedTool, Interval

"""
Remove everything past the first | and / in the name column
"""

def main(argv=None):
    if argv is None:
        argv = sys.argv

    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description="Cut names off at first | and or /")
    parser.add_argument("inBed", help="bed with chaux results to process")
    parser.add_argument("--keepSlash", help="dont strip anything after slash "
                        "ex: DNA/HELITRONY1C -> DNA", action="store_true",
                        default=False)
    parser.add_argument("--leaveNumbers", help="by default, numbers as the end"
                        " of names are trimmed off.  ex: DNA/HELITRONY1C -> "
                        " DNA/HELITRONY. This option disables this behaviour",
                        default=False)
    parser.add_argument("--mapPrefix", help="Rename all strings with given "
                        "prefix to just the prefix. ex: --mapPrefix DNA/HELI"
                        " would cause any instance of DNA/HELITRONY1C or "
                        "HELITRON2 to be mapped to just DNA/HELI.  This option"
                        " overrides --keepSlash and --leaveNumbers for the"
                        " elements to which it applies.  This option can be"
                        " specified more than once. ex --mapPrefix DNA/HELI "
                        "--maxPrefix DNA/ASINE.", action="append")
    parser.add_argument("--minScore", help="Minimum score value to not filter"
                        " out", default=100, type=float)

    args = parser.parse_args()
    assert os.path.exists(args.inBed)

    for interval in BedTool(args.inBed).sort():
        # filter score if exists
        if interval.score is not None and\
          float(interval.score) < args.minScore:
            continue
        prefix = findPrefix(interval.name, args.mapPrefix)
        if prefix is not None:
            # prefix was specified with --mapPrefix, that's what we use
            interval.name = prefix
        else:
            # otherwise, strip after |
            interval.name = interval.name[:interval.name.find("|")]
            # strip after "/" unless told not to
            if "/" in interval.name and args.keepSlash is False:
                interval.name = interval.name[:interval.name.find("/")]
            # strip trailing digits (and anything after) unless told not to
            if args.leaveNumbers is False:
                m = re.search("\d", interval.name)
                if m is not None:
                    interval.name = interval.name[:m.start()]
        
        sys.stdout.write(str(interval))


# should be a prefix tree...
def findPrefix(name, plist):
    if plist is None:
        return None
    for p in plist:
        if name[:len(p)] == p:
            return p
    return None

        
if __name__ == "__main__":
{'fraction_non_alphanumeric': 0.09659567331879926, 'fraction_numerical': 0.0031863156129465033, 'mean_word_length': 1.875602700096432, 'pattern_counts': {'":': 2, '<': 6, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '759764', 'n_tokens_mistral': 1644, 'n_tokens_neox': 1547, 'n_words': 526}