<commit_before>#!/usr/bin/env python
# encoding: utf-8

"""
make_levenshtein_tags.py

Created by <NAME> on 28 May 2010 23:27 PDT (-0700).
Copyright (c) 2010 <NAME>. All rights reserved.

USAGE:   python tag_maker.py --tag-length=8 --edit-distance=3 \
            --no-polybase --gc --comp --use-c --multiprocessing \
            --min-and-greater | tee 8_nt.txt
"""

import pdb
import os
import sys
import string
import itertools
import optparse
import numpy
import multiprocessing
import operator
import tempfile
import cPickle
from operator import itemgetter
# tested re2 DFA module in place of re2 here, but there were no substantial 
# performance gains for simple regex
import re
try:
    from Levenshtein import distance
    from Levenshtein import hamming
except:
    from lib.levenshtein import distance
    from lib.levenshtein import hamming


def interface():
    '''Command-line interface'''
    usage = "usage: %prog [options]"

    p = optparse.OptionParser(usage)
    
    p.add_option('--output', dest = 'output', action='store', type='string', 
default = None, help='The path to the file where you want to store the '+
'barcodes', metavar='FILE')

    p.add_option('--tag-length', dest = 'tl', action='store', 
type='int', default = 6, help='The desired tag length')
    
    p.add_option('--edit-distance', dest = 'ed', action='store', 
type='int', default = 3, help='The desired edit distance')

    p.add_option('--multiprocessing', dest = 'multiprocessing', 
action='store_true', default=False, help='Use multiprocessing')

    p.add_option('--processors', dest = 'nprocs', action='store', 
type='int', default = 6, help='The number of processing cores to use when' +
'using multiprocessing')

    p.add_option('--no-polybase', dest = 'polybase', action='store_true', default=False, 
help='Remove tags with > 2 identical nucleotides in a row')

    p.add_option('--gc', dest = 'gc', action='store_true', default=False, 
help='Remove tags with GC content (%) 40 > x > 60')

    p.add_option('--comp', dest = 'comp', action='store_true', default=False, 
help='Remove tags that are perfect self-complements')
    
    p.add_option('--use-c', dest = 'clev', action='store_true', default=False, 
help='Use the C version of Levenshtein (faster)')

    p.add_option('--min-and-greater', dest = 'greater', action='store_true', default=False, 
help='Show tags at all integer values of edit distance > that specified')

    p.add_option('--rescan', dest = 'rescan', action='store', type='string', 
default = None, help='Rescan a file')

    p.add_option('--rescan-length', dest = 'rescan_length', action='store', type='int', 
default = 6, help='Rescan length')


    (options,arg) = p.parse_args()
    if not options.tl:
        p.print_help()
        sys.exit(2)
    return options, arg

def pickler(stuff_to_dump, count = None):
    """ multiprocessing.Queue() doesn't dig having a lot of shit shoved through
    it (in terms of data volume).  So, although our remaining data sets are
    usually small, pickle them, store them in a temp file, and we'll
    get to them after the multiprocessing run.
    """
    if count:
        fd, tf = tempfile.mkstemp(suffix='.{0}.temptext'.format(count))
    else:
        fd, tf = tempfile.mkstemp(suffix='.temptext')
    os.close(fd)
    dump_file = open(tf, 'w')
    cPickle.dump(stuff_to_dump, dump_file)
    dump_file.close()
    return tf

def worker(tasks, results, edit_distance, tag_length):
    """we are going to generate a summary array of edit distances from of one 
    tag against all other tags like so, where the index position is equiv. to 
    the edit distance:
    
    [1, 12, 124, 5, 76]<commit_msg>adding some error checking for nprocs and setting default proc count<commit_after>#!/usr/bin/env python
# encoding: utf-8

"""
make_levenshtein_tags.py

Created by <NAME> on 28 May 2010 23:27 PDT (-0700).
Copyright (c) 2010 <NAME>. All rights reserved.

USAGE:   python tag_maker.py --tag-length=8 --edit-distance=3 \
            --no-polybase --gc --comp --use-c --multiprocessing \
            --min-and-greater | tee 8_nt.txt
"""

import pdb
import os
import sys
import string
import itertools
import optparse
import numpy
import multiprocessing
import operator
import tempfile
import cPickle
from operator import itemgetter
# tested re2 DFA module in place of re2 here, but there were no substantial 
# performance gains for simple regex
import re
try:
    from Levenshtein import distance
    from Levenshtein import hamming
except:
    from lib.levenshtein import distance
    from lib.levenshtein import hamming


def interface():
    '''Command-line interface'''
    usage = "usage: %prog [options]"

    p = optparse.OptionParser(usage)
    
    p.add_option('--output', dest = 'output', action='store', type='string', 
default = None, help='The path to the file where you want to store the '+
'barcodes', metavar='FILE')

    p.add_option('--tag-length', dest = 'tl', action='store', 
type='int', default = 6, help='The desired tag length')
    
    p.add_option('--edit-distance', dest = 'ed', action='store', 
type='int', default = 3, help='The desired edit distance')

    p.add_option('--multiprocessing', dest = 'multiprocessing', 
action='store_true', default=False, help='Use multiprocessing')

    p.add_option('--processors', dest = 'nprocs', action='store', 
type='int', default = None, help='The number of processing cores to use when' +
' using multiprocessing.  Default is # of cores - 2')

    p.add_option('--no-polybase', dest = 'polybase', action='store_true', default=False, 
help='Remove tags with > 2 identical nucleotides in a row')

    p.add_option('--gc', dest = 'gc', action='store_true', default=False, 
help='Remove tags with GC content (%) 40 > x > 60')

    p.add_option('--comp', dest = 'comp', action='store_true', default=False, 
help='Remove tags that are perfect self-complements')
    
    # c is on by default now
    #p.add_option('--use-c', dest = 'clev', action='store_true', default=False, 
#help='Use the C version of Levenshtein (faster)')

    p.add_option('--min-and-greater', dest = 'greater', action='store_true', default=False, 
help='Show tags at all integer values of edit distance > that specified')

    p.add_option('--rescan', dest = 'rescan', action='store', type='string', 
default = None, help='Rescan a file')

    p.add_option('--rescan-length', dest = 'rescan_length', action='store', type='int', 
default = 6, help='Rescan length')

        

    (options,arg) = p.parse_args()
    assert options.nprocs <= multiprocessing.cpu_count(), \
        "Processors count must equal those available"
    # set the number of processors by default
    if options.multiprocessing and not options.nprocs:
        options.nprocs = multiprocessing.cpu_count() - 2
    if not options.tl:
        p.print_help()
        sys.exit(2)
    return options, arg

def pickler(stuff_to_dump, count = None):
    """ multiprocessing.Queue() doesn't dig having a lot of shit shoved through
    it (in terms of data volume).  So, although our remaining data sets are
    usually small, pickle them, store them in a temp file, and we'll
    get to them after the multiprocessing run.
    """
    if count:
        fd, tf = tempfile.mkstemp(suffix='.{0}.temptext'.format(count))
    else:
        fd, tf = tempfile.mkstemp(suffix='.temptext')
    os.close(fd)
    dump_file = open(tf, 'w')
    cPickle.dump(stuff_to_dump, dump_file)
    dump_file.close()
    return tf

def worker(tasks, results, edit_distance, tag_length):
    """we are going to generate a summary array of edit distances from of one 
    tag against all other tags like so, where the index position is equiv. to 
    the edit distance:
    
    [1, 12, 124, 5, 76]
{'fraction_non_alphanumeric': 0.13033939863208155, 'fraction_numerical': 0.011485352948767583, 'mean_word_length': 3.632396891811118, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 2, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2740950', 'n_tokens_mistral': 2559, 'n_tokens_neox': 2405, 'n_words': 796}