<commit_before>from obj.misc   import ranges_to_set
from obj.logger import log

import codecs

def clean_info(infile, outfile, ranges,
               ienc="utf-8", oenc="utf-8", verbose=False):
    allowed = ranges_to_set(ranges, codecs.open(infile, "rU", ienc).readline().strip().split())
    
    if verbose:
        log('Cleaning "%s"...' %infile)
        
    with codecs.open(outfile, "w", oenc) as O:
        for line in codecs.open(infile, "rU", ienc):
            line = line.strip().split()
            if line != []:
                tokens = [line[i] for i in xrange(len(line)) if i in allowed]
                O.write(u"\t".join(tokens))
            O.write(u"\n")
    
    if verbose:
        log(' Done.\n')

if __name__ == "__main__":
    import argparse, sys

    parser = argparse.ArgumentParser(description="...")

    parser.add_argument("infile",
                        help="The input file")
    parser.add_argument("outfile",
                        help="The output file ")
    parser.add_argument("ranges",
                        help="The ranges of indexes to keep in the file.")
    parser.add_argument("--input-encoding", dest="ienc",
                        help="Encoding of the input (default: UTF-8)")
    parser.add_argument("--output-encoding", dest="oenc",
                        help="Encoding of the input (default: UTF-8)")
    parser.add_argument("--encoding", dest="enc", default="UTF-8",
                        help="Encoding of both the input and the output (default: UTF-8)")
    parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
                        help="Basic feedback for user (default: False).")
    
    arguments = (sys.argv[2:] if __package__ else sys.argv)
    parser    = parser.parse_args(arguments)
    
    enrich(parser.infile, parser.outfile, parser.ranges,
           ienc=parser.ienc or parser.enc, oenc=parser.oenc or parser.enc, verbose=parser.verbose)
    sys.exit(0)<commit_msg>Corrected a bug in clean_info that prevented an user from calling it from command-line.<commit_after>from obj.misc   import ranges_to_set
from obj.logger import log

import codecs

def clean_info(infile, outfile, ranges,
               ienc="utf-8", oenc="utf-8", verbose=False):
    allowed = ranges_to_set(ranges, codecs.open(infile, "rU", ienc).readline().strip().split())
    
    if verbose:
        log('Cleaning "%s"...' %infile)
        
    with codecs.open(outfile, "w", oenc) as O:
        for line in codecs.open(infile, "rU", ienc):
            line = line.strip().split()
            if line != []:
                tokens = [line[i] for i in xrange(len(line)) if i in allowed]
                O.write(u"\t".join(tokens))
            O.write(u"\n")
    
    if verbose:
        log(' Done.\n')

if __name__ == "__main__":
    import argparse, sys

    parser = argparse.ArgumentParser(description="...")

    parser.add_argument("infile",
                        help="The input file")
    parser.add_argument("outfile",
                        help="The output file ")
    parser.add_argument("ranges",
                        help="The ranges of indexes to keep in the file.")
    parser.add_argument("--input-encoding", dest="ienc",
                        help="Encoding of the input (default: UTF-8)")
    parser.add_argument("--output-encoding", dest="oenc",
                        help="Encoding of the input (default: UTF-8)")
    parser.add_argument("--encoding", dest="enc", default="UTF-8",
                        help="Encoding of both the input and the output (default: UTF-8)")
    parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
                        help="Basic feedback for user (default: False).")
    
    arguments = (sys.argv[2:] if __package__ else sys.argv)
    parser    = parser.parse_args(arguments)
    
    clean_info(parser.infile, parser.outfile, parser.ranges,
               ienc=parser.ienc or parser.enc, oenc=parser.oenc or parser.enc, verbose=parser.verbose)
    sys.exit(0)
{'fraction_non_alphanumeric': 0.1400198609731877, 'fraction_numerical': 0.003972194637537239, 'mean_word_length': 2.264991896272285, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1341200', 'n_tokens_mistral': 1229, 'n_tokens_neox': 1186, 'n_words': 298}