<commit_before>#!/usr/bin/env python

#Copyright (C) 2013 by <NAME>
#
#Released under the MIT license, see LICENSE.txt
import sys
import os
import argparse
import copy
from pybedtools import BedTool, Interval

"""
Get some basic statistics about *NUMERIC* values in a particular column of a
text file (ex BED or WIG).  Non-numeric values are ignored. Information is
used for setting binning parameters in the track xml file
"""

def main(argv=None):
    if argv is None:
        argv = sys.argv

    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description="Compute statistics for *NUMERIC* value column of text"
        " file (ex bed or wig)")
    parser.add_argument("inFile", help="input text file")
    parser.add_argument("column", help="column (starting at 1) to process",
                        type=int)
    
    args = parser.parse_args()
    assert os.path.exists(args.inFile)
    assert args.column > 0

    f = open(args.inFile, "r")
    col = args.column - 1
    minVal = sys.maxint
    maxVal = -sys.maxint
    valDict = dict()
    numVals = 0
    numLines = 0
    total = 0.0
    for line in f:
        numLines += 1
        try:
            val = float(line.split()[col])
            minVal = min(val, minVal)
            maxVal = max(val, maxVal)
            numVals += 1
            valDict[val] = 1
            total += val
            totalSq += val * val
        except:
            continue

    if numVals == 0 and numLines > 0:
        raise RuntimeError("Zero values read out of %d lines.  Perhaps column "
                           "%d doesnt exist or is not numeric" % (
                               numLines, args.column))
    numVals = float(numVals)
    meanVal = total / numVals
    numUnique = len(valDict)
<commit_msg>remove totalSq now that weve switched to 2 pass solution for no good reason<commit_after>#!/usr/bin/env python

#Copyright (C) 2013 by <NAME>
#
#Released under the MIT license, see LICENSE.txt
import sys
import os
import argparse
import copy
from pybedtools import BedTool, Interval

"""
Get some basic statistics about *NUMERIC* values in a particular column of a
text file (ex BED or WIG).  Non-numeric values are ignored. Information is
used for setting binning parameters in the track xml file
"""

def main(argv=None):
    if argv is None:
        argv = sys.argv

    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description="Compute statistics for *NUMERIC* value column of text"
        " file (ex bed or wig)")
    parser.add_argument("inFile", help="input text file")
    parser.add_argument("column", help="column (starting at 1) to process",
                        type=int)
    
    args = parser.parse_args()
    assert os.path.exists(args.inFile)
    assert args.column > 0

    f = open(args.inFile, "r")
    col = args.column - 1
    minVal = sys.maxint
    maxVal = -sys.maxint
    valDict = dict()
    numVals = 0
    numLines = 0
    total = 0.0
    for line in f:
        numLines += 1
        try:
            val = float(line.split()[col])
            minVal = min(val, minVal)
            maxVal = max(val, maxVal)
            numVals += 1
            valDict[val] = 1
            total += val
        except:
            continue

    if numVals == 0 and numLines > 0:
        raise RuntimeError("Zero values read out of %d lines.  Perhaps column "
                           "%d doesnt exist or is not numeric" % (
                               numLines, args.column))
    numVals = float(numVals)
    meanVal = total / numVals
    numUnique = len(valDict)

{'fraction_non_alphanumeric': 0.08981708981708982, 'fraction_numerical': 0.009009009009009009, 'mean_word_length': 2.172294372294372, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1148737', 'n_tokens_mistral': 1111, 'n_tokens_neox': 1061, 'n_words': 372}