<commit_before>import os, sys, nrrd, json, gc, stat
import numpy as np

adjust_thresh = 0.0035

def AutoBalance(data,threshold=adjust_thresh,background=0):
    data = np.uint8(np.round(255.0*(np.double(data)/np.max(data)))) #scale to 8 bit
    bins=np.unique(data)
    binc=np.bincount(data.flat)
    histogram=binc[binc>0]
    del binc
    gc.collect()
    temp=1
    c = 0
    mid = int(np.round(np.shape(bins)[0] / 2.0))
    # low end threshold only uses data from low end
    if background in bins[:mid]:
        i = np.where(bins==background)
        v = bins[i][0]
        c = histogram[i][0]
        th=long((np.sum(histogram[:mid])-c)*threshold)
    else:
        th=long((np.sum(histogram[:mid]))*threshold)
    print 'low end balancing:'
    print 'number of background voxels: ' + str(c)
    print 'number of data voxels: ' + str(np.sum(histogram[:mid])-c)
    print 'threshold set to: ' + str(th)
    m=np.min(bins)
    M=np.max(bins)
    for x in range(1,np.shape(bins)[0]-1):
        if (np.sum(histogram[0:x])-c) > th:
            m = bins[x-1]
            temp = x-1
            break
    print 'number of low end voxels cut: ' + str(np.sum(histogram[0:temp])-c)
    # high end threshold only uses data from high end
    c = 0
    if background in bins[mid:]:
        i = np.where(bins==background)
        v = bins[i][0]
        c = histogram[i][0]
        th=long((np.sum(histogram[mid:])-c)*threshold)
    else:
        th=long((np.sum(histogram[mid:]))*threshold)
    print 'high end balancing:'
    print 'number of background voxels: ' + str(c)
    print 'number of data voxels: ' + str(np.sum(histogram[mid:])-c)
    print 'threshold set to: ' + str(th)
    for x in range(np.shape(bins)[0]-1,0,-1):
        if (np.sum(histogram[x:])) > th:
            M = bins[x]
            temp = x + 1
            break
    print 'number of high end voxels cut: ' + str(np.sum(histogram[temp:]))
    del temp
    gc.collect()
    data[data>M]=M
    data[data<m]=m
    dataA=np.round((data-m)*(255.0/(M-m)))
    hist=np.zeros(255, dtype=long)
    for i in range(0,np.shape(bins)[0]-1):
        hist[bins[i]]=histogram[i]
    return (dataA, {'min': int(m),'max': int(M)}, hist)
    
if __name__ == "__main__":
  if (len(sys.argv) < 2):
    print 'Error: missing arguments!' 
    print 'e.g. python autoBalance.py imageIn.nrrd [imageOut.nrrd] [0] [0.0035]'
    print '0 = Background level'
    print '0.0035 = adjustment threshold - max percentage of voxels that can be clipped'
  else:
    outFile = str(sys.argv[1])
    if (len(sys.argv) > 2):
      outFile = str(sys.argv[2])
    BG = 0
    if (len(sys.argv) > 3):
      BG = int(sys.argv[3])
    TH = adjust_thresh
    if (len(sys.argv) > 4):
      TH = int(sys.argv[4])
    data, header = nrrd.read(str(sys.argv[1]))
    chan, Nbound, hist = AutoBalance(data,threshold=TH,background=BG)
    print 'New boundaries %s'% (str(Nbound))
    print 'Saving result to %s...'% (outFile)
    nrrd.write(outFile,chan, options=header)
<commit_msg>correcting type<commit_after>import os, sys, nrrd, json, gc, stat
import numpy as np

adjust_thresh = 0.0035

def AutoBalance(data,threshold=adjust_thresh,background=0):
    data = np.uint8(np.round(255.0*(np.double(data)/np.max(data)))) #scale to 8 bit
    bins=np.unique(data)
    binc=np.bincount(data.flat)
    histogram=binc[binc>0]
    del binc
    gc.collect()
    temp=1
    c = 0
    mid = int(np.round(np.shape(bins)[0] / 2.0))
    # low end threshold only uses data from low end
    if background in bins[:mid]:
        i = np.where(bins==background)
        v = bins[i][0]
        c = histogram[i][0]
        th=long((np.sum(histogram[:mid])-c)*threshold)
    else:
        th=long((np.sum(histogram[:mid]))*threshold)
    print 'low end balancing:'
    print 'number of background voxels: ' + str(c)
    print 'number of data voxels: ' + str(np.sum(histogram[:mid])-c)
    print 'threshold set to: ' + str(th)
    m=np.min(bins)
    M=np.max(bins)
    for x in range(1,np.shape(bins)[0]-1):
        if (np.sum(histogram[0:x])-c) > th:
            m = bins[x-1]
            temp = x-1
            break
    print 'number of low end voxels cut: ' + str(np.sum(histogram[0:temp])-c)
    # high end threshold only uses data from high end
    c = 0
    if background in bins[mid:]:
        i = np.where(bins==background)
        v = bins[i][0]
        c = histogram[i][0]
        th=long((np.sum(histogram[mid:])-c)*threshold)
    else:
        th=long((np.sum(histogram[mid:]))*threshold)
    print 'high end balancing:'
    print 'number of background voxels: ' + str(c)
    print 'number of data voxels: ' + str(np.sum(histogram[mid:])-c)
    print 'threshold set to: ' + str(th)
    for x in range(np.shape(bins)[0]-1,0,-1):
        if (np.sum(histogram[x:])) > th:
            M = bins[x]
            temp = x + 1
            break
    print 'number of high end voxels cut: ' + str(np.sum(histogram[temp:]))
    del temp
    gc.collect()
    data[data>M]=M
    data[data<m]=m
    dataA=np.round((data-m)*(255.0/(M-m)))
    hist=np.zeros(255, dtype=long)
    for i in range(0,np.shape(bins)[0]-1):
        hist[bins[i]]=histogram[i]
    return (dataA, {'min': int(m),'max': int(M)}, hist)
    
if __name__ == "__main__":
  if (len(sys.argv) < 2):
    print 'Error: missing arguments!' 
    print 'e.g. python autoBalance.py imageIn.nrrd [imageOut.nrrd] [0] [0.0035]'
    print '0 = Background level'
    print '0.0035 = adjustment threshold - max percentage of voxels that can be clipped'
  else:
    outFile = str(sys.argv[1])
    if (len(sys.argv) > 2):
      outFile = str(sys.argv[2])
    BG = 0
    if (len(sys.argv) > 3):
      BG = int(sys.argv[3])
    TH = adjust_thresh
    if (len(sys.argv) > 4):
      TH = int(sys.argv[4])
    data, header = nrrd.read(str(sys.argv[1]))
    chan, Nbound, hist = AutoBalance(data,threshold=TH,background=BG)
    print 'New boundaries %s'% (str(Nbound))
    print 'Saving result to %s...'% (outFile)
    nrrd.write(outFile,np.uint8(chan), options=header)

{'fraction_non_alphanumeric': 0.16661118508655126, 'fraction_numerical': 0.022470039946737682, 'mean_word_length': 2.7346177750155376, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5061026', 'n_tokens_mistral': 2358, 'n_tokens_neox': 2233, 'n_words': 519}