<commit_before>from collections import Counter
import sys

def two_freq(tree, sec_queue, freq):
    tree.append([freq[0][1] + freq[1][1], freq[0], freq[1]])
    sec_queue.append([freq[0][1] + freq[1][1], [freq[0][1] + freq[1][1], freq[0], freq[1]]])    
    freq = freq[2:]
    return tree, sec_queue, freq

def one_to_one(tree, sec_queue, freq):
    for i in tree:
        if i == sec_queue[0][1]:
            hold = tree.index(i)
            tree[tree.index(i)] = [sec_queue[0][0] + freq[0][1], sec_queue[0][1], freq[0]]
            sec_queue.append([freq[0][1] + sec_queue[0][0], tree[hold]])
            sec_queue = sec_queue[1:]
            freq = freq[1:]
            break
    return tree, sec_queue, freq

def two_sec(tree, sec_queue):
    for i in tree:  
        if i == sec_queue[0][1]:
            hold = tree.index(i)
            tree[tree.index(i)] = [sec_queue[0][0] + sec_queue[1][0], sec_queue[0][1], sec_queue[1][1]]
            sec_queue.append([sec_queue[0][0] + sec_queue[1][0], tree[hold]])
            for j in tree:
                if j == sec_queue[1][1]:
                    tree.remove(j)
                    break
            sec_queue = sec_queue[2:]
            break
    return tree, sec_queue

def h_tree(data):
    tree = []
    sec_queue = []
    counts = Counter(data)
    freq = sorted([(i[0], i[1],) for i in zip(counts.keys(), counts.values())], key=lambda x: x[1])
    tree, sec_queue, freq = two_freq(tree, sec_queue, freq)
    while len(freq) > 1:
        if len(sec_queue) < 2:
            if freq[0][1] + freq[1][1] <= freq[0][1] + sec_queue[0][0]:
                tree, sec_queue, freq = two_freq(tree, sec_queue, freq)
            elif freq[0][1] + sec_queue[0][0] < freq[0][1] + freq[1][1]:
                tree, sec_queue, freq = one_to_one(tree, sec_queue, freq)
            if len(freq) < 2:
                break
        if len(sec_queue) > 1:
            if freq[0][1] + freq[1][1] <= freq[0][1] + sec_queue[0][0] and freq[0][1] + freq[1][1] <= sec_queue[0][0] + sec_queue[1][0]:
                tree, sec_queue, freq = two_freq(tree, sec_queue, freq)
            elif freq[0][1] + sec_queue[0][0] < freq[0][1] + freq[1][1] and freq[0][1] + sec_queue[0][0] <= sec_queue[0][0] + sec_queue[1][0]:
                tree, sec_queue, freq = one_to_one(tree, sec_queue, freq)
            elif sec_queue[0][0] + sec_queue[1][0] < freq[0][1] + freq[1][1] and \
                sec_queue[0][0] + sec_queue[1][0] < sec_queue[0][0] + freq[0][1]:
                tree, sec_queue = two_sec(tree, sec_queue)
    while len(sec_queue) > 1:
        if freq:
            if freq[0][1] + sec_queue[0][0] < sec_queue[0][0] + sec_queue[1][0]:
                tree, sec_queue, freq = one_to_one(tree, sec_queue, freq)
            else:
                tree, sec_queue = two_sec(tree, sec_queue)
        else:
            tree, sec_queue = two_sec(tree, sec_queue)
    if freq:
        tree, sec_queue, freq = one_to_one(tree, sec_queue, freq)
    return tree[0]

print h_tree(sys.argv[1])

<commit_msg>added in the binary compression func<commit_after>from collections import Counter
import sys

def two_freq(tree, sec_queue, freq):
    tree.append([freq[0][1] + freq[1][1], freq[0], freq[1]])
    sec_queue.append([freq[0][1] + freq[1][1], [freq[0][1] + freq[1][1], freq[0], freq[1]]])    
    freq = freq[2:]
    return tree, sec_queue, freq

def one_to_one(tree, sec_queue, freq):
    for i in tree:
        if i == sec_queue[0][1]:
            hold = tree.index(i)
            tree[tree.index(i)] = [sec_queue[0][0] + freq[0][1], sec_queue[0][1], freq[0]]
            sec_queue.append([freq[0][1] + sec_queue[0][0], tree[hold]])
            sec_queue = sec_queue[1:]
            freq = freq[1:]
            break
    return tree, sec_queue, freq

def two_sec(tree, sec_queue):
    for i in tree:  
        if i == sec_queue[0][1]:
            hold = tree.index(i)
            tree[tree.index(i)] = [sec_queue[0][0] + sec_queue[1][0], sec_queue[0][1], sec_queue[1][1]]
            sec_queue.append([sec_queue[0][0] + sec_queue[1][0], tree[hold]])
            for j in tree:
                if j == sec_queue[1][1]:
                    tree.remove(j)
                    break
            sec_queue = sec_queue[2:]
            break
    return tree, sec_queue

def huff_bin(tree, encode):
    for i in tree:
        if isinstance(i, list) or isinstance(i, tuple):
            if tree.index(i) == 1:
                old_encode = encode
                encode += '0'
                huff_bin(i, encode)
            if tree.index(i) == 2:
                encode = old_encode
                encode += '1'
                huff_bin(i, encode)
        if isinstance(i, str):
            print i, encode

def h_tree(data):
    tree = []
    sec_queue = []
    counts = Counter(data)
    freq = sorted([(i[0], i[1],) for i in zip(counts.keys(), counts.values())], key=lambda x: x[1])
    tree, sec_queue, freq = two_freq(tree, sec_queue, freq)
    while len(freq) > 1:
        if len(sec_queue) < 2:
            if freq[0][1] + freq[1][1] <= freq[0][1] + sec_queue[0][0]:
                tree, sec_queue, freq = two_freq(tree, sec_queue, freq)
            elif freq[0][1] + sec_queue[0][0] < freq[0][1] + freq[1][1]:
                tree, sec_queue, freq = one_to_one(tree, sec_queue, freq)
            if len(freq) < 2:
                break
        if len(sec_queue) > 1:
            if freq[0][1] + freq[1][1] <= freq[0][1] + sec_queue[0][0] and freq[0][1] + freq[1][1] <= sec_queue[0][0] + sec_queue[1][0]:
                tree, sec_queue, freq = two_freq(tree, sec_queue, freq)
            elif freq[0][1] + sec_queue[0][0] < freq[0][1] + freq[1][1] and freq[0][1] + sec_queue[0][0] <= sec_queue[0][0] + sec_queue[1][0]:
                tree, sec_queue, freq = one_to_one(tree, sec_queue, freq)
            elif sec_queue[0][0] + sec_queue[1][0] < freq[0][1] + freq[1][1] and \
                sec_queue[0][0] + sec_queue[1][0] < sec_queue[0][0] + freq[0][1]:
                tree, sec_queue = two_sec(tree, sec_queue)
    while len(sec_queue) > 1:
        if freq:
            if freq[0][1] + sec_queue[0][0] < sec_queue[0][0] + sec_queue[1][0]:
                tree, sec_queue, freq = one_to_one(tree, sec_queue, freq)
            else:
                tree, sec_queue = two_sec(tree, sec_queue)
        else:
            tree, sec_queue = two_sec(tree, sec_queue)
    if freq:
        tree, sec_queue, freq = one_to_one(tree, sec_queue, freq)
    return tree[0]




print h_tree(sys.argv[1])
huff_bin(h_tree(sys.argv[1]), '')

{'fraction_non_alphanumeric': 0.16915802318486883, 'fraction_numerical': 0.04072605247101892, 'mean_word_length': 2.260566882148185, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4388928', 'n_tokens_mistral': 2555, 'n_tokens_neox': 2499, 'n_words': 509}