<commit_before>#!/usr/bin/python

from tokenizer import *
import sys
import subprocess
import timeit

def exportToDisk(wordcounts,diskFile,keepThreshold=5):
    commonwords = dict()
    for key in wordcounts.iterkeys():
        if wordcounts[key] < keepThreshold:
            key = key
            output = key + " " + str(wordcounts[key]) + "\n"
            diskFile.write(output)
        else:
            commonwords[key] = wordcounts[key]
    return commonwords
    print "export done"
    
def WordsTableCreate(maxDictionaryLength=1000000, maxMemoryStorage=20000000):
    database = open('files/texts/wordlist/raw.txt','w')
    start_time = timeit.default_timer()
    n = 1
    #When flushing the dictionary to disk, they are kept in memory if there count is
    #greater than keepThreshold to avoid overhead reinitializing and rewriting.
    keepThreshold=2

    wordcounts = dict()
    for row in sys.stdin:
        for item in row.split(" "):
            n+=1
            if n % 100000000==0:
                elapsed = timeit.default_timer() - start_time
                print str(float(len(wordcounts))/1000000) + " million distinct entries at " + str(float(n)/1000000000) + " billion words---" + str(elapsed) + " seconds since last print"
                start_time = timeit.default_timer()
            item = item.rstrip("\n")
            try: 
                wordcounts[item] += 1
            except KeyError:
                wordcounts[item] = 1

            while len(wordcounts) > maxMemoryStorage:
                print "exporting to disk at " + str(float(len(wordcounts))/1000000) + " million words"
                wordcounts = exportToDisk(wordcounts,diskFile=database,keepThreshold=keepThreshold)
                print "after export, it's " + str(float(len(wordcounts))/1000000) + " million words"
                if len(wordcounts) > .66 * maxMemoryStorage:
                    #If that's not enough to get down to a small dictionary,
                    #try again with a new higher limit.
                    keepThreshold = keepThreshold*2
                    print "upping the keep threshold to " + str(keepThreshold)

    #Write all remaining items to disk
    nothing = exportToDisk(wordcounts,diskFile=database,keepThreshold=float("inf"))
    database.close()
    sortWordlist(maxDictionaryLength=maxDictionaryLength)

def sortWordlist(maxDictionaryLength=1000000):

    print("Sorting full word counts\n")
    #This LC_COLLATE here seems to be extremely necessary, because otherwise alphabetical order isn't preserved across different orderings.
    subprocess.call(["export LC_COLLATE='C';export LC_ALL='C'; sort -k1 files/texts/wordlist/raw.txt > files/texts/wordlist/sorted.txt"], shell=True)
    
    print("Collapsing word counts\n")
    
    #This is in perl, using bignum, because it's possible to get integer overflows on a really huge text set (like Google ngrams).

    subprocess.call(["""
           perl -ne '
           BEGIN {use bignum; $last=""; $count=0} <commit_msg>tweaking the limit at which dictionary dumps to disk<commit_after>#!/usr/bin/python

from tokenizer import *
import sys
import subprocess
import timeit

def exportToDisk(wordcounts,diskFile,keepThreshold=5):

    """
    Periodically, the wordcounter writes what it knows to disk for dictionary values below a certain frequency:
    this lets us keep the most common words continually in memory, and only write out (say) 'the' once, at the end.
    """
    commonwords = dict()
    for key in wordcounts.iterkeys():
        if wordcounts[key] < keepThreshold:
            key = key
            output = key + " " + str(wordcounts[key]) + "\n"
            diskFile.write(output)
        else:
            commonwords[key] = wordcounts[key]
    return commonwords
    print "export done"
    
def WordsTableCreate(maxDictionaryLength=1000000, maxMemoryStorage=20000000):
    database = open('files/texts/wordlist/raw.txt','w')
    start_time = timeit.default_timer()
    n = 1
    #When flushing the dictionary to disk, they are kept in memory if there count is
    #greater than keepThreshold to avoid overhead reinitializing and rewriting.
    keepThreshold=2

    wordcounts = dict()
    for row in sys.stdin:
        for item in row.split(" "):
            n+=1
            if n % 100000000==0:
                elapsed = timeit.default_timer() - start_time
                print str(float(len(wordcounts))/1000000) + " million distinct entries at " + str(float(n)/1000000000) + " billion words---" + str(elapsed) + " seconds since last print"
                start_time = timeit.default_timer()
            item = item.rstrip("\n")
            try: 
                wordcounts[item] += 1
            except KeyError:
                wordcounts[item] = 1

            while len(wordcounts) > maxMemoryStorage:
                print "exporting to disk at " + str(float(len(wordcounts))/1000000) + " million words"
                wordcounts = exportToDisk(wordcounts,diskFile=database,keepThreshold=keepThreshold)
                print "after export, it's " + str(float(len(wordcounts))/1000000) + " million words"
                if len(wordcounts) > .8*float(maxMemoryStorage):
                    #If that's not enough to get down to a small dictionary,
                    #try again with a new higher limit.
                    keepThreshold = keepThreshold*2
                    print "upping the keep threshold to " + str(keepThreshold)

    #Write all remaining items to disk
    nothing = exportToDisk(wordcounts,diskFile=database,keepThreshold=float("inf"))
    database.close()
    sortWordlist(maxDictionaryLength=maxDictionaryLength)

def sortWordlist(maxDictionaryLength=1000000):

    print("Sorting full word counts\n")
    #This LC_COLLATE here seems to be extremely necessary, because otherwise alphabetical order isn't preserved across different orderings.
    subprocess.call(["export LC_COLLATE='C';export LC_ALL='C'; sort -k1 files/texts/wordlist/raw.txt > files/texts/wordlist/sorted.txt"], shell=True)
    
    print("Collapsing word counts\n")
    
    #This is in perl, using bignum, because it's possible to get integer overflows on a really huge text set (like Google ngrams).

    subprocess.call(["""
           perl -ne '
           BEGIN {use bignum; $last=""; $count=0} 
{'fraction_non_alphanumeric': 0.09575815131370687, 'fraction_numerical': 0.023266856600189935, 'mean_word_length': 2.6483833718244805, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 6, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '776057', 'n_tokens_mistral': 1917, 'n_tokens_neox': 1738, 'n_words': 518}