<commit_before>import MySQLdb
import re
import sys
import json
import os
from subprocess import call

from ParseDate import DateParser

#These three libraries define the Bookworm-specific methods.
from CreateDatabase import *
from ImportNewLibrary import *
from WordsTableCreate import WordsTableCreate
from tokenizeAndEncodeFiles import bookidlist

#First off: what are we using? Pull a dbname from command line input.                                                                 
#As well as the username and password. 
dbname = sys.argv[1]
dbuser = sys.argv[2]
dbpassword = sys.argv[3]

print "Parsing the dates to a native format"
DateParser()

Bookworm = BookwormSQLDatabase(dbname,dbuser,dbpassword)

print "Writing metadata to new catalog file..."

write_metadata(Bookworm.variables)

#These are imported with ImportNewLibrary
#Some sort of check here might make things a little faster on subsequent builds.
CopyDirectoryStructuresFromRawDirectory()

bookidList = bookidlist()

bookidList.clean()
bookidList.tokenize('unigrams')
bookidList.tokenize('bigrams')

#bookidList.tokenize('trigrams')

print "Creating a master wordlist" #These values shouldn't be hard-coded in, probably:
WordsTableCreate(maxDictionaryLength=1000000,maxMemoryStorage = 15000000)

bookidList.encodeUnigrams()
bookidList.encodeBigrams()
#bookidList.encodeTrigrams()


"""
Most of these commands are inside CreateNewDatabase.py. For manual constructions or 
stop-and-start operations, it's very helpful to have them spread out like this.
The class initialization should be quick, but--fair warning--does create a database if it's not there.
"""

Bookworm.load_word_list()
Bookworm.create_unigram_book_counts()
Bookworm.create_bigram_book_counts()
Bookworm.load_book_list()
Bookworm.create_memory_table_script()
Bookworm.jsonify_data()
Bookworm.create_API_settings()<commit_msg>Improving upon comments.<commit_after>import MySQLdb
import re
import sys
import json
import os
from subprocess import call

# These three libraries define the Bookworm-specific methods.
from CreateDatabase import *
from ImportNewLibrary import *
from WordsTableCreate import WordsTableCreate
from tokenizeAndEncodeFiles import bookidlist

# Import our data parser.
import ParseDate

# Pull a dbname from command line input.
dbname = sys.argv[1]
dbuser = sys.argv[2]
dbpassword = sys.argv[3]

print "Parsing the dates to a native format"
ParseDate.DateParser()

# Initiate MySQL connection.
Bookworm = BookwormSQLDatabase(dbname,dbuser,dbpassword)

# This creates helper files in the /metadata/ folder.
print "Writing metadata to new catalog file..."
write_metadata(Bookworm.variables)

#These are imported with ImportNewLibrary
CopyDirectoryStructuresFromRawDirectory() # rsync is very slow when the number of files in /texts/raw/ is large (1000000+)
bookidList = bookidlist()
bookidList.clean()
bookidList.tokenize('unigrams')
bookidList.tokenize('bigrams')
#bookidList.tokenize('trigrams')
print "Creating a master wordlist"
WordsTableCreate(maxDictionaryLength=1000000,maxMemoryStorage = 15000000)
bookidList.encodeUnigrams()
bookidList.encodeBigrams()
#bookidList.encodeTrigrams()


# Most of these commands are inside /Presidio/CreateNewDatabase.py.
Bookworm.load_word_list()
Bookworm.create_unigram_book_counts()
Bookworm.create_bigram_book_counts()
Bookworm.load_book_list()

# This needs to be run if the database resets. It builds a temporary MySQL table and the GUI will not work if this table not built.
Bookworm.create_memory_table_script()

# Create the dbname.json file in the root directory. Move this to the $project/dbname/static/ folder of the web server.
Bookworm.jsonify_data()

Bookworm.create_API_settings()
{'fraction_non_alphanumeric': 0.07260904903820103, 'fraction_numerical': 0.01164995936060688, 'mean_word_length': 5.640287769784172, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '443814', 'n_tokens_mistral': 1136, 'n_tokens_neox': 1045, 'n_words': 351}