<commit_before>"""The main program
"""

import time
import Queue
import MySQLdb as mysqldb
import MySQLdb.cursors
from wikiparser import WorkingThread
from wikiparser import ReadingThread
from preparation import ResolveThread
from preparation import PrepareThread

class Creator():
    def __init__(self, xml_path, max_queue_size=20, num_threads=1, 
            db_host='localhost', db_user='wikiwsd', db_pass='<PASSWORD>', action='learn'):
        self._queue = Queue.Queue(maxsize=max_queue_size)
        if action == 'learn':
            self._reading_thread = ReadingThread(xml_path, self._queue)
            self._worker_threads = []
            for i in range (0, num_threads):
                con = mysqldb.connect(db_host, db_user, db_pass, '<PASSWORD>', charset='utf8', use_unicode=True)
                self._worker_threads.append(WorkingThread(self._queue, con))
        elif action == 'prepare':
            self._reading_thread = ResolveThread(xml_path, self._queue)
            self._worker_threads = []
            for i in range (0, num_threads):
                con = mysqldb.connect(db_host, db_user, db_pass, '<PASSWORD>', charset='utf8', use_unicode=True)
                self._worker_threads.append(PrepareThread(self._queue, con))

    def run(self):
        self._reading_thread.start()
        for worker in self._worker_threads:
            worker.start()
	# wait until all articles are read
	self._reading_thread.join()

        # wait for all articles to be processed
        self._queue.join()
        for worker in self._worker_threads:
            worker.end()
        for worker in self._worker_threads:
            worker.join()


if __name__ == '__main__':
    try:
        # first action 'prepare' to learn links
        # then action 'learn' to learn
        prog = Creator('/home/paul/data/wikipedia/enwiki-20130102-pages-articles.xml', num_threads=26, max_queue_size=300, action='prepare')
        #prog = Creator('/home/paul/data/wikipedia/enwiki-20130102-pages-articles.xml', num_threads=26, max_queue_size=300, action='learn')
        #prog = Creator('data/training.xml', num_threads=25, max_queue_size=300, db_host='10.11.0.103', action='learn')
        time.clock()<commit_msg>updated db version to 3 also ;-)<commit_after>"""The main program
"""

import time
import Queue
import MySQLdb as mysqldb
import MySQLdb.cursors
from wikiparser import WorkingThread
from wikiparser import ReadingThread
from preparation import ResolveThread
from preparation import PrepareThread

class Creator():
    def __init__(self, xml_path, max_queue_size=20, num_threads=1, 
            db_host='localhost', db_user='wikiwsd', db_pass='<PASSWORD>', action='learn'):
        self._queue = Queue.Queue(maxsize=max_queue_size)
        if action == 'learn':
            self._reading_thread = ReadingThread(xml_path, self._queue)
            self._worker_threads = []
            for i in range (0, num_threads):
                con = mysqldb.connect(db_host, db_user, db_pass, '<PASSWORD>', charset='utf8', use_unicode=True)
                self._worker_threads.append(WorkingThread(self._queue, con))
        elif action == 'prepare':
            self._reading_thread = ResolveThread(xml_path, self._queue)
            self._worker_threads = []
            for i in range (0, num_threads):
                con = mysqldb.connect(db_host, db_user, db_pass, '<PASSWORD>', charset='utf8', use_unicode=True)
                self._worker_threads.append(PrepareThread(self._queue, con))

    def run(self):
        self._reading_thread.start()
        for worker in self._worker_threads:
            worker.start()
	# wait until all articles are read
	self._reading_thread.join()

        # wait for all articles to be processed
        self._queue.join()
        for worker in self._worker_threads:
            worker.end()
        for worker in self._worker_threads:
            worker.join()


if __name__ == '__main__':
    try:
        # first action 'prepare' to learn links
        # then action 'learn' to learn
        prog = Creator('/home/paul/data/wikipedia/enwiki-20130102-pages-articles.xml', num_threads=26, max_queue_size=300, action='prepare')
        #prog = Creator('/home/paul/data/wikipedia/enwiki-20130102-pages-articles.xml', num_threads=26, max_queue_size=300, action='learn')
        #prog = Creator('data/training.xml', num_threads=25, max_queue_size=300, db_host='10.11.0.103', action='learn')
        time.clock()
{'fraction_non_alphanumeric': 0.11195871662553288, 'fraction_numerical': 0.02086605339914741, 'mean_word_length': 3.0380434782608696, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2581132', 'n_tokens_mistral': 1411, 'n_tokens_neox': 1295, 'n_words': 307}