# Path: src/backend/tma_utils.py
# class TextCleaner:
#     def __init__(self, not_valid_reg=re.compile('\\S*[^\\w\\s]+\\S*'), remove_chars='[-\\d.,?;:\'")(!`}{\]\[]', stem = lib.porter2.stem, stopword_file=None):        
# 
#         self.not_valid_reg = not_valid_reg  
#         
#         if remove_chars and not remove_chars == '':
#             self.remove_chars_re = re.compile(remove_chars) 
#         else:
#             self.remove_chars_re = None
# 
#         self.stem = stem
#         
#         if stopword_file: 
#             self.stopwords = {}
#             stopfile = open(stopword_file,'r') 
#             for sw in stopfile:
#                 self.stopwords[sw.strip()] = 0
#             stopfile.close()
#         else: 
#             self.stopwords = None
#     
#     def parse_text(self, text):
#         clean_text = [] 
#         for word in text.strip().split():
#             word = self.parse_word(word)   
#             if not word == '':
#                 clean_text.append(word)
#         return clean_text 
#         
#     def parse_word(self, word):
#         if self.remove_chars_re:
#             retword = re.sub(self.remove_chars_re,'',word.strip()).lower() # remove punctuation and such
#         
#         if self.not_valid_reg:
#             if self.not_valid_reg.match(retword):
#                 return ''
#             
#         if self.stopwords:
#             if self.stopwords.has_key(retword):  
#                 return '' 
#         if self.stem:
#             retword = self.stem(retword)
#                 
#         return retword
# 
# def ids_to_key(id1, id2):
#     if type(id1) != int:
#         id1 = int(id1)
#     if type(id2) != int:
#         id2 = int(id2)
#     ids = sorted([id1,id2])
#     return ids[0], ids[1]
# 
# Path: lib/porter2.py
# def stem(word):
#     if len(word) <= 2:
#         return word
#     word = remove_initial_apostrophe(word)
# 
#     # handle some exceptional forms
#     if word in exceptional_forms:
#         return exceptional_forms[word]
# 
#     word = capitalize_consonant_ys(word)
#     r1 = get_r1(word)
#     r2 = get_r2(word)
#     word = step_0(word)
#     word = step_1a(word)
# 
#     # handle some more exceptional forms
#     if word in exceptional_early_exit_post_1a:
#         return word
# 
#     word = step_1b(word, r1)
#     word = step_1c(word)
#     word = step_2(word, r1)
#     word = step_3(word, r1, r2)
#     word = step_4(word, r2)
#     word = step_5(word, r1, r2)
#     word = normalize_ys(word)
# 
#     return word


import pdb 
import os 
import re 
import cPickle as pickle
import sqlite3 as sqlite                
import bsddb  
import random
from src.backend.tma_utils import TextCleaner, ids_to_key 
from lib.porter2 import stem
from time import time  
            if termterm_db.has_key(ikey):
                termterm_db[ikey] = str(int(termterm_db[ikey]) + termterm_dict[t1][t2])
            else:
                termterm_db[ikey] = str(termterm_dict[t1][t2])


if __name__ == '__main__':
    # DATA
    outdata_dir = '/Users/cradreed/Research/TMBrowse/develarea/'
    wikfile = outdata_dir + 'enwiki_abstracts-20120307.dat' #'/Users/cradreed/Research/TMBrowse/develarea/enwiki-latest-abstract18.xml'#

    # use bsd to create to cooccurence file then write to sqlite to maintain database consistency and reduce dependencies
    # set up the dbase
    #dbfile = '/Users/cradreed/Research/TMBrowse/develarea/wiki-terms.sqlite'
    # wikivocab_file = outdata_dir + 'wikivocab_full.bdb'
    # wikivocab_ct_file =  outdata_dir + 'wikivocab_ct_full.bdb'
    wiki_termterm_file = outdata_dir + 'wiki_termterm_full_100percent.bdb'

    # os.remove(dbfile) # TESTING
    # if os.path.exists(wikivocab_file):
    #     os.remove(wikivocab_file)
    # if os.path.exists(wikivocab_ct_file):
    #     os.remove(wikivocab_ct_file)
#    if os.path.exists(wiki_termterm_file):
#        os.remove(wiki_termterm_file)

    vocab_dict = pickle.load(open(outdata_dir + 'wiki_vocab_dic_full_100percent.obj','rb'))#{}#bsddb.hashopen(wikivocab_file)
    vocab_ct_dict = pickle.load(open(outdata_dir + 'wiki_vocab_ct_full_100percent.obj','rb'))#{}#bsddb.hashopen(wikivocab_ct_file)
    termterm_db = bsddb.btopen(wiki_termterm_file)
    termterm_dict = {}

# Path: src/backend/tma_utils.py
# class TextCleaner:
#     def __init__(self, not_valid_reg=re.compile('\\S*[^\\w\\s]+\\S*'), remove_chars='[-\\d.,?;:\'")(!`}{\]\[]', stem = lib.porter2.stem, stopword_file=None):        
# 
#         self.not_valid_reg = not_valid_reg  
#         
#         if remove_chars and not remove_chars == '':
#             self.remove_chars_re = re.compile(remove_chars) 
#         else:
#             self.remove_chars_re = None
# 
#         self.stem = stem
#         
#         if stopword_file: 
#             self.stopwords = {}
#             stopfile = open(stopword_file,'r') 
#             for sw in stopfile:
#                 self.stopwords[sw.strip()] = 0
#             stopfile.close()
#         else: 
#             self.stopwords = None
#     
#     def parse_text(self, text):
#         clean_text = [] 
#         for word in text.strip().split():
#             word = self.parse_word(word)   
#             if not word == '':
#                 clean_text.append(word)
#         return clean_text 
#         
#     def parse_word(self, word):
#         if self.remove_chars_re:
#             retword = re.sub(self.remove_chars_re,'',word.strip()).lower() # remove punctuation and such
#         
#         if self.not_valid_reg:
#             if self.not_valid_reg.match(retword):
#                 return ''
#             
#         if self.stopwords:
#             if self.stopwords.has_key(retword):  
#                 return '' 
#         if self.stem:
#             retword = self.stem(retword)
#                 
#         return retword
# 
# def ids_to_key(id1, id2):
#     if type(id1) != int:
#         id1 = int(id1)
#     if type(id2) != int:
#         id2 = int(id2)
#     ids = sorted([id1,id2])
#     return ids[0], ids[1]
# 
# Path: lib/porter2.py
# def stem(word):
#     if len(word) <= 2:
#         return word
#     word = remove_initial_apostrophe(word)
# 
#     # handle some exceptional forms
#     if word in exceptional_forms:
#         return exceptional_forms[word]
# 
#     word = capitalize_consonant_ys(word)
#     r1 = get_r1(word)
#     r2 = get_r2(word)
#     word = step_0(word)
#     word = step_1a(word)
# 
#     # handle some more exceptional forms
#     if word in exceptional_early_exit_post_1a:
#         return word
# 
#     word = step_1b(word, r1)
#     word = step_1c(word)
#     word = step_2(word, r1)
#     word = step_3(word, r1, r2)
#     word = step_4(word, r2)
#     word = step_5(word, r1, r2)
#     word = normalize_ys(word)
# 
#     return word


# Path: src/backend/aux/create_wiki_cooccurence.py
import pdb 
import os 
import re 
import cPickle as pickle
import sqlite3 as sqlite                
import bsddb  
import random
from src.backend.tma_utils import TextCleaner, ids_to_key 
from lib.porter2 import stem
from time import time  

            if termterm_db.has_key(ikey):
                termterm_db[ikey] = str(int(termterm_db[ikey]) + termterm_dict[t1][t2])
            else:
                termterm_db[ikey] = str(termterm_dict[t1][t2])


if __name__ == '__main__':
    # DATA
    outdata_dir = '/Users/cradreed/Research/TMBrowse/develarea/'
    wikfile = outdata_dir + 'enwiki_abstracts-20120307.dat' #'/Users/cradreed/Research/TMBrowse/develarea/enwiki-latest-abstract18.xml'#

    # use bsd to create to cooccurence file then write to sqlite to maintain database consistency and reduce dependencies
    # set up the dbase
    #dbfile = '/Users/cradreed/Research/TMBrowse/develarea/wiki-terms.sqlite'
    # wikivocab_file = outdata_dir + 'wikivocab_full.bdb'
    # wikivocab_ct_file =  outdata_dir + 'wikivocab_ct_full.bdb'
    wiki_termterm_file = outdata_dir + 'wiki_termterm_full_100percent.bdb'

    # os.remove(dbfile) # TESTING
    # if os.path.exists(wikivocab_file):
    #     os.remove(wikivocab_file)
    # if os.path.exists(wikivocab_ct_file):
    #     os.remove(wikivocab_ct_file)
#    if os.path.exists(wiki_termterm_file):
#        os.remove(wiki_termterm_file)

    vocab_dict = pickle.load(open(outdata_dir + 'wiki_vocab_dic_full_100percent.obj','rb'))#{}#bsddb.hashopen(wikivocab_file)
    vocab_ct_dict = pickle.load(open(outdata_dir + 'wiki_vocab_ct_full_100percent.obj','rb'))#{}#bsddb.hashopen(wikivocab_ct_file)
    termterm_db = bsddb.btopen(wiki_termterm_file)
    termterm_dict = {}

    text_cleaner = TextCleaner(stopword_file='/Users/cradreed/Research/TMBrowse/trunk/src/backend/aux/stop_words.txt')