# Path: src/backend/tma_utils.py
# class TextCleaner:
#     def __init__(self, not_valid_reg=re.compile('\\S*[^\\w\\s]+\\S*'), remove_chars='[-\\d.,?;:\'")(!`}{\]\[]', stem = lib.porter2.stem, stopword_file=None):        
# 
#         self.not_valid_reg = not_valid_reg  
#         
#         if remove_chars and not remove_chars == '':
#             self.remove_chars_re = re.compile(remove_chars) 
#         else:
#             self.remove_chars_re = None
# 
#         self.stem = stem
#         
#         if stopword_file: 
#             self.stopwords = {}
#             stopfile = open(stopword_file,'r') 
#             for sw in stopfile:
#                 self.stopwords[sw.strip()] = 0
#             stopfile.close()
#         else: 
#             self.stopwords = None
#     
#     def parse_text(self, text):
#         clean_text = [] 
#         for word in text.strip().split():
#             word = self.parse_word(word)   
#             if not word == '':
#                 clean_text.append(word)
#         return clean_text 
#         
#     def parse_word(self, word):
#         if self.remove_chars_re:
#             retword = re.sub(self.remove_chars_re,'',word.strip()).lower() # remove punctuation and such
#         
#         if self.not_valid_reg:
#             if self.not_valid_reg.match(retword):
#                 return ''
#             
#         if self.stopwords:
#             if self.stopwords.has_key(retword):  
#                 return '' 
#         if self.stem:
#             retword = self.stem(retword)
#                 
#         return retword
# 
# def ids_to_key(id1, id2):
#     if type(id1) != int:
#         id1 = int(id1)
#     if type(id2) != int:
#         id2 = int(id2)
#     ids = sorted([id1,id2])
#     return ids[0], ids[1]
# 
# Path: lib/porter2.py
# def stem(word):
#     if len(word) <= 2:
#         return word
#     word = remove_initial_apostrophe(word)
# 
#     # handle some exceptional forms
#     if word in exceptional_forms:
#         return exceptional_forms[word]
# 
#     word = capitalize_consonant_ys(word)
#     r1 = get_r1(word)
#     r2 = get_r2(word)
#     word = step_0(word)
#     word = step_1a(word)
# 
#     # handle some more exceptional forms
#     if word in exceptional_early_exit_post_1a:
#         return word
# 
#     word = step_1b(word, r1)
#     word = step_1c(word)
#     word = step_2(word, r1)
#     word = step_3(word, r1, r2)
#     word = step_4(word, r2)
#     word = step_5(word, r1, r2)
#     word = normalize_ys(word)
# 
#     return word


import pdb 
import os 
import re 
import cPickle as pickle
import sqlite3 as sqlite                
import bsddb  
import random
from src.backend.tma_utils import TextCleaner, ids_to_key 
from lib.porter2 import stem
from time import time  
                    del(termterm_dict)
                    termterm_dict = {}
                    print '---- %i transfer complete, took %0.1f seconds ----' % (dep_no, (time() - ltime))
                    ltime = time()
                    
            text = line.strip() # remove the abstract tags
            text = text_cleaner.parse_text(text)
            text = list(set(text))
            tot_ab_len += len(text)

            for nt1, term1 in enumerate(text):
                if not vocab_dict.has_key(term1):
                    t1_id = term_ct
                    vocab_dict[term1] = t1_id
                    vocab_ct_dict[t1_id] = 1
                    term_ct += 1
                else:
                    t1_id = vocab_dict[term1]
                    vocab_ct_dict[t1_id] += 1

                for nt2 in xrange(nt1+1, len(text)): # 173.271281 vs 185s TODO make sure the counting is correct
                    term2 = text[nt2]
                    if not vocab_dict.has_key(term2):
                        t2_id = term_ct
                        vocab_dict[term2] = t2_id
                        vocab_ct_dict[t2_id] = 0 # avoid overcounting
                        term_ct += 1
                    else:
                        t2_id = vocab_dict[term2]


# Path: src/backend/tma_utils.py
# class TextCleaner:
#     def __init__(self, not_valid_reg=re.compile('\\S*[^\\w\\s]+\\S*'), remove_chars='[-\\d.,?;:\'")(!`}{\]\[]', stem = lib.porter2.stem, stopword_file=None):        
# 
#         self.not_valid_reg = not_valid_reg  
#         
#         if remove_chars and not remove_chars == '':
#             self.remove_chars_re = re.compile(remove_chars) 
#         else:
#             self.remove_chars_re = None
# 
#         self.stem = stem
#         
#         if stopword_file: 
#             self.stopwords = {}
#             stopfile = open(stopword_file,'r') 
#             for sw in stopfile:
#                 self.stopwords[sw.strip()] = 0
#             stopfile.close()
#         else: 
#             self.stopwords = None
#     
#     def parse_text(self, text):
#         clean_text = [] 
#         for word in text.strip().split():
#             word = self.parse_word(word)   
#             if not word == '':
#                 clean_text.append(word)
#         return clean_text 
#         
#     def parse_word(self, word):
#         if self.remove_chars_re:
#             retword = re.sub(self.remove_chars_re,'',word.strip()).lower() # remove punctuation and such
#         
#         if self.not_valid_reg:
#             if self.not_valid_reg.match(retword):
#                 return ''
#             
#         if self.stopwords:
#             if self.stopwords.has_key(retword):  
#                 return '' 
#         if self.stem:
#             retword = self.stem(retword)
#                 
#         return retword
# 
# def ids_to_key(id1, id2):
#     if type(id1) != int:
#         id1 = int(id1)
#     if type(id2) != int:
#         id2 = int(id2)
#     ids = sorted([id1,id2])
#     return ids[0], ids[1]
# 
# Path: lib/porter2.py
# def stem(word):
#     if len(word) <= 2:
#         return word
#     word = remove_initial_apostrophe(word)
# 
#     # handle some exceptional forms
#     if word in exceptional_forms:
#         return exceptional_forms[word]
# 
#     word = capitalize_consonant_ys(word)
#     r1 = get_r1(word)
#     r2 = get_r2(word)
#     word = step_0(word)
#     word = step_1a(word)
# 
#     # handle some more exceptional forms
#     if word in exceptional_early_exit_post_1a:
#         return word
# 
#     word = step_1b(word, r1)
#     word = step_1c(word)
#     word = step_2(word, r1)
#     word = step_3(word, r1, r2)
#     word = step_4(word, r2)
#     word = step_5(word, r1, r2)
#     word = normalize_ys(word)
# 
#     return word


# Path: src/backend/aux/create_wiki_cooccurence.py
import pdb 
import os 
import re 
import cPickle as pickle
import sqlite3 as sqlite                
import bsddb  
import random
from src.backend.tma_utils import TextCleaner, ids_to_key 
from lib.porter2 import stem
from time import time  

                    del(termterm_dict)
                    termterm_dict = {}
                    print '---- %i transfer complete, took %0.1f seconds ----' % (dep_no, (time() - ltime))
                    ltime = time()
                    
            text = line.strip() # remove the abstract tags
            text = text_cleaner.parse_text(text)
            text = list(set(text))
            tot_ab_len += len(text)

            for nt1, term1 in enumerate(text):
                if not vocab_dict.has_key(term1):
                    t1_id = term_ct
                    vocab_dict[term1] = t1_id
                    vocab_ct_dict[t1_id] = 1
                    term_ct += 1
                else:
                    t1_id = vocab_dict[term1]
                    vocab_ct_dict[t1_id] += 1

                for nt2 in xrange(nt1+1, len(text)): # 173.271281 vs 185s TODO make sure the counting is correct
                    term2 = text[nt2]
                    if not vocab_dict.has_key(term2):
                        t2_id = term_ct
                        vocab_dict[term2] = t2_id
                        vocab_ct_dict[t2_id] = 0 # avoid overcounting
                        term_ct += 1
                    else:
                        t2_id = vocab_dict[term2]


                    t_keys = ids_to_key(t1_id, t2_id)