<commit_before>#!/usr/bin/python
"""
This is a general grammar file that assumes the following grammar:
W -> <english-word>L | <name>L
D -> <date> | <phone-no> | [0-9]+
Y -> [^\W]+  # symbol
K -> <keyboard-sequence>
R -> repeat
S -> sequence # 123456, abcdef, ABCDEFG
L -> Capitalize | ALL-UPPER | all-lower | l33t
G -> <some-combination-of-those-NonTs>
"""

import os, sys, string
BASE_DIR = os.getcwd()
sys.path.append(BASE_DIR)
from dawg import IntDAWG, DAWG
import marisa_trie
import struct, json, bz2, re
from lexer_helper import Date, KeyBoard, RuleSet
from helper.helper import open_, getIndex
import honeyvault_config as hny_config
from honeyvault_config import NONTERMINAL, TERMINAL, MIN_COUNT
from collections import OrderedDict, defaultdict
from pprint import pprint

# Dictionaries:
# English_30000.dawg
# facebook-firstnames-withcount.dawg
# facebook-lastnames-withcount.dawg            

class NonT(object): # baseclass
    def __init__(self):
        self.sym = 'G'
        self.prob = 0.0
        self.prod = ''
    
    def symbol(self):
        return self.sym

    def probability(self):
        return self.prob

    def production(self):
        return self.prod

    def __str__(self):
        p_str = [str(p) for p in self.prod] \
            if isinstance(self.prod, list) \
            else self.prod
        
        return '%s: %s (%g)' % (self.sym, p_str,
                                self.prob)

    def parse_tree(self):
        p_tree = OrderedDict([('G', OrderedDict())])
        if isinstance(self.prod, basestring):
            return self.prod
        elif isinstance(self.prod, list):
            for i,c in enumerate(zip(self.sym, self.prod)):
                k, p = c
                k = '%s:%d' % (k, i)
                if isinstance(p, basestring):
                    p_tree['G'][k] = p
                else:
                    p_tree['G'][k] = p.parse_tree()
        else:
            return self.prod.parse_tree()
        return p_tree

    def rule_set(self):
        rs = RuleSet()
        pass

    def __nonzero__(self):
        return bool(self.prod) and bool(self.prob)
    __bool__ = __nonzero__


class NonT_L(NonT):
    sym, prod, prob  = 'L', '', 0.0
    def __init__(self, w, v):
        super(NonT_L, self).__init__()
        self.prod = None if not w.isalpha \
            else 'Caps' if w.istitle()  \
            else 'lower' if w.islower() \
            else 'UPPER' if w.isupper() \
            else 'l33t'
        self.r = w
        self.l = v
        if self.prod == 'l33t':
            c = len([c for c,d in zip(self.l,self.r)<commit_msg>use of ParseTree class<commit_after>#!/usr/bin/python
"""
This is a general grammar file that assumes the following grammar:
W -> <english-word>L | <name>L
D -> <date> | <phone-no> | [0-9]+
Y -> [^\W]+  # symbol
K -> <keyboard-sequence>
R -> repeat
S -> sequence # 123456, abcdef, ABCDEFG
L -> Capitalize | ALL-UPPER | all-lower | l33t
G -> <some-combination-of-those-NonTs>
"""

import os, sys, string
BASE_DIR = os.getcwd()
sys.path.append(BASE_DIR)
from dawg import IntDAWG, DAWG
import marisa_trie
import struct, json, bz2, re
from lexer_helper import Date, KeyBoard, RuleSet
from helper.helper import open_, getIndex
import honeyvault_config as hny_config
from honeyvault_config import NONTERMINAL, TERMINAL, MIN_COUNT
from collections import OrderedDict, defaultdict
from pprint import pprint

# Dictionaries:
# English_30000.dawg
# facebook-firstnames-withcount.dawg
# facebook-lastnames-withcount.dawg            

class NonT(object): # baseclass
    def __init__(self):
        self.sym = 'G'
        self.prob = 0.0
        self.prod = ''
    
    def symbol(self):
        return self.sym

    def probability(self):
        return self.prob

    def production(self):
        return self.prod

    def __str__(self):
        p_str = [str(p) for p in self.prod] \
            if isinstance(self.prod, list) \
            else self.prod
        
        return '%s: %s (%g)' % (self.sym, p_str,
                                self.prob)

    def parse_tree(self):
        p_tree = ParseTree()
        if isinstance(self.prod, basestring):
            return self.prod
        elif isinstance(self.prod, list):
            for c, p in zip(self.sym, self.prod):
                if isinstance(p, basestring):
                    p_tree.add_rule((c, p))
                else:
                    p_tree.add_rule((c,p.parse_tree()))
        else:
            return self.prod.parse_tree()
        return p_tree

    def rule_set(self):
        rs = RuleSet()
        pass

    def __nonzero__(self):
        return bool(self.prod) and bool(self.prob)
    __bool__ = __nonzero__


class NonT_L(NonT):
    sym, prod, prob  = 'L', '', 0.0
    def __init__(self, w, v):
        super(NonT_L, self).__init__()
        self.prod = None if not w.isalpha \
            else 'Caps' if w.istitle()  \
            else 'lower' if w.islower() \
            else 'UPPER' if w.isupper() \
            else 'l33t'
        self.r = w
        self.l = v
        if self.prod == 'l33t':
            c = len([c for c,d in zip(self.l,self.r)
{'fraction_non_alphanumeric': 0.12061574434918161, 'fraction_numerical': 0.009353078721745909, 'mean_word_length': 2.053539559785842, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3460612', 'n_tokens_mistral': 1764, 'n_tokens_neox': 1697, 'n_words': 452}