<commit_before>import os
import sys
import re
import math
import codecs
import itertools
import cPickle
import json
import time

from collections import Counter, defaultdict
from subprocess import Popen, PIPE


try:
    from lxml import etree
except ImportError:
    try:
        import xml.etree.cElementTree as etree
    except ImportError:
        import xml.etree.ElementTree as etree


def init_paths(logonroot=None):
    """
    Set various paths. If logonroot argument omitted, uses LOGONROOT
    environment variable.
    """
    global LOGONROOT
    global LOGONBIN
    global TSDBHOME
    global ACEBIN
    global LOGONREGPATH

    LOGONROOT = logonroot

    if LOGONROOT is None:
        LOGONROOT = os.environ.get('LOGONROOT')

    if LOGONROOT is None:
        home = os.environ.get('HOME')
        if home is not None:
            LOGONROOT = os.path.join(home, 'logon')
        else:
            sys.stderr.write("Warining: LOGONROOT environment variable not set.")
            LOGONROOT = ''

    LOGONBIN = os.path.join(LOGONROOT, 'bin')
    ACEBIN = os.path.join(LOGONROOT, 'lingo', 'answer', 'bin', 'linux.x86.64', 'ace')
    TSDBHOME = os.environ.get('TSDBHOME')
    LOGONREGPATH = os.path.join(LOGONROOT, 'etc', 'registry')
    
    if TSDBHOME is None:
        TSDBHOME = os.path.join(LOGONROOT, 'lingo', 'lkb', 'src', 'tsdb', 'home') 


GRAMMAR_NAMES = {
    'erg'  : 'ERG',
    'terg' : 'Trunk ERG',
    'gg'   : 'GG',
    'jacy' : 'Jacy',
    'srg'  : 'SRG',
    'hag'  : 'Hausa Gramma',
}


# Virtual profiles found in the logon repository
# TODO write a funciton to actually read virtual profiles
VIRTUAL_PROFILES = {
    'ws' : ('WeScience', ['ws01', 'ws02', 'ws03', 'ws04', 'ws05', 'ws06', 'ws07',
                          'ws08', 'ws09', 'ws10', 'ws11', 'ws12', 'ws13']),
    'vm' : ('Verbmobil', ['vm6', 'vm13', 'vm31', 'vm32']),
    'tc' : ('Tanaka Corpus', ['tc-000', 'tc-001', 'tc-002', 'tc-003', 'tc-004', 
                              'tc-005', 'tc-006', 'tc-007', 'tc-008', 'tc-009', 
                              'tc-010', 'tc-011', 'tc-012', 'tc-013', 'tc-014', 
                              'tc-015', 'tc-016', 'tc-017', 'tc-018', 'tc-019', 
                              'tc-020', 'tc-021', 'tc-022', 'tc-023', 'tc-024', 
                              'tc-025', 'tc-026', 'tc-027', 'tc-028', 'tc-029', 
                              'tc-030', 'tc-031', 'tc-032', 'tc-033', 'tc-034', 
                              'tc-035', 'tc-036', 'tc-037', 'tc-038', 'tc-039', 
                              'tc-040', 'tc-041', 'tc-042', 'tc-043', 'tc-044', 
                              'tc-045', 'tc-046', 'tc-047', 'tc-048', 'tc-049', 
                              'tc-050', 'tc-051', 'tc-052', 'tc-053', 'tc-054', 
                              'tc-055', 'tc-056', 'tc-057', 'tc-058', 'tc-059', 
                              'tc-060', 'tc-061', 'tc-062', 'tc-063', 'tc-064', 
                              'tc-065', 'tc-066', 'tc-067', 'tc-068', 'tc-069', 
                              'tc-070', 'tc-071', 'tc-072', 'tc-073', 'tc-074', 
                              'tc-075', 'tc-076', 'tc-077', 'tc-078', 'tc-079', 
                              'tc-080', 'tc-081', 'tc-082', 'tc-083', 'tc-084', 
                              'tc-085', 'tc-086', 'tc-087', 'tc-088', 'tc-089', 
                              'tc-090', 'tc-091', 'tc-092', 'tc-093', 'tc-094', 
                              'tc-095', 'tc-096', 'tc-097', 'tc-098', 'tc-099', 
                              'tc-100']),
    'db' : ('DeepBank', ['wsj00a', 'wsj01a', 'wsj02a', 'wsj03a', 'wsj04b', 
                         'wsj05a', 'wsj05e', 'wsj06d', 'wsj07d', 'wsj09b', 
                         'wsj10b', 'wsj11b', 'wsj12a', 'wsj13a', 'wsj13e', 
                         'wsj14d', 'wsj15c', 'wsj16b', 'wsj16f', 'wsj17d', 
                         'wsj18d', 'wsj19c', 'wsj20c', 'wsj21b', 'wsj00b', 
                         'wsj01b', 'wsj02b', 'wsj03b', 'wsj04c', 'wsj05b', 
                         'wsj06a', 'wsj07a', 'wsj07e', 'wsj09c', 'wsj10c', 
                         'wsj11c', 'wsj12b', 'wsj13b', 'wsj14a', 'wsj14e', 
                         'wsj15d', 'wsj16c', 'wsj17a', 'wsj18a', 'wsj18e', 
                         'wsj19d', 'wsj20d', 'wsj21c', 'wsj00c', 'wsj01c', 
                         'wsj02c', 'wsj03c', 'wsj04d', 'wsj05c', 'wsj06b', 
                         'wsj07b', 'wsj08a', 'wsj09d', 'wsj10d', 'wsj11d', 
                         'wsj12c', 'wsj13c', 'wsj14b', 'wsj15a', 'wsj15e', 
                         'wsj16d', 'wsj17b', 'wsj18b', 'wsj19a', 'wsj20a', 
                         'wsj21', ' wsj21d', 'wsj00d', 'wsj01d', 'wsj02d', 
                         'wsj04a', 'wsj04e', 'wsj05d', 'wsj06c', 'wsj07c', 
                         'wsj09a', 'wsj10a', 'wsj11a', 'wsj11e', 'wsj12d', 
                         'wsj13d', 'wsj14c', 'wsj15b', 'wsj16a', 'wsj16e', 
                         'wsj17c', 'wsj18c', 'wsj19b', 'wsj20b', 'wsj21a']),
            }


class TsdbError(Exception):
    def __init__(self, msg):
        self.msg = msg

    def __str__(self):
        return self.msg


class LexLookupError(Exception):
    def __init__(self, msg):
        self.msg = msg


class DerivationError(Exception):
    def __init__(self, msg):
        self.msg = msg


class TypeNotFoundError(Exception):
    def __init__(self, t):
        self.type = t

    def __str__(self):
        return "{} not found in the hierarchy.\n".format(self.type)


class AceError(Exception):
    def __init__(self, prog, msg):
        self.prog = prog
        self.msg = msg

    def __str__(self):
        return "{} returned:\n{}".format(self.prog, self.msg)


class TypeStats(object):
    
    def __init__(self):
        self.counts = 0
        self.items = 0

    def update(self, counts):
        self.items += 1
        self.counts += counts


class Treebank(object):
    def __init__(self, params):
        for param, val in params.items():
            setattr(self, param, val)       


class JSONEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, set):
            return list(obj)
        if isinstance(obj, ConfigGrammar):
            return obj.json()
        if isinstance(obj, Treebank):
            return obj.__dict__
        if isinstance(obj, Reading):
            data = obj.__dict__
            data['tree'] = data['json_tree']
            del data['json_tree']
            del data['grammar']
            return data
        if isinstance(obj, Tree):
            return obj.ptb()
        if isinstance(obj, Token):
            return obj.__dict__
        if isinstance(obj, Fragment):
            data = obj.__dict__
            data['grammar'] = data['grammar'].alias
            del data['log_lines']
            del data['logpath']
            return data
        if isinstance(obj, TypeStats):
            return obj.__dict__
        return json.JSONEncoder.default(self, obj)


class Fragment(object):
    def __init__(self, text, grammar, ace_path=None, dat_path=None, count=None, 
                 tnt=False, typifier=None, fragments=False, logpath=None, 
                 cache=False, lextypes=True):
        self.input = text
        self.logpath = logpath
        self.log_lines = [time.strftime("%Y-%m-%d %H:%M:%S")]

        try:
            # supplied grammar is a Grammar object
            alias = grammar.alias
            self.grammar = grammar
        except AttributeError:
            # supplied grammar was just an alias
            self.grammar = LogonGrammar(grammar)

        if dat_path is not None:
            self.grammar.dat_path = dat_path

        if ace_path is None:
            ace_path = ACEBIN

        if self.grammar.lex_entries is None:
            lextypes = False

        self.preprocess()
        self.parse(ace_path, self.grammar.dat_path, count, fragments, tnt, typifier, cache, lextypes)

        if logpath is not None:
            self.write_log()

    def parse(self, ace_path, dat_path, count, fragments, tnt, typifier, cache, lextypes):
        env = dict(os.environ)
        env['LC_ALL'] = 'en_US.UTF-8'
        args = [ace_path, '-g', dat_path]

        if tnt:
            # If we have a logon installation, set PATH and model path 
            # to use this. Otherwise use the tnt tagger packaged with grammalytics
            if os.path.exists(LOGONBIN):
                env['PATH'] = "{}:{}".format(os.environ['PATH'], LOGONBIN) 
                model_path = os.path.join(LOGONROOT, 'coli', 'tnt', 'models', 'wsj.tnt') 
            else:
                thisdir = os.path.dirname(os.path.realpath(__file__))
                taggerpath = os.path.join(thisdir, '..', 'tagger')
                taggerbin = os.path.join(taggerpath, 'bin')
                env['PATH'] = "{}:{}".format(os.environ['PATH'], taggerbin) 
                model_path = os.path.join(taggerpath, 'coli', 'tnt', 'models', 'wsj.tnt') 
            args.append('--tnt-model')
            args.append(model_path)

        if count is not None:
            args.append('-n')
            args.append(str(count))

        if self.yy_input:
            args.append('-y')
            input_str = self.yy_input
        else:
            input_str = self.input

        if not fragments:
            if self.grammar.alias.startswith('erg') or self.grammar.alias.startswith('terg'):
                args.append('-r')
                args.append('root_strict root_informal')
                
        process = Popen(args, stdout=PIPE, stderr=PIPE, stdin=PIPE, env=env)
        out, err = process.communicate(input=input_str.encode('utf8'))
        lines = out.decode('utf8').strip().splitlines()
        status = lines[0]
        readings = lines[1:]
        self.stderr = err

        if process.returncode != 0 or out.startswith('SKIP'):
            ace_error_str = 'u\n'.join([status, err])
            self.log_lines.append(ace_error_str + '\n\n')<commit_msg>previous bugfix was not a fix<commit_after>import os
import sys
import re
import math
import codecs
import itertools
import cPickle
import json
import time

from collections import Counter, defaultdict
from subprocess import Popen, PIPE


try:
    from lxml import etree
except ImportError:
    try:
        import xml.etree.cElementTree as etree
    except ImportError:
        import xml.etree.ElementTree as etree


def init_paths(logonroot=None):
    """
    Set various paths. If logonroot argument omitted, uses LOGONROOT
    environment variable.
    """
    global LOGONROOT
    global LOGONBIN
    global TSDBHOME
    global ACEBIN
    global LOGONREGPATH

    LOGONROOT = logonroot

    if LOGONROOT is None:
        LOGONROOT = os.environ.get('LOGONROOT')

    if LOGONROOT is None:
        home = os.environ.get('HOME')
        if home is not None:
            LOGONROOT = os.path.join(home, 'logon')
        else:
            sys.stderr.write("Warining: LOGONROOT environment variable not set.")
            LOGONROOT = ''

    LOGONBIN = os.path.join(LOGONROOT, 'bin')
    ACEBIN = os.path.join(LOGONROOT, 'lingo', 'answer', 'bin', 'linux.x86.64', 'ace')
    TSDBHOME = os.environ.get('TSDBHOME')
    LOGONREGPATH = os.path.join(LOGONROOT, 'etc', 'registry')
    
    if TSDBHOME is None:
        TSDBHOME = os.path.join(LOGONROOT, 'lingo', 'lkb', 'src', 'tsdb', 'home') 


GRAMMAR_NAMES = {
    'erg'  : 'ERG',
    'terg' : 'Trunk ERG',
    'gg'   : 'GG',
    'jacy' : 'Jacy',
    'srg'  : 'SRG',
    'hag'  : 'Hausa Gramma',
}


# Virtual profiles found in the logon repository
# TODO write a funciton to actually read virtual profiles
VIRTUAL_PROFILES = {
    'ws' : ('WeScience', ['ws01', 'ws02', 'ws03', 'ws04', 'ws05', 'ws06', 'ws07',
                          'ws08', 'ws09', 'ws10', 'ws11', 'ws12', 'ws13']),
    'vm' : ('Verbmobil', ['vm6', 'vm13', 'vm31', 'vm32']),
    'tc' : ('Tanaka Corpus', ['tc-000', 'tc-001', 'tc-002', 'tc-003', 'tc-004', 
                              'tc-005', 'tc-006', 'tc-007', 'tc-008', 'tc-009', 
                              'tc-010', 'tc-011', 'tc-012', 'tc-013', 'tc-014', 
                              'tc-015', 'tc-016', 'tc-017', 'tc-018', 'tc-019', 
                              'tc-020', 'tc-021', 'tc-022', 'tc-023', 'tc-024', 
                              'tc-025', 'tc-026', 'tc-027', 'tc-028', 'tc-029', 
                              'tc-030', 'tc-031', 'tc-032', 'tc-033', 'tc-034', 
                              'tc-035', 'tc-036', 'tc-037', 'tc-038', 'tc-039', 
                              'tc-040', 'tc-041', 'tc-042', 'tc-043', 'tc-044', 
                              'tc-045', 'tc-046', 'tc-047', 'tc-048', 'tc-049', 
                              'tc-050', 'tc-051', 'tc-052', 'tc-053', 'tc-054', 
                              'tc-055', 'tc-056', 'tc-057', 'tc-058', 'tc-059', 
                              'tc-060', 'tc-061', 'tc-062', 'tc-063', 'tc-064', 
                              'tc-065', 'tc-066', 'tc-067', 'tc-068', 'tc-069', 
                              'tc-070', 'tc-071', 'tc-072', 'tc-073', 'tc-074', 
                              'tc-075', 'tc-076', 'tc-077', 'tc-078', 'tc-079', 
                              'tc-080', 'tc-081', 'tc-082', 'tc-083', 'tc-084', 
                              'tc-085', 'tc-086', 'tc-087', 'tc-088', 'tc-089', 
                              'tc-090', 'tc-091', 'tc-092', 'tc-093', 'tc-094', 
                              'tc-095', 'tc-096', 'tc-097', 'tc-098', 'tc-099', 
                              'tc-100']),
    'db' : ('DeepBank', ['wsj00a', 'wsj01a', 'wsj02a', 'wsj03a', 'wsj04b', 
                         'wsj05a', 'wsj05e', 'wsj06d', 'wsj07d', 'wsj09b', 
                         'wsj10b', 'wsj11b', 'wsj12a', 'wsj13a', 'wsj13e', 
                         'wsj14d', 'wsj15c', 'wsj16b', 'wsj16f', 'wsj17d', 
                         'wsj18d', 'wsj19c', 'wsj20c', 'wsj21b', 'wsj00b', 
                         'wsj01b', 'wsj02b', 'wsj03b', 'wsj04c', 'wsj05b', 
                         'wsj06a', 'wsj07a', 'wsj07e', 'wsj09c', 'wsj10c', 
                         'wsj11c', 'wsj12b', 'wsj13b', 'wsj14a', 'wsj14e', 
                         'wsj15d', 'wsj16c', 'wsj17a', 'wsj18a', 'wsj18e', 
                         'wsj19d', 'wsj20d', 'wsj21c', 'wsj00c', 'wsj01c', 
                         'wsj02c', 'wsj03c', 'wsj04d', 'wsj05c', 'wsj06b', 
                         'wsj07b', 'wsj08a', 'wsj09d', 'wsj10d', 'wsj11d', 
                         'wsj12c', 'wsj13c', 'wsj14b', 'wsj15a', 'wsj15e', 
                         'wsj16d', 'wsj17b', 'wsj18b', 'wsj19a', 'wsj20a', 
                         'wsj21', ' wsj21d', 'wsj00d', 'wsj01d', 'wsj02d', 
                         'wsj04a', 'wsj04e', 'wsj05d', 'wsj06c', 'wsj07c', 
                         'wsj09a', 'wsj10a', 'wsj11a', 'wsj11e', 'wsj12d', 
                         'wsj13d', 'wsj14c', 'wsj15b', 'wsj16a', 'wsj16e', 
                         'wsj17c', 'wsj18c', 'wsj19b', 'wsj20b', 'wsj21a']),
            }


class TsdbError(Exception):
    def __init__(self, msg):
        self.msg = msg

    def __str__(self):
        return self.msg


class LexLookupError(Exception):
    def __init__(self, msg):
        self.msg = msg


class DerivationError(Exception):
    def __init__(self, msg):
        self.msg = msg


class TypeNotFoundError(Exception):
    def __init__(self, t):
        self.type = t

    def __str__(self):
        return "{} not found in the hierarchy.\n".format(self.type)


class AceError(Exception):
    def __init__(self, prog, msg):
        self.prog = prog
        self.msg = msg

    def __str__(self):
        return "{} returned:\n{}".format(self.prog, self.msg)


class TypeStats(object):
    
    def __init__(self):
        self.counts = 0
        self.items = 0

    def update(self, counts):
        self.items += 1
        self.counts += counts


class Treebank(object):
    def __init__(self, params):
        for param, val in params.items():
            setattr(self, param, val)       


class JSONEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, set):
            return list(obj)
        if isinstance(obj, ConfigGrammar):
            return obj.json()
        if isinstance(obj, Treebank):
            return obj.__dict__
        if isinstance(obj, Reading):
            data = obj.__dict__
            data['tree'] = data['json_tree']
            del data['json_tree']
            del data['grammar']
            return data
        if isinstance(obj, Tree):
            return obj.ptb()
        if isinstance(obj, Token):
            return obj.__dict__
        if isinstance(obj, Fragment):
            data = obj.__dict__
            data['grammar'] = data['grammar'].alias
            del data['log_lines']
            del data['logpath']
            return data
        if isinstance(obj, TypeStats):
            return obj.__dict__
        return json.JSONEncoder.default(self, obj)


class Fragment(object):
    def __init__(self, text, grammar, ace_path=None, dat_path=None, count=None, 
                 tnt=False, typifier=None, fragments=False, logpath=None, 
                 cache=False, lextypes=True):
        self.input = text
        self.logpath = logpath
        self.log_lines = [time.strftime("%Y-%m-%d %H:%M:%S")]

        try:
            # supplied grammar is a Grammar object
            alias = grammar.alias
            self.grammar = grammar
        except AttributeError:
            # supplied grammar was just an alias
            self.grammar = LogonGrammar(grammar)

        if dat_path is not None:
            self.grammar.dat_path = dat_path

        if ace_path is None:
            ace_path = ACEBIN

        if self.grammar.lex_entries is None:
            lextypes = False

        self.preprocess()
        self.parse(ace_path, self.grammar.dat_path, count, fragments, tnt, typifier, cache, lextypes)

        if logpath is not None:
            self.write_log()

    def parse(self, ace_path, dat_path, count, fragments, tnt, typifier, cache, lextypes):
        env = dict(os.environ)
        env['LC_ALL'] = 'en_US.UTF-8'
        args = [ace_path, '-g', dat_path]

        if tnt:
            # If we have a logon installation, set PATH and model path 
            # to use this. Otherwise use the tnt tagger packaged with grammalytics
            if os.path.exists(LOGONBIN):
                env['PATH'] = "{}:{}".format(os.environ['PATH'], LOGONBIN) 
                model_path = os.path.join(LOGONROOT, 'coli', 'tnt', 'models', 'wsj.tnt') 
            else:
                thisdir = os.path.dirname(os.path.realpath(__file__))
                taggerpath = os.path.join(thisdir, '..', 'tagger')
                taggerbin = os.path.join(taggerpath, 'bin')
                env['PATH'] = "{}:{}".format(os.environ['PATH'], taggerbin) 
                model_path = os.path.join(taggerpath, 'coli', 'tnt', 'models', 'wsj.tnt') 
            args.append('--tnt-model')
            args.append(model_path)

        if count is not None:
            args.append('-n')
            args.append(str(count))

        if self.yy_input:
            args.append('-y')
            input_str = self.yy_input
        else:
            input_str = self.input

        if not fragments:
            if self.grammar.alias.startswith('erg') or self.grammar.alias.startswith('terg'):
                args.append('-r')
                args.append('root_strict root_informal')
                
        process = Popen(args, stdout=PIPE, stderr=PIPE, stdin=PIPE, env=env)
        out, err = process.communicate(input=input_str.encode('utf8'))
        lines = out.decode('utf8').strip().splitlines()
        status = lines[0]
        readings = lines[1:]
        self.stderr = err

        if process.returncode != 0 or out.startswith('SKIP'):
            ace_error_str = u'\n'.join([status, err])
            self.log_lines.append(ace_error_str + '\n\n')
{'fraction_non_alphanumeric': 0.15540132770066384, 'fraction_numerical': 0.05421444377388855, 'mean_word_length': 1.767956570155902, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7507318', 'n_tokens_mistral': 7510, 'n_tokens_neox': 6601, 'n_words': 1462}