<commit_before>from collections import namedtuple
import re

LogLine = namedtuple('LogLine', ['ts', 'msg',
                                 'processed', 'dictionary',
                                 'template', 'templateId', 'templateDict'])

TemplateLine = namedtuple('TemplateLine', ['id', 'template', 'skipWords'])


TransformLine = namedtuple('TransformLine',
                           ['id', 'type', 'NAME', 'transform', 'compiled'])

def badWay(r1, r2):
    '''
    correct way:

    For each pair of regexes r and s for languages L(r) and L(s)
    Find the corresponding Deterministic Finite Automata M(r) and M(s)   [1]
    Compute the cross-product machine M(r x s) and assign accepting states
    so that it computes L(r) - L(s)
    Use a DFS or BFS of the the M(r x s) transition table to see if any
    accepting state can be reached from the start state
    If no, you can eliminate s because L(s) is a subset of L(r).
    Reassign accepting states so that M(r x s) computes L(s) - L(r)
    Repeat the steps above to see if it's possible to eliminate r
    '''

    return(len(r2)-len(r1))


def rankMatches(m):
    retval = sorted(m, cmp=badWay)
    return retval


def getWordSkipNames(s):
    '''
    find the skip word patterns

    Args:
        s(_sre.SRE_Pattern): compiled regex to match a logline

    Returns:
        retval(list(string)): list of the skip patterns found in s
    '''

    pattern = r'\(\(\?\:\\\ \{0,1\}\\S\+\)\{(\d)\,(\d)\}\)'
    matchObj = re.finditer(pattern, s.pattern, re.M | re.I)

    retVal = list()

    if matchObj:
        for m in matchObj:
            vals = m.groups()
            fpattern = r'((?:\ {0,1}\S+){%i,%i})' % (int(vals[0]), int(vals[1]))
            retVal.append(fpattern)

    return retVal


def readTemplates(sc, templateFile):
    '''
    returns a list of regex for replacement processing

    Args:
        sc(sparkContext): spark context
        templateFile(string): uri to the transform file in HDFS

    Returns:
        retval(list(string))
    '''

    # map the templateFile
    templates = sc.textFile(templateFile)

    templateRDD = templates.collect()

    matches = list()

    for t in templateRDD:
        stripped = r''+t.strip().rstrip()
        print stripped
        escaped = re.escape(stripped)
        replaced = unescapeSkips(escaped)
        matches.append(replaced)

    matches = rankMatches(matches)

    templateLines = list()
    for index, m in enumerate(matches):
        # match end of line too
        t = TemplateLine(index,
                         re.compile(m + '$'),
                         getWordSkipNames(re.compile(m)))
        templateLines.append(t)

    print templateLines
    return templateLines


def unescapeSkips(s):
    '''
    find an escaped version of skip{m,n} words
    replace with unescaped version
<commit_msg>removing prints<commit_after>from collections import namedtuple
import re

LogLine = namedtuple('LogLine', ['ts', 'msg',
                                 'processed', 'dictionary',
                                 'template', 'templateId', 'templateDict'])

TemplateLine = namedtuple('TemplateLine', ['id', 'template', 'skipWords'])


TransformLine = namedtuple('TransformLine',
                           ['id', 'type', 'NAME', 'transform', 'compiled'])

def badWay(r1, r2):
    '''
    correct way:

    For each pair of regexes r and s for languages L(r) and L(s)
    Find the corresponding Deterministic Finite Automata M(r) and M(s)   [1]
    Compute the cross-product machine M(r x s) and assign accepting states
    so that it computes L(r) - L(s)
    Use a DFS or BFS of the the M(r x s) transition table to see if any
    accepting state can be reached from the start state
    If no, you can eliminate s because L(s) is a subset of L(r).
    Reassign accepting states so that M(r x s) computes L(s) - L(r)
    Repeat the steps above to see if it's possible to eliminate r
    '''

    return(len(r2)-len(r1))


def rankMatches(m):
    retval = sorted(m, cmp=badWay)
    return retval


def getWordSkipNames(s):
    '''
    find the skip word patterns

    Args:
        s(_sre.SRE_Pattern): compiled regex to match a logline

    Returns:
        retval(list(string)): list of the skip patterns found in s
    '''

    pattern = r'\(\(\?\:\\\ \{0,1\}\\S\+\)\{(\d)\,(\d)\}\)'
    matchObj = re.finditer(pattern, s.pattern, re.M | re.I)

    retVal = list()

    if matchObj:
        for m in matchObj:
            vals = m.groups()
            fpattern = r'((?:\ {0,1}\S+){%i,%i})' % (int(vals[0]), int(vals[1]))
            retVal.append(fpattern)

    return retVal


def readTemplates(sc, templateFile):
    '''
    returns a list of regex for replacement processing

    Args:
        sc(sparkContext): spark context
        templateFile(string): uri to the transform file in HDFS

    Returns:
        retval(list(string))
    '''

    # map the templateFile
    templates = sc.textFile(templateFile)

    templateRDD = templates.collect()

    matches = list()

    for t in templateRDD:
        stripped = r''+t.strip().rstrip()
        escaped = re.escape(stripped)
        replaced = unescapeSkips(escaped)
        matches.append(replaced)

    matches = rankMatches(matches)

    templateLines = list()
    for index, m in enumerate(matches):
        # match end of line too
        t = TemplateLine(index,
                         re.compile(m + '$'),
                         getWordSkipNames(re.compile(m)))
        templateLines.append(t)

    return templateLines


def unescapeSkips(s):
    '''
    find an escaped version of skip{m,n} words
    replace with unescaped version

{'fraction_non_alphanumeric': 0.11931315277040185, 'fraction_numerical': 0.0038944946008143033, 'mean_word_length': 2.367103694874851, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1365196', 'n_tokens_mistral': 1752, 'n_tokens_neox': 1675, 'n_words': 549}