<commit_before>import sys
from langtools.utils import readkr
import matcher
import np_grammar

def parse_rule(rule):
    right = rule.split('->')[1].strip()
    matchers = []
    for kr in right.split():        
            kr_dict = readkr.kr_to_dictionary('stem/' + kr)
            pattermatch = matcher.PatternMatcher(kr_dict)
            matchers.append(pattermatch)
    return matchers

def parse_chunk(chunk):
    # Chunk constructions are run here to form the phrase machines.
    change = True
    while change:
        change = False
        try:
            for length in xrange(2, len(chunk) + 1):
                for begin, end in _subsequence_index(chunk, length):
                    part = chunk[begin:end]
                    for c in np_grammar.np_rules:
                        if c.check(part):
                            c_res = c.act(part)
                            if c_res is not None:
                                change = True
                                chunk[begin:end] = c_res  # c_res should contain a single machine here (?)
                                raise ValueError  # == break outer
        except ValueError:
            pass

def _subsequence_index(seq, length):
    """
    Returns the start and end indices of all subsequences of @p seq of length
    @p length as an iterator.
    """
    l = len(seq)<commit_msg>parse_chunk() now returns the NP. 	modified:   np_parser.py<commit_after>import sys
from langtools.utils import readkr
import matcher
import np_grammar

def parse_rule(rule):
    right = rule.split('->')[1].strip()
    matchers = []
    for kr in right.split():        
            kr_dict = readkr.kr_to_dictionary('stem/' + kr)
            pattermatch = matcher.PatternMatcher(kr_dict)
            matchers.append(pattermatch)
    return matchers

def parse_chunk(chunk):
    # Chunk constructions are run here to form the phrase machines.
    change = True
    while change:
        change = False
        try:
            for length in xrange(2, len(chunk) + 1):
                for begin, end in _subsequence_index(chunk, length):
                    part = chunk[begin:end]
                    for c in np_grammar.np_rules:
                        if c.check(part):
                            c_res = c.act(part)
                            if c_res is not None:
                                change = True
                                chunk[begin:end] = c_res  # c_res should contain a single machine here (?)
                                raise ValueError  # == break outer
        except ValueError:
            pass
    return chunk

def _subsequence_index(seq, length):
    """
    Returns the start and end indices of all subsequences of @p seq of length
    @p length as an iterator.
    """
    l = len(seq)
{'fraction_non_alphanumeric': 0.07835420393559929, 'fraction_numerical': 0.002146690518783542, 'mean_word_length': 1.6130841121495327, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7083043', 'n_tokens_mistral': 764, 'n_tokens_neox': 743, 'n_words': 237}