<commit_before># -*- coding: utf-8 -*-

import os

import sys

from lcs import longest_common_substring
from operator import itemgetter


relpath = os.path.relpath


def clean_path(path):
    path = relpath(
        path.strip()
            .replace('**/', '')
            .replace('\r', '')
            .replace('\\ ', ' ')
            .replace('\\', '/')
    )
    return path


def _slash_pattern(pattern):
    """
    Checks if pattern ends with a slash and appends one if slash is not present

    :pattern (str) pattern added/removed

    returns a pattern with slash
    """
    if pattern.endswith('/'):
        return pattern
    else:
        return '%s/' % pattern


def _extract_match(toc, index):
    """
    Extracts a path between seperators (,)

    :toc (str) Table of contents
    :index (int) Index of match

    returns full path from match
    """
    length = len(toc)
    start_index = index
    while toc[start_index] != ',' and start_index >= 0:
        start_index -= 1
    end_index = index
    while toc[end_index] != ',' and end_index < length - 1:
        end_index += 1
    if end_index == length - 1:
        end_index += 1
    return toc[start_index+1:end_index]


def _resolve_path(toc, path, resolvers, ancestors=None):
    """
    Resolve a path

    :toc (str) Table of contents
    :path (str) The path to be resolved
    :resolvers (list) Resolved changes

    returns new_path (str), pattern (list)
    """
    path = clean_path(path)

    _pattern = ',{}{},'.format
    # direct match
    if _pattern(path, '') in toc:
        return path, None

    # known changes
    _path_startswith = path.startswith
    for (remove, add) in resolvers:
        if _path_startswith(remove):
            _path = _pattern(add, path[len(remove):])
            if _path in toc:
                return _path[1:-1], None<commit_msg>quicker exit if there is no possible match<commit_after># -*- coding: utf-8 -*-

import os

import sys

from lcs import longest_common_substring
from operator import itemgetter


relpath = os.path.relpath


def clean_path(path):
    path = relpath(
        path.strip()
            .replace('**/', '')
            .replace('\r', '')
            .replace('\\ ', ' ')
            .replace('\\', '/')
    )
    return path


def _slash_pattern(pattern):
    """
    Checks if pattern ends with a slash and appends one if slash is not present

    :pattern (str) pattern added/removed

    returns a pattern with slash
    """
    if pattern.endswith('/'):
        return pattern
    else:
        return '%s/' % pattern


def _extract_match(toc, index):
    """
    Extracts a path between seperators (,)

    :toc (str) Table of contents
    :index (int) Index of match

    returns full path from match
    """
    length = len(toc)
    start_index = index
    while toc[start_index] != ',' and start_index >= 0:
        start_index -= 1
    end_index = index
    while toc[end_index] != ',' and end_index < length - 1:
        end_index += 1
    if end_index == length - 1:
        end_index += 1
    return toc[start_index+1:end_index]


def _resolve_path(toc, path, resolvers, ancestors=None):
    """
    Resolve a path

    :toc (str) Table of contents
    :path (str) The path to be resolved
    :resolvers (list) Resolved changes

    returns new_path (str), pattern (list)
    """
    path = clean_path(path)

    _pattern = ',{}{},'.format
    # direct match
    if _pattern(path, '') in toc:
        return path, None

    # will not resolve - no possible matches
    if ('%s,' % '/'.join(path.rsplit('/', (ancestors or 0) + 1)[1:])) not in toc:
        return None, None

    # known changes
    _path_startswith = path.startswith
    for (remove, add) in resolvers:
        if _path_startswith(remove):
            _path = _pattern(add, path[len(remove):])
            if _path in toc:
                return _path[1:-1], None
{'fraction_non_alphanumeric': 0.1218195836545875, 'fraction_numerical': 0.005911076843998972, 'mean_word_length': 2.278854254422915, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4468024', 'n_tokens_mistral': 1260, 'n_tokens_neox': 1198, 'n_words': 379}