<commit_before>import sys

from .error import err_add

## unicode literal support
if sys.version < '3':
    import codecs
    def u(x):
        return codecs.unicode_escape_decode(x)[0]
else:
    def u(x):
        return x

def attrsearch(tag, attr, list):
    for x in list:
        if getattr(x, attr) == tag:
            return x
    return None

def keysearch(tag, n, list):
    for x in list:
        if x[n] == tag:
            return x
    return None

def dictsearch(val, dict):
    if sys.version < '3':
        n = dict.iteritems()
        try:
            while True:
                (k,v) = n.next()
                if v == val:
                    return k
        except StopIteration:
            return None
    else:
        n = iter(dict.items())
        try:
            while True:
                (k,v) = next(n)
                if v == val:
                    return k
        except StopIteration:
            return None

def is_prefixed(identifier):
    return type(identifier) == type(()) and len(identifier) == 2

def is_local(identifier):
    return type(identifier) == type('') or type(identifier) == type(u(''))

def keyword_to_str(keyword):
    if keyword == '__tmp_augment__':
        return "undefined"
    elif is_prefixed(keyword):
        (prefix, keyword) = keyword
        return prefix + ":" + keyword
    else:
        return keyword

def guess_format(text):
    """Guess YANG/YIN format

    If the first non-whitespace character is '<' then it is XML.
    Return 'yang' or 'yin'"""
    format = 'yang'
    i = 0
    while i < len(text) and text[i].isspace():
        i += 1
    if i < len(text):
        if text[i] == '<':<commit_msg>Simplified dictsearch is more efficient<commit_after>import sys

from .error import err_add

## unicode literal support
if sys.version < '3':
    import codecs
    def u(x):
        return codecs.unicode_escape_decode(x)[0]
else:
    def u(x):
        return x

def attrsearch(tag, attr, list):
    for x in list:
        if getattr(x, attr) == tag:
            return x
    return None

def keysearch(tag, n, list):
    for x in list:
        if x[n] == tag:
            return x
    return None

def dictsearch(val, dict):
    for key in dict:
        if dict[key] == val:
            return key
    return None

def is_prefixed(identifier):
    return type(identifier) == type(()) and len(identifier) == 2

def is_local(identifier):
    return type(identifier) == type('') or type(identifier) == type(u(''))

def keyword_to_str(keyword):
    if keyword == '__tmp_augment__':
        return "undefined"
    elif is_prefixed(keyword):
        (prefix, keyword) = keyword
        return prefix + ":" + keyword
    else:
        return keyword

def guess_format(text):
    """Guess YANG/YIN format

    If the first non-whitespace character is '<' then it is XML.
    Return 'yang' or 'yin'"""
    format = 'yang'
    i = 0
    while i < len(text) and text[i].isspace():
        i += 1
    if i < len(text):
        if text[i] == '<':
{'fraction_non_alphanumeric': 0.1186046511627907, 'fraction_numerical': 0.003654485049833887, 'mean_word_length': 2.0537525354969572, 'pattern_counts': {'":': 2, '<': 14, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1299440', 'n_tokens_mistral': 958, 'n_tokens_neox': 935, 'n_words': 303}