<commit_before>from base64 import b64encode
from binascii import hexlify
from collections import Counter, defaultdict
from string import ascii_letters
from typing import Dict
from pathlib import Path
import os

current_dir = str(Path(__file__).parent)


def get_most_common_from_counter(counter: Counter, n: int):
    """Like counter.most_common(n), but includes elements with
    equal counts, which means the returned
    list may be longer than n. Elements with the same count are ordered in
    reverse alphabetical order (easier to code), but all elements with a
    given count will be returned if one is returned.

    >>> get_most_common_from_counter(Counter('test'), 1)
    [('t', 2)]
    >>> get_most_common_from_counter(Counter('test'), 2)
    [('t', 2), ('s', 1), ('e', 1)]
    >>> get_most_common_from_counter(Counter('test'), 1)
    [('t', 2)]
    >>> get_most_common_from_counter(Counter('teest'), 1)
    [('t', 2), ('e', 2)]
    >>> get_most_common_from_counter(Counter('teest'), 9)
    [('t', 2), ('e', 2), ('s', 1)]"""
    if n < 0:
        return []
    elems = counter.most_common()
    # nums holds all the nums we have seen so far, including duplicates
    nums = [elems[0][1]]
    i = 1
    for elem, num in elems[1:]:
        if len(nums) >= n and num < nums[-1]:
            break
        # we either haven't got enough nums or we haven't finished collecting
        # the same num
        nums.append(num)
        i += 1
    return sorted(elems[:i], key=lambda tup: str(
        tup[1]) + tup[0], reverse=True)
<commit_msg>Optimise get_most_common_from_counter func<commit_after>from base64 import b64encode
from binascii import hexlify
from collections import Counter, defaultdict
from string import ascii_letters
from typing import Dict
from pathlib import Path
import os

current_dir = str(Path(__file__).parent)


def get_most_common_from_counter(counter: Counter, n: int):
    """Like counter.most_common(n), but includes elements with
    equal counts, which means the returned
    list may be longer than n. Elements with the same count are ordered in
    reverse alphabetical order (easier to code), but all elements with a
    given count will be returned if one is returned.

    >>> get_most_common_from_counter(Counter('test'), 1)
    [('t', 2)]
    >>> get_most_common_from_counter(Counter('test'), 2)
    [('t', 2), ('s', 1), ('e', 1)]
    >>> get_most_common_from_counter(Counter('test'), 1)
    [('t', 2)]
    >>> get_most_common_from_counter(Counter('teest'), 1)
    [('t', 2), ('e', 2)]
    >>> get_most_common_from_counter(Counter('teest'), 9)
    [('t', 2), ('e', 2), ('s', 1)]"""
    if n < 0:
        return []
    elems = counter.most_common()
    # nums holds all the nums we have seen so far, including duplicates
    last_num = elems[0][1]
    total_nums_so_far = i = 1
    for elem, num in elems[1:]:
        if total_nums_so_far >= n and num < last_num:
            break
        # we either haven't got enough nums or we haven't finished collecting
        # elems with the same count
        last_num = num
        total_nums_so_far += 1
        i += 1
    return sorted(elems[:i], key=lambda tup: str(
        tup[1]) + tup[0], reverse=True)

{'fraction_non_alphanumeric': 0.1262894654579556, 'fraction_numerical': 0.0175054704595186, 'mean_word_length': 3.4077134986225897, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6698575', 'n_tokens_mistral': 1084, 'n_tokens_neox': 1014, 'n_words': 346}