<commit_before>import hashlib
import pygments
import pygments.lexers as lexers
import pygments.formatters as formatters
from pygments.util import ClassNotFound
from tempfile import NamedTemporaryFile


def expand_dateformat(dateformat):
    """
    Takes the dateformat string from ~/.noterc
    and adds the '%'-signs.

    Y.m.d => %Y.%m.%d

    """
    format_str = ''

    for v in dateformat:
        if v.isalpha():
            format_str += '%' + v
        else:
            format_str += v
    return format_str


def create_tempfile():
    tmp_file = NamedTemporaryFile(delete=False)
    tmp_file.close()
    return tmp_file.name


def exit_not_exists():
    print(_('Error: This note does not exist!'))
    exit(1)


def highlight(data, lang):
    from pynote import config  # avoid circular importing

    try:
        lexer = lexers.get_lexer_by_name(lang)
    except ClassNotFound:
        print(_('Lexer not found!'))
        exit(1)

    try:
        formatter = formatters.Terminal256Formatter(style=config.PYGMENTS_THEME)
    except ClassNotFound:
        styles = list(get_all_styles())
        print(_('Theme {} not found!').format(config.PYGMENTS_THEME))
        print(_("Please correct pygments_theme in your '~/.noterc'!"))
        print(_('Supported themes are:'))
        print()
        for style in styles:
            print(style)
        exit(1)

    data = pygments.highlight(data, lexer, formatter)
    return data


def get_md5(item):
    return hashlib.md5(item.encode('UTF-8')).digest()
<commit_msg>Missing import<commit_after>import hashlib
import pygments
import pygments.lexers as lexers
import pygments.formatters as formatters
from pygments.util import ClassNotFound
from pygments.styles import get_all_styles
from tempfile import NamedTemporaryFile


def expand_dateformat(dateformat):
    """
    Takes the dateformat string from ~/.noterc
    and adds the '%'-signs.

    Y.m.d => %Y.%m.%d

    """
    format_str = ''

    for v in dateformat:
        if v.isalpha():
            format_str += '%' + v
        else:
            format_str += v
    return format_str


def create_tempfile():
    tmp_file = NamedTemporaryFile(delete=False)
    tmp_file.close()
    return tmp_file.name


def exit_not_exists():
    print(_('Error: This note does not exist!'))
    exit(1)


def highlight(data, lang):
    from pynote import config  # avoid circular importing

    try:
        lexer = lexers.get_lexer_by_name(lang)
    except ClassNotFound:
        print(_('Lexer not found!'))
        exit(1)

    try:
        formatter = formatters.Terminal256Formatter(style=config.PYGMENTS_THEME)
    except ClassNotFound:
        styles = list(get_all_styles())
        print(_('Theme {} not found!').format(config.PYGMENTS_THEME))
        print(_("Please correct pygments_theme in your '~/.noterc'!"))
        print(_('Supported themes are:'))
        print()
        for style in styles:
            print(style)
        exit(1)

    data = pygments.highlight(data, lexer, formatter)
    return data


def get_md5(item):
    return hashlib.md5(item.encode('UTF-8')).digest()

{'fraction_non_alphanumeric': 0.10714285714285714, 'fraction_numerical': 0.005791505791505791, 'mean_word_length': 2.8335388409371145, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2080162', 'n_tokens_mistral': 1015, 'n_tokens_neox': 947, 'n_words': 243}