<commit_before>import re

import sublime
from sublime_plugin import TextCommand, EventListener


SETTINGS_REGEX = r'(?:sublimeconf):\s+filetype=([\w][\w\./ ]+)'

DEFAULT_FILETYPE_MAP = {
    'python': 'Packages/Python/Python.tmLanguage',
    'shell': 'Packages/ShellScript/Shell-Unix-Generic.tmLanguage',
    'markdown': 'Packages/Markdown/Markdown.tmLanguage'
}


def get_settings():
    filename = __name__.split('.')[-1] + '.sublime-settings'
    return sublime.load_settings(filename)


def get_setting(name, default=None):
    return get_settings().get(name, default)


def find_filetype(view):
    # find the region for the first filetype conf snippet
    region = view.find(SETTINGS_REGEX, 0)
    # if there is one, extract the filetype value from the snippet
    match = re.match(SETTINGS_REGEX, view.substr(region))
    return match.group(1) if match else None


def filetype_to_package(filetype):
    filetype_map = get_setting('filetype_package_map')
    if filetype_map and filetype in filetype_map:
        return filetype_map[filetype]
    else:
        return DEFAULT_FILETYPE_MAP.get(filetype)


def update_filetype(view, filetype):
    try:
        new_filetype_path = DEFAULT_FILETYPE_MAP[filetype.lower()]
    except AttributeError:
        # use it as is, on the assumption that the user gave a complete path
        # like 'Packages/Python/Python.tmLanguage' instead of just 'python'
        new_filetype_path = filetype

    # only set the syntax if it would change it from the current syntax
    curr_filetype_path = view.settings().get('syntax')
    if new_filetype_path != curr_filetype_path:
        view.set_syntax_file(new_filetype_path)


def process_view_filetype(view):
    filetype = find_filetype(view)
    if filetype:<commit_msg>rest of commit<commit_after>import re

import sublime
from sublime_plugin import TextCommand, EventListener


SETTINGS_REGEX = r'(?:sublimeconf):\s+filetype=([\w][\w\./ ]+)'

DEFAULT_FILETYPE_MAP = {
    'python': 'Packages/Python/Python.tmLanguage',
    'shell': 'Packages/ShellScript/Shell-Unix-Generic.tmLanguage',
    'markdown': 'Packages/Markdown/Markdown.tmLanguage'
}


def get_settings():
    filename = __name__.split('.')[-1] + '.sublime-settings'
    return sublime.load_settings(filename)


def get_setting(name, default=None):
    return get_settings().get(name, default)


def find_filetype(view):
    # find the region for the first filetype conf snippet
    region = view.find(SETTINGS_REGEX, 0)
    # if there is one, extract the filetype value from the snippet
    match = re.match(SETTINGS_REGEX, view.substr(region))
    return match.group(1) if match else None


def filetype_to_package(filetype):
    filetype_map = get_setting('filetype_package_map')
    if filetype_map and filetype in filetype_map:
        return filetype_map[filetype]
    else:
        return DEFAULT_FILETYPE_MAP.get(filetype)


def update_filetype(view, filetype):
    new_filetype_path = filetype_to_package(filetype)
    # only set the syntax if it would change it from the current syntax
    curr_filetype_path = view.settings().get('syntax')
    if new_filetype_path and new_filetype_path != curr_filetype_path:
        view.set_syntax_file(new_filetype_path)


def process_view_filetype(view):
    filetype = find_filetype(view)
    if filetype:
{'fraction_non_alphanumeric': 0.09966777408637874, 'fraction_numerical': 0.001812141347025068, 'mean_word_length': 4.257142857142857, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1642084', 'n_tokens_mistral': 1087, 'n_tokens_neox': 1065, 'n_words': 248}