<commit_before># -*- coding: utf-8 -*-

"""
    TODO:
    - customize rendered HTML to ease choosing a new theme
"""

import desktop
import os
import pygments
import re
import sublime
import sublime_plugin
import subprocess
import tempfile

from pygments import highlight
from pygments.lexers import *
from pygments.formatters import *
from pygments.styles import STYLE_MAP

if desktop.get_desktop() == 'Windows':
    import winclip

# Don't judge me. Just don't. If you knew, you wouldn't.
__lexers = ['_asybuiltins', '_clbuiltins', '_lassobuiltins', '_luabuiltins',
    '_mapping', '_openedgebuiltins', '_phpbuiltins', '_postgres_builtins',
    '_scilab_builtins', '_sourcemodbuiltins', '_stan_builtins', '_vimbuiltins']
for l in __lexers:
    __import__('pygments.lexers.%s' % l)
for s in STYLE_MAP:
    __import__('pygments.styles.%s' % s)

DEFAULT_STYLE = "default"
FORMATS = ('html', 'rtf',)
WIN_CR_RE = re.compile(r"\r(?!\n)|(?<!\r)\n")

settings = sublime.load_settings('%s.sublime-settings' % __name__)


class SublimeHighlightCommand(sublime_plugin.TextCommand):
    """Code highlighter command."""

    @property
    def code(self):
        return self.view.substr(self.region)

    @property
    def encoding(self):
        encoding = self.view.encoding()
        if encoding == 'Undefined':
            encoding = 'utf-8'
        elif encoding == 'Western (Windows 1252)':
            encoding = 'windows-1252'
        else:
            i = encoding.find(' with BOM')

            if i != -1:
                encoding = encoding[:i]

        return encoding

    @property
    def region(self):
        regions = self.view.sel()
        if len(regions) > 0 and regions[0].size() > 0:
            return regions[0]
        else:
            return sublime.Region(0, self.view.size())

    def guess_lexer_from_syntax(self):
        syntax = self.view.settings().get('syntax')
        if not syntax:
            return
        match = re.match(r"Packages/.*/(.*?)\.tmLanguage$", syntax)<commit_msg>Fixing Sublime Text 3 syntax detection<commit_after># -*- coding: utf-8 -*-

"""
    TODO:
    - customize rendered HTML to ease choosing a new theme
"""

import desktop
import os
import pygments
import re
import sublime
import sublime_plugin
import subprocess
import tempfile

from pygments import highlight
from pygments.lexers import *
from pygments.formatters import *
from pygments.styles import STYLE_MAP

if desktop.get_desktop() == 'Windows':
    import winclip

# Don't judge me. Just don't. If you knew, you wouldn't.
__lexers = ['_asybuiltins', '_clbuiltins', '_lassobuiltins', '_luabuiltins',
    '_mapping', '_openedgebuiltins', '_phpbuiltins', '_postgres_builtins',
    '_scilab_builtins', '_sourcemodbuiltins', '_stan_builtins', '_vimbuiltins']
for l in __lexers:
    __import__('pygments.lexers.%s' % l)
for s in STYLE_MAP:
    __import__('pygments.styles.%s' % s)

DEFAULT_STYLE = "default"
FORMATS = ('html', 'rtf',)
WIN_CR_RE = re.compile(r"\r(?!\n)|(?<!\r)\n")

settings = sublime.load_settings('%s.sublime-settings' % __name__)


class SublimeHighlightCommand(sublime_plugin.TextCommand):
    """Code highlighter command."""

    @property
    def code(self):
        return self.view.substr(self.region)

    @property
    def encoding(self):
        encoding = self.view.encoding()
        if encoding == 'Undefined':
            encoding = 'utf-8'
        elif encoding == 'Western (Windows 1252)':
            encoding = 'windows-1252'
        else:
            i = encoding.find(' with BOM')

            if i != -1:
                encoding = encoding[:i]

        return encoding

    @property
    def region(self):
        regions = self.view.sel()
        if len(regions) > 0 and regions[0].size() > 0:
            return regions[0]
        else:
            return sublime.Region(0, self.view.size())

    def guess_lexer_from_syntax(self):
        syntax = self.view.settings().get('syntax')
        if not syntax:
            return
        match = re.match(r"Packages/.*/(.*?)\.(?:tmLanguage|sublime-syntax)$", syntax)
{'fraction_non_alphanumeric': 0.13244707040866568, 'fraction_numerical': 0.00812407680945347, 'mean_word_length': 2.9408341416100874, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7443623', 'n_tokens_mistral': 1356, 'n_tokens_neox': 1260, 'n_words': 324}