<commit_before>
import sublime, sublime_plugin, re

def matches(needle, haystack, search_type):
    if search_type is "regex":
        return re.match(needle, haystack)
    else:
        return (needle in haystack)


def filter(view, edit, needle, search_type):
    # get non-empty selections
    regions = [s for s in view.sel() if not s.empty()]

    # no selections? filter the whole document
    if len(regions) == 0:
        regions = [ sublime.Region(0, view.size()) ]

    for region in reversed(regions):
        lines = view.split_by_newlines(region)

        for line in reversed(lines):

            if not matches(needle, view.substr(line), search_type):
                view.erase(edit, view.full_line(line))


class FilterCommand(sublime_plugin.WindowCommand):

    def run(self):
        cb = sublime.get_clipboard()
        sublime.active_window().show_input_panel("Filter file for lines containing: ", cb, self.on_done, None, None)

    def on_done(self, text):
        if self.window.active_view():
            self.window.active_view().run_command("filter_lines", { "needle": text })


class FilterUsingRegularExpressionCommand(sublime_plugin.WindowCommand):

    def run(self):
        cb = sublime.get_clipboard()
        sublime.active_window().show_input_panel("Filter file for lines matching: ", cb, self.on_done, None, None)

    def on_done(self, text):
        if self.window.active_view():
            self.window.active_view().run_command("filter_lines", { "needle": text, "search_type": "regex" })


class FilterLinesCommand(sublime_plugin.TextCommand):

    def run(self, edit, needle, search_type = "string"):
        filter(self.view, edit, needle, search_type)<commit_msg>Uses re.search to find matches anywhere in a line<commit_after>
import sublime, sublime_plugin, re

def matches(needle, haystack, search_type):
    if search_type == "regex":
        return re.search(needle, haystack)
    else:
        return (needle in haystack)


def filter(view, edit, needle, search_type):
    # get non-empty selections
    regions = [s for s in view.sel() if not s.empty()]

    # no selections? filter the whole document
    if len(regions) == 0:
        regions = [ sublime.Region(0, view.size()) ]

    for region in reversed(regions):
        lines = view.split_by_newlines(region)

        for line in reversed(lines):

            if not matches(needle, view.substr(line), search_type):
                view.erase(edit, view.full_line(line))


class FilterCommand(sublime_plugin.WindowCommand):

    def run(self):
        sublime.active_window().show_input_panel("Filter file for lines containing: ", "", self.on_done, None, None)

    def on_done(self, text):
        if self.window.active_view():
            self.window.active_view().run_command("filter_lines", { "needle": text, "search_type": "string" })


class FilterUsingRegularExpressionCommand(sublime_plugin.WindowCommand):

    def run(self):
        sublime.active_window().show_input_panel("Filter file for lines matching: ", "", self.on_done, None, None)

    def on_done(self, text):
        if self.window.active_view():
            self.window.active_view().run_command("filter_lines", { "needle": text, "search_type": "regex" })


class FilterLinesCommand(sublime_plugin.TextCommand):

    def run(self, edit, needle, search_type):
        filter(self.view, edit, needle, search_type)
{'fraction_non_alphanumeric': 0.11979935084095604, 'fraction_numerical': 0.0011802891708468574, 'mean_word_length': 3.43717277486911, 'pattern_counts': {'":': 9, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4390694', 'n_tokens_mistral': 1068, 'n_tokens_neox': 1019, 'n_words': 255}