<commit_before>import dxr.plugins
import os, sys
import fnmatch
import clang.tokenizers as tokenizers
import urllib, re

from dxr.utils import search_url


class ClangHtmlifier(object):
    """ Pygmentizer add syntax regions for file """
    def __init__(self, tree, conn, path, text, file_id):
        self.tree    = tree
        self.conn    = conn
        self.path    = path
        self.text    = text
        self.file_id = file_id

    def regions(self):
        tokenizer = tokenizers.CppTokenizer(self.text)
        for token in tokenizer.getTokens():
            if token.token_type in [tokenizer.KEYWORD, tokenizer.STRING,
                                    tokenizer.COMMENT, tokenizer.PREPROCESSOR]:
                # "operator" is going to be part of something bigger like
                # "operator++" or "operator new" so we don't want to create
                # a region here that only covers part of that
                if token.name != "operator":
                    yield (token.start, token.end, '')


    def refs(self):
        """ Generate reference menus """
        # We'll need this argument for all queries here
        args = (self.file_id,)

        # Extents for functions defined here
        sql = """
            SELECT extent_start, extent_end, qualname
                FROM functions
              WHERE file_id = ?
        """
        for start, end, qualname in self.conn.execute(sql, args):
            yield start, end, self.function_menu(qualname)

        # Extents for functions declared here<commit_msg>Don't yield anything for regions<commit_after>import dxr.plugins
import os, sys
import fnmatch
import clang.tokenizers as tokenizers
import urllib, re

from dxr.utils import search_url


class ClangHtmlifier(object):
    """ Pygmentizer add syntax regions for file """
    def __init__(self, tree, conn, path, text, file_id):
        self.tree    = tree
        self.conn    = conn
        self.path    = path
        self.text    = text
        self.file_id = file_id

    def regions(self):
        return []

    def refs(self):
        """ Generate reference menus """
        # We'll need this argument for all queries here
        args = (self.file_id,)

        # Extents for functions defined here
        sql = """
            SELECT extent_start, extent_end, qualname
                FROM functions
              WHERE file_id = ?
        """
        for start, end, qualname in self.conn.execute(sql, args):
            yield start, end, self.function_menu(qualname)

        # Extents for functions declared here
{'fraction_non_alphanumeric': 0.0811335403726708, 'fraction_numerical': 0.0, 'mean_word_length': 2.021101992966002, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6790249', 'n_tokens_mistral': 709, 'n_tokens_neox': 679, 'n_words': 232}