<commit_before>import os
from collections import namedtuple
import cchardet
from hotdoc.parsers.c_comment_scanner.c_comment_scanner import extract_comments

from hotdoc.core.symbols import *
from hotdoc.utils.loggable import debug


RawMacro = namedtuple('RawMacro', ['raw', 'filename'])


def unicode_dammit(data):
    encoding = cchardet.detect(data)['encoding']
    try:
        return data.decode(encoding, errors='replace')
    except LookupError:
        return data.decode('utf8', errors='replace')


class CCommentExtractor:
    def __init__(self, extension, comment_parser):
        self.extension = extension
        self.app = extension.app
        self.project = extension.project
        self.__raw_comment_parser = comment_parser
        self.__raw_macros = []

    def parse_comments(self, filenames):
        for filename in filenames:
            with open(filename, 'rb') as f:
                debug('Getting comments in %s' % filename)
                lines = []
                header = filename.endswith('.h')
                skip_next_symbol = header
                # FIXME Use the lexer for that!
                for l in f.readlines():
                    l = unicode_dammit(l)
                    lines.append(l)
                    if skip_next_symbol and l.startswith("#pragma once"):
                        skip_next_symbol = False

                cs = extract_comments(''.join(lines))
                for c in cs:
                    if c[3]:
                        line = lines[c[1] - 1]

                        comment = (len(line) - len(line.lstrip(' '))
                                   ) * ' ' + c[0]
                        block = self.__raw_comment_parser.parse_comment(comment,
                                                                        filename, c[1], c[2], self.project.include_paths)
                        if block is not None:
                            self.app.database.add_comment(block)
                    elif not skip_next_symbol:
                        if header:
                            self.__raw_macros.append(RawMacro(c, filename))
                            # self.__create_macro_from_raw_text(c, filename, filter_names)
                    else:
                        skip_next_symbol = False

    def create_macro_symbols(self, filter_names=None, filenames=None):
        filenames = filenames or set()
        filter_names = filter_names or {}
        for raw_macro in self.__raw_macros:
            self.__create_macro_from_raw_text(
                raw_macro.raw, raw_macro.filename, filter_names, filenames)

    def __get_comment_filename(self, name, filenames):
        comment = self.app.database.get_comment(name)
        if comment and comment.filename:
            header = '%s.h' % os.path.splitext(comment.filename)[0]
            if header in filenames:
                return header
            elif comment.filename in filenames:
                return comment.filename

        return ""

    def __create_macro_from_raw_text(self, raw, filename, filter_names, filenames):
        mcontent = raw[0].replace('\t', ' ')
        mcontent = mcontent.split(' ', 1)[1]
        split = mcontent.split('(', 1)
        name = split[0]

        if name.strip() in filter_names:
            return None

        # `#define foo (x) ...` is *not* a function macro
        if not (' ' in name or '\t' in name) and len(split) == 2:
            args = split[1].split(')', 1)[0].split(',')
            if args:
                stripped_name = name.strip()
                filename = self.__get_comment_filename(stripped_name, filenames)
                return self.__create_function_macro_symbol(stripped_name,
                                                           filename, raw[1], raw[0])

        name = mcontent.split(' ', 1)[0]
        stripped_name = name.strip()
        filename = self.__get_comment_filename(stripped_name, filenames)

        return self.__create_constant_symbol(stripped_name, filename, raw[1], raw[0])

    def __create_function_macro_symbol(self, name, filename, lineno, original_text):
        if name in self.extension.created_symbols:
            return None<commit_msg>c: utils: Fallback to macro definition file for macros<commit_after>import os
from collections import namedtuple
import cchardet
from hotdoc.parsers.c_comment_scanner.c_comment_scanner import extract_comments

from hotdoc.core.symbols import *
from hotdoc.utils.loggable import debug


RawMacro = namedtuple('RawMacro', ['raw', 'filename'])


def unicode_dammit(data):
    encoding = cchardet.detect(data)['encoding']
    try:
        return data.decode(encoding, errors='replace')
    except LookupError:
        return data.decode('utf8', errors='replace')


class CCommentExtractor:
    def __init__(self, extension, comment_parser):
        self.extension = extension
        self.app = extension.app
        self.project = extension.project
        self.__raw_comment_parser = comment_parser
        self.__raw_macros = []

    def parse_comments(self, filenames):
        for filename in filenames:
            with open(filename, 'rb') as f:
                debug('Getting comments in %s' % filename)
                lines = []
                header = filename.endswith('.h')
                skip_next_symbol = header
                # FIXME Use the lexer for that!
                for l in f.readlines():
                    l = unicode_dammit(l)
                    lines.append(l)
                    if skip_next_symbol and l.startswith("#pragma once"):
                        skip_next_symbol = False

                cs = extract_comments(''.join(lines))
                for c in cs:
                    if c[3]:
                        line = lines[c[1] - 1]

                        comment = (len(line) - len(line.lstrip(' '))
                                   ) * ' ' + c[0]
                        block = self.__raw_comment_parser.parse_comment(comment,
                                                                        filename, c[1], c[2], self.project.include_paths)
                        if block is not None:
                            self.app.database.add_comment(block)
                    elif not skip_next_symbol:
                        if header:
                            self.__raw_macros.append(RawMacro(c, filename))
                            # self.__create_macro_from_raw_text(c, filename, filter_names)
                    else:
                        skip_next_symbol = False

    def create_macro_symbols(self, filter_names=None, filenames=None):
        filenames = filenames or set()
        filter_names = filter_names or {}
        for raw_macro in self.__raw_macros:
            self.__create_macro_from_raw_text(
                raw_macro.raw, raw_macro.filename, filter_names, filenames)

    def __get_comment_filename(self, name, filenames):
        comment = self.app.database.get_comment(name)
        if comment and comment.filename:
            header = '%s.h' % os.path.splitext(comment.filename)[0]
            if header in filenames:
                return header
            elif comment.filename in filenames:
                return comment.filename

        return ""

    def __create_macro_from_raw_text(self, raw, filename, filter_names, filenames):
        mcontent = raw[0].replace('\t', ' ')
        mcontent = mcontent.split(' ', 1)[1]
        split = mcontent.split('(', 1)
        name = split[0]

        if name.strip() in filter_names:
            return None

        # `#define foo (x) ...` is *not* a function macro
        if not (' ' in name or '\t' in name) and len(split) == 2:
            args = split[1].split(')', 1)[0].split(',')
            if args:
                stripped_name = name.strip()
                filename = self.__get_comment_filename(stripped_name, filenames) or filename
                return self.__create_function_macro_symbol(stripped_name,
                                                           filename, raw[1], raw[0])

        name = mcontent.split(' ', 1)[0]
        stripped_name = name.strip()
        filename = self.__get_comment_filename(stripped_name, filenames) or filename

        return self.__create_constant_symbol(stripped_name, filename, raw[1], raw[0])

    def __create_function_macro_symbol(self, name, filename, lineno, original_text):
        if name in self.extension.created_symbols:
            return None
{'fraction_non_alphanumeric': 0.0932213415355495, 'fraction_numerical': 0.005441854962735124, 'mean_word_length': 1.7708947885939037, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4412514', 'n_tokens_mistral': 2345, 'n_tokens_neox': 2222, 'n_words': 531}