<commit_before>#!/usr/bin/env python
"""Small libclang based app to fix gcov output.

Fix gcov output with templates.  This is done by first parsing in the .cpp files
(compilation units) with libclang.  The AST is then parsed and all lines within
composite statements ({ stmt; stmt; ... }) are memoized as 'interesting' lines.
The resulting interesting lines are serialized to a location file with pickle.
Finally, gcov output files are read and updated.  If a line is interesting but
not marked as covered or uncovered (marker '-'), it is marked as uncovered
(marker '#####').

USAGE: fixgcov.py -i $include_dir -g $gcov_file
USAGE: fixgcov.py -i $include_dir -s $source_file

Copyright: (c) 2010, <NAME>, FU Berlin
License:   3-clause BSD (see LICENSE)
"""

from __future__ import with_statement

__author__ = '<NAME> <<EMAIL>>'

import optparse
import os
import pickle
import sys

import clang.cindex as ci


def _hasFileLocation(node):
    """Return True if node has a file lcoation."""
    if not hasattr(node, 'location'):
        return False
    if not hasattr(node.location, 'file'):
        return False
    if not node.location.file:
        return False
    if not hasattr(node.location.file, 'name'):
        return False
    if not node.location.file.name:
        return False
    return True


class CollectCompoundStatementNodeVisitor(object):
    """Visitor for AST nodes that collects compound statements."""
    
    def __init__(self, options):
        self.options = options
        self.stack = []
        self.ranges = []
    
    def enterNode(self, node):
        """Called when a node is entered ("pre-order" traversal)."""
        self.stack.append(node)
        ## print '  ' * len(self.stack), node.kind,
        num_children = len([x for x in node.get_children()])
        ## if _hasFileLocation(node):
        ##     print node.location.file.name, '%d-%d' % (node.extent.start.line, node.extent.end.line)
        ## else:
        ##     print
        # Only add range for statements that are no compound statements.  Add
        # for empty compounds.
        if not node.kind.is_statement():
            ## print 'skipping, no statement'
            return
        if node.kind == ci.CursorKind.COMPOUND_STMT and num_children > 0:
            ## print 'skipping, non-empty compound statement', num_children
            return
        # Only add if has file location.
        if _hasFileLocation(node):
            self.ranges.append((node.location.file.name, node.extent.start.line, node.extent.end.line))

    def exitNode(self, node):
        """Called when a node is left ("post-order" traversa)."""<commit_msg>Skipping declarations as uninteresting in fixgcov.<commit_after>#!/usr/bin/env python
"""Small libclang based app to fix gcov output.

Fix gcov output with templates.  This is done by first parsing in the .cpp files
(compilation units) with libclang.  The AST is then parsed and all lines within
composite statements ({ stmt; stmt; ... }) are memoized as 'interesting' lines.
The resulting interesting lines are serialized to a location file with pickle.
Finally, gcov output files are read and updated.  If a line is interesting but
not marked as covered or uncovered (marker '-'), it is marked as uncovered
(marker '#####').

USAGE: fixgcov.py -i $include_dir -g $gcov_file
USAGE: fixgcov.py -i $include_dir -s $source_file

Copyright: (c) 2010, <NAME>, FU Berlin
License:   3-clause BSD (see LICENSE)
"""

from __future__ import with_statement

__author__ = '<NAME> <<EMAIL>>'

import optparse
import os
import pickle
import sys

import clang.cindex as ci


def _hasFileLocation(node):
    """Return True if node has a file lcoation."""
    if not hasattr(node, 'location'):
        return False
    if not hasattr(node.location, 'file'):
        return False
    if not node.location.file:
        return False
    if not hasattr(node.location.file, 'name'):
        return False
    if not node.location.file.name:
        return False
    return True


class CollectCompoundStatementNodeVisitor(object):
    """Visitor for AST nodes that collects compound statements."""
    
    def __init__(self, options):
        self.options = options
        self.stack = []
        self.ranges = []
    
    def enterNode(self, node):
        """Called when a node is entered ("pre-order" traversal)."""
        self.stack.append(node)
        ## print '  ' * len(self.stack), node.kind,
        num_children = len([x for x in node.get_children()])
        ## if _hasFileLocation(node):
        ##     print node.location.file.name, '%d-%d' % (node.extent.start.line, node.extent.end.line)
        ## else:
        ##     print
        # Only add range for statements that are no compound statements.  Add
        # for empty compounds.
        if not node.kind.is_statement():
            ## print 'skipping, no statement'
            return
        if node.kind == ci.CursorKind.COMPOUND_STMT and num_children > 0:
            ## print 'skipping, non-empty compound statement', num_children
            return
        if node.kind == ci.CursorKind.DECL_STMT:
            return  # Skip declarations.
        # Only add if has file location.
        if _hasFileLocation(node):
            self.ranges.append((node.location.file.name, node.extent.start.line, node.extent.end.line))

    def exitNode(self, node):
        """Called when a node is left ("post-order" traversa)."""
{'fraction_non_alphanumeric': 0.10570005534034312, 'fraction_numerical': 0.002213613724405091, 'mean_word_length': 2.765277777777778, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5986556', 'n_tokens_mistral': 1588, 'n_tokens_neox': 1549, 'n_words': 554}