<commit_before>import os.path
import re

from astvisitor import parse, ParseError, ASTVisitor
from store import Module, Class, Function, TestModule, TestClass, TestMethod
from util import read_file_contents, python_sources_below


def is_test_module_path(path):
    return re.search(r'(^test_)|(_test.py$)', path)

def descend(tree, visitor_type):
    """Walk over the AST using a visitor of a given type and return the visitor
    object once done.
    """
    visitor = visitor_type()
    visitor.visit(tree)
    return visitor<commit_msg>  Modified collector.is_test_module_path to recognize test files inside subdirectories.<commit_after>import os.path
import re

from astvisitor import parse, ParseError, ASTVisitor
from store import Module, Class, Function, TestModule, TestClass, TestMethod
from util import read_file_contents, python_sources_below


def is_test_module_path(path):
    """Return True if given path points to a test module.

    >>> is_test_module_path("module.py")
    False
    >>> is_test_module_path("test_module.py")
    True
    >>> is_test_module_path("pythoscope-tests%stest_module.py" % os.path.sep)
    True
    """
    return re.search(r'((^|%s)test_)|(_test.py$)' % re.escape(os.path.sep), path) is not None

def descend(tree, visitor_type):
    """Walk over the AST using a visitor of a given type and return the visitor
    object once done.
    """
    visitor = visitor_type()
    visitor.visit(tree)
    return visitor
{'fraction_non_alphanumeric': 0.09602194787379972, 'fraction_numerical': 0.0, 'mean_word_length': 4.4440298507462686, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6779451', 'n_tokens_mistral': 447, 'n_tokens_neox': 438, 'n_words': 138}