<commit_before>"""
Scanners are responsible for locating and creating node objects from a data source. They must
also implement a method that provides the actual contents for the node.
"""

import os
import os.path


def empty_scanner(*args, **kwargs):
    """A stub scanner that returns zero items."""
    return []


def create_filesystem_provider(filename, *args, **kwargs):
    """Creates a new provider function for the specified file."""
    def provide():
        with open(filename, 'r') as f:
            return f.read()
    return provide


def filesystem_scanner(site, *args, **kwargs):
    """A scanner that finds items from the file system."""
    from stayput.items import Node
    all = []
    for directory, dirnames, filenames in os.walk(site.items_path):
        for filename in filenames:
            full = os.path.join(directory, filename)
            relative = os.path.relpath(full, site.items_path)
            normalized = relative.replace(os.path.pathsep, '/')
            all.append(Node(normalized, content_provider=create_filesystem_provider(full)))
    return all<commit_msg>Fingerprint filesystem items with the mtime<commit_after>"""
Scanners are responsible for locating and creating node objects from a data source. They must
also implement a method that provides the actual contents for the node.
"""

import os
import os.path


def empty_scanner(*args, **kwargs):
    """A stub scanner that returns zero items."""
    return []


def create_filesystem_provider(filename, *args, **kwargs):
    """Creates a new provider function for the specified file."""
    def provide():
        with open(filename, 'r') as f:
            return f.read()
    return provide


def filesystem_scanner(site, *args, **kwargs):
    """A scanner that finds items from the file system."""
    from stayput.items import Node
    all = []
    for directory, dirnames, filenames in os.walk(site.items_path):
        for filename in filenames:
            full = os.path.join(directory, filename)
            relative = os.path.relpath(full, site.items_path)
            normalized = relative.replace(os.path.pathsep, '/')
            all.append(Node(normalized, content_provider=create_filesystem_provider(full),
                            fingerprint=int(os.path.getmtime(full))))
    return all
{'fraction_non_alphanumeric': 0.10220874837592031, 'fraction_numerical': 0.0, 'mean_word_length': 3.192377495462795, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5961642', 'n_tokens_mistral': 628, 'n_tokens_neox': 614, 'n_words': 219}