<commit_before>from __future__ import print_function
import multiprocessing
import time
from rx import Observable
from walker import Walker
from root_folder_info import RootFolderInfo

class CsvWalker(Walker):
    
    def __init__(self, config, storage):
        self._config = config
        self._storage = storage

    def walk(self):
        start = time.time()

        # Create source stream
        folders = Observable.from_(self._storage.list_folders())
        if self._config.root_files:
           folders = folders.start_with(RootFolderInfo()) 
        if self._config.skip_files:
            print("Folder")
            folders.subscribe(on_next=lambda folder: print(folder.name) if folder else '',
                on_completed=lambda: self._print_summary(time.time() - start))
        else:
            print("Folder, Filename, Checksum")
            # Expand folder stream into file stream
            files = folders.concat_map(lambda folder: Observable.from_((fileinfo, folder) for fileinfo in self._storage.list_files(folder)))
            # Print each file
            if self._config.list_sort:
                files.to_sorted_list(key_selector=lambda (fileinfo, folder): "{} {}".format(folder.name, fileinfo.name)) \
                    .subscribe(on_next=lambda items: [self._print_file(folder, fileinfo) for fileinfo, folder in items],
                        on_completed=lambda: self._print_summary(time.time() - start))
            else:
                files.subscribe(on_next=lambda (fileinfo, folder): self._print_file(folder, fileinfo),
                    on_completed=lambda: self._print_summary(time.time() - start))
    
    def _print_file(self, folder, fileinfo):
        print("{}, {}, {}".format(folder.name if folder else '', fileinfo.name, fileinfo.checksum))<commit_msg>skip-files and list-sort in CsvWalker. Oh my.<commit_after>from __future__ import print_function
import multiprocessing
import time
from rx import Observable
from walker import Walker
from root_folder_info import RootFolderInfo

class CsvWalker(Walker):
    
    def __init__(self, config, storage):
        self._config = config
        self._storage = storage

    def walk(self):
        start = time.time()

        # Create source stream
        folders = Observable.from_(self._storage.list_folders())
        if self._config.root_files:
           folders = folders.start_with(RootFolderInfo()) 
        if self._config.skip_files:
            print("Folder")
            if self._config.list_sort:
                folders = folders.to_sorted_list(key_selector=lambda folder: folder.name) \
                    .flat_map(lambda x: x)
            folders.subscribe(on_next=lambda folder: print(folder.name) if folder else '',
                on_completed=lambda: self._print_summary(time.time() - start))
        else:
            print("Folder, Filename, Checksum")
            # Expand folder stream into file stream
            files = folders.concat_map(lambda folder: Observable.from_((fileinfo, folder) for fileinfo in self._storage.list_files(folder)))
            # Print each file
            if self._config.list_sort:
                folders = folders.to_sorted_list(key_selector=lambda (fileinfo, folder): "{} {}".format(folder.name, fileinfo.name)) \
                    .flat_map(lambda x: x)
            files.subscribe(on_next=lambda (fileinfo, folder): self._print_file(folder, fileinfo),
                on_completed=lambda: self._print_summary(time.time() - start))
    
    def _print_file(self, folder, fileinfo):
        print("{}, {}, {}".format(folder.name if folder else '', fileinfo.name, fileinfo.checksum))
{'fraction_non_alphanumeric': 0.10265535176567205, 'fraction_numerical': 0.0, 'mean_word_length': 2.6576576576576576, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7217774', 'n_tokens_mistral': 1015, 'n_tokens_neox': 982, 'n_words': 237}