<commit_before>﻿import os
import argparse

if __name__ == "__main__" :
    parser = argparse.ArgumentParser(description='List the files in a folder.')
    parser.add_argument('folder_path', type=str, help='Path to the folder')
    parser.add_argument('--output_file', type=str, default='output.txt', help='Output file')

    args = parser.parse_args()
    path = args.folder_path
    fout = open(args.output_file, "w")
    for path, subdirs, files in os.walk(path):
        for filename in files:
            f = os.path.join(path, filename)
            print(filename)
            fout.write(str(f) + "\n")<commit_msg>update list_files.py: optionally specify a substring of file name to match<commit_after>﻿import os
import argparse

if __name__ == "__main__" :
    parser = argparse.ArgumentParser(description='List the files in a folder.')
    parser.add_argument('folder_path', type=str, help='Path to the folder')
    parser.add_argument('--output_file', type=str, default='output.txt', help='Output file')
    parser.add_argument('--partial_name', type=str, default=None, help='Part of file name')

    args = parser.parse_args()
    path = args.folder_path
    part = args.partial_name

    fout = open(args.output_file, "w")
    for path, subdirs, files in os.walk(path):
        for file_name in files:
            if part and part not in file_name:
                continue
            f = os.path.join(path, file_name)
            print(file_name)
            fout.write(str(f) + "\n")
{'fraction_non_alphanumeric': 0.12433155080213903, 'fraction_numerical': 0.0, 'mean_word_length': 3.16991643454039, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5157815', 'n_tokens_mistral': 464, 'n_tokens_neox': 460, 'n_words': 122}