<jupyter_start><jupyter_text>Python + OS Package: List and remove files A short script to list and delete duplicated files after Google Drive synchronization<jupyter_code># Import "os" package (full doc at https://docs.python.org/3/library/os.html)
import os
# Move to test directory
#os.chdir('C:\\Users\\s11as6\\Documents\\test_folder')
# Double-check we moved in the wanted directory
# os.getcwd()
# Move to real directory
os.chdir("C:\\Users\\s11as6\\Google Drive\\Research Library") #2,661 files before - 1,473 after
# Obtain the list of file and store in a var for later
file_list = os.listdir('.')
# Print number of files in the directory
str(len(file_list)) + " files in " + str(os.getcwd())
# Arrange an empty list that will contain the filenames of duplicated files
duplicated_list = [] 

# List suspect duplicated files (they have a length of 51 chars for some reasons, probably truncated by the OS)
potential_duplicate = [filename for filename in file_list if len(filename) == 51]
"Found " + str(len(potential_duplicate)) + " potential duplicates"
# See potential duplicates
potential_duplicate
# Remove duplicated files
for duplo in potential_duplicate:
    os.remove(duplo)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09860312243221035, 'fraction_numerical': 0.015612161051766639, 'mean_word_length': 5.09, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '518110', 'license': 'MIT', 'n_tokens_mistral': 385, 'n_tokens_neox': 358, 'n_words': 136, 'path': 'Python-Automate_deletion_duplicate_files-v.2.1.ipynb', 'repo_name': 'andscalco/Python_OS_duplicate_files', 'size': '1217'}