<commit_before>"""
Script to compare the contents of Ivans HD with the stuff present on AFS.
Due to space shortage, some folders might have been missed during copying.

And since a large part of the folders were already processed, we cannot just
use `rsync` or that will copy over all the files that already were deleted.
So we use a little routine to find missing folders and rsync only those one
after the other.
This should also take care of the speed issue should the script to have to run
several times.
"""

import os
import subprocess
import errno


def readit(InputFolder):
    '''
    Get all folders (Experiment) and ExperimentIDs inside StartingFolder
    '''
    print 'Looking for Experiment ID folders in', InputFolder
    print 'This will take a while'
    Folder = []
    ExperimentID = []
    for root, dirs, files in os.walk(InputFolder):
        # Go through all the directories
        try:
            # If the last foldername is a number,
            int(os.path.basename(root))
            # see if it's in the list of IDs above,
            if int(os.path.basename(root)):
                Folder.append(root)
                ExperimentID.append(int(os.path.basename(root)))
        # otherwise just continue
        except:
            continue
    return Folder, ExperimentID

# Read *all* experiment IDs from HD
RootFolderHD = ('/media/WINDOWS/Aptina')
FolderHD, IDHD = readit(RootFolderHD)

# Read *all* experiment IDs from AFS
RootFolderAFS = ('/afs/psi.ch/project/EssentialMed/MasterArbeitBFH/XrayImages')
FolderAFS, IDAFS = readit(RootFolderAFS)

for counter, experiment in enumerate(IDHD):
    print 5 * '-', '|', str(counter + 1) + '/' + str(len(IDHD)), '|', 65 * '-'
    if experiment not in IDAFS:
        InputPath = FolderHD[IDHD.index(experiment)]<commit_msg>Don't spill to next line<commit_after>"""
Script to compare the contents of Ivans HD with the stuff present on AFS.
Due to space shortage, some folders might have been missed during copying.

And since a large part of the folders were already processed, we cannot just
use `rsync` or that will copy over all the files that already were deleted.
So we use a little routine to find missing folders and rsync only those one
after the other.
This should also take care of the speed issue should the script to have to run
several times.
"""

import os
import subprocess
import errno


def readit(InputFolder):
    '''
    Get all folders (Experiment) and ExperimentIDs inside StartingFolder
    '''
    print 'Looking for Experiment ID folders in', InputFolder
    print 'This will take a while'
    Folder = []
    ExperimentID = []
    for root, dirs, files in os.walk(InputFolder):
        # Go through all the directories
        try:
            # If the last foldername is a number,
            int(os.path.basename(root))
            # see if it's in the list of IDs above,
            if int(os.path.basename(root)):
                Folder.append(root)
                ExperimentID.append(int(os.path.basename(root)))
        # otherwise just continue
        except:
            continue
    return Folder, ExperimentID

# Read *all* experiment IDs from HD
RootFolderHD = ('/media/WINDOWS/Aptina')
FolderHD, IDHD = readit(RootFolderHD)

# Read *all* experiment IDs from AFS
RootFolderAFS = ('/afs/psi.ch/project/EssentialMed/MasterArbeitBFH/XrayImages')
FolderAFS, IDAFS = readit(RootFolderAFS)

for counter, experiment in enumerate(IDHD):
    print 5 * '-', '|', str(counter + 1) + '/' + str(len(IDHD)), '|', 60 * '-'
    if experiment not in IDAFS:
        InputPath = FolderHD[IDHD.index(experiment)]
{'fraction_non_alphanumeric': 0.08687205106855399, 'fraction_numerical': 0.0022203719122953097, 'mean_word_length': 3.0539932508436447, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7519641', 'n_tokens_mistral': 1076, 'n_tokens_neox': 1011, 'n_words': 413}