<commit_before>#! /usr/bin/python

import json, subprocess
from argparse import ArgumentParser
from os import walk
from os.path import join, getsize
from datetime import datetime

parser = ArgumentParser(description="Tracks any changes in a specified directory [default is the current dir] \n \
                                     Additions, deletions, changes of files and subdirs are tracked and recorded in\n \
                                     a log file. If user-defined thresholds are exceeded an alert is also created.\n")
parser.add_argument("-d","--dir", dest="dir", default=".", help="The directory monitored")
parser.add_argument("-p","--period", dest="period", default=430000, type=int, help="The period of checking, in secs. (default = 60 secs)")
parser.add_argument("-s","--sizeabs", dest="size_abs", default=30, type=float, help="Number of MB of change in size to trigger an alert")
parser.add_argument("-r","--sizerel", dest="size_rel", default=0.05, type=float, help="Fraction of change in size to trigger an alert (default=0.05 = 5%")
parser.add_argument("-n","--numabs", dest="num_abs", default=50, type=int, help="Number files+dirs that need to be added/deleted to trigger an alert")
parser.add_argument("-q","--numrel", dest="num_rel", default=0.05, type=float, help="Fraction of files+dirs that need to be added/deleted to trigger an alert (default=0.05 = 5%")
parser.add_argument("-l","--logdir", dest="log_dir", default="/var/log/", help="The directory where log files and status information is kept (default /var/log/)")
parser.add_argument("--once", action='store_true', help="If this flag is set then the script performs the comparison to previous state only once and exits")

args = parser.parse_args()

'''
In order to know all the changes done to a folder we have to keep a detailed snapshop of the dir tree state.
This means we need to know all directories and files the main directory contains (along with individual item sizes) 
as well as the same information for all subdirs (recursively). Note that we only rely on file size to determine that
a file has changed or not, not on a hash generated by the file. This is acceptabe because the main application of
this script is to track size changes. The script is supposed to run as a daemon, and persits shut downs and restarts.
Hence we use two files to store needed information. 
One is a .json file recording the last state snapshop:  <log_dir>/Track<dir path>.json
The other is a log of the changes we find over time:    <log_dir>/Track<dir path>changes.log
'''

class Tracker:

    def __init__(self, root, log_dir='/var/log/'):
        
        self.root = root # the directory that we will monitor and track changes
        self.log_dir = log_dir # the directory where the logs and json files will be kept

        self.previous_state = self.readPrevState()  # a structure to hold all dir and file info of our previous snapshot
        self.current_state = {}  # a structure to hold all dir and file info of the current state
        
        self.added_dirs = []    # a list of directories added along with file and subdir info
        self.deleted_dirs = []  # a list of directories deleted along with the total size and number of files in them
        self.added_files = {}   # a dictionary of files added. filename is the key, size is the value
        self.deleted_files = {} # a dictionary of files deleted. filename is the key, size is the value
        self.changed_files = {} # a dictionary of files changed. filename is the key, a tuple (old_size, new_size) is the value 

        self.added_total_size = 0   # The total size in bytes of all files added
        self.deleted_total_size = 0 # The total size in bytes of all files deleted
        self.changed_total_size = 0 # The total size in bytes of all files changes
        self.added_total_num = 0   # The total number of all files added
        self.deleted_total_num = 0 # The total number of all files deleted
        self.changed_total_num = 0 # The total number of all files changed

        self.current_total_size = 0
        self.current_total_file_num = 0
        self.current_total_dir_num = 0 
    
    '''
    Read the previous state of the root directory from a special file. If the file does not exist return an empty dict
    '''
    def readPrevState(self):

        prev_state_filename = 'track{}.json'.format(self.root.replace('/','_'))
        try:
            with open(join(self.log_dir, prev_state_filename)) as state_file:
                return json.load(state_file)

        except (IOError, ValueError):
            return {}

    '''
    Write the current state of the root directory to the special file. Overwrite file.
    '''
    def writeCurrentState(self):
        prev_state_filename = 'track{}.json'.format(self.root.replace('/','_'))
        with open(join(self.log_dir, prev_state_filename), 'w') as state_file:
            state_file.write(json.dumps(self.current_state, separators=(',', ':')))

    '''
    A function to find the total size of files and number of files in a directory based on the previous state
    Returns the total files size and total files number
    '''
    def getSizeAndNum(self, pathname):

        if pathname not in self.previous_state: return (0, 0)

        dirs, files_with_sizes = self.previous_state[pathname]
        total_size = sum(files_with_sizes.values())
        total_num = len(files_with_sizes)
        # Recursively visit all subdirs
        for d in dirs:
            subdir_size, subdir_num = self.getSizeAndNum(join(pathname, d))
            total_size += subdir_size
            total_num += subdir_num

        return (total_size, total_num)


    '''
    A function to find all additions, deletions, and changes in all files and subdirs
    '''
    def findChanges(self):

        # reset the current state
        self.current_state = {}

        for path, curr_dirs, curr_files in walk(self.root):

            # find the sizes of all the files in this directory
            curr_files_and_sizes = {}
            for fname in curr_files:
                curr_files_and_sizes[fname] = getsize(join(path, fname))

            # update the total counts
            self.current_total_size += sum(curr_files_and_sizes.values())
            self.current_total_file_num += len(curr_files_and_sizes)
            self.current_total_dir_num += 1 

            # update the current state
            self.current_state[path] = [curr_dirs, curr_files_and_sizes]


            if path in self.previous_state:
                prev_dirs, prev_files_and_sizes = self.previous_state[path]

                # check if sub dirs are the same
                if prev_dirs != curr_dirs:
                    # we only want to get the deleted dirs here, since the 
                    # added ones will appear in the path as we walk the tree
                    deleted_dirs_list = list(set(prev_dirs)-set(curr_dirs))
                    for dname in deleted_dirs_list:
                        # use a recursive function to get the total size and file number in the deleted dir
                        dir_size , dir_file_num = self.getSizeAndNum(join(path, dname))
                        self.deleted_dirs.append([join(path, dname), dir_size, dir_file_num])


                # check if files are the same
                if prev_files_and_sizes != curr_files_and_sizes:
                    # find the differences
                    deleted_files_set = set(prev_files_and_sizes) - set(curr_files_and_sizes)
                    for f in deleted_files_set:
                        self.deleted_files[path+f] = prev_files_and_sizes[f]

                    added_files_set = set(curr_files_and_sizes) - set(prev_files_and_sizes)
                    for f in added_files_set:
                        self.added_files[path+f] = curr_files_and_sizes[f]

                    common_files_set = set(curr_files_and_sizes) - added_files_set
                    for f in common_files_set:
                        if prev_files_and_sizes[f] != curr_files_and_sizes[f]:
                            self.changed_files[path+f] = (prev_files_and_sizes[f], curr_files_and_sizes[f])

            else:
                self.added_dirs.append([path, curr_dirs, curr_files_and_sizes])

        # Finally calculate various aggregates:

        # sum up the deleted files sizes, and count the files
        self.deleted_total_size = sum(self.deleted_files.values())
        self.deleted_total_num = len(self.deleted_files)
        # add the total sizes and numbers of deleted directories
        self.deleted_total_size += sum([size for p, size, num in self.deleted_dirs])
        self.deleted_total_num  += sum([num for p, size, num in self.deleted_dirs])

        # sum up the added files sizes, and count the files
        self.added_total_size = sum(self.added_files.values())
        self.added_total_num = len(self.added_files)
        # add the total sizes and numbers of added directories
        self.added_total_size += sum([ sum(f.values()) for p, d, f in self.added_dirs])
        self.added_total_num += sum([ len(f) for p, d, f in self.added_dirs])

        # for changed files sum up the differences between old and new sizes for every changed file
        self.changed_total_size = sum(map(lambda (old_size, new_size):abs(old_size-new_size), self.changed_files.values()))
        self.changed_total_num = len(self.changed_files)

       

    def writeChanges(self):
        change_log_filename = 'track{}changes.log'.format(self.root.replace('/','_') )
        with open(join(self.log_dir, change_log_filename), 'a') as log_file: 
            # write a timestamp
            log_file.write('----------------  {}  ----------------\n'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
            # check to see if we had no previous state, in which case we do not want to record all additions
            if self.previous_state == {}:
                log_file.write('NEW directory tracked.\n(Or .json file holding the previous state was deleted or corrupted)\n')
                log_file.write('{} dirs and {} files occupying {}\n'.format(self.current_total_dir_num,
                                                                            self.current_total_file_num,
                                                                            self.humanReadableSize(self.current_total_size)))    
            else:
                # provide a summary of the changes
                report_for_added = ''; report_for_deleted =''; report_for_changed = ''
                if self.added_total_size > 0: 
                    report_for_added = '- Added {} files totalling {} -'.format(self.added_total_num, self.humanReadableSize(self.added_total_size))
                if self.deleted_total_size > 0: 
                    report_for_deleted = '- Deleted {} files totalling {} -'.format(self.deleted_total_num, self.humanReadableSize(self.deleted_total_size))
                if self.changed_total_size > 0: 
                    report_for_changed = '- {} files changed by {} -'.format(self.changed_total_num, self.humanReadableSize(self.changed_total_size))

                log_file.write(report_for_added + report_for_deleted + report_for_changed + '\n')

                # provide a detailed list of all changes
                # start with deleted dirs, 
                for d in self.deleted_dirs:
                    path, size, files_num = d
                    log_file.write('Deleted dir: {} contained {} in {} files\n'.format(path, self.humanReadableSize(size), files_num))

                # then added dirs    
                for d in self.added_dirs:
                    path, dirs, files_with_sizes = d
                    size = sum(files_with_sizes.values()); 
                    files_num = len(files_with_sizes)
                    log_file.write('Added dir: {}, contains {} in {} files\n'.format(path, self.humanReadableSize(size), files_num))

                # continuing with deleted/added/changed files
                for f, size in self.deleted_files.iteritems():
                    log_file.write('Deleted file: {} was {} bytes\n'.format(f, size))

                for f, size in self.added_files.iteritems():
                    log_file.write('Added file: {} is {} bytes\n'.format(f, size))     

                for f, (old_size, new_size) in self.changed_files.iteritems():
                    log_file.write('Changed file: {} from {} to {} bytes\n'.format(f, old_size, new_size)) 


    def alertUser(self, size_abs, size_rel, num_abs, num_rel):

        if self.previous_state == {}: return 

        size = self.added_total_size + self.deleted_total_size + self.changed_total_size
        num  = self.added_total_num  + self.deleted_total_num  + self.changed_total_num
        
        if (size > size_abs or 
            size > size_rel * self.current_total_size or 
            num  > num_abs or 
            num  > num_rel * self.current_total_file_num):

            applescript = 'display notification "Added/Deleted/Changed: {}, {} files" with title "Boulis Directory Tracker"'.format(self.humanReadableSize(size), num)
            alt_applescript = 'display dialog "Added/Deleted/Changed: {}, {} files" with title "Boulis Directory Tracker" with icon caution buttons {{"OK"}}'.format(self.humanReadableSize(size), num)

            subprocess.call("osascript -e '{}'".format(applescript), shell=True)
            #subprocess.call("osascript -e '{}'".format(alt_applescript), shell=True)

    '''
    A function to print file sizes in a more human readable form (using KB, MB, GB)
    '''
    def humanReadableSize(self, num):

        for unit, decimals_printed in zip(['bytes','KB','MB', 'GB'], [0, 0, 2, 3]):
            if abs(num) < 1024.0:
                if decimals_printed == 0:
                    return '{} {}'.format(int(round(num)), unit)
                else:
                    # create the format string to fit the desired decimal precision
                    # use {{ or }} to escape the special character { or }. for decimals= 3 this will return '{:.3f} {}'
                    format_string = '{{:.{}f}} {{}}'.format(decimals_printed) 
                    return format_string.format(num, unit)
            num /= 1024.0
        # if the num is bigger than 1024 after all divisions, just use the larger unit     
        return '{.3f} GB'.format(num)


def singleRun(root, log_dir, size_abs, size_rel, num_abs, num_rel):

    t = Tracker(root, log_dir)
    t.findChanges()
    t.writeChanges()
    t.writeCurrentState()
    t.alertUser(size_abs, size_rel, num_abs, num_rel)



def main_loop():
    singleRun(args.dir, args.log_dir, args.size_abs *1024*1024, args.size_rel, args.num_abs, args.num_rel)

if __name__ == '__main__':
     main_loop() 
        
        
        <commit_msg>fixed bug in argparse by removing percent char from the help lines, made default period 12 hours<commit_after>#! /usr/bin/python

import json, subprocess
from argparse import ArgumentParser
from os import walk
from os.path import join, getsize
from datetime import datetime

parser = ArgumentParser(description="Tracks any changes in a specified directory [default is the current dir] \n \
                                     Additions, deletions, changes of files and subdirs are tracked and recorded in\n \
                                     a log file. If user-defined thresholds are exceeded an alert is also created.\n")
parser.add_argument("-d","--dir", dest="dir", default=".", help="The directory monitored")
parser.add_argument("-p","--period", dest="period", default=43200, type=int, help="The period of checking, in secs. default=43200 (= 12 hours)")
parser.add_argument("-s","--sizeabs", dest="size_abs", default=30, type=float, help="Number of MB of change in size to trigger an alert")
parser.add_argument("-r","--sizerel", dest="size_rel", default=0.05, type=float, help="Fraction of change in size to trigger an alert (default=0.05)")
parser.add_argument("-n","--numabs", dest="num_abs", default=50, type=int, help="Number files+dirs that need to be added/deleted to trigger an alert")
parser.add_argument("-q","--numrel", dest="num_rel", default=0.05, type=float, help="Fraction of files+dirs that need to be added/deleted to trigger an alert (default=0.05)")
parser.add_argument("-l","--logdir", dest="log_dir", default="/var/log/", help="The directory where log files and status information is kept (default /var/log/)")
parser.add_argument("--once", action='store_true', help="If this flag is set then the script performs the comparison to previous state only once and exits")

args = parser.parse_args()

'''
In order to know all the changes done to a folder we have to keep a detailed snapshop of the dir tree state.
This means we need to know all directories and files the main directory contains (along with individual item sizes) 
as well as the same information for all subdirs (recursively). Note that we only rely on file size to determine that
a file has changed or not, not on a hash generated by the file. This is acceptabe because the main application of
this script is to track size changes. The script is supposed to run as a daemon, and persits shut downs and restarts.
Hence we use two files to store needed information. 
One is a .json file recording the last state snapshop:  <log_dir>/Track<dir path>.json
The other is a log of the changes we find over time:    <log_dir>/Track<dir path>changes.log
'''

class Tracker:

    def __init__(self, root, log_dir='/var/log/'):
        
        self.root = root # the directory that we will monitor and track changes
        self.log_dir = log_dir # the directory where the logs and json files will be kept

        self.previous_state = self.readPrevState()  # a structure to hold all dir and file info of our previous snapshot
        self.current_state = {}  # a structure to hold all dir and file info of the current state
        
        self.added_dirs = []    # a list of directories added along with file and subdir info
        self.deleted_dirs = []  # a list of directories deleted along with the total size and number of files in them
        self.added_files = {}   # a dictionary of files added. filename is the key, size is the value
        self.deleted_files = {} # a dictionary of files deleted. filename is the key, size is the value
        self.changed_files = {} # a dictionary of files changed. filename is the key, a tuple (old_size, new_size) is the value 

        self.added_total_size = 0   # The total size in bytes of all files added
        self.deleted_total_size = 0 # The total size in bytes of all files deleted
        self.changed_total_size = 0 # The total size in bytes of all files changes
        self.added_total_num = 0   # The total number of all files added
        self.deleted_total_num = 0 # The total number of all files deleted
        self.changed_total_num = 0 # The total number of all files changed

        self.current_total_size = 0
        self.current_total_file_num = 0
        self.current_total_dir_num = 0 
    
    '''
    Read the previous state of the root directory from a special file. If the file does not exist return an empty dict
    '''
    def readPrevState(self):

        prev_state_filename = 'track{}.json'.format(self.root.replace('/','_'))
        try:
            with open(join(self.log_dir, prev_state_filename)) as state_file:
                return json.load(state_file)

        except (IOError, ValueError):
            return {}

    '''
    Write the current state of the root directory to the special file. Overwrite file.
    '''
    def writeCurrentState(self):
        prev_state_filename = 'track{}.json'.format(self.root.replace('/','_'))
        with open(join(self.log_dir, prev_state_filename), 'w') as state_file:
            state_file.write(json.dumps(self.current_state, separators=(',', ':')))

    '''
    A function to find the total size of files and number of files in a directory based on the previous state
    Returns the total files size and total files number
    '''
    def getSizeAndNum(self, pathname):

        if pathname not in self.previous_state: return (0, 0)

        dirs, files_with_sizes = self.previous_state[pathname]
        total_size = sum(files_with_sizes.values())
        total_num = len(files_with_sizes)
        # Recursively visit all subdirs
        for d in dirs:
            subdir_size, subdir_num = self.getSizeAndNum(join(pathname, d))
            total_size += subdir_size
            total_num += subdir_num

        return (total_size, total_num)


    '''
    A function to find all additions, deletions, and changes in all files and subdirs
    '''
    def findChanges(self):

        # reset the current state
        self.current_state = {}

        for path, curr_dirs, curr_files in walk(self.root):

            # find the sizes of all the files in this directory
            curr_files_and_sizes = {}
            for fname in curr_files:
                curr_files_and_sizes[fname] = getsize(join(path, fname))

            # update the total counts
            self.current_total_size += sum(curr_files_and_sizes.values())
            self.current_total_file_num += len(curr_files_and_sizes)
            self.current_total_dir_num += 1 

            # update the current state
            self.current_state[path] = [curr_dirs, curr_files_and_sizes]


            if path in self.previous_state:
                prev_dirs, prev_files_and_sizes = self.previous_state[path]

                # check if sub dirs are the same
                if prev_dirs != curr_dirs:
                    # we only want to get the deleted dirs here, since the 
                    # added ones will appear in the path as we walk the tree
                    deleted_dirs_list = list(set(prev_dirs)-set(curr_dirs))
                    for dname in deleted_dirs_list:
                        # use a recursive function to get the total size and file number in the deleted dir
                        dir_size , dir_file_num = self.getSizeAndNum(join(path, dname))
                        self.deleted_dirs.append([join(path, dname), dir_size, dir_file_num])


                # check if files are the same
                if prev_files_and_sizes != curr_files_and_sizes:
                    # find the differences
                    deleted_files_set = set(prev_files_and_sizes) - set(curr_files_and_sizes)
                    for f in deleted_files_set:
                        self.deleted_files[path+f] = prev_files_and_sizes[f]

                    added_files_set = set(curr_files_and_sizes) - set(prev_files_and_sizes)
                    for f in added_files_set:
                        self.added_files[path+f] = curr_files_and_sizes[f]

                    common_files_set = set(curr_files_and_sizes) - added_files_set
                    for f in common_files_set:
                        if prev_files_and_sizes[f] != curr_files_and_sizes[f]:
                            self.changed_files[path+f] = (prev_files_and_sizes[f], curr_files_and_sizes[f])

            else:
                self.added_dirs.append([path, curr_dirs, curr_files_and_sizes])

        # Finally calculate various aggregates:

        # sum up the deleted files sizes, and count the files
        self.deleted_total_size = sum(self.deleted_files.values())
        self.deleted_total_num = len(self.deleted_files)
        # add the total sizes and numbers of deleted directories
        self.deleted_total_size += sum([size for p, size, num in self.deleted_dirs])
        self.deleted_total_num  += sum([num for p, size, num in self.deleted_dirs])

        # sum up the added files sizes, and count the files
        self.added_total_size = sum(self.added_files.values())
        self.added_total_num = len(self.added_files)
        # add the total sizes and numbers of added directories
        self.added_total_size += sum([ sum(f.values()) for p, d, f in self.added_dirs])
        self.added_total_num += sum([ len(f) for p, d, f in self.added_dirs])

        # for changed files sum up the differences between old and new sizes for every changed file
        self.changed_total_size = sum(map(lambda (old_size, new_size):abs(old_size-new_size), self.changed_files.values()))
        self.changed_total_num = len(self.changed_files)

       

    def writeChanges(self):
        change_log_filename = 'track{}changes.log'.format(self.root.replace('/','_') )
        with open(join(self.log_dir, change_log_filename), 'a') as log_file: 
            # write a timestamp
            log_file.write('----------------  {}  ----------------\n'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
            # check to see if we had no previous state, in which case we do not want to record all additions
            if self.previous_state == {}:
                log_file.write('NEW directory tracked.\n(Or .json file holding the previous state was deleted or corrupted)\n')
                log_file.write('{} dirs and {} files occupying {}\n'.format(self.current_total_dir_num,
                                                                            self.current_total_file_num,
                                                                            self.humanReadableSize(self.current_total_size)))    
            else:
                # provide a summary of the changes
                report_for_added = ''; report_for_deleted =''; report_for_changed = ''
                if self.added_total_size > 0: 
                    report_for_added = '- Added {} files totalling {} -'.format(self.added_total_num, self.humanReadableSize(self.added_total_size))
                if self.deleted_total_size > 0: 
                    report_for_deleted = '- Deleted {} files totalling {} -'.format(self.deleted_total_num, self.humanReadableSize(self.deleted_total_size))
                if self.changed_total_size > 0: 
                    report_for_changed = '- {} files changed by {} -'.format(self.changed_total_num, self.humanReadableSize(self.changed_total_size))

                log_file.write(report_for_added + report_for_deleted + report_for_changed + '\n')

                # provide a detailed list of all changes
                # start with deleted dirs, 
                for d in self.deleted_dirs:
                    path, size, files_num = d
                    log_file.write('Deleted dir: {} contained {} in {} files\n'.format(path, self.humanReadableSize(size), files_num))

                # then added dirs    
                for d in self.added_dirs:
                    path, dirs, files_with_sizes = d
                    size = sum(files_with_sizes.values()); 
                    files_num = len(files_with_sizes)
                    log_file.write('Added dir: {}, contains {} in {} files\n'.format(path, self.humanReadableSize(size), files_num))

                # continuing with deleted/added/changed files
                for f, size in self.deleted_files.iteritems():
                    log_file.write('Deleted file: {} was {} bytes\n'.format(f, size))

                for f, size in self.added_files.iteritems():
                    log_file.write('Added file: {} is {} bytes\n'.format(f, size))     

                for f, (old_size, new_size) in self.changed_files.iteritems():
                    log_file.write('Changed file: {} from {} to {} bytes\n'.format(f, old_size, new_size)) 


    def alertUser(self, size_abs, size_rel, num_abs, num_rel):

        if self.previous_state == {}: return 

        size = self.added_total_size + self.deleted_total_size + self.changed_total_size
        num  = self.added_total_num  + self.deleted_total_num  + self.changed_total_num
        
        if (size > size_abs or 
            size > size_rel * self.current_total_size or 
            num  > num_abs or 
            num  > num_rel * self.current_total_file_num):

            applescript = 'display notification "Added/Deleted/Changed: {}, {} files" with title "Boulis Directory Tracker"'.format(self.humanReadableSize(size), num)
            alt_applescript = 'display dialog "Added/Deleted/Changed: {}, {} files" with title "Boulis Directory Tracker" with icon caution buttons {{"OK"}}'.format(self.humanReadableSize(size), num)

            subprocess.call("osascript -e '{}'".format(applescript), shell=True)
            #subprocess.call("osascript -e '{}'".format(alt_applescript), shell=True)

    '''
    A function to print file sizes in a more human readable form (using KB, MB, GB)
    '''
    def humanReadableSize(self, num):

        for unit, decimals_printed in zip(['bytes','KB','MB', 'GB'], [0, 0, 2, 3]):
            if abs(num) < 1024.0:
                if decimals_printed == 0:
                    return '{} {}'.format(int(round(num)), unit)
                else:
                    # create the format string to fit the desired decimal precision
                    # use {{ or }} to escape the special character { or }. for decimals= 3 this will return '{:.3f} {}'
                    format_string = '{{:.{}f}} {{}}'.format(decimals_printed) 
                    return format_string.format(num, unit)
            num /= 1024.0
        # if the num is bigger than 1024 after all divisions, just use the larger unit     
        return '{.3f} GB'.format(num)


def singleRun(root, log_dir, size_abs, size_rel, num_abs, num_rel):

    t = Tracker(root, log_dir)
    t.findChanges()
    t.writeChanges()
    t.writeCurrentState()
    t.alertUser(size_abs, size_rel, num_abs, num_rel)



def main_loop():
    singleRun(args.dir, args.log_dir, args.size_abs *1024*1024, args.size_rel, args.num_abs, args.num_rel)

if __name__ == '__main__':
     main_loop() 
        
        
        
{'fraction_non_alphanumeric': 0.09555101082911456, 'fraction_numerical': 0.004894893888087974, 'mean_word_length': 2.4085247400297107, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3063622', 'n_tokens_mistral': 8633, 'n_tokens_neox': 8289, 'n_words': 2632}