<commit_before>import os
from pydispatch import dispatcher
from media.monitor.events import OrganizeFile, NewFile, DeleteFile
from media.monitor.log import Loggable
import media.monitor.pure as mmp

class Bootstrapper(Loggable):
    """
    Bootstrapper reads all the info in the filesystem flushes organize
    events and watch events
    """
    def __init__(self,db,last_run,org_channels,watch_channels):
        self.db = db
        self.org_channels = org_channels
        self.watch_channels = watch_channels
        self.last_run = last_run

    def flush_organize(self):
        """
        walks the organize directories and sends an organize event for every file manually
        """
        flushed = 0
        for pc in self.org_channels:
            for f in mmp.walk_supported(pc.path, clean_empties=True):
                self.logger.info("Bootstrapping: File in 'organize' directory: '%s'" % f)
                dispatcher.send(signal=pc.signal, sender=self, event=OrganizeFile(f))
                flushed += 1
        self.logger.info("Flushed organized directory with %d files" % flushed)

    def flush_watch(self):
        """
        Syncs the file system into the database. Walks over deleted/new/modified files since
        the last run in mediamonitor and sends requests to make the database consistent with
        file system
        """
        songs = set()
        modded = deleted = 0
        for pc in self.watch_channels:
            for f in mmp.walk_supported(pc.path, clean_empties=False):
                songs.add(f)
                # We decide whether to update a file's metadata by checking
                # its system modification date. If it's above the value
                # self.last_run which is passed to us that means media monitor
                # wasn't aware when this changes occured in the filesystem
                # hence it will send the correct events to sync the database
                # with the filesystem
                if os.path.getmtime(f) > self.last_run:
                    modded += 1
                    dispatcher.send(signal=pc.signal, sender=self, event=DeleteFile(f))
                    dispatcher.send(signal=pc.signal, sender=self, event=NewFile(f))
        # Want all files in the database that are not in the filesystem
        for to_delete in self.db.difference(songs):
            for pc in self.watch_channels:
                if os.path.commonprefix([pc.path, to_delete]) == pc.path:
                    dispatcher.send(signal=pc.signal, sender=self, event=DeleteFile(f))
                    os.remove(to_delete)
                    deleted += 1
                    break
            else:
                self.logger.info("Error, could not find watch directory of would be deleted file '%s'" % to_delete)
        self.logger.info( "Flushed watch directories. (modified, deleted) = (%d, %d)"
                         % (modded, deleted) )


<commit_msg>cc-4105: refactored Bootstrapper to work with SyncDB's api.<commit_after>import os
from pydispatch import dispatcher
from media.monitor.events import OrganizeFile, NewFile, DeleteFile
from media.monitor.log import Loggable
import media.monitor.pure as mmp

class Bootstrapper(Loggable):
    """
    Bootstrapper reads all the info in the filesystem flushes organize
    events and watch events
    """
    def __init__(self,db,last_run,org_channels,watch_channels):
        self.db = db
        self.org_channels = org_channels
        self.watch_channels = watch_channels
        self.last_run = last_run

    def flush_organize(self):
        """
        walks the organize directories and sends an organize event for every file manually
        """
        flushed = 0
        for pc in self.org_channels:
            for f in mmp.walk_supported(pc.path, clean_empties=True):
                self.logger.info("Bootstrapping: File in 'organize' directory: '%s'" % f)
                dispatcher.send(signal=pc.signal, sender=self, event=OrganizeFile(f))
                flushed += 1
        self.logger.info("Flushed organized directory with %d files" % flushed)

    def flush_watch(self):
        """
        Syncs the file system into the database. Walks over deleted/new/modified files since
        the last run in mediamonitor and sends requests to make the database consistent with
        file system
        """
        # Songs is a dictionary where every key is the watched the directory
        # and the value is a set with all the files in that directory.
        songs = {}
        modded = deleted = 0
        signal_by_path = dict( (pc.signal, pc.path) for pc in self.watch_channels )
        for pc in self.watch_channels:
            # TODO : fixed the inconsistencies regarding normpath
            songs[ pc.path ] = set()
            for f in mmp.walk_supported(pc.path, clean_empties=False):
                songs[ pc.path ].add(os.path.normpath(f))
                # We decide whether to update a file's metadata by checking
                # its system modification date. If it's above the value
                # self.last_run which is passed to us that means media monitor
                # wasn't aware when this changes occured in the filesystem
                # hence it will send the correct events to sync the database
                # with the filesystem
                if os.path.getmtime(f) > self.last_run:
                    modded += 1
                    dispatcher.send(signal=pc.signal, sender=self, event=DeleteFile(f))
                    dispatcher.send(signal=pc.signal, sender=self, event=NewFile(f))
        # Want all files in the database that are not in the filesystem
        for watch_dir in self.db.list_directories():
            db_songs = self.db.directory_get_files(watch_dir)
            # Get all the files that are in the database but in the file
            # system. These are the files marked for deletions
            for to_delete in db_songs.difference(songs[watch_dir]):
                # need the correct watch channel signal to call delete
                if watch_dir in signal_by_path:
                    dispatcher.send(signal=signal_by_path[watch_dir], sender=self, event=DeleteFile(f))
                    os.remove(to_delete)
                    deleted += 1
                else:
                    self.logger.error("Could not find the signal corresponding to path: '%s'" % watch_dir)

        for to_delete in self.db.difference(songs):
            for pc in self.watch_channels:
                if os.path.commonprefix([pc.path, to_delete]) == pc.path:
                    dispatcher.send(signal=pc.signal, sender=self, event=DeleteFile(f))
                    os.remove(to_delete)
                    deleted += 1
                    break
            else:
                self.logger.info("Error, could not find watch directory of would be deleted file '%s'" % to_delete)
        self.logger.info( "Flushed watch directories. (modified, deleted) = (%d, %d)"
                         % (modded, deleted) )



{'fraction_non_alphanumeric': 0.07631092795225237, 'fraction_numerical': 0.002131590166264033, 'mean_word_length': 2.0653310104529616, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3397165', 'n_tokens_mistral': 1894, 'n_tokens_neox': 1797, 'n_words': 619}