<commit_before>#!/usr/bin/env python

"""
@package mi.dataset.harvester A collection of polling routines that pull
data out of some source (an external file?) and put them into the ION system
via data set agents.
@file mi/dataset/harvester.py
@author <NAME>
@brief Base classes for data set agent pollers
"""

__author__ = '<NAME>, <NAME>, <NAME>'
__license__ = 'Apache 2.0'

from mi.core.log import get_logger ; log = get_logger()
from ooi.poller import DirectoryPoller


class Harvester(object):
    """ abstract class to show API needed for plugin poller objects """
    def __init__(self, config, memento, data_callback, exception_callback):  
        pass

    def start(self): 
        pass

    def shutdown(self): 
        pass
    
    def connect_to_source(self):
        """
        Trigger a connection to the data source using the configuration
        information for this stream. 
        """
        pass

## other pollers that check HTTP, FTP or other methods of finding data may be
## added here down the road

class AdditiveSequentialFileHarvester(DirectoryPoller, Harvester):
    def __init__(self, config, memento, file_callback, exception_callback):
        if not isinstance(config, dict):
            raise TypeError("Config object must be a dict")

        self.callback = file_callback
        self.last_file_completed = memento
        DirectoryPoller.__init__(self,
                                 config['directory'],
                                 config['pattern'],
                                 self.on_new_files,
                                 exception_callback,
                                 config.get('frequency', 300))

    def on_new_files(self, files):
        for file in files:
            if file>self.last_file_completed:
                with open(file,'rb') as f:
                    self.callback(f, file)<commit_msg>fixed default timeout.  was 300 seconds, made it 1.<commit_after>#!/usr/bin/env python

"""
@package mi.dataset.harvester A collection of polling routines that pull
data out of some source (an external file?) and put them into the ION system
via data set agents.
@file mi/dataset/harvester.py
@author <NAME>
@brief Base classes for data set agent pollers
"""

__author__ = '<NAME>, <NAME>, <NAME>'
__license__ = 'Apache 2.0'

from mi.core.log import get_logger ; log = get_logger()
from ooi.poller import DirectoryPoller


class Harvester(object):
    """ abstract class to show API needed for plugin poller objects """
    def __init__(self, config, memento, data_callback, exception_callback):  
        pass

    def start(self): 
        pass

    def shutdown(self): 
        pass
    
    def connect_to_source(self):
        """
        Trigger a connection to the data source using the configuration
        information for this stream. 
        """
        pass

## other pollers that check HTTP, FTP or other methods of finding data may be
## added here down the road

class AdditiveSequentialFileHarvester(DirectoryPoller, Harvester):
    def __init__(self, config, memento, file_callback, exception_callback):
        if not isinstance(config, dict):
            raise TypeError("Config object must be a dict")

        self.callback = file_callback
        self.last_file_completed = memento
        DirectoryPoller.__init__(self,
                                 config['directory'],
                                 config['pattern'],
                                 self.on_new_files,
                                 exception_callback,
                                 config.get('frequency', 1))

    def on_new_files(self, files):
        for file in files:
            if file>self.last_file_completed:
                with open(file,'rb') as f:
                    self.callback(f, file)
{'fraction_non_alphanumeric': 0.08113107822410148, 'fraction_numerical': 0.003171247357293869, 'mean_word_length': 2.2405821917808217, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2230090', 'n_tokens_mistral': 1030, 'n_tokens_neox': 964, 'n_words': 339}