<commit_before>from jcudc24ingesterapi.models.sampling import _Sampling
from jcudc24ingesterapi import typed, APIDomainObject

"""
    Defines all possible data sources or in other words data input methods that can be provisioned.
"""

__author__ = '<NAME>'

class _DataSource(APIDomainObject):
    """
    Base data source class that does nothing beyond defining a known type.

    Data sources are known types that provide a known set of information but are unrelated to the data type.
    The ingester platform will need to implement data type specific ingesters for each data source.
    """
    processing_script = typed("_processing_script", str, "Script to run after download")

class DatasetDataSource(_DataSource):
    """
    Uses the resulting data_entry from another dataset and processes it further.
    """
    __xmlrpc_class__ = "dataset_data_source"
    dataset_id = typed("_dataset_id", int, "")
    
    def __init__(self, dataset_id):
        self.dataset_id = dataset_id

class PullDataSource(_DataSource):
    """
    A data source that polls a URI for data of the dataset's data type.
    """
    __xmlrpc_class__ = "pull_data_source"
    url = typed("_url", (str,unicode), "URL of the directory to scan")
    pattern = typed("_pattern", (str,unicode), "Pattern for identifying files, regex")
    recursive = typed("_recursive", bool, "Should the URL be treated as an index page")
    mime_type = typed("_mime_type", (str,unicode), "Mime type of the file")
    field = typed("_field", (str,unicode), "Field name to ingest into")
    sampling_script = typed("_sampling_script", (str,unicode), "Script to run to determine when to sample")
    def __init__(self, url=None, pattern=None, recursive=False, mime_type=None, field=None, processing_script=None, sampling_script=None):
        """Initialise the PullDataSource with a URI for the source file, and the field that 
        the uri will be saved to.
        """
        self.url = url
        self.field = field
        self.pattern = pattern
        self.mime_type = mime_type
        self.processing_script = processing_script
        self.sampling_script = sampling_script
        self.recursive = recursive

class PushDataSource(_DataSource):
    """
    A data source where the external application will use the ingester platform API to pass data into.
    """
    pass

class SOSDataSource(_DataSource):<commit_msg>Changed sampling script to sampling<commit_after>from jcudc24ingesterapi.models.sampling import _Sampling
from jcudc24ingesterapi import typed, APIDomainObject

"""
    Defines all possible data sources or in other words data input methods that can be provisioned.
"""

__author__ = '<NAME>'

class _DataSource(APIDomainObject):
    """
    Base data source class that does nothing beyond defining a known type.

    Data sources are known types that provide a known set of information but are unrelated to the data type.
    The ingester platform will need to implement data type specific ingesters for each data source.
    """
    processing_script = typed("_processing_script", str, "Script to run after download")

class DatasetDataSource(_DataSource):
    """
    Uses the resulting data_entry from another dataset and processes it further.
    """
    __xmlrpc_class__ = "dataset_data_source"
    dataset_id = typed("_dataset_id", int, "")
    
    def __init__(self, dataset_id):
        self.dataset_id = dataset_id

class PullDataSource(_DataSource):
    """
    A data source that polls a URI for data of the dataset's data type.
    """
    __xmlrpc_class__ = "pull_data_source"
    url = typed("_url", (str,unicode), "URL of the directory to scan")
    pattern = typed("_pattern", (str,unicode), "Pattern for identifying files, regex")
    recursive = typed("_recursive", bool, "Should the URL be treated as an index page")
    mime_type = typed("_mime_type", (str,unicode), "Mime type of the file")
    field = typed("_field", (str,unicode), "Field name to ingest into")
    sampling = typed("_sampling", _Sampling, "Script to run to determine when to sample")
    def __init__(self, url=None, pattern=None, recursive=False, mime_type=None, field=None, processing_script=None, sampling=None):
        """Initialise the PullDataSource with a URI for the source file, and the field that 
        the uri will be saved to.
        """
        self.url = url
        self.field = field
        self.pattern = pattern
        self.mime_type = mime_type
        self.processing_script = processing_script
        self.sampling = sampling
        self.recursive = recursive

class PushDataSource(_DataSource):
    """
    A data source where the external application will use the ingester platform API to pass data into.
    """
    pass

class SOSDataSource(_DataSource):
{'fraction_non_alphanumeric': 0.0843979057591623, 'fraction_numerical': 0.001675392670157068, 'mean_word_length': 3.6869479882237486, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '514754', 'n_tokens_mistral': 1323, 'n_tokens_neox': 1275, 'n_words': 493}