<commit_before>class MissingValuesProbe(object):
    """Data quality statistics for a dataset field
    
    :Attributes:
        * `count`: total count of null records
    """    
    def __init__(self):
        self.count = 0
        
    def probe(self, value):
        """Probe the value.
        """
        
        if value is None:
            self.count += 1

class StatisticsProbe(object):
    """Data quality statistics for a dataset field

    :Attributes:
        * `record_count`: total count of records in dataset. This should be set explicitly on
          finalisation. Seet :meth:`FieldStatistics.finalize`. In relational database this should be the
          same as `value_count`.

        * `fields`: 
        * `count`: number of non-null objects
    """    
    def __init__(self):
        self.min = None
        self.max = None
        self.sum = None
        self.count = 0
    @property
    def average(self):
        return self.sum / self.count
        
    def probe(self, value):
        self.count += 1
        if value is not None:
            if self.sum is None:
                self.sum = value
                self.min = value
                self.max = value
            else:
                self.sum += value
                self.min = min(self.min, value)
                self.max = max(self.max, value)

class DistinctProbe(object):
    def __init__(self, threshold = None):
        self.distinct_values = set([])
        self.overflow = False
        self.threshold = threshold

    def probe(self, value):
        if self.threshold and len(self.distinct_values) >= threshold:
            self.overflow = True
            return

        self.distinct_values.add(value)

class StorageTypeProbe(object):
    """Probe for guessing field data type

    Attributes:
        * field: name of a field which statistics are being presented
        * storage_types: found storage types
        * unique_storage_type: if there is only one storage type, then this is set to that type
    """    
    def __init__(self):
        self.storage_types = set()

    def probe(self, value):
        storage_type = value.__class__
        self.storage_types.add(storage_type.__name__)

    @property
    def unique_storage_type(self):
        """Return storage type if there is only one. This should always return a type in relational
        databases, but does not have to in databases such as MongoDB."""

        if len(self.storage_types) == 1:
            return list(self.storage_types)[0]
        else:
            return None
    <commit_msg>Preliminary implementation of data probes (just concept, API not decided yet for 100%)<commit_after>"""Data probes"""

import utils
import re

__all__ = [
    "MissingValuesProbe",
    "StatisticsProbe",
    "DistinctProbe",
    "StorageTypeProbe",
    "MultiProbe",
    "CompletenessProbe"
]

class MultiProbe(object):
    """Probe with multiple probes"""
    def __init__(self, probes = None):
        if probes:
            self.probes = probes
        else:
            self.probes = []
        
    def probe(self, value):
        for probe in self.probes:
            probe.probe(value)
        

    def to_dict(self):
        d = {}
        for probe in self.probes:
            name = utils.to_identifier(utils.decamelize(probe.__class__.__name__))
            re.sub('_probe$', name, '')
            d[name] = probe.to_dict()
            
        return d

class MissingValuesProbe(object):
    """Data quality statistics for a dataset field
    
    :Attributes:
        * `count`: total count of null records
    """    
    def __init__(self):
        self.count = 0
        
    def probe(self, value):
        """Probe the value.
        """
        
        if value is None:
            self.count += 1

    def to_dict(self):
        return {"count": self.count}

class CompletenessProbe(object):
    """Data quality statistics for a dataset field

    :Attributes:
        * `count`: total count of records
        * `unknown`: number of unknown records (NULL, None, nil, ...)
    """    
    def __init__(self):
        self.count = 0
        self.unknown = 0

    def probe(self, value):
        """Probe the value.
        """
        self.count += 1
        if value is None:
            self.unknown += 1
            
    def to_dict(self):
        return {"count": self.count, "unknown": self.unknown}

class StatisticsProbe(object):
    """Data quality statistics for a dataset field

    :Attributes:
        * `min` - minimum value found
        * `max` - maxumum value found
        * `sum` - sum of values
        * `count` - count of values
        * `average` - average value
    """    
    def __init__(self):
        self.min = None
        self.max = None
        self.sum = None
        self.count = 0
    @property
    def average(self):
        return self.sum / self.count
        
    def probe(self, value):
        self.count += 1
        if value is not None:
            if self.sum is None:
                self.sum = value
                self.min = value
                self.max = value
            else:
                self.sum += value
                self.min = min(self.min, value)
                self.max = max(self.max, value)

    def to_dict(self):
        return {"count": self.count, "min": self.min, "max": self.max,
                "sum": self.sum, "average": self.average }

class DistinctProbe(object):
    """Probe for distinct values."""
    def __init__(self, threshold = None):
        self.distinct_values = set([])
        self.overflow = False
        self.threshold = threshold

    def probe(self, value):
        if self.threshold and len(self.distinct_values) >= threshold:
            self.overflow = True
            return

        self.distinct_values.add(value)

class StorageTypeProbe(object):
    """Probe for guessing field data type

    Attributes:
        * field: name of a field which statistics are being presented
        * storage_types: found storage types
        * unique_storage_type: if there is only one storage type, then this is set to that type
    """    
    def __init__(self):
        self.storage_types = set()

    def probe(self, value):
        storage_type = value.__class__
        self.storage_types.add(storage_type.__name__)

    @property
    def unique_storage_type(self):
        """Return storage type if there is only one. This should always return a type in relational
        databases, but does not have to in databases such as MongoDB."""

        if len(self.storage_types) == 1:
            return list(self.storage_types)[0]
        else:
            return None

    def to_dict(self):
        d = {
            "storage_types": [str(st) for st in self.storage_types],
            "unique_storage_type": self.unique_storage_type
        }
        return d
{'fraction_non_alphanumeric': 0.09566865976374508, 'fraction_numerical': 0.0027708910602304215, 'mean_word_length': 1.8694560669456066, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5957577', 'n_tokens_mistral': 1938, 'n_tokens_neox': 1855, 'n_words': 594}