<commit_before>"""
Module containting the interface definitions and base implementations
of the DataSource and related objects

Interface Field
  datasource : DataSource
  concept : Concept
  getConceptMapping : Mapping

Interface Mapping
  a, b : Field
  geCost(bool reverse=False) : float
  map(a', bool reverse=False) : sequence of b'   # a' = object corresponding to concept a
 
Interface DataSource
  getMappings : sequence of Mapping
  
Interface DataModel
  register(datasource) 
  getConcepts() : Concept
  getMappings(): sequence of Mapping

Interface Concept:
  pass (opaque)
"""

import collections
from toolkit import Identity, IDLabel
import project, toolkit, article, sources
#import association, amcatxapian, dbtoolkit,article
################### Base implementations ##############################

class Field(Identity):
    """
    Fields represent fields within datasources, ie concrete instantiations of Concepts
    A field has a datasource and concept, and it creates the Mapping from this Concept to itself.
    This mappping allows a conversion between Concept values (e.g. Article objects) and datasource
    specific values (e.g. article ids).
    """
    def __init__(self, datasource, concept):
        Identity.__init__(self, datasource, concept)
        self.datasource = datasource
        self.concept = concept
    def getConceptMapping(self):
        return FieldConceptMapping(self.concept, self)
    def __str__(self):
        return '%s::%s' % (self.datasource, self.concept)
    __repr__ = __str__

        
class Mapping(Identity):
    """
    Mappings represent the edges in the data graph and link two Fields or a
    Field and a Concept (see FieldConceptMapping). A mapping has references
    to two Field/Concepts (a and b) and can give the estimated cost for
    traversing it. It can also execute the mapping of concrete values
    """
    def __init__(self, a, b, cost=1.0, reversecost=10.0):
        Identity.__init__(self, a, b)
        self.a = a
        self.b = b
        self._cost = cost
        self._reversecost = reversecost
    def getCost(self, reverse=False):
        return self._reversecost if reverse else self._cost
    def map(self, value, reverse=False, memo=None):
        abstract
    def startMapping(self, values, reverse=False):
        """
        Allows a mapping to prepare the mapping of the given values and
        return a 'memo' object that will be passed on to the map call
        """
        pass
    def __str__(self):
        return "%s -> %s" % (self.a, self.b)
    __repr__ = __str__
    
    
class DataSource(object):
    """ 
    Datasource represents the datasource (e.g. articles, sentiment, associations). 
    A datasource contains mapping(s) of fields to Concepts or other fields.
    """
    def __init__(self, mappings = None):
        if mappings:
            self.setMappings(*mappings)
    def setMappings(self, *mappings):
        self._mappings = mappings or set()
    def getMappings(self):
        return self._mappings
    def deserialize(self, concept, value):
        return None
    def getPossibleValues(self, concept):
        return None

    def __str__(self):
        return "<%s>" % (self.__class__.__name__)
        
class FieldConceptMapping(Mapping):
    """ 
    Maps a Field to a Concept.
    """
    def __init__(self, concept, field, func=None):
        Mapping.__init__(self, concept, field, -2.0, -1.0)
        self.func = func
    def map(self, value, reverse=False, memo=None):
        if self.func:
            return [self.func(value, reverse)]
        else:
            return [value]
    def __str__(self):
        return "%s => %s" % (self.a, self.b)
    __repr__ = __str__

class Concept(IDLabel):
    """ 
    Concept represents the concepts in the datamodel. The concepts can be used to create mappings between datasources.
    """
    def __init__(self, datamodel, label, id):
        IDLabel.__init__(self, id, label)
        self.datamodel = datamodel
    def getObject(self, id):
        return self.datamodel.deserialize(self, id)
    def __repr__(self):
        return "Concept(%s)" % self.label

# TODO: identifier -> id (list index) mapping for concepts. This is pretty ugly and should be either decentralized
#       (but we need to guarantee consistency across datasources/models) or put into database?
CONCEPTS = ["article", "batch", "headline",  "date", "week", "year", "source", "url", "project", "sourcetype", "storedresult", "sentiment", "search", "brand", "property", "brandproperty", "associationcooc", "customer"]

class DataModel(object):
    """ 
    The datamodel contains all the datasources, 
    which in their turn contain fields and concepts that can be either mapped or not. 
    """<commit_msg>[svn r2202] propertycluster<commit_after>"""
Module containting the interface definitions and base implementations
of the DataSource and related objects

Interface Field
  datasource : DataSource
  concept : Concept
  getConceptMapping : Mapping

Interface Mapping
  a, b : Field
  geCost(bool reverse=False) : float
  map(a', bool reverse=False) : sequence of b'   # a' = object corresponding to concept a
 
Interface DataSource
  getMappings : sequence of Mapping
  
Interface DataModel
  register(datasource) 
  getConcepts() : Concept
  getMappings(): sequence of Mapping

Interface Concept:
  pass (opaque)
"""

import collections
from toolkit import Identity, IDLabel
import project, toolkit, article, sources
#import association, amcatxapian, dbtoolkit,article
################### Base implementations ##############################

class Field(Identity):
    """
    Fields represent fields within datasources, ie concrete instantiations of Concepts
    A field has a datasource and concept, and it creates the Mapping from this Concept to itself.
    This mappping allows a conversion between Concept values (e.g. Article objects) and datasource
    specific values (e.g. article ids).
    """
    def __init__(self, datasource, concept):
        Identity.__init__(self, datasource, concept)
        self.datasource = datasource
        self.concept = concept
    def getConceptMapping(self):
        return FieldConceptMapping(self.concept, self)
    def __str__(self):
        return '%s::%s' % (self.datasource, self.concept)
    __repr__ = __str__

        
class Mapping(Identity):
    """
    Mappings represent the edges in the data graph and link two Fields or a
    Field and a Concept (see FieldConceptMapping). A mapping has references
    to two Field/Concepts (a and b) and can give the estimated cost for
    traversing it. It can also execute the mapping of concrete values
    """
    def __init__(self, a, b, cost=1.0, reversecost=10.0):
        Identity.__init__(self, a, b)
        self.a = a
        self.b = b
        self._cost = cost
        self._reversecost = reversecost
    def getCost(self, reverse=False):
        return self._reversecost if reverse else self._cost
    def map(self, value, reverse=False, memo=None):
        abstract
    def startMapping(self, values, reverse=False):
        """
        Allows a mapping to prepare the mapping of the given values and
        return a 'memo' object that will be passed on to the map call
        """
        pass
    def __str__(self):
        return "%s -> %s" % (self.a, self.b)
    __repr__ = __str__
    
    
class DataSource(object):
    """ 
    Datasource represents the datasource (e.g. articles, sentiment, associations). 
    A datasource contains mapping(s) of fields to Concepts or other fields.
    """
    def __init__(self, mappings = None):
        if mappings:
            self.setMappings(*mappings)
    def setMappings(self, *mappings):
        self._mappings = mappings or set()
    def getMappings(self):
        return self._mappings
    def deserialize(self, concept, value):
        return None
    def getPossibleValues(self, concept):
        return None

    def __str__(self):
        return "<%s>" % (self.__class__.__name__)
        
class FieldConceptMapping(Mapping):
    """ 
    Maps a Field to a Concept.
    """
    def __init__(self, concept, field, func=None):
        Mapping.__init__(self, concept, field, -2.0, -1.0)
        self.func = func
    def map(self, value, reverse=False, memo=None):
        if self.func:
            return [self.func(value, reverse)]
        else:
            return [value]
    def __str__(self):
        return "%s => %s" % (self.a, self.b)
    __repr__ = __str__

class Concept(IDLabel):
    """ 
    Concept represents the concepts in the datamodel. The concepts can be used to create mappings between datasources.
    """
    def __init__(self, datamodel, label, id):
        IDLabel.__init__(self, id, label)
        self.datamodel = datamodel
    def getObject(self, id):
        return self.datamodel.deserialize(self, id)
    def __repr__(self):
        return "Concept(%s)" % self.label

# TODO: identifier -> id (list index) mapping for concepts. This is pretty ugly and should be either decentralized
#       (but we need to guarantee consistency across datasources/models) or put into database?
CONCEPTS = ["article", "batch", "headline",  "date", "week", "year", "source", "url", "project", "sourcetype", "storedresult", "sentiment", "search", "brand", "property", "brandproperty", "associationcooc", "customer", "propertycluster"]

class DataModel(object):
    """ 
    The datamodel contains all the datasources, 
    which in their turn contain fields and concepts that can be either mapped or not. 
    """
{'fraction_non_alphanumeric': 0.10346996540517873, 'fraction_numerical': 0.0023063214173393437, 'mean_word_length': 3.083904109589041, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6095243', 'n_tokens_mistral': 2773, 'n_tokens_neox': 2655, 'n_words': 963}