<commit_before>"""PROV-RDF serializers for ProvDocument

@author: <NAME> <<EMAIL>>
@copyright: University of Southampton 2014
"""
import logging
logger = logging.getLogger(__name__)

import base64
import datetime
import dateutil.parser
from prov import Serializer, Error
from prov.constants import *
from prov.model import (Literal, Identifier, QualifiedName, Namespace,
                        ProvRecord, ProvDocument, XSDQName)
import prov.model as pm

attr2rdf = lambda attr: URIRef(PROV[PROV_ID_ATTRIBUTES_MAP[attr].split('prov:')[1]].uri)

from rdflib.term import URIRef, BNode
from rdflib.term import Literal as RDFLiteral
from rdflib.graph import ConjunctiveGraph
from rdflib.namespace import RDF, RDFS, XSD

class ProvRDFException(Error):
    pass


class AnonymousIDGenerator():
    def __init__(self):
        self._cache = {}
        self._count = 0

    def get_anon_id(self, obj, local_prefix="id"):
        if obj not in self._cache:
            self._count += 1
            self._cache[obj] = Identifier('_:%s%d' % (local_prefix,
                                                      self._count)).uri
        return self._cache[obj]


# Reverse map for prov.model.XSD_DATATYPE_PARSERS
LITERAL_XSDTYPE_MAP = {
    float: XSD['double'],
    long: XSD['long'],
    int: XSD['int'],
    # boolean, string values are supported natively by PROV-JSON
    # datetime values are converted separately
}

def valid_qualified_name(bundle, value, xsd_qname=False):
    if value is None:
        return None
    qualified_name = bundle.valid_qualified_name(value)
    return qualified_name if not xsd_qname else XSDQName(qualified_name)


class ProvRDFSerializer(Serializer):
    def serialize(self, stream=None, **kwargs):
        container = self.encode_document(self.document)
        newargs = kwargs.copy()
        if newargs and 'rdf_format' in newargs:
            newargs['format'] = newargs['rdf_format']
            del newargs['rdf_format']
        container.serialize(stream, **newargs)

    def deserialize(self, stream, **kwargs):
        newargs = kwargs.copy()
        if newargs and 'rdf_format' in newargs:
            newargs['format'] = newargs['rdf_format']
            del newargs['rdf_format']
        container = ConjunctiveGraph().parse(stream, **newargs)
        document = ProvDocument()
        self.document = document
        self.decode_document(container, document)
        return document

    def valid_identifier(self, value):
        return self.document.valid_qualified_name(value)

    def encode_rdf_representation(self, value):
        #print value, type(value) #dbg
        if isinstance(value, URIRef):
            return value
        elif isinstance(value, Literal):
            return literal_rdf_representation(value)
        elif isinstance(value, datetime.datetime):
            return RDFLiteral(value.isoformat(), datatype=XSD['dateTime'])
        elif isinstance(value, QualifiedName):
            return URIRef(value.uri) #, datatype=XSD['QName'])
        elif isinstance(value, XSDQName):
            return URIRef(value.uri) #, datatype=XSD['QName'])
        elif isinstance(value, Identifier):
            return URIRef(value.uri)
        elif type(value) in LITERAL_XSDTYPE_MAP:
            return RDFLiteral(value, datatype=LITERAL_XSDTYPE_MAP[type(value)])
        else:
            return RDFLiteral(value)

    """
    def decode_rdf_representation(self, literal):
        if isinstance(literal, RDFLiteral):
            # complex type
            value = literal.value if literal.value is not None else literal
            datatype = literal.datatype if hasattr(literal, 'datatype') else None
            langtag = literal.language if hasattr(literal, 'language') else None
            datatype = valid_qualified_name(self.document, datatype)
            if datatype == XSD_ANYURI:
                return Identifier(value)
            elif datatype == XSD_QNAME:
                return valid_qualified_name(self.document, value, xsd_qname=True)
            elif datatype == PROV_QUALIFIEDNAME:
                return valid_qualified_name(self.document, value)
            else:
                # The literal of standard Python types is not converted here
                # It will be automatically converted when added to a record by _auto_literal_conversion()
                return Literal(value, datatype, langtag)
        elif isinstance(literal, URIRef):
            val = unicode(literal)
            return Identifier(val)
        else:
            # simple type, just return it
            return literal

    """
    def decode_rdf_representation(self, literal):
        #print(('Decode', literal))
        if isinstance(literal, RDFLiteral):
            value = literal.value if literal.value is not None else literal
            datatype = literal.datatype if hasattr(literal, 'datatype') else None
            langtag = literal.language if hasattr(literal, 'language') else None
            if datatype and 'base64Binary' in datatype:
                value = base64.standard_b64encode(value)
            #print((value, datatype, langtag)) #dbg
            '''
            if datatype == XSD['anyURI']:
                return Identifier(value)
            elif datatype == PROV['QualifiedName']:
                return self.valid_identifier(value)
            '''
            if datatype == XSD['QName']:
                for ns in self.document.namespaces:
                    if literal.startswith(ns.prefix):
                        return pm.XSDQName(QualifiedName(ns,
                                                         literal.replace(ns.prefix + ':',
                                                                         '')))
                raise Exception('No namespace found for: %s' % literal)
            if datatype == XSD['dateTime']:
                return dateutil.parser.parse(literal)
            else:
                # The literal of standard Python types is not converted here
                # It will be automatically converted when added to a record by _auto_literal_conversion()
                return Literal(value, self.valid_identifier(datatype), langtag)
        elif isinstance(literal, URIRef):
            val = unicode(literal)
            return Identifier(val)
        else:
            # simple type, just return it
            return literal

    def encode_document(self, document):
        container = self.encode_container(document)
        for b_id, b in document._bundles.items():
            #  encoding the sub-bundle
            bundle = self.encode_container(b, identifier=b_id.uri)
            container.addN(bundle.quads())
        return container

    def encode_container(self, bundle, container=None, identifier=None):
        if container is None:
            container = ConjunctiveGraph(identifier=identifier)
            nm = container.namespace_manager
            nm.bind('prov', PROV.uri)
        prefixes = {}
        for namespace in bundle._namespaces.get_registered_namespaces():
            container.bind(namespace.prefix, namespace.uri)
        if bundle._namespaces._default:
            prefixes['default'] = bundle._namespaces._default.uri

        id_generator = AnonymousIDGenerator()
        real_or_anon_id = lambda record: record._identifier.uri if \
            record._identifier else id_generator.get_anon_id(record)

        for record in bundle._records:
            rec_type = record.get_type()
            rec_label = PROV[PROV_N_MAP[rec_type]].uri
            if hasattr(record, 'identifier') and record.identifier: #record.is_relation():
                identifier = URIRef(unicode(real_or_anon_id(record)))
                container.add((identifier, RDF.type, URIRef(rec_type.uri)))
            else:
                identifier = None
            if record.attributes:
                bnode = None
                formal_objects = []
                used_objects = []
                all_attributes = set(record.formal_attributes).union(set(record.attributes))
                for idx, (attr, value) in enumerate(all_attributes):
                    #print identifier, idx, attr, value
                    #print record, rec_type.uri
                    if record.is_relation():
                        pred = URIRef(PROV[PROV_N_MAP[rec_type]].uri)
                        # create bnode relation
                        if bnode is None:
                            for key, val in record.formal_attributes:
                                formal_objects.append(key)
                            used_objects = [record.formal_attributes[0][0]]
                            subj = None
                            if record.formal_attributes[0][1]:
                                subj = URIRef(record.formal_attributes[0][1].uri)
                            if identifier is None and subj is not None:
                                try:
                                    obj_val = record.formal_attributes[1][1]
                                    obj_attr = URIRef(record.formal_attributes[1][0].uri)
                                except IndexError:
                                    obj_val = None
                                if obj_val:
                                    used_objects.append(record.formal_attributes[1][0])
                                    obj_val = self.encode_rdf_representation(obj_val)
                                    container.add((subj, pred, obj_val))
                                #print identifier, pred, obj_val
                            if rec_type in [PROV_ALTERNATE]: #, PROV_ASSOCIATION]:
                                continue
                            if subj and identifier:
                                QRole = URIRef(PROV['qualified' +
                                                    rec_type._localpart].uri)
                                container.add((subj, QRole, identifier))

                            '''
                            for key, val in record.formal_attributes:
                                formal_objects.append(key)
                            used_objects = [record.formal_attributes[0][0]]
                            if record.formal_attributes[0][1]:
                                identifier = URIRef(record.formal_attributes[0][1].uri)
                                try:
                                    obj_val = record.formal_attributes[1][1]
                                    obj_attr = URIRef(record.formal_attributes[1][0].uri)
                                except IndexError:
                                    obj_val = None
                                if obj_val:
                                    used_objects.append(record.formal_attributes[1][0])
                                    obj_val = self.encode_rdf_representation(obj_val)
                                    container.add((identifier, pred, obj_val))
                                print identifier, pred, obj_val
                                if rec_type in [PROV_ALTERNATE]: #, PROV_ASSOCIATION]:
                                    continue
                                QRole = URIRef(PROV['qualified' +
                                                    rec_type._localpart].uri)
                                if hasattr(record, 'identifier') and record.identifier:
                                    bnode = URIRef(record.identifier.uri)
                                else:
                                    bnode = BNode()
                                container.add((identifier, QRole, bnode))
                                container.add((bnode, RDF.type,
                                               URIRef(rec_type.uri)))
                                # reset identifier to BNode
                                identifier = bnode
                                print identifier, obj_attr, obj_val #dbg
                                if obj_val:
                                    container.add((identifier, obj_attr, obj_val))
                            '''
                        if value is not None and attr not in used_objects:
                            #print 'attr', attr #dbg
                            if attr in formal_objects:
                                pred = attr2rdf(attr)
                            elif attr == PROV['role']:
                                pred = URIRef(PROV['hadRole'].uri)
                            elif attr == PROV['plan']:
                                pred = URIRef(PROV['hadPlan'].uri)
                            elif attr == PROV['type']:
                                pred = RDF.type
                            elif attr == PROV['label']:
                                pred = RDFS.label
                            elif isinstance(attr, QualifiedName):
                                pred = URIRef(attr.uri)
                            else:
                                pred = self.encode_rdf_representation(attr)
                            if PROV['plan'].uri in pred:
                                pred = URIRef(PROV['hadPlan'].uri)
                            #print pred #dbg
                            container.add((identifier, pred,
                                           self.encode_rdf_representation(value)))
                        continue
                    if value is None:
                        continue
                    if isinstance(value, ProvRecord):
                        obj = URIRef(unicode(real_or_anon_id(value)))
                    else:
                        #  Assuming this is a datetime value
                        obj = self.encode_rdf_representation(value)
                    if attr == PROV['location']:
                        pred = URIRef(PROV['atLocation'].uri)
                        if isinstance(value, (URIRef, QualifiedName)):
                            if isinstance(value, QualifiedName):
                                #value = RDFLiteral(unicode(value), datatype=XSD['QName'])
                                value = URIRef(value.uri)
                            container.add((identifier, pred, value))
                            #container.add((value, RDF.type,
                            #               URIRef(PROV['Location'].uri)))
                        else:
                            container.add((identifier, pred,
                                           self.encode_rdf_representation(obj)))
                        continue
                    #pred = attr2rdf(attr)
                    if attr == PROV['type']:
                        pred = RDF.type
                    elif attr == PROV['label']:
                        pred = RDFS.label
                    else:
                        pred = self.encode_rdf_representation(attr)
                    container.add((identifier, pred, obj))
        return container

    def decode_document(self, content, document):
        for prefix, url in content.namespaces():
            #if prefix in ['rdf', 'rdfs', 'xml']:
            #    continue
            document.add_namespace(prefix, unicode(url))
        if hasattr(content, 'contexts'):
            for graph in content.contexts():
                bundle_id = unicode(graph.identifier)
                bundle = document.bundle(bundle_id)
                self.decode_container(graph, bundle)
        else:
            self.decode_container(content, document)

    def decode_container(self, graph, bundle):
        ids = {}
        PROV_CLS_MAP = {}
        for key, val in PROV_N_MAP.items():<commit_msg>current state<commit_after>"""PROV-RDF serializers for ProvDocument

@author: <NAME> <<EMAIL>>
@copyright: University of Southampton 2014
"""
import logging
logger = logging.getLogger(__name__)

import base64
import datetime
import dateutil.parser
from prov import Serializer, Error
from prov.constants import *
from prov.model import (Literal, Identifier, QualifiedName, Namespace,
                        ProvRecord, ProvDocument, XSDQName)
import prov.model as pm

attr2rdf = lambda attr: URIRef(PROV[PROV_ID_ATTRIBUTES_MAP[attr].split('prov:')[1]].uri)

from rdflib.term import URIRef, BNode
from rdflib.term import Literal as RDFLiteral
from rdflib.graph import ConjunctiveGraph
from rdflib.namespace import RDF, RDFS, XSD

class ProvRDFException(Error):
    pass


class AnonymousIDGenerator():
    def __init__(self):
        self._cache = {}
        self._count = 0

    def get_anon_id(self, obj, local_prefix="id"):
        if obj not in self._cache:
            self._count += 1
            self._cache[obj] = Identifier('_:%s%d' % (local_prefix,
                                                      self._count)).uri
        return self._cache[obj]


# Reverse map for prov.model.XSD_DATATYPE_PARSERS
LITERAL_XSDTYPE_MAP = {
    float: XSD['double'],
    long: XSD['long'],
    int: XSD['int'],
    # boolean, string values are supported natively by PROV-JSON
    # datetime values are converted separately
}

def valid_qualified_name(bundle, value, xsd_qname=False):
    if value is None:
        return None
    qualified_name = bundle.valid_qualified_name(value)
    return qualified_name if not xsd_qname else XSDQName(qualified_name)


class ProvRDFSerializer(Serializer):
    def serialize(self, stream=None, **kwargs):
        container = self.encode_document(self.document)
        newargs = kwargs.copy()
        if newargs and 'rdf_format' in newargs:
            newargs['format'] = newargs['rdf_format']
            del newargs['rdf_format']
        container.serialize(stream, **newargs)

    def deserialize(self, stream, **kwargs):
        newargs = kwargs.copy()
        if newargs and 'rdf_format' in newargs:
            newargs['format'] = newargs['rdf_format']
            del newargs['rdf_format']
        container = ConjunctiveGraph().parse(stream, **newargs)
        document = ProvDocument()
        self.document = document
        self.decode_document(container, document)
        return document

    def valid_identifier(self, value):
        return self.document.valid_qualified_name(value)

    def encode_rdf_representation(self, value):
        #print value, type(value) #dbg
        if isinstance(value, URIRef):
            return value
        elif isinstance(value, Literal):
            return literal_rdf_representation(value)
        elif isinstance(value, datetime.datetime):
            return RDFLiteral(value.isoformat(), datatype=XSD['dateTime'])
        elif isinstance(value, QualifiedName):
            #if value.namespace == PROV:
                return URIRef(value.uri) #, datatype=XSD['QName'])
            #else:
            #    return RDFLiteral(value, datatype=XSD['QName'])
        elif isinstance(value, XSDQName):
            return RDFLiteral(value, datatype=XSD['QName'])
        elif isinstance(value, Identifier):
            return URIRef(value.uri)
        elif type(value) in LITERAL_XSDTYPE_MAP:
            return RDFLiteral(value, datatype=LITERAL_XSDTYPE_MAP[type(value)])
        else:
            return RDFLiteral(value)

    """
    def decode_rdf_representation(self, literal):
        if isinstance(literal, RDFLiteral):
            # complex type
            value = literal.value if literal.value is not None else literal
            datatype = literal.datatype if hasattr(literal, 'datatype') else None
            langtag = literal.language if hasattr(literal, 'language') else None
            datatype = valid_qualified_name(self.document, datatype)
            if datatype == XSD_ANYURI:
                return Identifier(value)
            elif datatype == XSD_QNAME:
                return valid_qualified_name(self.document, value, xsd_qname=True)
            elif datatype == PROV_QUALIFIEDNAME:
                return valid_qualified_name(self.document, value)
            else:
                # The literal of standard Python types is not converted here
                # It will be automatically converted when added to a record by _auto_literal_conversion()
                return Literal(value, datatype, langtag)
        elif isinstance(literal, URIRef):
            val = unicode(literal)
            return Identifier(val)
        else:
            # simple type, just return it
            return literal

    """
    def decode_rdf_representation(self, literal):
        #print(('Decode', literal))
        if isinstance(literal, RDFLiteral):
            value = literal.value if literal.value is not None else literal
            datatype = literal.datatype if hasattr(literal, 'datatype') else None
            langtag = literal.language if hasattr(literal, 'language') else None
            if datatype and 'base64Binary' in datatype:
                value = base64.standard_b64encode(value)
            #print((value, datatype, langtag)) #dbg
            '''
            if datatype == XSD['anyURI']:
                return Identifier(value)
            elif datatype == PROV['QualifiedName']:
                return self.valid_identifier(value)
            '''
            if datatype == XSD['QName']:
                for ns in self.document.namespaces:
                    if literal.startswith(ns.prefix):
                        return pm.XSDQName(QualifiedName(ns,
                                                         literal.replace(ns.prefix + ':',
                                                                         '')))
                raise Exception('No namespace found for: %s' % literal)
            if datatype == XSD['dateTime']:
                return dateutil.parser.parse(literal)
            else:
                # The literal of standard Python types is not converted here
                # It will be automatically converted when added to a record by _auto_literal_conversion()
                return Literal(value, self.valid_identifier(datatype), langtag)
        elif isinstance(literal, URIRef):
            val = unicode(literal)
            return Identifier(val)
        else:
            # simple type, just return it
            return literal

    def encode_document(self, document):
        container = self.encode_container(document)
        for b_id, b in document._bundles.items():
            #  encoding the sub-bundle
            bundle = self.encode_container(b, identifier=b_id.uri)
            container.addN(bundle.quads())
        return container

    def encode_container(self, bundle, container=None, identifier=None):
        if container is None:
            container = ConjunctiveGraph(identifier=identifier)
            nm = container.namespace_manager
            nm.bind('prov', PROV.uri)
        prefixes = {}
        for namespace in bundle._namespaces.get_registered_namespaces():
            container.bind(namespace.prefix, namespace.uri)
        if bundle._namespaces._default:
            prefixes['default'] = bundle._namespaces._default.uri

        id_generator = AnonymousIDGenerator()
        real_or_anon_id = lambda record: record._identifier.uri if \
            record._identifier else id_generator.get_anon_id(record)

        for record in bundle._records:
            rec_type = record.get_type()
            rec_label = PROV[PROV_N_MAP[rec_type]].uri
            if hasattr(record, 'identifier') and record.identifier: #record.is_relation():
                identifier = URIRef(unicode(real_or_anon_id(record)))
                container.add((identifier, RDF.type, URIRef(rec_type.uri)))
            else:
                identifier = None
            if record.attributes:
                bnode = None
                formal_objects = []
                used_objects = []
                all_attributes = list(record.formal_attributes) + list(record.attributes)
                #print all_attributes
                #all_attributes = set(record.formal_attributes).union(set(record.attributes))
                for idx, (attr, value) in enumerate(all_attributes):
                    #print identifier, idx, attr, value
                    #print record, rec_type.uri
                    if record.is_relation():
                        pred = URIRef(PROV[PROV_N_MAP[rec_type]].uri)
                        # create bnode relation
                        if bnode is None:
                            for key, val in record.formal_attributes:
                                formal_objects.append(key)
                            used_objects = [record.formal_attributes[0][0]]
                            subj = None
                            if record.formal_attributes[0][1]:
                                subj = URIRef(record.formal_attributes[0][1].uri)
                            if identifier is None and subj is not None:
                                try:
                                    obj_val = record.formal_attributes[1][1]
                                    obj_attr = URIRef(record.formal_attributes[1][0].uri)
                                except IndexError:
                                    obj_val = None
                                if obj_val:
                                    used_objects.append(record.formal_attributes[1][0])
                                    obj_val = self.encode_rdf_representation(obj_val)
                                    container.add((subj, pred, obj_val))
                                #print identifier, pred, obj_val
                            if rec_type in [PROV_ALTERNATE]: #, PROV_ASSOCIATION]:
                                continue
                            if subj:
                                QRole = URIRef(PROV['qualified' +
                                                    rec_type._localpart].uri)
                                if identifier is not None:
                                    container.add((subj, QRole, identifier))
                                else:
                                    identifier = BNode()
                                    container.add((subj, QRole, identifier))
                                    container.add((identifier, RDF.type,
                                                   URIRef(rec_type.uri)))
                                               # reset identifier to BNode
                            '''
                            for key, val in record.formal_attributes:
                                formal_objects.append(key)
                            used_objects = [record.formal_attributes[0][0]]
                            if record.formal_attributes[0][1]:
                                identifier = URIRef(record.formal_attributes[0][1].uri)
                                try:
                                    obj_val = record.formal_attributes[1][1]
                                    obj_attr = URIRef(record.formal_attributes[1][0].uri)
                                except IndexError:
                                    obj_val = None
                                if obj_val:
                                    used_objects.append(record.formal_attributes[1][0])
                                    obj_val = self.encode_rdf_representation(obj_val)
                                    container.add((identifier, pred, obj_val))
                                print identifier, pred, obj_val
                                if rec_type in [PROV_ALTERNATE]: #, PROV_ASSOCIATION]:
                                    continue
                                QRole = URIRef(PROV['qualified' +
                                                    rec_type._localpart].uri)
                                if hasattr(record, 'identifier') and record.identifier:
                                    bnode = URIRef(record.identifier.uri)
                                else:
                                    bnode = BNode()
                                container.add((identifier, QRole, bnode))
                                container.add((bnode, RDF.type,
                                               URIRef(rec_type.uri)))
                                # reset identifier to BNode
                                identifier = bnode
                                print identifier, obj_attr, obj_val #dbg
                                if obj_val:
                                    container.add((identifier, obj_attr, obj_val))
                            '''
                        if value is not None and attr not in used_objects:
                            #print 'attr', attr #dbg
                            if attr in formal_objects:
                                pred = attr2rdf(attr)
                            elif attr == PROV['role']:
                                pred = URIRef(PROV['hadRole'].uri)
                            elif attr == PROV['plan']:
                                pred = URIRef(PROV['hadPlan'].uri)
                            elif attr == PROV['type']:
                                pred = RDF.type
                            elif attr == PROV['label']:
                                pred = RDFS.label
                            elif isinstance(attr, QualifiedName):
                                pred = URIRef(attr.uri)
                            else:
                                pred = self.encode_rdf_representation(attr)
                            if PROV['plan'].uri in pred:
                                pred = URIRef(PROV['hadPlan'].uri)
                            #print identifier, pred, value #dbg
                            container.add((identifier, pred,
                                           self.encode_rdf_representation(value)))
                        continue
                    if value is None:
                        continue
                    if isinstance(value, ProvRecord):
                        obj = URIRef(unicode(real_or_anon_id(value)))
                    else:
                        #  Assuming this is a datetime value
                        obj = self.encode_rdf_representation(value)
                    #print type(value), type(obj)
                    if attr == PROV['location']:
                        pred = URIRef(PROV['atLocation'].uri)
                        if False and isinstance(value, (URIRef, QualifiedName)):
                            if isinstance(value, QualifiedName):
                                #value = RDFLiteral(unicode(value), datatype=XSD['QName'])
                                value = URIRef(value.uri)
                            container.add((identifier, pred, value))
                            #container.add((value, RDF.type,
                            #               URIRef(PROV['Location'].uri)))
                        else:
                            container.add((identifier, pred,
                                           self.encode_rdf_representation(obj)))
                        continue
                    #pred = attr2rdf(attr)
                    if attr == PROV['type']:
                        pred = RDF.type
                    elif attr == PROV['label']:
                        pred = RDFS.label
                    else:
                        pred = self.encode_rdf_representation(attr)
                    container.add((identifier, pred, obj))
        return container

    def decode_document(self, content, document):
        for prefix, url in content.namespaces():
            #if prefix in ['rdf', 'rdfs', 'xml']:
            #    continue
            document.add_namespace(prefix, unicode(url))
        for bundle_stmt in content.triples((None, RDF.type,
                                            URIRef(pm.PROV['bundle'].uri))):
            bundle_id = unicode(bundle_stmt[0])
        if hasattr(content, 'contexts'):
            for graph in content.contexts():
                bundle_id = unicode(graph.identifier)
                bundle = document.bundle(bundle_id)
                self.decode_container(graph, bundle)
        else:
            self.decode_container(content, document)

    def decode_container(self, graph, bundle):
        ids = {}
        PROV_CLS_MAP = {}
        for key, val in PROV_N_MAP.items():
{'fraction_non_alphanumeric': 0.08159386877687083, 'fraction_numerical': 0.002648140071032463, 'mean_word_length': 1.3881407633360614, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 93, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4010362', 'n_tokens_mistral': 7897, 'n_tokens_neox': 7555, 'n_words': 1789}