<commit_before>from cPickle import dumps
import json
import os
import pwd
from socket import getfqdn
from uuid import uuid1

try:
    import prov.model as pm
except ImportError:
    from ..external import provcopy as pm

from .. import get_info
from .. import logging
iflogger = logging.getLogger('interface')

foaf = pm.Namespace("foaf", "http://xmlns.com/foaf/0.1/")
dcterms = pm.Namespace("dcterms", "http://purl.org/dc/terms/")
nipype_ns = pm.Namespace("nipype", "http://nipy.org/nipype/terms/")
niiri = pm.Namespace("niiri", "http://nidm.nidash.org/iri/")

get_id = lambda: niiri[uuid1().hex]

def safe_encode(x):
    """Encodes a python value for prov
    """
    if x is None:
        return pm.Literal("Unknown", pm.XSD['string'])
    try:
        if isinstance(x, (str, unicode)):
            if os.path.exists(x):
                try:
                    return pm.URIRef('file://%s%s' % (getfqdn(), x))
                except AttributeError:
                    return pm.Literal('file://%s%s' % (getfqdn(), x),
                                      pm.XSD['anyURI'])
            else:
                return pm.Literal(x, pm.XSD['string'])
        if isinstance(x, (int,)):
            return pm.Literal(int(x), pm.XSD['integer'])
        if isinstance(x, (float,)):
            return pm.Literal(x, pm.XSD['float'])
        if isinstance(x, dict):
            outdict = {}
            for key, value in x.items():
                encoded_value = safe_encode(value)
                if isinstance(encoded_value, (pm.Literal,)):
                    outdict[key] = encoded_value.json_representation()<commit_msg>fix: limit text literal in provenance to 1MB<commit_after>from cPickle import dumps
import json
import os
import pwd
from socket import getfqdn
from uuid import uuid1

try:
    import prov.model as pm
except ImportError:
    from ..external import provcopy as pm

from .. import get_info
from .. import logging
iflogger = logging.getLogger('interface')

foaf = pm.Namespace("foaf", "http://xmlns.com/foaf/0.1/")
dcterms = pm.Namespace("dcterms", "http://purl.org/dc/terms/")
nipype_ns = pm.Namespace("nipype", "http://nipy.org/nipype/terms/")
niiri = pm.Namespace("niiri", "http://nidm.nidash.org/iri/")

get_id = lambda: niiri[uuid1().hex]

max_text_len = 1024000

def safe_encode(x):
    """Encodes a python value for prov
    """
    if x is None:
        return pm.Literal("Unknown", pm.XSD['string'])
    try:
        if isinstance(x, (str, unicode)):
            if os.path.exists(x):
                try:
                    return pm.URIRef('file://%s%s' % (getfqdn(), x))
                except AttributeError:
                    return pm.Literal('file://%s%s' % (getfqdn(), x),
                                      pm.XSD['anyURI'])
            else:
                if len(x) > max_text_len:
                    return pm.Literal(x[:max_text_len - 13] + ['...Clipped...'],
                                      pm.XSD['string'])
                else:
                    return pm.Literal(x, pm.XSD['string'])
        if isinstance(x, (int,)):
            return pm.Literal(int(x), pm.XSD['integer'])
        if isinstance(x, (float,)):
            return pm.Literal(x, pm.XSD['float'])
        if isinstance(x, dict):
            outdict = {}
            for key, value in x.items():
                encoded_value = safe_encode(value)
                if isinstance(encoded_value, (pm.Literal,)):
                    outdict[key] = encoded_value.json_representation()
{'fraction_non_alphanumeric': 0.14420331239291834, 'fraction_numerical': 0.005139920045688178, 'mean_word_length': 2.0487380330722367, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7430306', 'n_tokens_mistral': 1124, 'n_tokens_neox': 1080, 'n_words': 257}