<commit_before>'''Python implemetation of the W3C Provenance Data Model (PROV-DM)

Support for PROV-JSON import/export 

References:

PROV-DM: http://www.w3.org/TR/prov-dm/

@author: <NAME> <<EMAIL>>
@copyright: University of Southampton 2012
'''

import logging
import datetime
import json
import re
import collections
from collections import defaultdict
try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict
logger = logging.getLogger(__name__)

## PROV record constants - PROV-DM LC
# C1. Entities/Activities
PROV_REC_ENTITY                 = 1
PROV_REC_ACTIVITY               = 2
PROV_REC_GENERATION             = 11
PROV_REC_USAGE                  = 12
PROV_REC_COMMUNICATION          = 13
PROV_REC_START                  = 14
PROV_REC_END                    = 15
PROV_REC_INVALIDATION           = 16

# C2. Derivations
PROV_REC_DERIVATION             = 21

# C3. Agents/Responsibility
PROV_REC_AGENT                  = 3
PROV_REC_ATTRIBUTION            = 31
PROV_REC_ASSOCIATION            = 32
PROV_REC_DELEGATION             = 33
PROV_REC_INFLUENCE              = 34
# C4. Bundles
PROV_REC_BUNDLE                 = 4 # This is the lowest value, so bundle(s) in JSON will be decoded first 
# C5. Alternate
PROV_REC_ALTERNATE              = 51
PROV_REC_SPECIALIZATION         = 52
PROV_REC_MENTION                = 53
# C6. Collections
PROV_REC_MEMBERSHIP             = 61

PROV_RECORD_TYPES = (
    (PROV_REC_ENTITY,               u'Entity'),
    (PROV_REC_ACTIVITY,             u'Activity'),
    (PROV_REC_GENERATION,           u'Generation'),
    (PROV_REC_USAGE,                u'Usage'),
    (PROV_REC_COMMUNICATION,        u'Communication'),
    (PROV_REC_START,                u'Start'),
    (PROV_REC_END,                  u'End'),
    (PROV_REC_INVALIDATION,         u'Invalidation'),
    (PROV_REC_DERIVATION,           u'Derivation'),
    (PROV_REC_AGENT,                u'Agent'),
    (PROV_REC_ATTRIBUTION,          u'Attribution'),
    (PROV_REC_ASSOCIATION,          u'Association'),
    (PROV_REC_DELEGATION,           u'Delegation'),
    (PROV_REC_INFLUENCE,            u'Influence'),
    (PROV_REC_BUNDLE,               u'Bundle'),
    (PROV_REC_ALTERNATE,            u'Alternate'),
    (PROV_REC_SPECIALIZATION,       u'Specialization'),
    (PROV_REC_MENTION,              u'Mention'),
    (PROV_REC_MEMBERSHIP,           u'Membership'),
    (PROV_REC_BUNDLE,               u'Bundle'),
)

PROV_N_MAP = {
    PROV_REC_ENTITY:               u'entity',
    PROV_REC_ACTIVITY:             u'activity',
    PROV_REC_GENERATION:           u'wasGeneratedBy',
    PROV_REC_USAGE:                u'used',
    PROV_REC_COMMUNICATION:        u'wasInformedBy',
    PROV_REC_START:                u'wasStartedBy',
    PROV_REC_END:                  u'wasEndedBy',
    PROV_REC_INVALIDATION:         u'wasInvalidatedBy',
    PROV_REC_DERIVATION:           u'wasDerivedFrom',
    PROV_REC_AGENT:                u'agent',<commit_msg>Fixed: Removed the duplicated Bundle definition in PROV_RECORD_TYPES<commit_after>'''Python implemetation of the W3C Provenance Data Model (PROV-DM)

Support for PROV-JSON import/export 

References:

PROV-DM: http://www.w3.org/TR/prov-dm/

@author: <NAME> <<EMAIL>>
@copyright: University of Southampton 2012
'''

import logging
import datetime
import json
import re
import collections
from collections import defaultdict
try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict
logger = logging.getLogger(__name__)

## PROV record constants - PROV-DM LC
# C1. Entities/Activities
PROV_REC_ENTITY                 = 1
PROV_REC_ACTIVITY               = 2
PROV_REC_GENERATION             = 11
PROV_REC_USAGE                  = 12
PROV_REC_COMMUNICATION          = 13
PROV_REC_START                  = 14
PROV_REC_END                    = 15
PROV_REC_INVALIDATION           = 16

# C2. Derivations
PROV_REC_DERIVATION             = 21

# C3. Agents/Responsibility
PROV_REC_AGENT                  = 3
PROV_REC_ATTRIBUTION            = 31
PROV_REC_ASSOCIATION            = 32
PROV_REC_DELEGATION             = 33
PROV_REC_INFLUENCE              = 34
# C4. Bundles
PROV_REC_BUNDLE                 = 4 # This is the lowest value, so bundle(s) in JSON will be decoded first 
# C5. Alternate
PROV_REC_ALTERNATE              = 51
PROV_REC_SPECIALIZATION         = 52
PROV_REC_MENTION                = 53
# C6. Collections
PROV_REC_MEMBERSHIP             = 61

PROV_RECORD_TYPES = (
    (PROV_REC_ENTITY,               u'Entity'),
    (PROV_REC_ACTIVITY,             u'Activity'),
    (PROV_REC_GENERATION,           u'Generation'),
    (PROV_REC_USAGE,                u'Usage'),
    (PROV_REC_COMMUNICATION,        u'Communication'),
    (PROV_REC_START,                u'Start'),
    (PROV_REC_END,                  u'End'),
    (PROV_REC_INVALIDATION,         u'Invalidation'),
    (PROV_REC_DERIVATION,           u'Derivation'),
    (PROV_REC_AGENT,                u'Agent'),
    (PROV_REC_ATTRIBUTION,          u'Attribution'),
    (PROV_REC_ASSOCIATION,          u'Association'),
    (PROV_REC_DELEGATION,           u'Delegation'),
    (PROV_REC_INFLUENCE,            u'Influence'),
    (PROV_REC_BUNDLE,               u'Bundle'),
    (PROV_REC_ALTERNATE,            u'Alternate'),
    (PROV_REC_SPECIALIZATION,       u'Specialization'),
    (PROV_REC_MENTION,              u'Mention'),
    (PROV_REC_MEMBERSHIP,           u'Membership'),
)

PROV_N_MAP = {
    PROV_REC_ENTITY:               u'entity',
    PROV_REC_ACTIVITY:             u'activity',
    PROV_REC_GENERATION:           u'wasGeneratedBy',
    PROV_REC_USAGE:                u'used',
    PROV_REC_COMMUNICATION:        u'wasInformedBy',
    PROV_REC_START:                u'wasStartedBy',
    PROV_REC_END:                  u'wasEndedBy',
    PROV_REC_INVALIDATION:         u'wasInvalidatedBy',
    PROV_REC_DERIVATION:           u'wasDerivedFrom',
    PROV_REC_AGENT:                u'agent',
{'fraction_non_alphanumeric': 0.08190158465387823, 'fraction_numerical': 0.015346121768140117, 'mean_word_length': 2.062308478038815, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1269572', 'n_tokens_mistral': 2108, 'n_tokens_neox': 1985, 'n_words': 366}