<commit_before># Copyright (c) 2013 <NAME>
# See the file license.txt for copying permission.

import urlparse
import itertools
import curie
import link
import UserDict
from functools import wraps

class Relationships(UserDict.DictMixin):
    def __init__(self, links, embedded, curie):
        self.rels = {}

        item_urls = set()
        for key, values in itertools.chain(embedded.iteritems(),
                                           links.iteritems()):
            rel_key = curie.expand(key)
            if not isinstance(values, list):
                values = [values]

            for value in values:
                url = value.url()
                if url is not None and url in item_urls:
                    continue
                item_urls.add(url)
                
                self.rels.setdefault(rel_key, []).append(value)

    def __getitem__(self, key):
        return self.rels[key]

    def keys(self):
        return self.rels.keys()
         

def mutator(fn):
    @wraps(fn)
    def deco(self, *args, **kwargs):
        try:
            return fn(self, *args, **kwargs)
        finally:
            self.prepare_cache()

    return deco


class Document(object):
    def __init__(self, o, relative_to_url, parent_curie=None):
        self.o = o
        self.parent_curie = parent_curie
        self.relative_to_url = relative_to_url
        self.prepare_cache()

    def prepare_cache(self):
        def attrs_cache():
            attrs = dict(self.o)
            attrs['_links'] = None
            del attrs['_links']
            attrs['_embedded'] = None
            del attrs['_embedded']
            return attrs

        def links_cache():
            links = {}

            for key, value in self.o.get("_links", {}).iteritems():
                if key == 'curie':
                    continue
                links[key] = link.Link.from_object(value, self.relative_to_url)

            return links

        def load_curie_collection():
            result = curie.CurieCollection()
            if self.parent_curie is not None:
                result.update(self.parent_curie)

            curies = link.Link.from_object(
                self.o.get('_links', {}).get('curie', []),
                self.relative_to_url)

            if not isinstance(curies, list):
                curies = [curies]

            for curie_link in curies:
                result[curie_link.name] = curie_link

            return result

        def embedded_cache():
            embedded = {}
            for key, value in self.o.get("_embedded", {}).iteritems():
                embedded[key] = self.from_object(value,
                                                 self.relative_to_url,
                                                 self.curie)
            return embedded

        self.attrs = attrs_cache()
        self.links = links_cache()
        self.curie = load_curie_collection()
        self.embedded = embedded_cache()
        self.rels = Relationships(self.links, self.embedded, self.curie)

    def url(self):
        if not 'self' in self.links:
            return None

        self_link = self.links['self']

        if isinstance(self_link, list):
            for link in self_link:
                return link.url()

        return self_link.url()

    def expand_curie(self, link):
        return self.curie.expand(link)

    def as_object(self):
        return self.o

    @mutator
    def set_attribute(self, key, value):
        if key in ('_links', '_embedded'):
            return
        self.o[key] = value

    @mutator
    def delete_attribute(self, key):
        if key in ('_links', '_embedded'):
            raise KeyError(key)
        del self.o[key]

    def link(self, href, **kwargs):
        return link.Link(dict(href=href, **kwargs), self.relative_to_url)

    @mutator
    def add_link(self, rel, link):
        links = self.o.setdefault('_links', {})
        new_link = link.as_object()
        if rel not in links:
            links[rel] = new_link
            return

        current_links = links[rel]
        if isinstance(current_links, list):
            current_links.append(new_link)
        else:<commit_msg>Extract constant for reserved attributes names.<commit_after># Copyright (c) 2013 <NAME>
# See the file license.txt for copying permission.

import urlparse
import itertools
import curie
import link
import UserDict
from functools import wraps

class Relationships(UserDict.DictMixin):
    def __init__(self, links, embedded, curie):
        self.rels = {}

        item_urls = set()
        for key, values in itertools.chain(embedded.iteritems(),
                                           links.iteritems()):
            rel_key = curie.expand(key)
            if not isinstance(values, list):
                values = [values]

            for value in values:
                url = value.url()
                if url is not None and url in item_urls:
                    continue
                item_urls.add(url)
                
                self.rels.setdefault(rel_key, []).append(value)

    def __getitem__(self, key):
        return self.rels[key]

    def keys(self):
        return self.rels.keys()
         

def mutator(fn):
    @wraps(fn)
    def deco(self, *args, **kwargs):
        try:
            return fn(self, *args, **kwargs)
        finally:
            self.prepare_cache()

    return deco


class Document(object):
    def __init__(self, o, relative_to_url, parent_curie=None):
        self.o = o
        self.parent_curie = parent_curie
        self.relative_to_url = relative_to_url
        self.prepare_cache()

    RESERVED_ATTRIBUTE_NAMES = ('_links', '_embedded')

    def prepare_cache(self):
        def attrs_cache():
            attrs = dict(self.o)
            for name in self.RESERVED_ATTRIBUTE_NAMES:
                attrs[name] = None
                del attrs[name]
            return attrs

        def links_cache():
            links = {}

            for key, value in self.o.get("_links", {}).iteritems():
                if key == 'curie':
                    continue
                links[key] = link.Link.from_object(value, self.relative_to_url)

            return links

        def load_curie_collection():
            result = curie.CurieCollection()
            if self.parent_curie is not None:
                result.update(self.parent_curie)

            curies = link.Link.from_object(
                self.o.get('_links', {}).get('curie', []),
                self.relative_to_url)

            if not isinstance(curies, list):
                curies = [curies]

            for curie_link in curies:
                result[curie_link.name] = curie_link

            return result

        def embedded_cache():
            embedded = {}
            for key, value in self.o.get("_embedded", {}).iteritems():
                embedded[key] = self.from_object(value,
                                                 self.relative_to_url,
                                                 self.curie)
            return embedded

        self.attrs = attrs_cache()
        self.links = links_cache()
        self.curie = load_curie_collection()
        self.embedded = embedded_cache()
        self.rels = Relationships(self.links, self.embedded, self.curie)

    def url(self):
        if not 'self' in self.links:
            return None

        self_link = self.links['self']

        if isinstance(self_link, list):
            for link in self_link:
                return link.url()

        return self_link.url()

    def expand_curie(self, link):
        return self.curie.expand(link)

    def as_object(self):
        return self.o

    @mutator
    def set_attribute(self, key, value):
        if key in self.RESERVED_ATTRIBUTE_NAMES:
            return
        self.o[key] = value

    @mutator
    def delete_attribute(self, key):
        if key in self.RESERVED_ATTRIBUTE_NAMES:
            raise KeyError(key)
        del self.o[key]

    def link(self, href, **kwargs):
        return link.Link(dict(href=href, **kwargs), self.relative_to_url)

    @mutator
    def add_link(self, rel, link):
        links = self.o.setdefault('_links', {})
        new_link = link.as_object()
        if rel not in links:
            links[rel] = new_link
            return

        current_links = links[rel]
        if isinstance(current_links, list):
            current_links.append(new_link)
        else:
{'fraction_non_alphanumeric': 0.09754369390647143, 'fraction_numerical': 0.000944733112895607, 'mean_word_length': 1.6648835745752046, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4360712', 'n_tokens_mistral': 2443, 'n_tokens_neox': 2338, 'n_words': 579}