<commit_before># Copyright (c) 2013 Red Hat, Inc.
# Author:  <NAME> (<EMAIL>)

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import errors
from errors import fail
import itertools

def default_get(d, key, default=None):
    if d.has_key(key):
        return d[key]
    else: 
        return default

def default_getattr(o, key, default=None):
    if hasattr(o,key):
        return getattr(o,key)
    else: 
        return default

def uniq(ls):
    def gen_check_and_add(seen):
      def ca(elt):
        result = elt in seen
        seen.add(elt)
        return result
      return ca
    c_and_a = gen_check_and_add(set([]))
    return [x for x in ls if not c_and_a(x)]

def combine(old, new):
    [x for y in old, new for x in y]

def intersect_collections(f,s):
    return f & s

def union_collections(f, s):
    return f | s

def diff_collections(f, s):
    return (f | s) - (f & s)

def arcmethod(getfun, setfun, **kwargs):
    explain = default_get(kwargs, "explain", "have an arc to")
    preserve_order = default_get(kwargs, "preserve_order", False)
    heterogeneous = default_get(kwargs, "heterogeneous", False)
    
    def modify_arcs(self, command, dests, options):
        keyfun = self.name
        command = command.upper()
        what = default_get(kwargs, "what", self.__class__.__name__.lower())
        errwhat = default_getattr(errors, what.upper(), errors.UNKNOWN)
        
        if command == "ADD":
            old_dests = preserve_order and getfun() or set(getfun())
            new_dests = preserve_order and dests or set(dests)
            if keyfun in new_dests and not heterogeneous:
                fail(errors.make(errors.CIRCULAR_RELATIONSHIP, errors.INVALID_RELATIONSHIP, errwhat), "%s %s cannot %s itself" % (what, name, explain))
            
            setfun(self, uniq(itertools.chain(old, new)))
            self.update()
        elif command == "REPLACE":
            new_dests = preserve_order and dests or set(dests)
            if keyfun in new_dests and not heterogeneous:
                fail(errors.make(errors.CIRCULAR_RELATIONSHIP, errors.INVALID_RELATIONSHIP, errwhat), "%s %s cannot %s itself" % (what, name, explain))
            
            setfun(self, new_dests)
            self.update()
        elif command == "REMOVE":
            setfun(self, [dest for new_dests in getfun() if not dest in set(dests)])
            self.update()
        elif command in ["INTERSECT", "DIFF", "UNION"]:
            if preserve_order:
                fail(errors.make(errors.INTERNAL_ERROR, errors.NOT_IMPLEMENTED, errwhat), "%s not implemented for order-preserving relations" % command)
            
            op = globals()["%s_collections" % command.lower()]
            old_dests = getfun()
            supplied_dests = set(dests)
            new_dests = op(old_dests, supplied_dests)
            
            if keyfun() in new_dests and not heterogeneous:
                fail(errors.make(errors.CIRCULAR_RELATIONSHIP, errors.INVALID_RELATIONSHIP, errwhat), "%s %s cannot %s itself" % (what, name, explain))
            
            setfun(self, new_dests)<commit_msg>py: fixes to arc_utils<commit_after># Copyright (c) 2013 Red Hat, Inc.
# Author:  <NAME> (<EMAIL>)

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import errors
from errors import fail
import itertools

def default_get(d, key, default=None):
    if d.has_key(key):
        return d[key]
    else: 
        return default

def default_getattr(o, key, default=None):
    if hasattr(o,key):
        return getattr(o,key)
    else: 
        return default

def uniq(ls):
    def gen_check_and_add(seen):
      def ca(elt):
        result = elt in seen
        seen.add(elt)
        return result
      return ca
    c_and_a = gen_check_and_add(set([]))
    return [x for x in ls if not c_and_a(x)]

def combine(old, new):
    [x for y in old, new for x in y]

def intersect_collections(f,s):
    return f & s

def union_collections(f, s):
    return f | s

def diff_collections(f, s):
    return (f | s) - (f & s)

def arcmethod(getfun, setfun, **kwargs):
    explain = default_get(kwargs, "explain", "have an arc to")
    preserve_order = default_get(kwargs, "preserve_order", False)
    heterogeneous = default_get(kwargs, "heterogeneous", False)
    
    def modify_arcs(self, command, dests, options):
        keyfun = self.name
        command = command.upper()
        what = default_get(kwargs, "what", self.__class__.__name__.lower())
        errwhat = default_getattr(errors, what.upper(), errors.UNKNOWN)
        
        if command == "ADD":
            old_dests = preserve_order and getfun(self) or set(getfun(self))
            new_dests = preserve_order and dests or set(dests)
            if keyfun in new_dests and not heterogeneous:
                fail(errors.make(errors.CIRCULAR_RELATIONSHIP, errors.INVALID_RELATIONSHIP, errwhat), "%s %s cannot %s itself" % (what, name, explain))
            
            setfun(self, uniq(itertools.chain(old_dests, new_dests)))
            self.update()
        elif command == "REPLACE":
            new_dests = preserve_order and dests or set(dests)
            if keyfun in new_dests and not heterogeneous:
                fail(errors.make(errors.CIRCULAR_RELATIONSHIP, errors.INVALID_RELATIONSHIP, errwhat), "%s %s cannot %s itself" % (what, name, explain))
            
            setfun(self, new_dests)
            self.update()
        elif command == "REMOVE":
            setfun(self, [dest for new_dests in getfun(self) if not dest in set(dests)])
            self.update()
        elif command in ["INTERSECT", "DIFF", "UNION"]:
            if preserve_order:
                fail(errors.make(errors.INTERNAL_ERROR, errors.NOT_IMPLEMENTED, errwhat), "%s not implemented for order-preserving relations" % command)
            
            op = globals()["%s_collections" % command.lower()]
            old_dests = getfun(self)
            supplied_dests = set(dests)
            new_dests = op(old_dests, supplied_dests)
            
            if keyfun() in new_dests and not heterogeneous:
                fail(errors.make(errors.CIRCULAR_RELATIONSHIP, errors.INVALID_RELATIONSHIP, errwhat), "%s %s cannot %s itself" % (what, name, explain))
            
            setfun(self, new_dests)
{'fraction_non_alphanumeric': 0.10122235956599368, 'fraction_numerical': 0.002197500343359429, 'mean_word_length': 2.662977867203219, 'pattern_counts': {'":': 6, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4431207', 'n_tokens_mistral': 2234, 'n_tokens_neox': 2205, 'n_words': 692}