<commit_before>from iati.models import Activity
from traceability.models import Chain, ChainNode, ChainNodeError, ChainLink, ChainLinkRelation
from django.core.exceptions import ObjectDoesNotExist
from datetime import datetime
from django.db.models import Q, Count


def find(list, filter):
    for x in list:
        if filter(x):
            return x
    return False


class ChainRetriever():
    """
    Wrapper class for all chain building functionality

    """
    def __init__(self):
        self.links = []
        self.errors = []
        self.started_at = datetime.now()
        self.chain = None

    def retrieve_chains_by_publisher(self, publisher_iati_id):
        for activity in Activity.objects.filter(publisher__publisher_iati_id=publisher_iati_id, hierarchy=1):
            if self.chain_update_needed(activity):
                self.retrieve_chain(activity)

    def retrieve_chain_by_activity_id(self, activity_id):
        activity = Activity.objects.get(iati_identifier=activity_id)
        self.retrieve_chain(activity)

    def retrieve_chain_for_all_activities(self):
        #get all activities that other activities set as provider of funds
        for activity in Activity.objects.annotate(related_set_as_provider=Count('relatedactivity__ref_activity__transaction_provider_activity')).annotate(set_as_provider=Count('transaction_provider_activity')).filter(Q(related_set_as_provider__gt=0) | Q(set_as_provider__gt=0)):
            # filter all activities that have  (this was too slow to put in the query above) 
            if Activity.objects.filter(transaction_provider_activity__transaction__activity=activity).count() > 0:
                print 'skipped {}'.format(activity.iati_identifier)
                continue

            if self.chain_update_needed(activity):
                self.retrieve_chain(activity)
    
    def chain_update_needed(self, activity):
        chain = Chain.objects.filter(Q(chainlink__start_node__activity=activity) | Q(chainlink__end_node__activity=activity)).distinct()

        if len(chain) > 0 and chain[0].last_updated < self.started_at:
            # in a chain, only update if the chain is not created within this run
            return True 

        elif len(chain) is 0:
            # not in a chain yet, create the chain
            return True               
        return False

    def retrieve_chain(self, activity):

        # delete old chain
        if ChainNode.objects.filter(activity=activity, bol=True, treated_as_end_node=False, tier=0).exists():
            node = ChainNode.objects.filter(activity=activity, bol=True, treated_as_end_node=False, tier=0)

            if len(node) > 1:
                print 'Multiple mentions of this chain, should not happen'
                print node
            node[0].chain.delete()<commit_msg>further bugfixing on getting the root nodes of delivery chains<commit_after>from iati.models import Activity
from traceability.models import Chain, ChainNode, ChainNodeError, ChainLink, ChainLinkRelation
from django.core.exceptions import ObjectDoesNotExist
from datetime import datetime
from django.db.models import Q, Count


def find(list, filter):
    for x in list:
        if filter(x):
            return x
    return False


class ChainRetriever():
    """
    Wrapper class for all chain building functionality

    """
    def __init__(self):
        self.links = []
        self.errors = []
        self.started_at = datetime.now()
        self.chain = None

    def retrieve_chains_by_publisher(self, publisher_iati_id):
        for activity in Activity.objects.filter(publisher__publisher_iati_id=publisher_iati_id, hierarchy=1):
            if self.chain_update_needed(activity):
                self.retrieve_chain(activity)

    def retrieve_chain_by_activity_id(self, activity_id):
        activity = Activity.objects.get(iati_identifier=activity_id)
        self.retrieve_chain(activity)

    def retrieve_chain_for_all_activities(self):
        #get all activities that other activities set as provider of funds
        for activity in Activity.objects.annotate(related_set_as_provider=Count('relatedactivity__ref_activity__transaction_provider_activity')).annotate(set_as_provider=Count('transaction_provider_activity')).filter(Q(related_set_as_provider__gt=0) | Q(set_as_provider__gt=0)):
            # filter all activities that have  (this was too slow to put in the query above) 
            if Activity.objects.filter(transaction_provider_activity__transaction__activity=activity).count() > 0:
                print 'skipped {}'.format(activity.iati_identifier)
                continue

            if self.chain_update_needed(activity):
                self.retrieve_chain(activity)
    
    def chain_update_needed(self, activity):
        start_nodes = ChainNode.objects.filter(activity=activity, bol=True, treated_as_end_node=False, tier=0)

        if len(start_nodes) > 0 and start_nodes[0].chain.last_updated < self.started_at:
            # in a chain, only update if the chain is not created within this run
            return True 

        elif len(start_nodes) is 0:
            # not in a chain yet, create the chain
            return True               
        return False

    def retrieve_chain(self, activity):

        # delete old chain
        if ChainNode.objects.filter(activity=activity, bol=True, treated_as_end_node=False, tier=0).exists():
            node = ChainNode.objects.filter(activity=activity, bol=True, treated_as_end_node=False, tier=0)

            if len(node) > 1:
                print 'Multiple mentions of this chain, should not happen'
                print node
            node[0].chain.delete()
{'fraction_non_alphanumeric': 0.07728789741788161, 'fraction_numerical': 0.004040049183207448, 'mean_word_length': 3.0354358610914245, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4651684', 'n_tokens_mistral': 1576, 'n_tokens_neox': 1512, 'n_words': 390}