<commit_before>from IATI_2_01 import Parse as IATI_201_Parser
from IATI_1_05 import Parse as IATI_105_Parser
from deleter import Deleter
import gc
from iati.filegrabber import FileGrabber
from lxml import etree
from iati_synchroniser.exception_handler import exception_handler
import hashlib




class ParseIATI():

    def hashfile(self,afile, hasher, blocksize=65536):
        buf = afile.read(blocksize)
        while len(buf) > 0:
            hasher.update(buf)
            buf = afile.read(blocksize)
        return hasher.hexdigest()

    def parse_url(self, source ):
        url = source.source_url
        xml_source_ref = source.ref
        last_hash = source.last_hash
        
        try:
            #iterate through iati-activity tree
            file_grabber = FileGrabber()
            iati_file = file_grabber.get_the_file(url)
            #get the hash
            #hash  = self.hashfile(iati_file,hashlib.md5())
            #if hash == last_hash:
                #return
            if iati_file:

                # delete old activities
                try:
                    deleter = Deleter()
                    deleter.delete_by_source(xml_source_ref)
                except Exception as e:
                    exception_handler(e, "parse url", "delete by source")
                print 'activities deleted'
                # parse the new file
                self.xml_source_ref = xml_source_ref
                data = iati_file.read()
                #print data
                #print 'iati data is'
                #root = etree.fromstring(str(data))
                parser = None
                print root.xpath('@version')
                if root.xpath('@version')[0] == '2.01':
                    parser = IATI_201_Parser()
                else:
                    parser = IATI_105_Parser()
                    parser.VERSION = root.xpath('@version')[0]
                print 'before parsing'
                parser.load_and_parse(root)

                del iati_file
                gc.collect()

                # Throw away query logs when in debug mode to prevent memory from overflowing
                if settings.DEBUG:
                    from django import db
                    db.reset_queries()

        except Exception as e:
            exception_handler(e, "parse url", "parse_url")<commit_msg>removed to much<commit_after>from IATI_2_01 import Parse as IATI_201_Parser
from IATI_1_05 import Parse as IATI_105_Parser
from deleter import Deleter
import gc
from iati.filegrabber import FileGrabber
from lxml import etree
from iati_synchroniser.exception_handler import exception_handler
import hashlib




class ParseIATI():

    def hashfile(self,afile, hasher, blocksize=65536):
        buf = afile.read(blocksize)
        while len(buf) > 0:
            hasher.update(buf)
            buf = afile.read(blocksize)
        return hasher.hexdigest()

    def parse_url(self, source ):
        url = source.source_url
        xml_source_ref = source.ref
        last_hash = source.last_hash
        
        try:
            #iterate through iati-activity tree
            file_grabber = FileGrabber()
            iati_file = file_grabber.get_the_file(url)
            #get the hash
            #hash  = self.hashfile(iati_file,hashlib.md5())
            #if hash == last_hash:
                #return
            if iati_file:

                # delete old activities
                try:
                    deleter = Deleter()
                    deleter.delete_by_source(xml_source_ref)
                except Exception as e:
                    exception_handler(e, "parse url", "delete by source")
                print 'activities deleted'
                # parse the new file
                self.xml_source_ref = xml_source_ref
                data = iati_file.read()
                #print data
                #print 'iati data is'
                root = etree.fromstring(str(data))
                parser = None
                print root.xpath('@version')
                if root.xpath('@version')[0] == '2.01':
                    parser = IATI_201_Parser()
                else:
                    parser = IATI_105_Parser()
                    parser.VERSION = root.xpath('@version')[0]
                print 'before parsing'
                parser.load_and_parse(root)

                del iati_file
                gc.collect()

                # Throw away query logs when in debug mode to prevent memory from overflowing
                if settings.DEBUG:
                    from django import db
                    db.reset_queries()

        except Exception as e:
            exception_handler(e, "parse url", "parse_url")
{'fraction_non_alphanumeric': 0.07083599234205488, 'fraction_numerical': 0.012763241863433313, 'mean_word_length': 1.5568243610657966, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6852819', 'n_tokens_mistral': 1376, 'n_tokens_neox': 1258, 'n_words': 325}