<commit_before>from django.db import models
import datetime
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _
from IATI_synchroniser.dataset_syncer import DatasetSyncer
from IATI_synchroniser.codelist_importer import CodeListImporter
from IATI.parser import Parser



INTERVAL_CHOICES = (
    (u'YEARLY', _(u"Parse yearly")),
    (u'MONTHLY', _(u"Parse monthly")),
    (u'WEEKLY', _(u"Parse weekly")),
    (u'DAILY', _(u"Parse daily")),
)
class Publisher(models.Model):
    org_id = models.CharField(max_length=100, blank=True, null=True)
    org_abbreviate = models.CharField(max_length=55, blank=True, null=True)
    org_name = models.CharField(max_length=255)
    default_interval = models.CharField(verbose_name=_(u"Interval"), max_length=55, choices=INTERVAL_CHOICES, default=u'MONTHLY')

    def __unicode__(self):
        if self.org_abbreviate:
            return self.org_abbreviate
        return self.org_name

    class Meta:
        ordering = ["org_name"]


class iati_xml_source(models.Model):
    TYPE_CHOICES = (
        (1, _(u"Activity Files")),
        (2, _(u"Organisation Files")),
    )
    INTERVAL_CHOICES = (
        ("day", _(u"Day")),
        ("week", _(u"Week")),
        ("month", _(u"Month")),
        ("year", _(u"Year")),
    )
    ref = models.CharField(verbose_name=_(u"Reference"), max_length=70, help_text=_(u"Reference for the XML file. Preferred usage: 'collection' or single country or region name"))
    title = models.CharField(max_length=255, null=True)
    type = models.IntegerField(choices=TYPE_CHOICES, default=1)
    publisher = models.ForeignKey(Publisher)
    source_url = models.CharField(max_length=255, unique=True, help_text=_(u"Hyperlink to an IATI activity or organisation XML file."))
    date_created = models.DateTimeField(auto_now_add=True, editable=False)
    date_updated = models.DateTimeField(auto_now=True, editable=False)
    update_interval = models.CharField(max_length=20, choices=INTERVAL_CHOICES, default="month", null=True, blank=True)
    last_found_in_registry = models.DateTimeField(default=None, null=True)

    class Meta:
        verbose_name_plural = "IATI XML sources"
        ordering = ["ref"]

    def __unicode__(self):
        return self.ref

    def get_parse_status(self):
        return mark_safe("<img class='loading' src='/static/img/loading.gif' alt='loading' style='display:none;' /><a data-xml='xml_%i' class='parse'><img src='/static/img/utils.parse.png' style='cursor:pointer;' /></a>") % self.id
    get_parse_status.allow_tags = True
    get_parse_status.short_description = _(u"Parse status")

    def process(self):
        parser = Parser()
        parser.parse_url(self.source_url, self.ref)
        self.date_updated = datetime.datetime.now()

    def save(self, process=True, *args, **kwargs):
        super(iati_xml_source, self).save()
        if process:
            self.process()
            self.date_updated = datetime.datetime.now()



class dataset_sync(models.Model):
    TYPE_CHOICES = (
        (1, _(u"Activity Files")),<commit_msg>updated last_updated field on xml source<commit_after>from django.db import models
import datetime
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _
from IATI_synchroniser.dataset_syncer import DatasetSyncer
from IATI_synchroniser.codelist_importer import CodeListImporter
from IATI.parser import Parser



INTERVAL_CHOICES = (
    (u'YEARLY', _(u"Parse yearly")),
    (u'MONTHLY', _(u"Parse monthly")),
    (u'WEEKLY', _(u"Parse weekly")),
    (u'DAILY', _(u"Parse daily")),
)
class Publisher(models.Model):
    org_id = models.CharField(max_length=100, blank=True, null=True)
    org_abbreviate = models.CharField(max_length=55, blank=True, null=True)
    org_name = models.CharField(max_length=255)
    default_interval = models.CharField(verbose_name=_(u"Interval"), max_length=55, choices=INTERVAL_CHOICES, default=u'MONTHLY')

    def __unicode__(self):
        if self.org_abbreviate:
            return self.org_abbreviate
        return self.org_name

    class Meta:
        ordering = ["org_name"]


class iati_xml_source(models.Model):
    TYPE_CHOICES = (
        (1, _(u"Activity Files")),
        (2, _(u"Organisation Files")),
    )
    INTERVAL_CHOICES = (
        ("day", _(u"Day")),
        ("week", _(u"Week")),
        ("month", _(u"Month")),
        ("year", _(u"Year")),
    )
    ref = models.CharField(verbose_name=_(u"Reference"), max_length=70, help_text=_(u"Reference for the XML file. Preferred usage: 'collection' or single country or region name"))
    title = models.CharField(max_length=255, null=True)
    type = models.IntegerField(choices=TYPE_CHOICES, default=1)
    publisher = models.ForeignKey(Publisher)
    source_url = models.CharField(max_length=255, unique=True, help_text=_(u"Hyperlink to an IATI activity or organisation XML file."))
    date_created = models.DateTimeField(auto_now_add=True, editable=False)
    date_updated = models.DateTimeField(auto_now_add=True, editable=False)
    update_interval = models.CharField(max_length=20, choices=INTERVAL_CHOICES, default="month", null=True, blank=True)
    last_found_in_registry = models.DateTimeField(default=None, null=True)

    class Meta:
        verbose_name_plural = "IATI XML sources"
        ordering = ["ref"]

    def __unicode__(self):
        return self.ref

    def get_parse_status(self):
        return mark_safe("<img class='loading' src='/static/img/loading.gif' alt='loading' style='display:none;' /><a data-xml='xml_%i' class='parse'><img src='/static/img/utils.parse.png' style='cursor:pointer;' /></a>") % self.id
    get_parse_status.allow_tags = True
    get_parse_status.short_description = _(u"Parse status")

    def process(self):
        parser = Parser()
        parser.parse_url(self.source_url, self.ref)


    def save(self, process=True, *args, **kwargs):
        super(iati_xml_source, self).save()
        if process:
            self.process()
            self.date_updated = datetime.datetime.now()



class dataset_sync(models.Model):
    TYPE_CHOICES = (
        (1, _(u"Activity Files")),
{'fraction_non_alphanumeric': 0.1297314784859269, 'fraction_numerical': 0.007764477515367195, 'mean_word_length': 4.063882063882064, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 9}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4417644', 'n_tokens_mistral': 2077, 'n_tokens_neox': 1985, 'n_words': 396}