<commit_before>import json, requests
from collections import OrderedDict

from django.db import models
from django.core.cache import cache
from django.conf import settings
from django.utils.text import slugify

from wagtail.wagtailcore.models import Page, Orderable
from wagtail.wagtailcore.fields import RichTextField
from wagtail.wagtailadmin.edit_handlers import FieldPanel, InlinePanel
from wagtail.wagtailimages.models import Image
from wagtail.wagtailimages.edit_handlers import ImageChooserPanel
from wagtail.wagtailsearch import index

from modelcluster.fields import ParentalKey


class ProjectPage(Orderable, Page):
    STATUSES = (
        ('discovery', 'Discovery'),
        ('alpha', 'Alpha'),
        ('beta', 'Beta'),
        ('live', 'LIVE')
    )
    short_description = models.TextField()
    full_description = RichTextField(blank=True)
    image = models.ForeignKey(
        'wagtailimages.Image',
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name='+',
    )
    status = models.CharField(max_length=20, choices=STATUSES, default='discovery')
    piwik_id = models.IntegerField(blank=True, null=True)
    uptimerobot_name = models.CharField(blank=True, null=True, max_length=100)

    def save(self, *args, **kwargs):
        if not self.title:
            if self.project:
                self.title = self.project.name
        return super().save(*args, **kwargs)

    search_fields = Page.search_fields + (
        index.SearchField('short_description'),
        index.SearchField('full_description'),
    )

    content_panels = Page.content_panels + [
        FieldPanel('status'),
        FieldPanel('short_description'),
        FieldPanel('full_description'),
        ImageChooserPanel('image'),
        FieldPanel('piwik_id'),
        FieldPanel('uptimerobot_name'),
        InlinePanel('kpis', label="Key performance indicators"),
        InlinePanel('roles', label="Roles"),
        InlinePanel('links', label="Links"),
    ]

    def piwik_data(self):
        data = cache.get('piwik_' + slugify(self.title))
        if not data and self.piwik_id:
            response = requests.get(
                'https://analytics.hel.ninja/piwik/?idSite=' +
                str(self.piwik_id) +
                '&module=API&period=day&date=last30&method=VisitsSummary.get&format=json&token_auth=' +
                settings.PIWIK_API_TOKEN)
            if response.status_code == 200:
                # piwik_id was valid
                data = response.text
                # data = json.loads(response.text, object_pairs_hook=OrderedDict)
                cache.add('piwik_' + slugify(self.title), data, 3600)
        return data

    def uptime_data(self):
        complete_data = cache.get('uptime')
        if not complete_data and self.uptimerobot_name:
            response = requests.get(
                'https://api.uptimerobot.com/getMonitors?apiKey=' +
                settings.UPTIMEROBOT_API_TOKEN +
                '&format=json&noJsonCallback=1')
            if response.status_code == 200:
                # returns all monitors, we still have to check for matching name
                complete_data = response.json()
                cache.add('uptime', complete_data, 3600)
        try:
            data = [single_data for single_data in
                    complete_data['monitors']['monitor'] if
                    single_data['friendlyname'] == self.uptimerobot_name][0]
            # coffeescript wants the data in json
            data = json.dumps(data)
        # if name doesn't match, return None
        except (IndexError, TypeError):
            data = None
        return data

class ProjectRoleType(Orderable, models.Model):
    name = models.CharField(max_length=50)<commit_msg>Catch KeyError when parsing Uptime Robot data<commit_after>import json, requests
from collections import OrderedDict

from django.db import models
from django.core.cache import cache
from django.conf import settings
from django.utils.text import slugify

from wagtail.wagtailcore.models import Page, Orderable
from wagtail.wagtailcore.fields import RichTextField
from wagtail.wagtailadmin.edit_handlers import FieldPanel, InlinePanel
from wagtail.wagtailimages.models import Image
from wagtail.wagtailimages.edit_handlers import ImageChooserPanel
from wagtail.wagtailsearch import index

from modelcluster.fields import ParentalKey


class ProjectPage(Orderable, Page):
    STATUSES = (
        ('discovery', 'Discovery'),
        ('alpha', 'Alpha'),
        ('beta', 'Beta'),
        ('live', 'LIVE')
    )
    short_description = models.TextField()
    full_description = RichTextField(blank=True)
    image = models.ForeignKey(
        'wagtailimages.Image',
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name='+',
    )
    status = models.CharField(max_length=20, choices=STATUSES, default='discovery')
    piwik_id = models.IntegerField(blank=True, null=True)
    uptimerobot_name = models.CharField(blank=True, null=True, max_length=100)

    def save(self, *args, **kwargs):
        if not self.title:
            if self.project:
                self.title = self.project.name
        return super().save(*args, **kwargs)

    search_fields = Page.search_fields + (
        index.SearchField('short_description'),
        index.SearchField('full_description'),
    )

    content_panels = Page.content_panels + [
        FieldPanel('status'),
        FieldPanel('short_description'),
        FieldPanel('full_description'),
        ImageChooserPanel('image'),
        FieldPanel('piwik_id'),
        FieldPanel('uptimerobot_name'),
        InlinePanel('kpis', label="Key performance indicators"),
        InlinePanel('roles', label="Roles"),
        InlinePanel('links', label="Links"),
    ]

    def piwik_data(self):
        data = cache.get('piwik_' + slugify(self.title))
        if not data and self.piwik_id:
            response = requests.get(
                'https://analytics.hel.ninja/piwik/?idSite=' +
                str(self.piwik_id) +
                '&module=API&period=day&date=last30&method=VisitsSummary.get&format=json&token_auth=' +
                settings.PIWIK_API_TOKEN)
            if response.status_code == 200:
                # piwik_id was valid
                data = response.text
                # data = json.loads(response.text, object_pairs_hook=OrderedDict)
                cache.add('piwik_' + slugify(self.title), data, 3600)
        return data

    def uptime_data(self):
        complete_data = cache.get('uptime')
        if not complete_data and self.uptimerobot_name:
            response = requests.get(
                'https://api.uptimerobot.com/getMonitors?apiKey=' +
                settings.UPTIMEROBOT_API_TOKEN +
                '&format=json&noJsonCallback=1')
            if response.status_code == 200:
                # returns all monitors, we still have to check for matching name
                complete_data = response.json()
                cache.add('uptime', complete_data, 3600)
        try:
            data = [single_data for single_data in
                    complete_data['monitors']['monitor'] if
                    single_data['friendlyname'] == self.uptimerobot_name][0]
            # coffeescript wants the data in json
            data = json.dumps(data)
        # if name doesn't match, return None
        except (IndexError, TypeError, KeyError):
            data = None
        return data


class ProjectRoleType(Orderable, models.Model):
    name = models.CharField(max_length=50)
{'fraction_non_alphanumeric': 0.10382945124358468, 'fraction_numerical': 0.006579813133307014, 'mean_word_length': 2.596781826786559, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5351619', 'n_tokens_mistral': 2251, 'n_tokens_neox': 2145, 'n_words': 452}