<commit_before>import json
from django.db import models


SEQUENCE_FORMATS = (
    ('text', 'text'),
    ('json', 'json'),
)


class DjangoPillowCheckpoint(models.Model):

    checkpoint_id = models.CharField(primary_key=True, max_length=100)
    sequence = models.TextField()
    timestamp = models.DateTimeField(auto_now=True)
    old_sequence = models.TextField(null=True, blank=True)
    sequence_format = models.CharField(max_length=20, choices=SEQUENCE_FORMATS, default='text')

    @property
    def wrapped_sequence(self):
        if self.sequence_format == 'json':
            seq = json.loads(self.sequence)
            # deconstruct tuple keys
            marshaled_seq = {}
            for key, val in seq.items():
                topic, partition = key.split(',')
                marshaled_seq[(topic, int(partition))] = val
            return marshaled_seq
        else:
            return self.sequence

    class Meta:
        app_label = "pillowtop"

    @staticmethod<commit_msg>use named tuple for convenience<commit_after>import json
from django.db import models
from kafka.common import TopicAndPartition

SEQUENCE_FORMATS = (
    ('text', 'text'),
    ('json', 'json'),
)


class DjangoPillowCheckpoint(models.Model):

    checkpoint_id = models.CharField(primary_key=True, max_length=100)
    sequence = models.TextField()
    timestamp = models.DateTimeField(auto_now=True)
    old_sequence = models.TextField(null=True, blank=True)
    sequence_format = models.CharField(max_length=20, choices=SEQUENCE_FORMATS, default='text')

    @property
    def wrapped_sequence(self):
        if self.sequence_format == 'json':
            seq = json.loads(self.sequence)
            # deconstruct tuple keys
            marshaled_seq = {}
            for key, val in seq.items():
                topic, partition = key.split(',')
                marshaled_seq[TopicAndPartition(topic, int(partition))] = val
            return marshaled_seq
        else:
            return self.sequence

    class Meta:
        app_label = "pillowtop"

    @staticmethod
{'fraction_non_alphanumeric': 0.10478029937228392, 'fraction_numerical': 0.004828585224529213, 'mean_word_length': 2.8018348623853213, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4656013', 'n_tokens_mistral': 613, 'n_tokens_neox': 589, 'n_words': 127}