<commit_before>from django.core.urlresolvers import reverse
from django.db import models


def parse_as_dict(parse):
    return {
        label: parse[i]
        for i, label in enumerate([
            "person", "tense", "voice", "mood", "case", "number", "gender", "degree"
        ])
        if parse[i] != "-"
    }


class Word(models.Model):

    word_id = models.CharField(max_length=11)
    verse_id = models.CharField(max_length=6)
    paragraph_id = models.CharField(max_length=5)
    sentence_id = models.CharField(max_length=6)
    pos = models.CharField(max_length=2)
    parse = models.CharField(max_length=8)

    crit_text = models.CharField(max_length=50)
    text = models.CharField(max_length=50)
    word = models.CharField(max_length=50)
    norm = models.CharField(max_length=50)
    lemma = models.CharField(max_length=50)

    dep_type = models.CharField(max_length=4)
    head = models.CharField(max_length=11, null=True)

    def to_dict(self):
        d = {
            "@id": reverse("word", args=[self.word_id]),
            "@type": "word",
            "verse_id": reverse("verse", args=[self.verse_id]),
            "paragraph_id": reverse("paragraph", args=[self.paragraph_id]),
            "sentence_id": reverse("sentence", args=[self.sentence_id]),
            "pos": self.pos.strip("-"),
            "crit_text": self.crit_text,
            "text": self.text,
            "word": self.word,
            "norm": self.norm,
            "lemma": self.lemma,
            "dep_type": self.dep_type,
            "head": reverse("word", args=[self.head]) if self.head else None,
        }
        d.update(parse_as_dict(self.parse))
        return d


class Paragraph(models.Model):

    paragraph_id = models.CharField(max_length=5)
    book_osis_id = models.CharField(max_length=6)
    prev_paragraph = models.CharField(max_length=5, null=True)
    next_paragraph = models.CharField(max_length=5, null=True)

    def words(self):
        return Word.objects.filter(paragraph_id=self.paragraph_id).order_by("word_id")

    def to_dict(self):
        return {
            "@id": reverse("paragraph", args=[self.paragraph_id]),
            "@type": "paragraph",
            "prev": reverse("paragraph", args=[self.prev_paragraph]) if self.prev_paragraph else None,
            "next": reverse("paragraph", args=[self.next_paragraph]) if self.next_paragraph else None,
            "book": reverse("book", args=[self.book_osis_id]),
            "words": [w.to_dict() for w in self.words()],
        }


class Sentence(models.Model):

    sentence_id = models.CharField(max_length=6)
    book_osis_id = models.CharField(max_length=6)
    prev_sentence = models.CharField(max_length=6, null=True)
    next_sentence = models.CharField(max_length=6, null=True)

    def words(self):
        return Word.objects.filter(sentence_id=self.sentence_id).order_by("word_id")

    def to_dict(self):
        return {
            "@id": reverse("sentence", args=[self.sentence_id]),
            "@type": "sentence",
            "prev": reverse("sentence", args=[self.prev_sentence]) if self.prev_sentence else None,
            "next": reverse("sentence", args=[self.next_sentence]) if self.next_sentence else None,
            "book": reverse("book", args=[self.book_osis_id]),
            "words": [w.to_dict() for w in self.words()],
        }


class Verse(models.Model):

    verse_id = models.CharField(max_length=6)
    book_osis_id = models.CharField(max_length=6)
    prev_verse = models.CharField(max_length=6, null=True)
    next_verse = models.CharField(max_length=6, null=True)

    def words(self):
        return Word.objects.filter(verse_id=self.verse_id).order_by("word_id")

    def to_dict(self):
        return {
            "@id": reverse("verse", args=[self.verse_id]),
            "@type": "verse",
            "book": reverse("book", args=[self.book_osis_id]),
            "prev": reverse("verse", args=[self.prev_verse]) if self.prev_verse else None,
            "next": reverse("verse", args=[self.next_verse]) if self.next_verse else None,
            "words": [w.to_dict() for w in self.words()],
        }


class Book(models.Model):

    book_osis_id = models.CharField(max_length=6)
    name = models.CharField(max_length=20)
    sblgnt_id = models.CharField(max_length=2)

    def first_verse(self):
        return Verse.objects.filter(book_osis_id=self.book_osis_id).order_by("verse_id")[0]

    def first_sentence(self):
        return Sentence.objects.filter(book_osis_id=self.book_osis_id).order_by("sentence_id")[0]

    def first_paragraph(self):
        return Paragraph.objects.filter(book_osis_id=self.book_osis_id).order_by("paragraph_id")[0]

    def to_dict(self):
        return {
            "@id": reverse("book", args=[self.book_osis_id]),
            "@type": "book",
            "name": self.name,
            "root": reverse("root"),
            "first_verse": reverse("verse", args=[self.first_verse().verse_id]),
            "first_sentence": reverse("sentence", args=[self.first_sentence().sentence_id]),
            "first_paragraph": reverse("paragraph", args=[self.first_paragraph().paragraph_id]),
        }<commit_msg>cleaned up code a little more<commit_after>from django.core.urlresolvers import reverse
from django.db import models


def parse_as_dict(parse):
    return {
        label: parse[i]
        for i, label in enumerate([
            "person", "tense", "voice", "mood", "case", "number", "gender", "degree"
        ])
        if parse[i] != "-"
    }


class Word(models.Model):

    word_id = models.CharField(max_length=11)
    verse_id = models.CharField(max_length=6)
    paragraph_id = models.CharField(max_length=5)
    sentence_id = models.CharField(max_length=6)
    pos = models.CharField(max_length=2)
    parse = models.CharField(max_length=8)

    crit_text = models.CharField(max_length=50)
    text = models.CharField(max_length=50)
    word = models.CharField(max_length=50)
    norm = models.CharField(max_length=50)
    lemma = models.CharField(max_length=50)

    dep_type = models.CharField(max_length=4)
    head = models.CharField(max_length=11, null=True)

    class Meta:
        ordering = ["word_id"]

    @staticmethod
    def get_full_id(word_id):
        return reverse("word", args=[word_id]) if word_id else None

    def to_dict(self):
        d = {
            "@id": Word.get_full_id(self.word_id),
            "@type": "word",
            "verse_id": Verse.get_full_id(self.verse_id),
            "paragraph_id": Paragraph.get_full_id(self.paragraph_id),
            "sentence_id": Sentence.get_full_id(self.sentence_id),
            "pos": self.pos.strip("-"),
            "crit_text": self.crit_text,
            "text": self.text,
            "word": self.word,
            "norm": self.norm,
            "lemma": self.lemma,
            "dep_type": self.dep_type,
            "head": Word.get_full_id(self.head),
        }
        d.update(parse_as_dict(self.parse))
        return d


class Paragraph(models.Model):

    paragraph_id = models.CharField(max_length=5)
    book_osis_id = models.CharField(max_length=6)
    prev_paragraph = models.CharField(max_length=5, null=True)
    next_paragraph = models.CharField(max_length=5, null=True)

    class Meta:
        ordering = ["paragraph_id"]

    @staticmethod
    def get_full_id(paragraph_id):
        return reverse("paragraph", args=[paragraph_id]) if paragraph_id else None

    def words(self):
        return Word.objects.filter(paragraph_id=self.paragraph_id)

    def to_dict(self):
        return {
            "@id": Paragraph.get_full_id(self.paragraph_id),
            "@type": "paragraph",
            "prev": Paragraph.get_full_id(self.prev_paragraph),
            "next": Paragraph.get_full_id(self.next_paragraph),
            "book": Book.get_full_id(self.book_osis_id),
            "words": [w.to_dict() for w in self.words()],
        }


class Sentence(models.Model):

    sentence_id = models.CharField(max_length=6)
    book_osis_id = models.CharField(max_length=6)
    prev_sentence = models.CharField(max_length=6, null=True)
    next_sentence = models.CharField(max_length=6, null=True)

    class Meta:
        ordering = ["sentence_id"]

    @staticmethod
    def get_full_id(sentence_id):
        return reverse("sentence", args=[sentence_id]) if sentence_id else None

    def words(self):
        return Word.objects.filter(sentence_id=self.sentence_id)

    def to_dict(self):
        return {
            "@id": Sentence.get_full_id(self.sentence_id),
            "@type": "sentence",
            "prev": Sentence.get_full_id(self.prev_sentence),
            "next": Sentence.get_full_id(self.next_sentence),
            "book": Book.get_full_id(self.book_osis_id),
            "words": [w.to_dict() for w in self.words()],
        }


class Verse(models.Model):

    verse_id = models.CharField(max_length=6)
    book_osis_id = models.CharField(max_length=6)
    prev_verse = models.CharField(max_length=6, null=True)
    next_verse = models.CharField(max_length=6, null=True)

    class Meta:
        ordering = ["verse_id"]

    @staticmethod
    def get_full_id(verse_id):
        return reverse("verse", args=[verse_id]) if verse_id else None

    def words(self):
        return Word.objects.filter(verse_id=self.verse_id)

    def to_dict(self):
        return {
            "@id": reverse("verse", args=[self.verse_id]),
            "@type": "verse",
            "book": reverse("book", args=[self.book_osis_id]),
            "prev": Verse.get_full_id(self.prev_verse),
            "next": Verse.get_full_id(self.next_verse),
            "words": [w.to_dict() for w in self.words()],
        }


class Book(models.Model):

    book_osis_id = models.CharField(max_length=6)
    name = models.CharField(max_length=20)
    sblgnt_id = models.CharField(max_length=2)

    class Meta:
        ordering = ["sblgnt_id"]

    @staticmethod
    def get_full_id(book_osis_id):
        return reverse("book", args=[book_osis_id]) if book_osis_id else None

    def first_verse(self):
        return Verse.objects.filter(book_osis_id=self.book_osis_id)[0]

    def first_sentence(self):
        return Sentence.objects.filter(book_osis_id=self.book_osis_id)[0]

    def first_paragraph(self):
        return Paragraph.objects.filter(book_osis_id=self.book_osis_id)[0]

    def to_dict(self):
        return {
            "@id": Book.get_full_id(self.book_osis_id),
            "@type": "book",
            "name": self.name,
            "root": reverse("root"),
            "first_verse": Verse.get_full_id(self.first_verse().verse_id),
            "first_sentence": Sentence.get_full_id(self.first_sentence().sentence_id),
            "first_paragraph": Paragraph.get_full_id(self.first_paragraph().paragraph_id),
        }
{'fraction_non_alphanumeric': 0.13719878127596713, 'fraction_numerical': 0.00720155110331456, 'mean_word_length': 2.964860907759883, 'pattern_counts': {'":': 76, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4855405', 'n_tokens_mistral': 3586, 'n_tokens_neox': 3479, 'n_words': 588}