<commit_before>from __future__ import unicode_literals

from django.db import models

class Keyphrase(models.Model):
    id = models.AutoField(primary_key=True)
    method = models.ForeignKey('KeyphraseMethod')
    term = models.TextField()
    val = models.DecimalField(max_digits=20, decimal_places=10)
    class Meta:
        db_table = 'keyphrase'

class KeyphraseMethod(models.Model):
    id = models.AutoField(primary_key=True)
    name = models.CharField(max_length=50L)
    description = models.TextField()
    class Meta:
        db_table = 'keyphrase_method'

class Post(models.Model):
    id = models.AutoField(primary_key=True)
    fb_post_id = models.CharField(max_length=255L)
    posttype = models.CharField(max_length=25L, db_column='type')
    text = models.TextField()
    createtime = models.DateTimeField()
    likes = models.IntegerField()
    parent = models.ForeignKey('self', null=True, db_column='parent', blank=True)
    page = models.ForeignKey('Page', db_column='page')
    createuser = models.ForeignKey('User', db_column='createuser')
    class Meta:
        db_table = 'post'

class PostKeyphraseAssoc(models.Model):
    post = models.ForeignKey(Post)
    keyphrase = models.ForeignKey(Keyphrase)
    offset = models.IntegerField()
    length = models.IntegerField()
    class Meta:
        db_table = 'post_keyphrase_assoc'
        unique_together = ('post', 'keyphrase', 'offset', 'length')

class User(models.Model):
    id = models.BigIntegerField(primary_key=True)
    fullname = models.CharField(max_length=500L)
    alias = models.CharField(max_length=50L)
    class Meta:
        db_table = 'user'

class Page(models.Model):
    id = models.AutoField(primary_key=True)
    fb_page_id = models.CharField(max_length=100L)
    fb_page_name = models.CharField(max_length=2048L)
    last_updated = models.DateTimeField()
    owner = models.ForeignKey('User', db_column='owner')
    class Meta:
        db_table = 'page'

<commit_msg>Add normalized text column to the keyphrase model<commit_after>from __future__ import unicode_literals

from django.db import models

class Keyphrase(models.Model):
    id = models.AutoField(primary_key=True)
    method = models.ForeignKey('KeyphraseMethod')
    term = models.TextField()
    normalized = models.TextField(blank=True)
    val = models.DecimalField(max_digits=20, decimal_places=10)
    class Meta:
        db_table = 'keyphrase'

class KeyphraseMethod(models.Model):
    id = models.AutoField(primary_key=True)
    name = models.CharField(max_length=50L)
    description = models.TextField()
    class Meta:
        db_table = 'keyphrase_method'

class Post(models.Model):
    id = models.AutoField(primary_key=True)
    fb_post_id = models.CharField(max_length=255L)
    posttype = models.CharField(max_length=25L, db_column='type')
    text = models.TextField()
    createtime = models.DateTimeField()
    likes = models.IntegerField()
    parent = models.ForeignKey('self', null=True, db_column='parent', blank=True)
    page = models.ForeignKey('Page', db_column='page')
    createuser = models.ForeignKey('User', db_column='createuser')
    class Meta:
        db_table = 'post'

class PostKeyphraseAssoc(models.Model):
    post = models.ForeignKey(Post)
    keyphrase = models.ForeignKey(Keyphrase)
    offset = models.IntegerField()
    length = models.IntegerField()
    class Meta:
        db_table = 'post_keyphrase_assoc'
        unique_together = ('post', 'keyphrase', 'offset', 'length')

class User(models.Model):
    id = models.BigIntegerField(primary_key=True)
    fullname = models.CharField(max_length=500L)
    alias = models.CharField(max_length=50L)
    class Meta:
        db_table = 'user'

class Page(models.Model):
    id = models.AutoField(primary_key=True)
    fb_page_id = models.CharField(max_length=100L)
    fb_page_name = models.CharField(max_length=2048L)
    last_updated = models.DateTimeField()
    owner = models.ForeignKey('User', db_column='owner')
    class Meta:
        db_table = 'page'


{'fraction_non_alphanumeric': 0.11429282868525896, 'fraction_numerical': 0.011454183266932271, 'mean_word_length': 4.1368286445012785, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2592839', 'n_tokens_mistral': 1297, 'n_tokens_neox': 1260, 'n_words': 235}