<commit_before>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

from amcat.tools.model import AmcatModel
from amcat.models import Word
from django.db import models
from collections import namedtuple

TripleValues = namedtuple("TripleValues", ["analysis_sentence", "child", "parent", "relation"])
TokenValues = namedtuple("TokenValues", ["analysis_sentence", "position", "word", "lemma", "pos", "major", "minor"])

class Pos(AmcatModel):
    id = models.AutoField(primary_key=True, db_column='pos_id')
    major = models.CharField(max_length=100)
    minor = models.CharField(max_length=500, null=True)
    pos =  models.CharField(max_length=1, null=True)

    class Meta():
        db_table = 'tokens_pos'
        app_label = 'amcat'

class Token(AmcatModel):
    __label__ = 'word'

    id = models.AutoField(primary_key=True, db_column='token_id')

    sentence = models.ForeignKey("amcat.AnalysisSentence", related_name="tokens")
    word = models.ForeignKey(Word)
    position = models.IntegerField()
    pos = models.ForeignKey(Pos, related_name="+")

    class Meta():
        db_table = 'tokens'
        app_label = 'amcat'
        unique_together = ("sentence", "position")
        ordering = ['sentence', 'position']
        
    def __unicode__(self):
        return unicode(self.word)
    
class Relation(AmcatModel):
    id = models.AutoField(db_column='relation_id', primary_key=True)
    label = models.CharField(max_length=100)

    class Meta():
        db_table = 'tokens_triples_relations'
        app_label = 'amcat'

class Triple(AmcatModel):
    id = models.AutoField(primary_key=True, db_column='triple_id')

    parent = models.ForeignKey(Token, related_name="+")
    child = models.ForeignKey(Token, related_name="+")
    relation = models.ForeignKey(Relation)

    class Meta():
        db_table = 'tokens_triples'
        app_label = 'amcat'
        unique_together = ('parent', 'child')


###########################################################################
#                          U N I T   T E S T S                            #
###########################################################################

from amcat.tools import amcattest

class TestTokens(amcattest.PolicyTestCase):
    def test_get_tokens_order(self):<commit_msg>add coreference<commit_after>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

from amcat.tools.model import AmcatModel
from amcat.models import Word
from django.db import models
from collections import namedtuple

TripleValues = namedtuple("TripleValues", ["analysis_sentence", "child", "parent", "relation"])
TokenValues = namedtuple("TokenValues", ["analysis_sentence", "position", "word", "lemma", "pos", "major", "minor", "namedentity"])

class Pos(AmcatModel):
    id = models.AutoField(primary_key=True, db_column='pos_id')
    major = models.CharField(max_length=100)
    minor = models.CharField(max_length=500, null=True)
    pos =  models.CharField(max_length=1, null=True)

    class Meta():
        db_table = 'tokens_pos'
        app_label = 'amcat'

class Token(AmcatModel):
    __label__ = 'word'

    id = models.AutoField(primary_key=True, db_column='token_id')

    sentence = models.ForeignKey("amcat.AnalysisSentence", related_name="tokens")
    word = models.ForeignKey(Word)
    position = models.IntegerField()
    pos = models.ForeignKey(Pos, related_name="+")
    namedentity = models.CharField(max_length=1, null=True, blank=True)

    class Meta():
        db_table = 'tokens'
        app_label = 'amcat'
        unique_together = ("sentence", "position")
        ordering = ['sentence', 'position']
        
    def __unicode__(self):
        return unicode(self.word)

class Relation(AmcatModel):
    id = models.AutoField(db_column='relation_id', primary_key=True)
    label = models.CharField(max_length=100)

    class Meta():
        db_table = 'tokens_triples_relations'
        app_label = 'amcat'

class Triple(AmcatModel):
    id = models.AutoField(primary_key=True, db_column='triple_id')

    parent = models.ForeignKey(Token, related_name="+")
    child = models.ForeignKey(Token, related_name="+")
    relation = models.ForeignKey(Relation)

    class Meta():
        db_table = 'tokens_triples'
        app_label = 'amcat'
        unique_together = ('parent', 'child')


class CoreferenceSet(AmcatModel):
    id = models.AutoField(primary_key=True, db_column='coreference_set_id')
    analysis_article = models.ForeignKey("amcat.AnalysisArticle", related_name='coreferencesets')
    tokens = models.ManyToManyField(Token)

    class Meta():
        db_table = 'coreferencesets'
        app_label = 'amcat'

###########################################################################
#                          U N I T   T E S T S                            #
###########################################################################

from amcat.tools import amcattest

class TestTokens(amcattest.PolicyTestCase):
    def test_get_tokens_order(self):
{'fraction_non_alphanumeric': 0.168310322156476, 'fraction_numerical': 0.0030243261012491782, 'mean_word_length': 2.5180388529139686, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5747646', 'n_tokens_mistral': 1943, 'n_tokens_neox': 1870, 'n_words': 525}