<commit_before>import datetime
from mongoengine import *
from mongoengine.fields import (
    BooleanField,
    DateTimeField,
    DictField,
    EmbeddedDocumentField,
    IntField,
    ListField,
    StringField,
)
# Below import is necessary to instantiate connection object
import settings


class Office(EmbeddedDocument):
    state = StringField()
    name = StringField()
    district = StringField()


class Contest(DynamicDocument):
    created = DateTimeField()
    updated = DateTimeField()
    source = StringField(required=True, help_text="Name of data source for this file, preferably standardized filename from datasource.py")
    election_id = StringField(required=True, help_text="election id, e.g. md-2012-11-06-general")
    slug = StringField(required=True, help_text="Slugified office name, plus district and party if relevant")
    state = StringField(required=True)
    start_date = DateTimeField(required=True)
    end_date = DateTimeField(required=True)
    election_type = StringField(help_text="general, primary, etc. from OpenElex metadata")
    result_type = StringField(required=True)
    special = BooleanField(default=False)
    raw_office = StringField(required=True)
    raw_district = StringField()
    raw_party = StringField(help_text="This should only be assigned for closed primaries, where voters must be registered in party to vote in the contest")

    # FIELDS FOR TRANSFORMED/LINKED DATA
    office = EmbeddedDocumentField(Office)
    district = StringField()
    party = StringField(help_text="This should only be assigned for closed primaries, where voters must be registered in party to vote in the contest")

    def __unicode__(self):
        return u'%s-%s' % (self.election_id, self.slug)


class Candidate(DynamicDocument):
    """
    State is included because in nearly all cases, a candidate is unique to a state (presidential candidates run in multiple states,
    but hail from a single state). This would help with lookups and prevent duplicates. Identifiers is a DictField because a candidate
    may have 0 or more identifiers, including state-level IDs.

    parties = ['Democratic', 'Republican'] 

    identifiers = {
        'bioguide' : <bioguide_id>,
        'fec' : [<fecid_1>, <fecid_2>, ...],
        'votesmart' : <votesmart_id>,
        ...
    }

    """
    source = StringField(required=True, help_text="Name of data source for this file, preferably standardized filename from datasource.py")
    election_id = StringField(required=True, help_text="election id, e.g. md-2012-11-06-general")
    contest = ReferenceField(Contest, reverse_delete_rule=CASCADE, required=True)
    contest_slug = StringField(required=True, help_text="Denormalized contest slug for easier querying and obj repr")
    # Keep state denormalized to ease querying?
    state = StringField(max_length=2, required=True, help_text="Capitalized postal code")
    #TODO: Add validation to require raw_full_name or raw_family_name
    raw_full_name = StringField(max_length=300)
    slug = StringField(max_length=300, required=True, help_text="Slugified name for easier querying and obj repr")
    raw_given_name = StringField(max_length=200)
    raw_family_name = StringField(max_length=200)
    raw_suffix = StringField(max_length=200)
    raw_parties = ListField(StringField(), default=list)

    # FIELDS FOR TRANSFORMED/CLEANED DATA or LINKS TO OTHER DATA SETS
    #name = StringField(max_length=300, required=True)
    given_name = StringField(max_length=200)
    family_name = StringField(max_length=200)
    suffix = StringField(max_length=200)
    additional_name = StringField(max_length=200)
    other_names = ListField(StringField(), default=list)
    parties = ListField(StringField(), default=list) # normalized? abbreviations?
    identifiers = DictField()

    def __unicode__(self):
        name =  u'%s - %s' % (self.contest_slug, self.name)
        parties = ""
        if self.raw_parties:
            parties = ", ".join([party for party in self.raw_parties])
            if parties:
                name += " (%s)" % paries
        return name

    @property<commit_msg>added raw_additional_name to models<commit_after>import datetime
from mongoengine import *
from mongoengine.fields import (
    BooleanField,
    DateTimeField,
    DictField,
    EmbeddedDocumentField,
    IntField,
    ListField,
    StringField,
)
# Below import is necessary to instantiate connection object
import settings


class Office(EmbeddedDocument):
    state = StringField()
    name = StringField()
    district = StringField()


class Contest(DynamicDocument):
    created = DateTimeField()
    updated = DateTimeField()
    source = StringField(required=True, help_text="Name of data source for this file, preferably standardized filename from datasource.py")
    election_id = StringField(required=True, help_text="election id, e.g. md-2012-11-06-general")
    slug = StringField(required=True, help_text="Slugified office name, plus district and party if relevant")
    state = StringField(required=True)
    start_date = DateTimeField(required=True)
    end_date = DateTimeField(required=True)
    election_type = StringField(help_text="general, primary, etc. from OpenElex metadata")
    result_type = StringField(required=True)
    special = BooleanField(default=False)
    raw_office = StringField(required=True)
    raw_district = StringField()
    raw_party = StringField(help_text="This should only be assigned for closed primaries, where voters must be registered in party to vote in the contest")

    # FIELDS FOR TRANSFORMED/LINKED DATA
    office = EmbeddedDocumentField(Office)
    district = StringField()
    party = StringField(help_text="This should only be assigned for closed primaries, where voters must be registered in party to vote in the contest")

    def __unicode__(self):
        return u'%s-%s' % (self.election_id, self.slug)


class Candidate(DynamicDocument):
    """
    State is included because in nearly all cases, a candidate is unique to a state (presidential candidates run in multiple states,
    but hail from a single state). This would help with lookups and prevent duplicates. Identifiers is a DictField because a candidate
    may have 0 or more identifiers, including state-level IDs.

    parties = ['Democratic', 'Republican'] 

    identifiers = {
        'bioguide' : <bioguide_id>,
        'fec' : [<fecid_1>, <fecid_2>, ...],
        'votesmart' : <votesmart_id>,
        ...
    }

    """
    source = StringField(required=True, help_text="Name of data source for this file, preferably standardized filename from datasource.py")
    election_id = StringField(required=True, help_text="election id, e.g. md-2012-11-06-general")
    contest = ReferenceField(Contest, reverse_delete_rule=CASCADE, required=True)
    contest_slug = StringField(required=True, help_text="Denormalized contest slug for easier querying and obj repr")
    # Keep state denormalized to ease querying?
    state = StringField(max_length=2, required=True, help_text="Capitalized postal code")
    #TODO: Add validation to require raw_full_name or raw_family_name
    raw_full_name = StringField(max_length=300)
    slug = StringField(max_length=300, required=True, help_text="Slugified name for easier querying and obj repr")
    raw_given_name = StringField(max_length=200)
    raw_family_name = StringField(max_length=200)
    raw_suffix = StringField(max_length=200)
    raw_additional_name = StringField(max_length=200, help_text="For middle names, nicknames, etc")
    raw_parties = ListField(StringField(), default=list)

    # FIELDS FOR TRANSFORMED/CLEANED DATA or LINKS TO OTHER DATA SETS
    #name = StringField(max_length=300, required=True)
    given_name = StringField(max_length=200)
    family_name = StringField(max_length=200)
    suffix = StringField(max_length=200)
    additional_name = StringField(max_length=200, help_text="For middle names, nicknames, etc")
    other_names = ListField(StringField(), default=list)
    parties = ListField(StringField(), default=list) # normalized? abbreviations?
    identifiers = DictField()

    def __unicode__(self):
        name =  u'%s - %s' % (self.contest_slug, self.name)
        parties = ""
        if self.raw_parties:
            parties = ", ".join([party for party in self.raw_parties])
            if parties:
                name += " (%s)" % paries
        return name

    @property
{'fraction_non_alphanumeric': 0.0903271692745377, 'fraction_numerical': 0.012209577999051683, 'mean_word_length': 4.079470198675497, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7442646', 'n_tokens_mistral': 2547, 'n_tokens_neox': 2404, 'n_words': 700}