<commit_before>from . import base
from . import mixins

from datetime import date


class TransformedRecord(
        mixins.GenericCompensationMixin,
        mixins.GenericDepartmentMixin, mixins.GenericIdentifierMixin,
        mixins.GenericJobTitleMixin, mixins.GenericPersonMixin,
        mixins.MembershipMixin, mixins.OrganizationMixin, mixins.PostMixin,
        mixins.RaceMixin, mixins.LinkMixin, base.BaseTransformedRecord):

    MAP = {
        'last_name': 'LASTNAME',
        'first_name': 'FIRSTNAME',
        'middle_name': 'MIDDLE',
        'department': 'DEPT',
        'job_title': 'TITLE',
        'hire_date': 'HIREDATE',
        'compensation': 'GROSS',
        'employee_type': 'EMPLSTATUS',
        'gender': 'GENDER',
        'race': 'ETHNICITY',
    }

    NAME_FIELDS = ('first_name', 'middle_name', 'last_name', )

    ORGANIZATION_NAME = 'Midwestern State University'

    ORGANIZATION_CLASSIFICATION = 'University'

    DATE_PROVIDED = date(2016, 3, 24)

    URL = ('http://raw.texastribune.org.s3.amazonaws.com/'
           'midwestern_state_university/salaries/2016-03/'
           'midwestern.xlsx')

    @property
    def is_valid(self):
        # Adjust to return False on invalid fields.  For example:
        return self.last_name.strip() != ''

    @property
    def compensation_type(self):
        emptype = self.get_mapped_value('employee_type')

        if emptype == 'P':
            return 'PT'
        else:
            return 'FT'

    @property
    def description(self):
        emptype = self.get_mapped_value('employee_type')
        title = self.get_mapped_value('job_title')

        if emptype == 'P':
            return 'Part-time 2015 gross annual salary'
        elif title == 'President':
            return 'Budgeted salary'
        else:
            return '2015 gross annual salary'

    @property
    def person(self):
        name = self.get_name()
        r = {
            'family_name': name.last,
            'given_name': name.first,
            'additional_name': name.middle,
            'name': unicode(name),
            'gender': self.gender.strip()
        }

        return r

transform = base.transform_factory(TransformedRecord)
<commit_msg>get rid of retirees<commit_after>from . import base
from . import mixins

from datetime import date


class TransformedRecord(
        mixins.GenericCompensationMixin,
        mixins.GenericDepartmentMixin, mixins.GenericIdentifierMixin,
        mixins.GenericJobTitleMixin, mixins.GenericPersonMixin,
        mixins.MembershipMixin, mixins.OrganizationMixin, mixins.PostMixin,
        mixins.RaceMixin, mixins.LinkMixin, base.BaseTransformedRecord):

    MAP = {
        'last_name': 'LASTNAME',
        'first_name': 'FIRSTNAME',
        'middle_name': 'MIDDLE',
        'department': 'DEPT',
        'job_title': 'TITLE',
        'hire_date': 'HIREDATE',
        'compensation': 'GROSS',
        'employee_type': 'EMPLSTATUS',
        'gender': 'GENDER',
        'race': 'ETHNICITY',
    }

    NAME_FIELDS = ('first_name', 'middle_name', 'last_name', )

    ORGANIZATION_NAME = 'Midwestern State University'

    ORGANIZATION_CLASSIFICATION = 'University'

    DATE_PROVIDED = date(2016, 3, 24)

    URL = ('http://raw.texastribune.org.s3.amazonaws.com/'
           'midwestern_state_university/salaries/2016-03/'
           'midwestern.xlsx')

    # There are some people listed as Other status, because they retired in 2015, which the data represents
    REJECT_ALL_IF_INVALID_RECORD_EXISTS = False

    @property
    def is_valid(self):
        # Adjust to return False on invalid fields.  For example:
        return self.employee_type.strip() != 'O'

    @property
    def compensation_type(self):
        emptype = self.get_mapped_value('employee_type')

        if emptype == 'P':
            return 'PT'
        else:
            return 'FT'

    @property
    def description(self):
        emptype = self.get_mapped_value('employee_type')
        title = self.get_mapped_value('job_title')

        if emptype == 'P':
            return 'Part-time 2015 gross annual salary'
        elif title == 'President':
            return 'Budgeted salary'
        else:
            return '2015 gross annual salary'

    @property
    def person(self):
        name = self.get_name()
        r = {
            'family_name': name.last,
            'given_name': name.first,
            'additional_name': name.middle,
            'name': unicode(name),
            'gender': self.gender.strip()
        }

        return r

transform = base.transform_factory(TransformedRecord)

{'fraction_non_alphanumeric': 0.10966057441253264, 'fraction_numerical': 0.010443864229765013, 'mean_word_length': 2.42548435171386, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1461720', 'n_tokens_mistral': 1486, 'n_tokens_neox': 1370, 'n_words': 297}