<commit_before>from . import base
from . import mixins

from datetime import date


class TransformedRecord(mixins.GenericCompensationMixin,
        mixins.GenericDepartmentMixin, mixins.GenericIdentifierMixin,
        mixins.GenericJobTitleMixin, mixins.GenericPersonMixin,
        mixins.MembershipMixin, mixins.OrganizationMixin, mixins.PostMixin,
        mixins.RaceMixin, mixins.LinkMixin, base.BaseTransformedRecord):

    MAP = {
        'last_name': '<NAME>',
        'first_name': '<NAME>',
        'middle_name': '<NAME>',
        'department': 'Department Name',
        'job_title': 'Title',
        'hire_date': 'Hire Date',
        'compensation': 'Gross Annual Salary',
        'gender': 'Gender',
        'race': 'Race',
    }

    NAME_FIELDS = ('first_name', 'middle_name', 'last_name', )

    ORGANIZATION_NAME = 'Houston'

    ORGANIZATION_CLASSIFICATION = 'City'

    DATE_PROVIDED = date(2015, 10, 20)
    # Y/M/D agency provided the data

    URL = "http://raw.texastribune.org.s3.amazonaws.com/houston/salaries/2015-10/houston.xlsx"

    gender_map = {'Female': 'F', 'Male': 'M'}

    compensation_type = 'FT'
    description = 'Annualized base pay'

    @property
    def is_valid(self):
        # Adjust to return False on invalid fields.  For example:
        return self.last_name.strip() != ''


    @property
    def person(self):
        name = self.get_name()
        r = {
            'family_name': name.last,
            'given_name': name.first,
            'additional_name': name.middle,
            'name': unicode(name),
            'gender': self.gender_map[self.gender.strip()]
        }

        return r

transform = base.transform_factory(TransformedRecord)
<commit_msg>trying to start on changes based on new pay data<commit_after>from . import base
from . import mixins

from datetime import date


class TransformedRecord(mixins.GenericCompensationMixin,
        mixins.GenericDepartmentMixin, mixins.GenericIdentifierMixin,
        mixins.GenericJobTitleMixin, mixins.GenericPersonMixin,
        mixins.MembershipMixin, mixins.OrganizationMixin, mixins.PostMixin,
        mixins.RaceMixin, mixins.LinkMixin, base.BaseTransformedRecord):

    MAP = {
        'last_name': '<NAME>',
        'first_name': '<NAME>',
        'middle_name': '<NAME>',
        'department': 'Department',
        'job_title': 'Title',
        'hire_date': 'Hire Date',
        'employee_type': 'Status',
        'wage_type': 'Base Pay Wage Type',
        'compensation': 'Base Pay Rate',
        'gender': 'Gender',
        'race': 'Race',
    }

    NAME_FIELDS = ('first_name', 'middle_name', 'last_name', )

    ORGANIZATION_NAME = 'Houston'

    ORGANIZATION_CLASSIFICATION = 'City'

    DATE_PROVIDED = date(2017, 5, 3)

    URL = "http://raw.texastribune.org.s3.amazonaws.com/houston/salaries/2017-05/houston.xlsx"

    gender_map = {'Female': 'F', 'Male': 'M'}

    department_map = {
        '1000': 'Houston Police',
        '1100': 'Neighborhood',
        '1200': 'Houston Fire',
        '1500': 'Houston Emergency Center',
        '1600': 'Municipal Court',
        '2000': 'Public Works & Engineering',
        '2100': 'Solid Waste',
        '2500': 'General Service',
        '2800': 'Houston Airport System',
        '3200': 'Housing & Community Development',
        '3400': 'Houston Public Library',
        '3600': 'Parks & Recreation',
        '3800': 'Health & Human Services',
        '4200': 'Convention',
        '5000': 'Mayors Office',
        '5100': 'Office of Business Opportunity',
        '5500': 'City Council',
        '6000': 'Controllers',
        '6400': 'Finance',
        '6500': 'Admin. & Regulatory Affairs',
        '6700': 'Fleet',
        '6800': 'Houston IT Services',
        '7000': 'Planning',
        '7500': 'City Secretary',
        '8000': 'Human Resources',
        '9000': 'Legal',
    }

    @property
    def is_valid(self):
        return self.last_name.strip() != ''

    @property
    def compensation(self):
        pay = self.get_mapped_value('compensation')
        status = self.get_mapped_value('employee_type')
        wage_type = self.get_mapped_value('wage_type')

        if status == 'Full Time':
            if wage_type == 'Cadet Period Salary':
                # cadets salary is listed outright
                return pay
            else:
                # everyone else is two-week period, multiple by 26 for annual
                return pay * 26
        else:
            # part-timers are listed for 80 hours like full-timers, but we can't
            # say how many hours they work, so can't multiply. Instead, I divide
            # by 80 and just list their hourly rate
            return pay / 80


    @property
    def compensation_type(self):
        status = self.get_mapped_value('employee_type')
        wage_type = self.get_mapped_value('wage_type')

        # cadets are listed as full-timers but we dont want them in the calcs
        if status == 'Full Time' && wage_type != 'Cadet Period Salary':
            return 'FT'
        else:
            return 'PT'

    @property
    def description(self):
        status = self.get_mapped_value('employment_type')
        wage_type = self.get_mapped_value('wage_type')

        if status == 'Full Time':
            if wage_type == 'Cadet Period Salary':
                return 'Cadet Period Salary'
            else:
                return 'Annualized base pay'
        else:
            'Part-time hourly rate'

    @property
    def department(self):
        return self.department_map[self.department.strip()]


    @property
    def race(self):
        if self.race == '':
            return {
                'name': 'Not specified'
            }
        else:
            return {
                'name': self.race.strip()
            }

    @property
    def person(self):
        name = self.get_name()
        r = {
            'family_name': name.last,
            'given_name': name.first,
            'additional_name': name.middle,
            'name': unicode(name),
            'gender': self.gender_map[self.gender.strip()]
        }

        return r

transform = base.transform_factory(TransformedRecord)

{'fraction_non_alphanumeric': 0.12208926261319535, 'fraction_numerical': 0.02296248382923674, 'mean_word_length': 2.24501573976915, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1874808', 'n_tokens_mistral': 2036, 'n_tokens_neox': 1837, 'n_words': 449}