<commit_before>from . import base
from . import mixins

from datetime import date


class TransformedRecord(
        mixins.GenericCompensationMixin,
        mixins.GenericDepartmentMixin, mixins.GenericIdentifierMixin,
        mixins.GenericJobTitleMixin, mixins.GenericPersonMixin,
        mixins.MembershipMixin, mixins.OrganizationMixin, mixins.PostMixin,
        mixins.RaceMixin, mixins.LinkMixin, base.BaseTransformedRecord):

    MAP = {
        'last_name': 'LAST_NAME',
        'first_name': 'FIRST_NAME',
        'department': 'DEPARTMENT',
        'job_title': 'TITLE',
        'hire_date': 'ORIGINAL_DATE_OF_HIRE',
        'compensation': 'ANNUAL_SALARY',
        'gender': 'SEX',
        'nationality': 'ETHNICITY',
    }

    # The order of the name fields to build a full name.
    # If `full_name` is in MAP, you don't need this at all.
    NAME_FIELDS = ('first_name', 'last_name', )

    # The name of the organization this WILL SHOW UP ON THE SITE, so double check it!
    ORGANIZATION_NAME = 'Texas Woman\'s University'

    # What type of organization is this? This MUST match what we use on the site, double check against salaries.texastribune.org
    ORGANIZATION_CLASSIFICATION = 'University'

    # ???
    compensation_type = 'FT'

    # How would you describe the compensation field? We try to respect how they use their system.
    description = 'Annual salary'

    # When did you receive the data? NOT when we added it to the site.
    DATE_PROVIDED = date(2015, 7, 9)

    # The URL to find the raw data in our S3 bucket.
    URL = ('http://raw.texastribune.org.s3.amazonaws.com/'
           'texas_womans_university/salaries/2015-07/'
           'texas_womans_university.xls')

    race_map = {
        'Asian (Not Hispanic or Latino)': 'Asian (Not Hispanic or Latino)',
        'White (Not Hispanic or Latino)': 'White (Not Hispanic or Latino)',
        'American Indian or Alaskan Native (Not Hispanic or Latino)': 'American Indian or Alaskan Native (Not Hispanic or Latino)',
        'Native Hawiian or Other Pacific (Not Hispanic/Latino)': 'Native Hawiian or Other Pacific (Not Hispanic/Latino)',
        'Black or African American (Not Hispanic or Latino)': 'Black or African American (Not Hispanic or Latino)',
        'HAWAIIAN': 'Native Hawaiian/Other Pacific Islander',
        'UKN': 'Unknown',
        'Hispanic or Latino': 'Hispanic or Latino',
        'Not Known': 'Not Known',
    }

    # How do they track gender? We need to map what they use to `F` and `M`.
    gender_map = {'F': 'F', 'M': 'M'}

    # This is how the loader checks for valid people. Defaults to checking to see if `last_name` is empty.
    @property
    def is_valid(self):
        # Adjust to return False on invalid fields.  For example:
        return self.first_name.strip() != ''

    @property
    def race(self):
        return {
            'name': self.race_map[self.nationality.strip()]
        }<commit_msg>added map so UNK and Not Known both map to Not Known<commit_after>from . import base
from . import mixins

from datetime import date


class TransformedRecord(
        mixins.GenericCompensationMixin,
        mixins.GenericDepartmentMixin, mixins.GenericIdentifierMixin,
        mixins.GenericJobTitleMixin, mixins.GenericPersonMixin,
        mixins.MembershipMixin, mixins.OrganizationMixin, mixins.PostMixin,
        mixins.RaceMixin, mixins.LinkMixin, base.BaseTransformedRecord):

    MAP = {
        'last_name': 'LAST_NAME',
        'first_name': 'FIRST_NAME',
        'department': 'DEPARTMENT',
        'job_title': 'TITLE',
        'hire_date': 'ORIGINAL_DATE_OF_HIRE',
        'compensation': 'ANNUAL_SALARY',
        'gender': 'SEX',
        'nationality': 'ETHNICITY',
    }

    # The order of the name fields to build a full name.
    # If `full_name` is in MAP, you don't need this at all.
    NAME_FIELDS = ('first_name', 'last_name', )

    # The name of the organization this WILL SHOW UP ON THE SITE, so double check it!
    ORGANIZATION_NAME = 'Texas Woman\'s University'

    # What type of organization is this? This MUST match what we use on the site, double check against salaries.texastribune.org
    ORGANIZATION_CLASSIFICATION = 'University'

    # ???
    compensation_type = 'FT'

    # How would you describe the compensation field? We try to respect how they use their system.
    description = 'Annual salary'

    # When did you receive the data? NOT when we added it to the site.
    DATE_PROVIDED = date(2015, 7, 9)

    # The URL to find the raw data in our S3 bucket.
    URL = ('http://raw.texastribune.org.s3.amazonaws.com/'
           'texas_womans_university/salaries/2015-07/'
           'texas_womans_university.xls')

    race_map = {
        'Asian (Not Hispanic or Latino)': 'Asian (Not Hispanic or Latino)',
        'White (Not Hispanic or Latino)': 'White (Not Hispanic or Latino)',
        'American Indian or Alaskan Native (Not Hispanic or Latino)': 'American Indian or Alaskan Native (Not Hispanic or Latino)',
        'Native Hawiian or Other Pacific (Not Hispanic/Latino)': 'Native Hawiian or Other Pacific (Not Hispanic/Latino)',
        'Black or African American (Not Hispanic or Latino)': 'Black or African American (Not Hispanic or Latino)',
        'HAWAIIAN': 'Native Hawaiian/Other Pacific Islander',
        'UKN': 'Not Known',
        'Hispanic or Latino': 'Hispanic or Latino',
        'Not Known': 'Not Known',
    }

    # How do they track gender? We need to map what they use to `F` and `M`.
    gender_map = {'F': 'F', 'M': 'M'}

    # This is how the loader checks for valid people. Defaults to checking to see if `last_name` is empty.
    @property
    def is_valid(self):
        # Adjust to return False on invalid fields.  For example:
        return self.first_name.strip() != ''

    @property
    def race(self):
        return {
            'name': self.race_map[self.nationality.strip()]
        }
{'fraction_non_alphanumeric': 0.10011838322340605, 'fraction_numerical': 0.004735328936242178, 'mean_word_length': 2.9347970725216235, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2273023', 'n_tokens_mistral': 1885, 'n_tokens_neox': 1748, 'n_words': 622}