<commit_before>#!/usr/bin/env python
"""
Calculate several measures of segregation in a given data set.

The assumption is that the dataset will be a dictionary-like object
that we can index for one of several parameters to get the required
data.
"""
import sys
from nces_parser import NCESParser

# ==============================================================================
# Constants
# ==============================================================================

# ==============================================================================
# Utility Functions
# ==============================================================================

# ==============================================================================
class SegCalc(object):
    """
    A segregation calculating object.
    """
    def __init__(self, data_list, index_dict):
        """
        Set a dataset iterator object that we can step through
        and a dictionary of indexes for extracting the information
        from the dataobjects turned off by the dataset iterator
        """
        self.debug = 0
        self.data = data_list
        self.y_group_idx = index_dict['Y_GROUP']  # Minority Group Student Count
        self.z_group_idx = index_dict['Z_GROUP']  # Majority Group Student Count
        self.total_idx = index_dict['TOTAL']      # Total Student Count
        self.cat_idx = index_dict['CATEGORY']     # Index to Categorize along (state, district, etc)
        self.sub_cat_idx = index_dict['SUB_CAT']  # Index to Sub Categorize along (grades, district, zip, etc)

        # Skip items that don't match item[idx] == val
        try:
            self.match = True
            self.match_idx = index_dict['MATCH_IDX']
            self.match_val = index_dict['MATCH_VAL']
        except KeyError:
            self.match = False

    # ======================================
    # Support Functions
    # ======================================
    @property
    def filtered_data(self):
        """
        Filter the data per the requested matching
        data index and value.  Cache the results for
        later use
        """
        if self.match == False:
            return self.data
        else:
            try:
                return self._filtered_data
            except AttributeError:
                self._filtered_data = []
                for data in self.data:
                    if data[self.match_idx] == self.match_val:
                        self._filtered_data.append(data)
                return self._filtered_data

    # ======================================<commit_msg>Try to match the integer version of the match_val as well as the string version.<commit_after>#!/usr/bin/env python
"""
Calculate several measures of segregation in a given data set.

The assumption is that the dataset will be a dictionary-like object
that we can index for one of several parameters to get the required
data.
"""
import sys
from nces_parser import NCESParser

# ==============================================================================
# Constants
# ==============================================================================

# ==============================================================================
# Utility Functions
# ==============================================================================

# ==============================================================================
class SegCalc(object):
    """
    A segregation calculating object.
    """
    def __init__(self, data_list, index_dict):
        """
        Set a dataset iterator object that we can step through
        and a dictionary of indexes for extracting the information
        from the dataobjects turned off by the dataset iterator
        """
        self.debug = 0
        self.data = data_list
        self.y_group_idx = index_dict['Y_GROUP']  # Minority Group Student Count
        self.z_group_idx = index_dict['Z_GROUP']  # Majority Group Student Count
        self.total_idx = index_dict['TOTAL']      # Total Student Count
        self.cat_idx = index_dict['CATEGORY']     # Index to Categorize along (state, district, etc)
        self.sub_cat_idx = index_dict['SUB_CAT']  # Index to Sub Categorize along (grades, district, zip, etc)

        # Skip items that don't match item[idx] == val
        try:
            self.match = True
            self.match_idx = index_dict['MATCH_IDX']
            self.match_val = index_dict['MATCH_VAL']
        except KeyError:
            self.match = False

    # ======================================
    # Support Functions
    # ======================================
    @property
    def filtered_data(self):
        """
        Filter the data per the requested matching
        data index and value.  Cache the results for
        later use
        """
        if self.match == False:
            return self.data
        else:
            if self.match_val.isdigit():
                match_int_val = int(self.match_val)
            try:
                return self._filtered_data
            except AttributeError:
                self._filtered_data = []
                for data in self.data:
                    if data[self.match_idx] == self.match_val:
                        self._filtered_data.append(data)
                    if data[self.match_idx] == match_int_val:
                        self._filtered_data.append(data)
                return self._filtered_data

    # ======================================
{'fraction_non_alphanumeric': 0.24450499545867393, 'fraction_numerical': 0.00036330608537693006, 'mean_word_length': 2.700268817204301, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1136701', 'n_tokens_mistral': 1344, 'n_tokens_neox': 1221, 'n_words': 396}