<commit_before>#!/usr/bin/env python
"""
Pull together the NCES data and the Segregation calculator to produce
general reports on segregation in the USA.
"""
import sys
import argparse

from segcalc import SegCalc
from nces_parser import NCESParser

from xlwt import Workbook

from tuda import tuda_dist
from big import big_dist

# ==============================================================================
# Constants
# ==============================================================================

# ==============================================================================
# Functions
# ==============================================================================
def write_ws(worksheets, leaid, row, col, data):
    try:
        val = data[leaid]
    except KeyError:
        val = ""
    if val < 0.001:
        val = ""

    worksheets[leaid].write(row, col, val)

# -------------------------------------
# Parse the command line options
# -------------------------------------
def main(argv):
    parser = argparse.ArgumentParser(description='Segregation Report Generator')
    parser.add_argument('--outfile', action='store', dest='outfile', required=True,
            help='Report Filename')
    parser.add_argument('--minority', action='store', dest='minority', required=False,
            help='Override the default list of Minority Groups')
    parser.add_argument('--sec_minority', action='store', dest='sec_minority', required=False,
            help='Override the default list of Secondary Minority Groups')
    parser.add_argument('--majority', action='store', dest='majority', required=False,
            help='Override the default list of Majority Groups')
    parser.add_argument('--year', action='store', dest='year', required=False, type=int,
            help='Override the default list of years to report on')
    parser.add_argument('-big_dist', action='store_true', dest='big_dist', required=False,
            help='Big District Mode')
    parser.add_argument('-debug', action='store_true', dest='debug', required=False,
            help='Debug Mode')
    args = parser.parse_args()

    # Lets calculate all the data first
    if args.debug:
        year_range = range(2009,2011)
        minorities = ['BLACK']
        sec_minorities = [None]
        majorities = ['WHITE']
    else:
        year_range = range(1987, 2011)
        minorities     = ['WHITE', 'BLACK', 'HISP', 'BLACK', 'HISP', 'FRELCH', 'FRELCH']
        sec_minorities = [None, None, None, 'HISP', None, None, 'REDLCH']
        majorities     = ['WHITE', 'WHITE', 'WHITE', 'WHITE', 'BLACK', None, None]

    # Override the default years/groups per command line requests
    if args.year:
        year_range = [args.year]
    if args.minority:
        minorities = [args.minority]
    if args.sec_minority:
        sec_minorities = [args.sec_minorities]
    if args.majority:
        majorities = [args.majority]
    if args.majority:
        majorities = [args.majority]
    if args.big_dist:
        dist_list = big_dist
    else:
        dist_list = tuda_dist

    wb = Workbook()
    worksheets = {}
    for leaid, dist_name in dist_list.items():
        worksheets[leaid] = wb.add_sheet(dist_name)

    # --------------------------------------
    # Create the row labels
    # --------------------------------------
    for ws in worksheets.values():
        for j, year in enumerate(year_range):
            ws.write(j+1, 0, year)
        base_col_offset = 1

    # --------------------------------------
    # Create the column labels
    # --------------------------------------
    minority_headers = [
        'count', # 'Minority Student Count',
        'prop', # 'Minority Proportion',
    ]

    min_maj_headers = [
        'dis_idx', # 'Dissimilarity Index',
        'exp_idx', # 'Exposure Index',
        'iso_idx', # 'Isolation Index',
    ]

    common_headers = [
        'stu_count', # 'Student Count',
        'mag_prop', # 'Magnet Proportion',
        'cha_prop', # 'Charter Proportion',
        'cho_prop', # 'Choice Proportion'
    ]

    for ws in worksheets.values():
        col_offset=1

        # Common Data Across minorities
        # e.g. Total Students in a district
        for header in common_headers:
            ws.write(0, col_offset, header)
            col_offset += 1

        # Data unique to each minority group
        # e.g. Proportion of Students in the Minority Group
        for j, minority in enumerate(minorities):
            min_label = minority[:2].lower()
            if sec_minorities[j]:
                min_label += "_"+sec_minorities[j][:2].lower()

            maj_label = min_label
            if majorities[j]:
                maj_label += "_"+majorities[j][:2].lower()

            for header in minority_headers:
                ws.write(0, col_offset, min_label+"_"+header)
                col_offset += 1
            for header in min_maj_headers:
                ws.write(0, col_offset, maj_label+"_"+header)
                col_offset += 1
        row_offset = 1

    # --------------------------------------
    # Now fill in the static data data
    # --------------------------------------
    for i, year in enumerate(year_range):
        # Reset the column offset as we move to a new row
        col_offset = base_col_offset

        # Default SegCalc search query - for calculating basic totals
        idx = {
            'TOTAL': 'MEMBER',
            'CATEGORY': 'LEAID',
            'MINORITY':  'BLACK',
            'SEC_MINORITY': '',
            'MAJORITY': 'WHITE',
            'MATCH_IDX': 'CHARTR',
            'MATCH_VAL': '1'
        }

        print "Loading NCES Data from:  %d" % year
        nces = NCESParser(year=year)
        schools = nces.parse(make_dict=True)
        segcalc = SegCalc(schools, idx)<commit_msg>Modified to work with arbitrary district lists.<commit_after>#!/usr/bin/env python
"""
Pull together the NCES data and the Segregation calculator to produce
general reports on segregation in the USA.
"""
import sys
import argparse

from segcalc import SegCalc
from nces_parser import NCESParser

from xlwt import Workbook

from tuda import tuda_dist
from big import big_dist

# ==============================================================================
# Constants
# ==============================================================================

# ==============================================================================
# Functions
# ==============================================================================
def write_ws(worksheets, leaid, row, col, data):
    try:
        val = data[leaid]
    except KeyError:
        val = ""
    if val < 0.001:
        val = ""

    worksheets[leaid].write(row, col, val)

# -------------------------------------
# Parse the command line options
# -------------------------------------
def main(argv):
    parser = argparse.ArgumentParser(description='Segregation Report Generator')
    parser.add_argument('--outfile', action='store', dest='outfile', required=True,
            help='Report Filename')
    parser.add_argument('--minority', action='store', dest='minority', required=False,
            help='Override the default list of Minority Groups')
    parser.add_argument('--sec_minority', action='store', dest='sec_minority', required=False,
            help='Override the default list of Secondary Minority Groups')
    parser.add_argument('--majority', action='store', dest='majority', required=False,
            help='Override the default list of Majority Groups')
    parser.add_argument('--year', action='store', dest='year', required=False, type=int,
            help='Override the default list of years to report on')
    parser.add_argument('-all_dist', action='store_true', dest='all_dist', required=False,
            help='All Districts Found in the data Mode')
    parser.add_argument('-big_dist', action='store_true', dest='big_dist', required=False,
            help='Big District Mode')
    parser.add_argument('-debug', action='store_true', dest='debug', required=False,
            help='Debug Mode')
    args = parser.parse_args()

    # Lets calculate all the data first
    if args.debug:
        year_range = range(2009,2011)
        minorities = ['BLACK']
        sec_minorities = [None]
        majorities = ['WHITE']
    else:
        year_range = range(1987, 2011)
        minorities     = ['WHITE', 'BLACK', 'HISP', 'BLACK', 'HISP', 'FRELCH', 'FRELCH']
        sec_minorities = [None, None, None, 'HISP', None, None, 'REDLCH']
        majorities     = ['WHITE', 'WHITE', 'WHITE', 'WHITE', 'BLACK', None, None]

    # Override the default years/groups per command line requests
    if args.year:
        year_range = [args.year]
    if args.minority:
        minorities = [args.minority]
    if args.sec_minority:
        sec_minorities = [args.sec_minorities]
    if args.majority:
        majorities = [args.majority]
    if args.all_dist:
        nces = NCESParser(year=2010)
        schools = nces.parse(make_dict=True)
        dist_list = {}
        for school in schools:
            id = school["LEAID"]
            name = school['LEANM'][:28].title()
            name = name.replace("/", "_")
            if id not in dist_list.keys():
                if name + "_1" in dist_list.values():
                    name += "_2"
                    print name
                elif name in dist_list.values():
                    name += "_1"
                    print name
                dist_list[id] = name
        # print dist_list
        print "Found %d Districts" % (len(dist_list.keys()))
    elif args.big_dist:
        dist_list = big_dist
    else:
        dist_list = tuda_dist

    wb = Workbook()
    worksheets = {}
    for leaid, dist_name in dist_list.items():
        worksheets[leaid] = wb.add_sheet(dist_name)

    # --------------------------------------
    # Create the row labels
    # --------------------------------------
    for ws in worksheets.values():
        for j, year in enumerate(year_range):
            ws.write(j+1, 0, year)
        base_col_offset = 1

    # --------------------------------------
    # Create the column labels
    # --------------------------------------
    minority_headers = [
        'count', # 'Minority Student Count',
        'prop', # 'Minority Proportion',
    ]

    min_maj_headers = [
        'dis_idx', # 'Dissimilarity Index',
        'exp_idx', # 'Exposure Index',
        'iso_idx', # 'Isolation Index',
    ]

    common_headers = [
        'stu_count', # 'Student Count',
        'mag_prop', # 'Magnet Proportion',
        'cha_prop', # 'Charter Proportion',
        'cho_prop', # 'Choice Proportion'
    ]

    for ws in worksheets.values():
        col_offset=1

        # Common Data Across minorities
        # e.g. Total Students in a district
        for header in common_headers:
            ws.write(0, col_offset, header)
            col_offset += 1

        # Data unique to each minority group
        # e.g. Proportion of Students in the Minority Group
        for j, minority in enumerate(minorities):
            min_label = minority[:2].lower()
            if sec_minorities[j]:
                min_label += "_"+sec_minorities[j][:2].lower()

            maj_label = min_label
            if majorities[j]:
                maj_label += "_"+majorities[j][:2].lower()

            for header in minority_headers:
                ws.write(0, col_offset, min_label+"_"+header)
                col_offset += 1
            for header in min_maj_headers:
                ws.write(0, col_offset, maj_label+"_"+header)
                col_offset += 1
        row_offset = 1

    # --------------------------------------
    # Now fill in the static data data
    # --------------------------------------
    for i, year in enumerate(year_range):
        # Reset the column offset as we move to a new row
        col_offset = base_col_offset

        # Default SegCalc search query - for calculating basic totals
        idx = {
            'TOTAL': 'MEMBER',
            'CATEGORY': 'LEAID',
            'MINORITY':  'BLACK',
            'SEC_MINORITY': '',
            'MAJORITY': 'WHITE'
        }

        print "Loading NCES Data from:  %d" % year
        nces = NCESParser(year=year)
        schools = nces.parse(make_dict=True)
        segcalc = SegCalc(schools, idx)
{'fraction_non_alphanumeric': 0.20341451573751923, 'fraction_numerical': 0.006311190225746419, 'mean_word_length': 2.106308117617492, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1403217', 'n_tokens_mistral': 3580, 'n_tokens_neox': 3335, 'n_words': 902}