<commit_before>import json
from csv import DictReader as cdr

from paying_for_college.models import School

"""
# Data processing steps

This script was used to process a few xlsx files to look at expediture data based on income and region
- source: http://www.bls.gov/cex/
- files under Region of residence by income before taxes:
  - xregnmw.xlsx
  - xregnne.xlsx
  - xregns.xlsx
  - xregnw.xlsx

The xlsx were converted manually using the following steps:
- convert xlsx to csv
- delete the top 1 line, which are fake headings
- remove newline characters on the first line (the real headings)
*- Remove everything from the line "Addenda:" to the end

To run from the script from the Django shell:

```
./manage.py shell
from paying_for_college.data_sources.bls_processing import *
a = create_bls_json_file()
```

"""
BASE_DIR = 'paying_for_college/data_sources'
WE_CSVFILE = '{}/xregnw.csv'.format(BASE_DIR)
NE_CSVFILE = '{}/xregnne.csv'.format(BASE_DIR)
MW_CSVFILE = '{}/xregnmw.csv'.format(BASE_DIR)
SO_CSVFILE = '{}/xregns.csv'.format(BASE_DIR)
YEAR = 2014
<commit_msg>Fix comment on how to use this file<commit_after>import json
from csv import DictReader as cdr

from paying_for_college.models import School

"""
# Data processing steps

This script was used to process a few xlsx files to look at expediture data based on income and region
- source: http://www.bls.gov/cex/
- files under Region of residence by income before taxes:
  - xregnmw.xlsx
  - xregnne.xlsx
  - xregns.xlsx
  - xregnw.xlsx

The xlsx were converted manually using the following steps:
- convert xlsx to csv
- delete the top 1 line, which are fake headings
- remove newline characters on the first line (the real headings)
*- Remove everything from the line "Addenda:" to the end

To run from the script from the Django shell:

```
./manage.py shell
from paying_for_college.data_sources.bls_processing import *
create_bls_json_file()
```

This will create a new file in `paying_for_college/fixtures/bls_data_new.json`

"""

BASE_DIR = 'paying_for_college/data_sources'
WE_CSVFILE = '{}/xregnw.csv'.format(BASE_DIR)
NE_CSVFILE = '{}/xregnne.csv'.format(BASE_DIR)
MW_CSVFILE = '{}/xregnmw.csv'.format(BASE_DIR)
SO_CSVFILE = '{}/xregns.csv'.format(BASE_DIR)
YEAR = 2014

{'fraction_non_alphanumeric': 0.09337483325922633, 'fraction_numerical': 0.004446420631391729, 'mean_word_length': 4.474452554744525, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1783696', 'n_tokens_mistral': 801, 'n_tokens_neox': 766, 'n_words': 252}