<commit_before>#import json
import time
import psycopg2
import psycopg2.extras
from connector import connect_DB as connect
from builder import build_JSON as build
from selector import report_selector as selector
from constructor import report_construction
from file_check import check_file
from report_list import report_list_maker

connection = connect() #connects to the DB
selector = selector() #holds lists of reports to be generated for each MSA
cur = connection.connect() #creates cursor object connected to HMDAPub2013 sql database, locally hosted postgres
selector.get_report_lists('MSAinputs2013.csv') #fills the dictionary of lists of reports to be generated
build_msa = build() #instantiate the build object for file path, jekyll files
#build_msa.msas_in_state(cur, selector, 'aggregate') #creates a list of all MSAs in each state and places the file in the state's aggregate folder


#List of Alabama MSAs for test state case
#AL_MSAs = ['45180', '45980', '11500', '10760', '42460', '13820', '19460', '23460', '46740', '17980', '12220', '20020', '18980', '33860', '46260', '33660', '19300', '22840', '21460','10700','21640','42820','26620','22520','46220']
AL_MSAs = ['33660']

#report lists for testing
selector.reports_to_run = ['A 3-1', 'D 3-1']
#selector.reports_to_run = ['A 11-1']
#selector.reports_to_run = ['D 4-1', 'D 4-2', 'D 4-3', 'D 4-4', 'D 4-6', 'D 4-7']
#selector.reports_to_run = ['A 5-1', 'A 5-2', 'A 5-3', 'A 5-4', 'A 5-5', 'A 5-7']
#selector.reports_to_run = ['A 7-1', 'A 7-2', 'A 7-3', 'A 7-4', 'A 7-5', 'A 7-6', 'A 7-7']
#selector.reports_to_run = ['A 8-1', 'A 8-2', 'A 8-3', 'A 8-4', 'A 8-5', 'A 8-6', 'A 8-7']
#selector.reports_to_run = ['A 11-1', 'A 11-2', 'A 11-3', 'A 11-4', 'A 11-5', 'A 11-6', 'A 11-7', 'A 11-8', 'A 11-9', 'A 11-10']
#selector.reports_to_run = ['D 12-1', 'D 12-2']
#selector.reports_to_run = ['D A-1', 'D A-2', 'D A-3', 'D A-4']
#selector.reports_to_run = ['B']

#complete report list
#selector.reports_to_run = ['A 3-1', 'A 3-2', 'A 4-1', 'A 4-2', 'A 4-3', 'A 4-4', 'A 4-5', 'A 4-6', 'A 4-7', 'A 5-1', 'A 5-2', 'A 5-3', 'A 5-4', 'A 5-5', 'A 5-7', 'A 7-1', 'A 7-2', 'A 7-3', 'A 7-4', 'A 7-5', 'A 7-6', 'A 7-7', 'A 8-1', 'A 8-2', 'A 8-3', 'A 8-4', 'A 8-5', 'A 8-6', 'A 8-7', 'A 9', 'A 11-1', 'A 11-2', 'A 11-3', 'A 11-4', 'A 11-5', 'A 11-6', 'A 11-7', 'A 11-8', 'A 11-9', 'A 11-10', 'A 12-1', 'A 12-2', 'A A-1', 'A A-2', 'A A-3', 'A A-4', 'A B'] #this needs to be changed to read from the input file
#selector.reports_to_run = ['D 3-1', 'D 3-2', 'D 4-1', 'D 4-2', 'D 4-3', 'D 4-4', 'D 4-5', 'D 4-6', 'D 4-7', 'D 5-1', 'D 5-2', 'D 5-3', 'D 5-4', 'D 5-5', 'D 5-7', 'D 7-1', 'D 7-2', 'D 7-3', 'D 7-4', 'D 7-5', 'D 7-6', 'D 7-7', 'D 8-1', 'D 8-2', 'D 8-3', 'D 8-4', 'D 8-5', 'D 8-6', 'D 8-7', 'D 9', 'D 11-1', 'D 11-2', 'D 11-3', 'D 11-4', 'D 11-5', 'D 11-6', 'D 11-7', 'D 11-8', 'D 11-9', 'D 11-10', 'D 12-1', 'D 12-2', 'D A-1', 'D A-2', 'D A-3', 'D A-4', 'D B'] #this needs to be changed to read from the input file
#selector.reports_to_run = ['D 11-1']
for i in range(0, len(selector.reports_to_run)):
	selector.report_list[selector.reports_to_run[i]] = ['33660']
#control loop
total_time_start2 = time.time()
total_time_start = time.clock() #set start time for total report batch
logfile = open('processing_log.txt', 'w')
for report in selector.reports_to_run: #loop over a list of report names
	#if len(selector.report_list[report]) >0:
	start = time.clock() #set start for one report
	start2 = time.time()
	for MSA in selector.report_list[report]: #loop through MSAs flagged for report generation
		if report[0] == 'A':
			aggregate_report = report_construction(report, selector) #instantiate class and set function strings
			aggregate_report.aggregate_report(MSA, cur) #variabalize funciton inputs!!!!
		elif report[0] == 'D':
			disclosure_report = report_construction(report, selector)
			disclosure_report.disclosure_report(MSA, cur)
	end = time.clock() #set end for one report
	end2 = time.time()

	cpu_report_time = end-start
	clock_report_time = end2-start2<commit_msg>modified gitignore to include Alabama disclosure<commit_after>#import json
import time
import psycopg2
import psycopg2.extras
from connector import connect_DB as connect
from builder import build_JSON as build
from selector import report_selector as selector
from constructor import report_construction
from file_check import check_file
from report_list import report_list_maker

connection = connect() #connects to the DB
selector = selector() #holds lists of reports to be generated for each MSA
cur = connection.connect() #creates cursor object connected to HMDAPub2013 sql database, locally hosted postgres
selector.get_report_lists('MSAinputs2013.csv') #fills the dictionary of lists of reports to be generated
build_msa = build() #instantiate the build object for file path, jekyll files
#build_msa.msas_in_state(cur, selector, 'aggregate') #creates a list of all MSAs in each state and places the file in the state's aggregate folder


#List of Alabama MSAs for test state case
#AL_MSAs = ['45180', '45980', '11500', '10760', '42460', '13820', '19460', '23460', '46740', '17980', '12220', '20020', '18980', '33860', '46260', '33660', '19300', '22840', '21460','10700','21640','42820','26620','22520','46220']
AL_MSAs = ['33660']

#report lists for testing
#selector.reports_to_run = ['A 3-1', 'D 3-1']
#selector.reports_to_run = ['A 11-1']
#selector.reports_to_run = ['D 4-1', 'D 4-2', 'D 4-3', 'D 4-4', 'D 4-6', 'D 4-7']
#selector.reports_to_run = ['A 5-1', 'A 5-2', 'A 5-3', 'A 5-4', 'A 5-5', 'A 5-7']
#selector.reports_to_run = ['A 7-1', 'A 7-2', 'A 7-3', 'A 7-4', 'A 7-5', 'A 7-6', 'A 7-7']
#selector.reports_to_run = ['A 8-1', 'A 8-2', 'A 8-3', 'A 8-4', 'A 8-5', 'A 8-6', 'A 8-7']
#selector.reports_to_run = ['A 11-1', 'A 11-2', 'A 11-3', 'A 11-4', 'A 11-5', 'A 11-6', 'A 11-7', 'A 11-8', 'A 11-9', 'A 11-10']
#selector.reports_to_run = ['D 12-1', 'D 12-2']
#selector.reports_to_run = ['D A-1', 'D A-2', 'D A-3', 'D A-4']
#selector.reports_to_run = ['B']

#complete report list
selector.reports_to_run = ['A 3-1', 'A 3-2', 'A 4-1', 'A 4-2', 'A 4-3', 'A 4-4', 'A 4-5', 'A 4-6', 'A 4-7', 'A 5-1', 'A 5-2', 'A 5-3', 'A 5-4', 'A 5-5', 'A 5-7', 'A 7-1', 'A 7-2', 'A 7-3', 'A 7-4', 'A 7-5', 'A 7-6', 'A 7-7', 'A 8-1', 'A 8-2', 'A 8-3', 'A 8-4', 'A 8-5', 'A 8-6', 'A 8-7', 'A 9', 'A 11-1', 'A 11-2', 'A 11-3', 'A 11-4', 'A 11-5', 'A 11-6', 'A 11-7', 'A 11-8', 'A 11-9', 'A 11-10', 'A 12-1', 'A 12-2', 'A A-1', 'A A-2', 'A A-3', 'A A-4', 'A B'] #this needs to be changed to read from the input file
#selector.reports_to_run = ['D 3-1', 'D 3-2', 'D 4-1', 'D 4-2', 'D 4-3', 'D 4-4', 'D 4-5', 'D 4-6', 'D 4-7', 'D 5-1', 'D 5-2', 'D 5-3', 'D 5-4', 'D 5-5', 'D 5-7', 'D 7-1', 'D 7-2', 'D 7-3', 'D 7-4', 'D 7-5', 'D 7-6', 'D 7-7', 'D 8-1', 'D 8-2', 'D 8-3', 'D 8-4', 'D 8-5', 'D 8-6', 'D 8-7', 'D 9', 'D 11-1', 'D 11-2', 'D 11-3', 'D 11-4', 'D 11-5', 'D 11-6', 'D 11-7', 'D 11-8', 'D 11-9', 'D 11-10', 'D 12-1', 'D 12-2', 'D A-1', 'D A-2', 'D A-3', 'D A-4', 'D B'] #this needs to be changed to read from the input file
#selector.reports_to_run = ['D 11-1']
for i in range(0, len(selector.reports_to_run)):
	selector.report_list[selector.reports_to_run[i]] = ['33660']
#control loop
total_time_start2 = time.time()
total_time_start = time.clock() #set start time for total report batch
logfile = open('processing_log.txt', 'w')
for report in selector.reports_to_run: #loop over a list of report names
	#if len(selector.report_list[report]) >0:
	start = time.clock() #set start for one report
	start2 = time.time()
	for MSA in selector.report_list[report]: #loop through MSAs flagged for report generation
		if report[0] == 'A':
			aggregate_report = report_construction(report, selector) #instantiate class and set function strings
			aggregate_report.aggregate_report(MSA, cur) #variabalize funciton inputs!!!!
		elif report[0] == 'D':
			disclosure_report = report_construction(report, selector)
			disclosure_report.disclosure_report(MSA, cur)
	end = time.clock() #set end for one report
	end2 = time.time()

	cpu_report_time = end-start
	clock_report_time = end2-start2
{'fraction_non_alphanumeric': 0.20974706971005552, 'fraction_numerical': 0.11276989512646514, 'mean_word_length': 4.085319949811794, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1944109', 'n_tokens_mistral': 3952, 'n_tokens_neox': 3368, 'n_words': 1124}