<jupyter_start><jupyter_text>University of Michigan School of Information**Masters of Applied Data Science****Milestone 1**Affordable Housing Project**Submitted by**><NAME>  Get CoreLogic Data  Start Spark Session<jupyter_code>import pandas as pd
from config import basedir

from pyspark.sql import SparkSession
spark=SparkSession\
.builder\
.master("local[*]")\
.appName("TaxDataImport")\
.getOrCreate()

sc=spark.sparkContext

base=basedir()

data_dir = base + r"/data/core_logic_data/"
input_file = data_dir + r"/TaxData.txt"

lines=sc.textFile(fp_input_file)
TaxDf=spark.read.option("header","true").csv(fp_input_file,sep='|')<jupyter_output><empty_output><jupyter_text>Read CoreLogic Data Three Variables at a Time<jupyter_code>output_file1 = data_dir + r"/TaxData1.csv"

TaxDf2=TaxDf.filter(TaxDf['SITUS STATE']=='MI')
TaxDf3=TaxDf2['ORIGINAL APN','FIPS CODE','CENSUS TRACT'].collect()
TaxDf4=pd.DataFrame(TaxDf3)
TaxDf4.to_csv(output_file1)
output_file2 = data_dir + r"/TaxData2.csv"

TaxDf2=TaxDf.filter(TaxDf['SITUS STATE']=='MI')
TaxDf3=TaxDf2['ORIGINAL APN','COUNTY USE1','LAND USE'].collect()
TaxDf4=pd.DataFrame(TaxDf3)
TaxDf4.to_csv(output_file2)
output_file3 = data_dir + r"/TaxData3.csv"

TaxDf2=TaxDf.filter(TaxDf['SITUS STATE']=='MI')
TaxDf3=TaxDf2['ORIGINAL APN','PROPERTY INDICATOR','TOTAL VALUE CALCULATED'].collect()
TaxDf4=pd.DataFrame(TaxDf3)
TaxDf4.to_csv(output_file3)
output_file4 = data_dir + r"/TaxData4.csv"

TaxDf2=TaxDf.filter(TaxDf['SITUS STATE']=='MI')
TaxDf3=TaxDf2['ORIGINAL APN','MUNICIPALITY NAME','MUNICIPALITY CODE'].collect()
TaxDf4=pd.DataFrame(TaxDf3)
TaxDf4.to_csv(output_file4)
output_file5 = data_dir + r"/TaxData5.csv"

TaxDf2=TaxDf.filter(TaxDf['SITUS STATE']=='MI')
TaxDf3=TaxDf2['ORIGINAL APN','PROPERTY LEVEL LATITUDE','PROPERTY LEVEL LONGITUDE'].collect()
TaxDf4=pd.DataFrame(TaxDf3)
TaxDf4.to_csv(output_file5)
output_file6 = data_dir + r"/TaxData6.csv"

TaxDf2=TaxDf.filter(TaxDf['SITUS STATE']=='MI')
TaxDf3=TaxDf2['ORIGINAL APN','ACRES','YEAR BUILT'].collect()
TaxDf4=pd.DataFrame(TaxDf3)
TaxDf4.to_csv(output_file6)
output_file7 = data_dir + r"/TaxData7.csv"

TaxDf2=TaxDf.filter(TaxDf['SITUS STATE']=='MI')
TaxDf3=TaxDf2['ORIGINAL APN','LIVING SQUARE FEET','UNITS NUMBER'].collect()
TaxDf4=pd.DataFrame(TaxDf3)
TaxDf4.to_csv(output_file7)<jupyter_output><empty_output><jupyter_text>Concatenate All CoreLogic Data<jupyter_code>import pandas as pd
TaxData1=pd.read_csv(output_file1,index_col=1,names=['index_column','FIPS CODE','CENSUS TRACT'],header=None,skiprows=1)
TaxData2=pd.read_csv(output_file2,index_col=1,names=['index_column','COUNTY USE1','LAND USE'],header=None,skiprows=1)
TaxData3=pd.read_csv(output_file3,index_col=1,names=['index_column','PROPERTY INDICATOR','TOTAL VALUE CALCULATED'],header=None,skiprows=1)
TaxData3=pd.read_csv(output_file4,index_col=1,names=['index_column','MUNICIPALITY NAME','MUNICIPALITY CODE'],header=None,skiprows=1)
TaxData4=pd.read_csv(output_file5,index_col=1,names=['index_column','PROPERTY LEVEL LATITUDE','PROPERTY LEVEL LONGITUDE'],header=None,skiprows=1)
TaxData5=pd.read_csv(output_file6,index_col=1,names=['index_column','ACRES','YEAR BUILT'],header=None,skiprows=1)
TaxData6=pd.read_csv(output_file7,index_col=1,names=['index_column','LIVING SQUARE FEET','UNITS NUMBER'],header=None,skiprows=1)                                                                 
TaxData1=TaxData1.drop(['index_column'],axis=1)
TaxData2=TaxData2.drop(['index_column'],axis=1)
TaxData3=TaxData3.drop(['index_column'],axis=1)
TaxData4=TaxData4.drop(['index_column'],axis=1)
TaxData5=TaxData5.drop(['index_column'],axis=1)
TaxData6=TaxData6.drop(['index_column'],axis=1)
output_data_dir = base + r"/data/TaxData.csv"

TaxData=pd.concat([TaxData1,TaxData2,TaxData3,TaxData4,TaxData5,TaxData6],axis=1)
TaxData.index=TaxData.index.rename('ORIGINAL APN')
TaxData.to_csv(output_data_dir)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15960995637670003, 'fraction_numerical': 0.030536309982037466, 'mean_word_length': 9.918767507002801, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '540861', 'license': 'MIT', 'n_tokens_mistral': 1670, 'n_tokens_neox': 1610, 'n_words': 175, 'path': 'src/final/.ipynb_checkpoints/core_logic_data-checkpoint.ipynb', 'repo_name': 'siads591-fa20-001-affordable-housing/go_blue', 'size': '3930'}