<jupyter_start><jupyter_text>CSP ASEC<jupyter_code># CSP ASEC

# example query

# api.census.gov/data/timeseries/poverty/histpov2?get=PCTPOV&time=2018&RACE=1

# Cant get multiple years
# one year at a time

# RACE codes 1-12
# All Races
# White Alone
# White
# White Alone, Not Hispanic
# White, Not Hispanic
# Black Alone or in Combination
# Black Alone
# Black
# Asian Alone or in Combination
# Asian Alone
# Asian and Pacific Islander
# Hispanic (of any race)

# GET variables - comma sep
# PCTFAMPOV	People in Families!!All Families!!Below Poverty Level!!Percent	
# PCTFEMHHPOV	People in Families!!Female HH no HB!!Below Poverty Level!!Percent		
# PCTPOV	All People!!Below Poverty Level!!Percent	
# PCTUNRELPOV	People in Families!!Unrelated Individuals!!Below Poverty Level!!Percent	
# POP	All People!!Total NOTE: Numbers in thousands. People as of March of the following year		
# POV	All People!!Below Poverty Level!!Number NOTE: Numbers in thousands. People as of March of the following year
# UNREL	People in Families!!Unrelated Individuals!!Total NOTE: Numbers in thousands. People as of March of the following year		
# UNRELPOV	People in Families!!Unrelated Individuals!!Below Poverty Level!!Number NOTE: Numbers in thousands. People as of March of the following year

url = "https://api.census.gov/data/timeseries/poverty/histpov2?get=PCTPOV,POV&time=2019"

response = requests.get(url)
# data = pd.DataFrame(columns=response.json()[0], data=response.json()[1:])
response.json()
#  Small Area Income and Poverty Estimates (SAIPE) 
# looks like it ended in 2016? maybe?<jupyter_output><empty_output><jupyter_text>BLS<jupyter_code># BLS

# url = "https://api.bls.gov/publicAPI/v2/timeseries/data/LAUCN040010000000005/latest=true"

# response = requests.get(url)
# response

headers = {'Content-type': 'application/json'}
data = json.dumps({"seriesid": ['CUUR0000SA0','SUUR0000SA0'],"startyear":"2011", "endyear":"2014"})
p = requests.post('https://api.bls.gov/publicAPI/v2/timeseries/data/', data=data, headers=headers)
json_data = json.loads(p.text)<jupyter_output><empty_output><jupyter_text>ACS<jupyter_code>def get_data(year, 
             dsource, 
             dname, 
             variables, 
             state, 
             county, 
             tract,
            ):
    if variables:
        variable_names = [i for i,v in variables]
        variable_codes = [v for i,v in variables]
        codes = ',' + (',').join(variable_codes)
        
    if not variables:
        codes = ''
        
    codes = 'NAME' + codes

    base_url = f'https://api.census.gov/data/{year}/{dsource}/{dname}/'

    data_url = f'{base_url}?get={codes}&for=tract:{tract}&in=state:{state}&in=county:{county}'

    response = requests.get(data_url)
    data = pd.DataFrame(columns=response.json()[0], data=response.json()[1:])
    
    data['GEOID'] = data['state'] + data['county'] + data['tract']
    
    if not variables:
        data['NAME'] = data['NAME'].str.split(',')
        data['tract_name'] = data['NAME'].apply(lambda row: row[0])
        data['county_name'] = data['NAME'].apply(lambda row: row[1])

    data = data.drop(columns =['NAME','state','county','tract'])
    data = data.set_index('GEOID')
    
    if variables:
        data.columns = variable_names
        data = data.astype(int)
    
    data = data.sort_index()
    
    return data
def get_ACS(
    variable_dict,
    year = 2019,
    dsource = 'acs',
    dname = 'acs5',
    state = 18,
    county = '*',
    tract = '*'
):
    variables = [(i,v) for i,v in variable_dict.items()]
    
    if len(variables) > 15:
        groups = [variables[i:i + 15] for i in range(0, len(variables), 15)]
        df_list = []
        for grp in groups:
            tmp_data = get_data(
                year = year,
                variables = grp,
                dsource = dsource,
                dname = dname,
                state = state,
                county = county,
                tract = tract
            )
            df_list.append(tmp_data)
            
        data = pd.concat(df_list, axis=1)
        
    else:
        data = get_data(
            year = year,
            variables = variables,
            dsource = dsource,
            dname = dname,
            state = state,
            county = county,
            tract = tract
            )
        
    name_frame = get_data(
            year = year,
            variables = None,
            dsource = dsource,
            dname = dname,
            state = state,
            county = county,
            tract = tract
            )
    return data.join(name_frame)
out = get_ACS(
    variable_dict=population_variables,
    year = 2019,
    dsource = 'acs',
    dname = 'acs5',
    state = 18,
    county = '*',
    tract = '*'
)
# # data = get_data(
# #     year = 2019,
# #     dsource = 'acs',
# #     dname = 'acs5',
# #     cols = 'NAME,' + vars_struct['Total Population'],
# #     state = 18,
# #     county = '*',
# #     tract = '*',
# #     rename = ['Total Population']
# # )
# # I should pass the variable dict and then just have the function partse it 

# # if length is greater than 20, need to do in batches
# # if I am to do it in batches, don't want to duplicate tract and name, so should do a single pull of tract and name

# variable_names = [i for i,v in variables]
# variable_codes = [v for i,v in variables]

# codes = ',' + (',').join(variable_codes)



# year = 2014
# dsource = 'acs'
# dname = 'acs5'
# cols = 'NAME' + codes
# state = 18
# county = '*'
# tract = '*'

# base_url = f'https://api.census.gov/data/{year}/{dsource}/{dname}/'

# data_url = f'{base_url}?get={cols}&for=tract:{tract}&in=state:{state}&in=county:{county}'

# response = requests.get(data_url)
# data = pd.DataFrame(columns=response.json()[0], data=response.json()[1:])

# data['NAME'] = data['NAME'].str.split(',')

# data['tract_name'] = data['NAME'].apply(lambda row: row[0])
# data['county_name'] = data['NAME'].apply(lambda row: row[1])

# data['GEOID'] = data['state'] + data['county'] + data['tract']
# data = data.set_index('GEOID')
# data = data.drop(columns =['NAME','state','county','tract'])


# # data.columns = variable_names # + ['tract_name','county_name']
# data = data.astype(int)
# data = data.sort_index()
# population_variables
population_variables = {
    'Population Total' : "B01003_001E", # B01001_001E B03001_001E
    'Population White': 'B02001_002E',
    'Population Black': 'B02001_003E',
    'Population Arab': 'B04004_006E',
    'Population Asian':'B02001_005E',
    'Population Hispanic': 'B03003_003E',
    
    'Population American Indiana & Alaska Native': 'B02001_004E',
    'Population Hawaiian & Pacific Islander': 'B02001_006E',
    
    'Population Non Hispanic': 'B03001_002E',
    'Population White Not Hispanic': 'B03002_003E',

    'Population White Male': "B01001A_002E",
    'Population White Female': "B01001A_017E",
    
    'Population Black Male': 'B01001B_002E',
    'Population Black Female': 'B01001B_017E',
    
    'Population Asian Male': 'B01001D_002E',
    'Population Asian Female': 'B01001D_017E',
    
    'Population White Non Hispanic': 'B01001H_001E',
    'Population White Male Non Hispanic': 'B01001H_002E',
    'Population White Female Non Hispanic':'B01001H_017E',

    'Population Hispanic Male': 'B01001I_002E',
    'Population Hispanic Female':'B01001I_017E',
    
    'Population Male': 'B01001_002E',
    'Population Female': 'B01001_026E',
    
    'Population Male 0-5':'B01001_003E',
    'Population Male 5-9':'B01001_004E',
    'Population Male 10-14':'B01001_005E',
    'Population Male 15-17':'B01001_006E',
    'Population Male 18-19':'B01001_007E',
    'Population Male 20':'B01001_008E',
    'Population Male 21':'B01001_009E',
    'Population Male 22-24':'B01001_010E',
    'Population Male 25-29':'B01001_011E',
    'Population Male 30-34':'B01001_012E',
    'Population Male 35-39':'B01001_013E',
    'Population Male 40-44':'B01001_014E',
    'Population Male 45-49':'B01001_015E',
    'Population Male 50-54':'B01001_016E',
    'Population Male 55-59':'B01001_017E',
    'Population Male 60-61':'B01001_018E',
    'Population Male 62-64':'B01001_019E',
    'Population Male 65-66':'B01001_020E',
    'Population Male 67-69':'B01001_021E',
    'Population Male 70-74':'B01001_022E',
    'Population Male 75-79':'B01001_023E',
    'Population Male 80-84':'B01001_024E',
    
    'Population Female 0-5':'B01001_027E',
    'Population Female 5-9':'B01001_028E',
    'Population Female 10-14':'B01001_029E',
    'Population Female 15-17':'B01001_030E',
    'Population Female 18-19':'B01001_031E',
    'Population Female 20':'B01001_032E',
    'Population Female 21':'B01001_033E',
    'Population Female 22-24':'B01001_034E',
    'Population Female 25-29':'B01001_035E',
    'Population Female 30-34':'B01001_036E',
    'Population Female 35-39':'B01001_037E',
    'Population Female 40-44':'B01001_038E',
    'Population Female 45-49':'B01001_039E',
    'Population Female 50-54':'B01001_040E',
    'Population Female 55-59':'B01001_041E',
    'Population Female 60-61':'B01001_042E',
    'Population Female 62-64':'B01001_043E',
    'Population Female 65-66':'B01001_044E',
    'Population Female 67-69':'B01001_045E',
    'Population Female 70-74':'B01001_046E',
    'Population Female 75-79':'B01001_047E',
    'Population Female 80-84':'B01001_048E',
    'Population Female 85+':'B01001_049E',   
}
economic_variables = {
    'Income Total Population': 'B17001_001E',
    'Income Below Poverty Line': 'B17001_002E',
    'Median Income': 'B19013_001E',
    'Education Attainment 18 to 64 Total': 'B23006_001E',
    'Education Attainment Less than Highschool':'B23006_002E',
    'Education Attainment Highschool Graduate': 'B23006_009E',
    'Education Attainment Bachelors Degree or Higher': 'B23006_023E',
}<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14278481012658228, 'fraction_numerical': 0.08941772151898734, 'mean_word_length': 3.275324675324675, 'pattern_counts': {'":': 3, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '26729', 'license': 'MIT', 'n_tokens_mistral': 4127, 'n_tokens_neox': 3297, 'n_words': 768, 'path': 'examples/data_exploration.ipynb', 'repo_name': 'Vinnie-Palazeti/IndyData', 'size': '9875'}