<commit_before># -*- coding: utf-8 -*-
"""
    Clean raw RAIS data and output to CSV
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    The script is the first step in adding a new year of RAIS data to the 
    database. The script will output x CSV files that can then be added to
    the database by the add_to_db.py script.
    
    The user needs to specify the path to the file they are looking to use
    as input.
    
    0: Year; 1: Employee_ID; 2: Establishment_ID; 3: Municipality_ID;
    4: BrazilianOcupation_ID; 5: SBCLAS20; 6: CLASCNAE20; 7: WageReceived;
    8: EconomicActivity_ID_ISIC; 9: Average_monthly_wage
"""

''' Import statements '''
import csv, sys, os, argparse
from collections import defaultdict

basedir = os.path.abspath(os.path.dirname(__file__))
parent_dir = os.path.abspath(os.path.join(basedir, '..'))

def export(data, year, max_depth, csv_writer, keys, depth = 0):
    if 'wage' not in data:
        for k, v in data.items():
            # if len(keys) > max_depth:
            #     keys = set(list(keys)[-1:])
            # keys.add(k)
            # if len(keys) == max_depth:
            #     keys = []
            if len(keys) == depth:
                keys.append(k)
            else:
                keys[depth] = k
            export(v, year, max_depth, csv_writer, keys, depth+1)
    else:
        # print keys
        # raw_input('')
        # raw_input([year] + keys + [data['wage'], data['num_emp'], len(data['num_est'])])
        csv_writer.writerow([year] + keys + [data['wage'], len(data['num_emp']), len(data['num_est'])])

def clean(file_path):
    '''Initialize our data dictionaries'''
    yb = defaultdict(lambda: defaultdict(int))
    ybi = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
    ybio = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(int))))
    ybo = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
    yi = defaultdict(lambda: defaultdict(int))
    yio = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
    yo = defaultdict(lambda: defaultdict(int))
    
    '''Open CSV file'''
    csv_reader = csv.reader(open(file_path, 'rU'), delimiter=",", quotechar='"')
    header = [s.replace('\xef\xbb\xbf', '') for s in csv_reader.next()]
    
    '''Populate the data dictionaries'''
    print 'reading CSV file "' + file_path + '"'
    for i, line in enumerate(csv_reader):
        
        line = dict(zip(header, line))
        
        if i % 100000 == 0:
            sys.stdout.write('\r lines read: ' + str(i) + ' ' * 20)
            sys.stdout.flush() # important
        
        try:
            year = int(line["Year"])
        except:
            print "Error reading year on line {0}".format(i+1)
            continue
        
        try:
            est = line["Establishment_ID"]
        except:
            print "Error reading establishment ID on line {0}".format(i+1)
            continue

        try:
            emp = line["Employee_ID"]
        except:
            print "Error reading employee ID on line {0}".format(i+1)
            continue

        try:
            munic = line["Municipality_ID"]
        except:
            print "Error reading municipality ID on line {0}".format(i+1)
            continue
        
        try:
            # chop off last 2 digits since we're only using CBO_4
            occ = line["BrazilianOcupation_ID"][:-2]
        except:
            print "Error reading occupation ID on line {0}".format(i+1)
            continue
        
        try:
            isic = line["EconmicAtivity_ID_ISIC"]
        except:
            print "Error reading industry ID on line {0}".format(i+1)
            continue<commit_msg>updated for spelling error in ISIC column name<commit_after># -*- coding: utf-8 -*-
"""
    Clean raw RAIS data and output to CSV
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    The script is the first step in adding a new year of RAIS data to the 
    database. The script will output x CSV files that can then be added to
    the database by the add_to_db.py script.
    
    The user needs to specify the path to the file they are looking to use
    as input.
    
    0: Year; 1: Employee_ID; 2: Establishment_ID; 3: Municipality_ID;
    4: BrazilianOcupation_ID; 5: SBCLAS20; 6: CLASCNAE20; 7: WageReceived;
    8: EconomicActivity_ID_ISIC; 9: Average_monthly_wage
"""

''' Import statements '''
import csv, sys, os, argparse
from collections import defaultdict

basedir = os.path.abspath(os.path.dirname(__file__))
parent_dir = os.path.abspath(os.path.join(basedir, '..'))

def export(data, year, max_depth, csv_writer, keys, depth = 0):
    if 'wage' not in data:
        for k, v in data.items():
            # if len(keys) > max_depth:
            #     keys = set(list(keys)[-1:])
            # keys.add(k)
            # if len(keys) == max_depth:
            #     keys = []
            if len(keys) == depth:
                keys.append(k)
            else:
                keys[depth] = k
            export(v, year, max_depth, csv_writer, keys, depth+1)
    else:
        # print keys
        # raw_input('')
        # raw_input([year] + keys + [data['wage'], data['num_emp'], len(data['num_est'])])
        csv_writer.writerow([year] + keys + [data['wage'], len(data['num_emp']), len(data['num_est'])])

def clean(file_path):
    '''Initialize our data dictionaries'''
    yb = defaultdict(lambda: defaultdict(int))
    ybi = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
    ybio = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(int))))
    ybo = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
    yi = defaultdict(lambda: defaultdict(int))
    yio = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
    yo = defaultdict(lambda: defaultdict(int))
    
    '''Open CSV file'''
    csv_reader = csv.reader(open(file_path, 'rU'), delimiter=",", quotechar='"')
    header = [s.replace('\xef\xbb\xbf', '') for s in csv_reader.next()]
    
    '''Populate the data dictionaries'''
    print 'reading CSV file "' + file_path + '"'
    for i, line in enumerate(csv_reader):
        
        line = dict(zip(header, line))
        
        if i % 100000 == 0:
            sys.stdout.write('\r lines read: ' + str(i) + ' ' * 20)
            sys.stdout.flush() # important
        
        try:
            year = int(line["Year"])
        except:
            print "Error reading year on line {0}".format(i+1)
            continue
        
        try:
            est = line["Establishment_ID"]
        except:
            print "Error reading establishment ID on line {0}".format(i+1)
            continue

        try:
            emp = line["Employee_ID"]
        except:
            print "Error reading employee ID on line {0}".format(i+1)
            continue

        try:
            munic = line["Municipality_ID"]
        except:
            print "Error reading municipality ID on line {0}".format(i+1)
            continue
        
        try:
            # chop off last 2 digits since we're only using CBO_4
            occ = line["BrazilianOcupation_ID"][:-2]
        except:
            print "Error reading occupation ID on line {0}".format(i+1)
            continue
        
        try:
            isic = line["EconmicAtivity_ID_ISIC"]
        except:
            try:
                isic = line["EconomicAtivity_ID_ISIC"]
            except:
                print "Error reading industry ID on line {0}".format(i+1)
                continue
{'fraction_non_alphanumeric': 0.13089700996677742, 'fraction_numerical': 0.011162790697674419, 'mean_word_length': 2.206646783127397, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4609151', 'n_tokens_mistral': 2365, 'n_tokens_neox': 2257, 'n_words': 640}