<commit_before>import sys
import os
import fnmatch
from xml.dom import minidom
import happybase
import pprint

sys.path.append('../trulia-fetcher')
import TruliaDataFetcher

data_dir = '/home/ubuntu/data'
loc_type = 'city'


def get_file_list(data_dir, loc_type):
    file_list = []
    for root, dir_names, file_names in os.walk(data_dir + '/' + loc_type ):
        for file_name in fnmatch.filter(file_names, '*.xml'):
            file_list.append(os.path.join(root, file_name))
    return file_list


def parse_get_city_stats_resp(text): 

        head_dom = minidom.parseString(text)
        dom_list = head_dom.getElementsByTagName('listingStat')
        state_code = head_dom.getElementsByTagName('state')[0].firstChild.nodeValue
        city = head_dom.getElementsByTagName('city')[0].firstChild.nodeValue

        print city, state_code
        # No key, do not log                                                                                              
        if len(state_code) != 2 and len(city) > 0:
            return

        #res = dm.simple_select_query(dm.conn, "info_city", "latitude, longitude", "state_code = '" + state_code + "' and city = '" + city + "' LIMIT 1")
        #lat = res[0][0]
        #lon = res[0][1]

        # Base of HBase key, will append bedroom count
        hbase_base_key = state_code.lower() + '-' + '_'.join(city.lower().split(' '))

        # for batching to hbase
        dom_accum = {}

        for dom_i in dom_list:

            week_ending_date = dom_i.getElementsByTagName('weekEndingDate')[0].firstChild.nodeValue
            week_list = dom_i.getElementsByTagName('listingPrice')

            for week_dom_i in week_list:
                k_bed_list = week_dom_i.getElementsByTagName('subcategory')
                for k_bed_i in k_bed_list:
                    prop_list = k_bed_i.getElementsByTagName('type')[0].firstChild.nodeValue

                    # checking k_bed to be either a positive int
                    # don't record aggregated 'All Properties' stats
                    k_bed = prop_list.split(' ')[0]


                    if (TruliaDataFetcher.TruliaDataFetcher.is_str_positive_int(k_bed)):
                        #print k_bed
                        try:
                            ## HBase
                            key = hbase_base_key + '-' + str(k_bed)

                            num_list = k_bed_i.getElementsByTagName('numberOfProperties')[0].firstChild.nodeValue
                            avg_list = k_bed_i.getElementsByTagName('averageListingPrice')[0].firstChild.nodeValue
                            col_name = 'cf:' + week_ending_date
                            val = str({'a': avg_list, 'n' : num_list })
                            
                            if k_bed not in dom_accum:
                                dom_accum[k_bed] = {}
                            dom_accum[k_bed][col_name] = val

                            #table.put(key, {col_name : val} )

                            # city meta-data
                            # table.put(key, {'i:city': city, 'i:sc':state_code, 'i:lat:':str(lat),'i:lon':str(lon)})
                            # End Hbase
                            
                            ## Fluentd
                            city_dict = {}
                            city_dict['state_code'] = str(state_code)
                            city_dict['city'] = str(city)
                            city_dict['week_ending_date'] = str(week_ending_date)
                            city_dict['num_beds'] = int(k_bed)
                            
                            # carefully parsing of sub xml dom
                            listing_stat_dict = TruliaDataFetcher.TruliaDataFetcher.parse_listing_stat(k_bed_i)

                            # merge keys
                            city_dict = dict(city_dict.items() + listing_stat_dict.items())
                            #event.Event('city.all_list_stats', city_dict)
                            ## End Fluentd

                        except:
                            continue
            
        for key in dom_accum:
            table.put(hbase_base_key + '-' + key, dom_accum[key])

def get_dom_dict(dom):
    pass
<commit_msg>updates to getting offline stats into hdfs via fluentd and into hbase via happybase<commit_after>import sys
import os
import fnmatch
from xml.dom import minidom
import happybase
import pprint

sys.path.append('../trulia-fetcher')
import TruliaDataFetcher

data_dir = '/home/ubuntu/data'
loc_type = 'city'


def get_file_list(data_dir, loc_type):
    file_list = []
    for root, dir_names, file_names in os.walk(data_dir + '/' + loc_type ):
        for file_name in fnmatch.filter(file_names, '*.xml'):
            file_list.append(os.path.join(root, file_name))
    return file_list


def parse_get_city_stats_resp(text): 

        head_dom = minidom.parseString(text)
        dom_list = head_dom.getElementsByTagName('listingStat')
        state_code = head_dom.getElementsByTagName('state')[0].firstChild.nodeValue
        city = head_dom.getElementsByTagName('city')[0].firstChild.nodeValue

        print city, state_code
        # No key, do not log                                                                                              
        if len(state_code) != 2 and len(city) > 0:
            return

        res = dm.simple_select_query(dm.conn, "info_city", "latitude, longitude", "state_code = '" + state_code + "' and city = '" + city + "' LIMIT 1")
        lat = res[0][0]
        lon = res[0][1]

        # Base of HBase key, will append bedroom count
        hbase_base_key = state_code.lower() + '-' + '_'.join(city.lower().split(' '))

        # for batching to hbase
        dom_accum = {}

        for dom_i in dom_list:

            week_ending_date = dom_i.getElementsByTagName('weekEndingDate')[0].firstChild.nodeValue
            week_list = dom_i.getElementsByTagName('listingPrice')

            for week_dom_i in week_list:
                k_bed_list = week_dom_i.getElementsByTagName('subcategory')
                for k_bed_i in k_bed_list:
                    prop_list = k_bed_i.getElementsByTagName('type')[0].firstChild.nodeValue

                    # checking k_bed to be either a positive int
                    # don't record aggregated 'All Properties' stats
                    k_bed = prop_list.split(' ')[0]


                    if (TruliaDataFetcher.TruliaDataFetcher.is_str_positive_int(k_bed)):
                        #print k_bed
                        try:

                            num_list = k_bed_i.getElementsByTagName('numberOfProperties')[0].firstChild.nodeValue
                            avg_list = k_bed_i.getElementsByTagName('averageListingPrice')[0].firstChild.nodeValue
                            col_name = 'cf:' + week_ending_date
                            val = str({'a': avg_list, 'n' : num_list })
                            
                            if k_bed not in dom_accum:
                                dom_accum[k_bed] = {}
                            dom_accum[k_bed][col_name] = val

                            # End Hbase
                            
                            ## Fluentd
                            city_dict = {}
                            city_dict['state_code'] = str(state_code)
                            city_dict['city'] = str(city)
                            city_dict['week_ending_date'] = str(week_ending_date)
                            city_dict['num_beds'] = int(k_bed)
                            
                            # carefully parsing of sub xml dom
                            listing_stat_dict = TruliaDataFetcher.TruliaDataFetcher.parse_listing_stat(k_bed_i)

                            # merge keys
                            city_dict = dict(city_dict.items() + listing_stat_dict.items())
                            event.Event('city.all_list_stats', city_dict)
                            ## End Fluentd

                        except:
                            continue
            
        for key in dom_accum:
            table.put(hbase_base_key + '-' + key, dom_accum[key])
            table.put(hbase_base_key + '-' + key, {'i:city': str(city), 'i:sc':str(state_code), 'i:lat:':str(lat),'i:lon':str(lon)})

def get_dom_dict(dom):
    pass

{'fraction_non_alphanumeric': 0.09510120972571566, 'fraction_numerical': 0.0033536950532997963, 'mean_word_length': 1.5022475277195086, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '801359', 'n_tokens_mistral': 2440, 'n_tokens_neox': 2343, 'n_words': 435}