# Path: geotweet/mapreduce/state_county_wordcount.py
# class StateCountyWordCountJob(MRJob):
#     """
#     Count word occurences for US tweets by entire county, by State and County
#     
#     A geojson file of US counties is downloaded from an S3 bucket. A RTree index
#     is built using the bounding box of each county, and is used for determining
#     State and County for each tweet.
#     
#     """
# 
#     INPUT_PROTOCOL = JSONValueProtocol
#     INTERNAL_PROTOCOL = JSONProtocol
#     OUTPUT_PROTOCOL = RawValueProtocol
# 
#     def steps(self):
#         return [
#             MRStep(
#                 mapper_init=self.mapper_init,
#                 mapper=self.mapper,
#                 combiner=self.combiner,
#                 reducer=self.reducer
#             )
#         ]
# 
#     def mapper_init(self):
#         """ Download counties geojson from S3 and build spatial index and cache """
#         self.counties = CachedCountyLookup(precision=GEOHASH_PRECISION)
#         self.extractor = WordExtractor()
#     
#     def mapper(self, _, data):
#         # ignore HR geo-tweets for job postings
#         if data['description'] and self.hr_filter(data['description']):
#             return
#         lonlat = data['lonlat']
#         # spatial lookup for state and county
#         state, county = self.counties.get(lonlat)
#         if not state or not county:
#             return
#         # count words
#         for word in self.extractor.run(data['text']):
#             yield (word, ), 1
#             yield (word, state), 1
#             yield (word, state, county), 1
#     
#     def hr_filter(self, text):
#         """ check if description of twitter using contains job related key words """
#         expr = "|".join(["(job)", "(hiring)", "(career)"])
#         return re.findall(expr, text)
#   
#     def combiner(self, key, values):
#         yield key, sum(values)
# 
#     def reducer(self, key, values):
#         total = int(sum(values))
#         if total < MIN_WORD_COUNT:
#             return
#         word = state = county = None
#         word = key[0]
#         if len(key) >= 2:
#             state = key[1]
#         if len(key) >= 3:
#             county = key[2]
#         output = "{0}\t{1}\t{2}\t{3}"
#         word = word.encode('utf-8')
#         state = state.encode('utf-8') if state else None
#         county = county.encode('utf-8') if county else None
#         yield None, output.format(word, state, county, total)
# 
# Path: geotweet/mapreduce/state_county_wordcount.py
# GEOHASH_PRECISION = 7 


import unittest
import os
import sys
import json
import Geohash
from os.path import dirname
from mrjob.job import MRJob
from geotweet.mapreduce.state_county_wordcount import StateCountyWordCountJob
from geotweet.mapreduce.state_county_wordcount import GEOHASH_PRECISION


root = dirname(dirname(dirname(dirname(os.path.abspath(__file__)))))
sys.path.append(root)

GEOTWEET_DIR = root
DATA_DIR = os.path.join(root, 'data', 'geo')
COUNTIES_GEOJSON_LOCAL = os.path.join(DATA_DIR,'us_counties102005.geojson')
os.environ['COUNTIES_GEOJSON_LOCAL'] = COUNTIES_GEOJSON_LOCAL

# COUNTIES_GEOJSON_LOCAL environment variable must be set before import


def build_input(text, desc="My Account", lonlat=[-122.5, 45.4]):
    return dict(
        description=desc,
        text=text,
        lonlat=lonlat
    )


class MapperTweetTests(unittest.TestCase):

    def setUp(self):

# Path: geotweet/mapreduce/state_county_wordcount.py
# class StateCountyWordCountJob(MRJob):
#     """
#     Count word occurences for US tweets by entire county, by State and County
#     
#     A geojson file of US counties is downloaded from an S3 bucket. A RTree index
#     is built using the bounding box of each county, and is used for determining
#     State and County for each tweet.
#     
#     """
# 
#     INPUT_PROTOCOL = JSONValueProtocol
#     INTERNAL_PROTOCOL = JSONProtocol
#     OUTPUT_PROTOCOL = RawValueProtocol
# 
#     def steps(self):
#         return [
#             MRStep(
#                 mapper_init=self.mapper_init,
#                 mapper=self.mapper,
#                 combiner=self.combiner,
#                 reducer=self.reducer
#             )
#         ]
# 
#     def mapper_init(self):
#         """ Download counties geojson from S3 and build spatial index and cache """
#         self.counties = CachedCountyLookup(precision=GEOHASH_PRECISION)
#         self.extractor = WordExtractor()
#     
#     def mapper(self, _, data):
#         # ignore HR geo-tweets for job postings
#         if data['description'] and self.hr_filter(data['description']):
#             return
#         lonlat = data['lonlat']
#         # spatial lookup for state and county
#         state, county = self.counties.get(lonlat)
#         if not state or not county:
#             return
#         # count words
#         for word in self.extractor.run(data['text']):
#             yield (word, ), 1
#             yield (word, state), 1
#             yield (word, state, county), 1
#     
#     def hr_filter(self, text):
#         """ check if description of twitter using contains job related key words """
#         expr = "|".join(["(job)", "(hiring)", "(career)"])
#         return re.findall(expr, text)
#   
#     def combiner(self, key, values):
#         yield key, sum(values)
# 
#     def reducer(self, key, values):
#         total = int(sum(values))
#         if total < MIN_WORD_COUNT:
#             return
#         word = state = county = None
#         word = key[0]
#         if len(key) >= 2:
#             state = key[1]
#         if len(key) >= 3:
#             county = key[2]
#         output = "{0}\t{1}\t{2}\t{3}"
#         word = word.encode('utf-8')
#         state = state.encode('utf-8') if state else None
#         county = county.encode('utf-8') if county else None
#         yield None, output.format(word, state, county, total)
# 
# Path: geotweet/mapreduce/state_county_wordcount.py
# GEOHASH_PRECISION = 7 


# Path: geotweet/tests/integration/mapreduce/state-county_tests.py
import unittest
import os
import sys
import json
import Geohash
from os.path import dirname
from mrjob.job import MRJob
from geotweet.mapreduce.state_county_wordcount import StateCountyWordCountJob
from geotweet.mapreduce.state_county_wordcount import GEOHASH_PRECISION



root = dirname(dirname(dirname(dirname(os.path.abspath(__file__)))))
sys.path.append(root)

GEOTWEET_DIR = root
DATA_DIR = os.path.join(root, 'data', 'geo')
COUNTIES_GEOJSON_LOCAL = os.path.join(DATA_DIR,'us_counties102005.geojson')
os.environ['COUNTIES_GEOJSON_LOCAL'] = COUNTIES_GEOJSON_LOCAL

# COUNTIES_GEOJSON_LOCAL environment variable must be set before import


def build_input(text, desc="My Account", lonlat=[-122.5, 45.4]):
    return dict(
        description=desc,
        text=text,
        lonlat=lonlat
    )


class MapperTweetTests(unittest.TestCase):

    def setUp(self):

        self.mr = StateCountyWordCountJob()