# Path: hose_util.py
# def iterate(raw=False, filters=True):
# def lookup(myjson, k):
# def parse_date(twitter_lame_datetime_string):
# def get_date(myjson_object):
# def ws_norm(s):
# def unicodify(s, encoding='utf8', *args):
# def stringify(s, encoding='utf8', *args):
# def tabjoin(*args):
# def uniq_c(seq):
# def commaize(num):
# WS = re.compile(r'[ \t\r\n]+', re.U)


import sys,os,re
import twokenize
from hose_util import lookup, iterate, json
      #print "REJECT NO GEO REGEX\t" + json.dumps(record)
      continue
    lat,lon = m.groups()
    loc_type = 'REGEX'

  lat=float(lat); lon=float(lon)
  if (lat,lon)==(0,0) or lat < -90 or lat > 90 or lon < -180 or lon > 180:
    #print "REJECT JUNK GEO\t" + json.dumps([lat,lon]) + "\t" + json.dumps(record)
    continue

  ## For our applications we usually want to kill retweets
  if lookup(tweet, 'retweeted_status'):
    #print "REJECT OFFICIAL RT\t" + json.dumps(text)
    continue
  toks = twokenize.tokenize(text)
  if any(tok=='RT' for tok in toks):
    #print "REJECT TEXT RT\t" + json.dumps(text)
    continue

  # Build a "SmallTweet" format record
  record = {
      'id': lookup(tweet, 'id'),
      'user': lookup(tweet, 'user.screen_name'),
      'date': tweet['created_at_datetime'].strftime("%Y-%m-%dT%H:%M:%S"),
      'text': lookup(tweet,'text')
  }

  record['lonlat'] = [lon,lat]

  if '\t' in record['user']:

# Path: hose_util.py
# def iterate(raw=False, filters=True):
# def lookup(myjson, k):
# def parse_date(twitter_lame_datetime_string):
# def get_date(myjson_object):
# def ws_norm(s):
# def unicodify(s, encoding='utf8', *args):
# def stringify(s, encoding='utf8', *args):
# def tabjoin(*args):
# def uniq_c(seq):
# def commaize(num):
# WS = re.compile(r'[ \t\r\n]+', re.U)


# Path: geo_filter.py
import sys,os,re
import twokenize
from hose_util import lookup, iterate, json

      #print "REJECT NO GEO REGEX\t" + json.dumps(record)
      continue
    lat,lon = m.groups()
    loc_type = 'REGEX'

  lat=float(lat); lon=float(lon)
  if (lat,lon)==(0,0) or lat < -90 or lat > 90 or lon < -180 or lon > 180:
    #print "REJECT JUNK GEO\t" + json.dumps([lat,lon]) + "\t" + json.dumps(record)
    continue

  ## For our applications we usually want to kill retweets
  if lookup(tweet, 'retweeted_status'):
    #print "REJECT OFFICIAL RT\t" + json.dumps(text)
    continue
  toks = twokenize.tokenize(text)
  if any(tok=='RT' for tok in toks):
    #print "REJECT TEXT RT\t" + json.dumps(text)
    continue

  # Build a "SmallTweet" format record
  record = {
      'id': lookup(tweet, 'id'),
      'user': lookup(tweet, 'user.screen_name'),
      'date': tweet['created_at_datetime'].strftime("%Y-%m-%dT%H:%M:%S"),
      'text': lookup(tweet,'text')
  }

  record['lonlat'] = [lon,lat]

  if '\t' in record['user']:

    print>>sys.stderr, "WTF\t" + json.dumps(record)