<commit_before>import httplib2
import urllib
import simplejson
import logging
logging.basicConfig(level=logging.DEBUG)
import UserDict
import os
import pkg_resources
import csv
import unidecode
import datetime
import time

import config

US_BUS="s4OOB4"
API_KEY=config.conf.get('factual', 'api_key')

### Factual filter functions
# todo: move to module

def prep_term(term):
	# ensure dates are converted to epoch if needed
	if type(term)==datetime.datetime:
		term = time.mktime(term)
	return term

# generic filters
def or_(*args):
	return {"$or": args}
def and_(*args):
	return {"$and": args}
def field_filter(op, field, term):
	return {field: {op: prep_term(term)}}
# text and numeric ops
def eq_(field, term):
	return {field: prep_term(term)}
def neq_(field, term):
	return field_filter("$neq", field, term)
def in_(field, terms):
	if type(terms) != list:
		terms = [terms]
	return field_filter("$in", field, terms)
def blank_(field):
	return field_filter("$blank", field, True)
def not_blank_(field):
	return field_filter("$blank", field, False)
# text ops
def bw_(field, term):
	return field_filter("$bw", field, term)
def search_(term):
	return {"$search": term}
# geo ops
def within_(lat, lon, radius=1500):
	return {"$loc": {"$within": {"$center":[[lat, lon], radius]}}}
# numeric ops
def gt_(field, term):
	return field_filter("$gt", field, term)
def gte_(field, term):
	return field_filter("$gte", field, term)
def lt_(field, term):
	return field_filter("$lt", field, term)
def lte_(field, term):
	return field_filter("$lte", field, term)

class FactualRequest(object):
	def __init__(self, filters={}, limit=50, api_key=API_KEY, table_id=US_BUS):
		self._filters = filters
		self._limit = limit
		self.api_key = api_key
		self.table_id = table_id

	def get_url(self):
		filters = dict([(k,v) for k,v in self._filters.iteritems() if v != None])
		params = {
			"filters": urllib.quote(unidecode.unidecode(\
							simplejson.dumps(filters,ensure_ascii=False, \
									separators=(',',':'))), \
								'/{}"$:,[]'),
			"limit": self._limit
		}
		param_str = "&".join([("%s=%s" % (k,v)) for k,v in params.iteritems()])
		url = self._url_pat % ({
						'TABLE_ID':self.table_id, 
						'API_KEY':self.api_key,
						'PARAMS': param_str
						})
		return url
	
	def get_filters(self):
		return self._filters
	def set_filters(self, filters):
		self._filters = filters
	filters = property(get_filters, set_filters)

	def limit(self, limit):
		self._limit = limit
		return self #chainable
	
	def filter(self, filter):
		self._filters.update(filter)
		return self # chainable
	def search(self, search_term):
		'''Convenience method to apply a {"$search":...} filter'''
		return self.filter(search_(search_term))
	def within(self, lat, lon, radius):
		return self.filter(within_(lat, lon, radius))
	def name(self, term):
		return self.filter(eq_("name", "term"))
		
	def GET(self):
		url = self.get_url()
		logging.debug(url)
		h = httplib2.Http()
		resp, body = h.request(url)
		return simplejson.loads(body)
	_url_pat = "http://api.factual.com/v2/tables/%(TABLE_ID)s/read?api_key=%(API_KEY)s&%(PARAMS)s"

class FactualResponse(object):
	def __init__(self, body, field_map=None):
		self._status = body.get("status", None)
		self._version = body.get("version", None)
		self._response = body.get("response", None)
		self._data = self._response.get("data", None)
		self._fields = self._response.get("fields", None)
		if field_map == None:
			self._field_map = dict([(f,f) for f in self._fields])
			self._mapped_fields = self._fields<commit_msg>Add note about JsonExceptions if Factual doesn't send good JSON.<commit_after>import httplib2
import urllib
import simplejson
import logging
logging.basicConfig(level=logging.DEBUG)
import UserDict
import os
import pkg_resources
import csv
import unidecode
import datetime
import time

import config

US_BUS="s4OOB4"
API_KEY=config.conf.get('factual', 'api_key')

### Factual filter functions
# todo: move to module

def prep_term(term):
	# ensure dates are converted to epoch if needed
	if type(term)==datetime.datetime:
		term = time.mktime(term)
	return term

# generic filters
def or_(*args):
	return {"$or": args}
def and_(*args):
	return {"$and": args}
def field_filter(op, field, term):
	return {field: {op: prep_term(term)}}
# text and numeric ops
def eq_(field, term):
	return {field: prep_term(term)}
def neq_(field, term):
	return field_filter("$neq", field, term)
def in_(field, terms):
	if type(terms) != list:
		terms = [terms]
	return field_filter("$in", field, terms)
def blank_(field):
	return field_filter("$blank", field, True)
def not_blank_(field):
	return field_filter("$blank", field, False)
# text ops
def bw_(field, term):
	return field_filter("$bw", field, term)
def search_(term):
	return {"$search": term}
# geo ops
def within_(lat, lon, radius=1500):
	return {"$loc": {"$within": {"$center":[[lat, lon], radius]}}}
# numeric ops
def gt_(field, term):
	return field_filter("$gt", field, term)
def gte_(field, term):
	return field_filter("$gte", field, term)
def lt_(field, term):
	return field_filter("$lt", field, term)
def lte_(field, term):
	return field_filter("$lte", field, term)

class FactualRequest(object):
	def __init__(self, filters={}, limit=50, api_key=API_KEY, table_id=US_BUS):
		self._filters = filters
		self._limit = limit
		self.api_key = api_key
		self.table_id = table_id

	def get_url(self):
		filters = dict([(k,v) for k,v in self._filters.iteritems() if v != None])
		params = {
			"filters": urllib.quote(unidecode.unidecode(\
							simplejson.dumps(filters,ensure_ascii=False, \
									separators=(',',':'))), \
								'/{}"$:,[]'),
			"limit": self._limit
		}
		param_str = "&".join([("%s=%s" % (k,v)) for k,v in params.iteritems()])
		url = self._url_pat % ({
						'TABLE_ID':self.table_id, 
						'API_KEY':self.api_key,
						'PARAMS': param_str
						})
		return url
	
	def get_filters(self):
		return self._filters
	def set_filters(self, filters):
		self._filters = filters
	filters = property(get_filters, set_filters)

	def limit(self, limit):
		self._limit = limit
		return self #chainable
	
	def filter(self, filter):
		self._filters.update(filter)
		return self # chainable
	def search(self, search_term):
		'''Convenience method to apply a {"$search":...} filter'''
		return self.filter(search_(search_term))
	def within(self, lat, lon, radius):
		return self.filter(within_(lat, lon, radius))
	def name(self, term):
		return self.filter(eq_("name", "term"))
		
	def GET(self):
		'''
		Note: this function can raise JsonDecode exceptions if Factual doesn't provide 
		valid JSON – as when they are doing maintenance. 
		'''
		url = self.get_url()
		logging.debug(url)
		h = httplib2.Http()
		resp, body = h.request(url)
		return simplejson.loads(body)
	_url_pat = "http://api.factual.com/v2/tables/%(TABLE_ID)s/read?api_key=%(API_KEY)s&%(PARAMS)s"

class FactualResponse(object):
	def __init__(self, body, field_map=None):
		self._status = body.get("status", None)
		self._version = body.get("version", None)
		self._response = body.get("response", None)
		self._data = self._response.get("data", None)
		self._fields = self._response.get("fields", None)
		if field_map == None:
			self._field_map = dict([(f,f) for f in self._fields])
			self._mapped_fields = self._fields
{'fraction_non_alphanumeric': 0.15494309611956672, 'fraction_numerical': 0.0030165912518853697, 'mean_word_length': 4.853932584269663, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2051271', 'n_tokens_mistral': 2756, 'n_tokens_neox': 2535, 'n_words': 662}