<commit_before>from urllib import parse
from urllib.request import Request, urlopen
import json
import re

accepts = {
    "json": "application/json",
    "xml": "application/xml"
}


def under_to_title(s):
    return " ".join(word.capitalize() for word in re.split(r"_+", s))


def no_data(s):
    return re.match(r"(n/?a|no change|same)", s, re.I)


def make_request(domain, resource_id, token, fmt="json",
                 soql=None):
    if soql:
        qs = "?" + parse.urlencode({"$query": soql})
    else:
        qs = ""
    url = "https://{domain}/resource/{resource_id}.{fmt}{qs}"\
          .format(domain=domain,
                  resource_id=resource_id,
                  fmt=fmt,
                  qs=qs)
    data = None

    return Request(url, data, {"Accept": accepts[fmt],
                               "X-App-Token": token})


def get_json(req):
    f = urlopen(req)
    body = f.read().decode("utf-8")
    json_response = json.loads(body)
    f.close()

    return json_response


class Importer(object):
    region_name = "Cambridge, MA"
    domain = "data.cambridgema.gov"

    def __init__(self, api_key, resource="urfm-usws"):
        self.api_key = api_key
        self.resource_id = resource

    def query(self, soql):
        req = make_request(self.domain, self.resource_id, self.api_key,
                           soql=soql)
        json = get_json(req)
        return map(self.process_json, json)

    def updated_since(self, dt):
        soql = ("SELECT * WHERE applicationdate >= "
                "'{dt}' OR decisiondate >= '{dt}'")\
                .format(dt=dt.isoformat())<commit_msg>Fixed Cambridge importer arguments<commit_after>from urllib import parse
from urllib.request import Request, urlopen
import json
import re

accepts = {
    "json": "application/json",
    "xml": "application/xml"
}


def under_to_title(s):
    return " ".join(word.capitalize() for word in re.split(r"_+", s))


def no_data(s):
    return re.match(r"(n/?a|no change|same)", s, re.I)


def make_request(domain, resource_id, token, fmt="json",
                 soql=None):
    if soql:
        qs = "?" + parse.urlencode({"$query": soql})
    else:
        qs = ""
    url = "https://{domain}/resource/{resource_id}.{fmt}{qs}"\
          .format(domain=domain,
                  resource_id=resource_id,
                  fmt=fmt,
                  qs=qs)
    data = None

    return Request(url, data, {"Accept": accepts[fmt],
                               "X-App-Token": token})


def get_json(req):
    f = urlopen(req)
    body = f.read().decode("utf-8")
    json_response = json.loads(body)
    f.close()

    return json_response


class Importer(object):
    region_name = "Cambridge, MA"
    domain = "data.cambridgema.gov"

    def __init__(self, api_key, resource="urfm-usws"):
        self.api_key = api_key
        self.resource_id = resource

    def query(self, soql):
        req = make_request(self.domain, self.resource_id, self.api_key,
                           soql=soql)
        json = get_json(req)
        return map(self.process_json, json)

    def updated_since(self, dt, **kwargs):
        soql = ("SELECT * WHERE applicationdate >= "
                "'{dt}' OR decisiondate >= '{dt}'")\
                .format(dt=dt.isoformat())
{'fraction_non_alphanumeric': 0.14285714285714285, 'fraction_numerical': 0.0006091989034419738, 'mean_word_length': 2.2676616915422887, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7552265', 'n_tokens_mistral': 1098, 'n_tokens_neox': 1053, 'n_words': 229}