<commit_before>from flask import make_response, request, render_template, current_app, g, \
    Blueprint, abort
from flask.ext.cache import Cache
from functools import update_wrapper
import os
import re
import math
from datetime import date, datetime, timedelta
from dateutil.parser import parse
from datetime_truncate import truncate
import time
import json
from sqlalchemy import func, distinct, Column, Float, Table
from sqlalchemy.exc import NoSuchTableError
from sqlalchemy.types import NullType
from sqlalchemy.sql.expression import cast
from geoalchemy2 import Geometry
from operator import itemgetter
from itertools import groupby
from cStringIO import StringIO
import csv
from shapely.wkb import loads
from shapely.geometry import box, asShape
from collections import OrderedDict
from urlparse import urlparse
from hashlib import md5

from plenario.models import MasterTable, MetaTable
from plenario.database import session, app_engine as engine, Base
from plenario.utils.helpers import get_socrata_data_info, slugify, increment_datetime_aggregate
from plenario.tasks import add_dataset

API_VERSION = '/v1'
RESPONSE_LIMIT = 500
VALID_DATA_TYPE = ['csv', 'json']
VALID_AGG = ['day', 'week', 'month', 'quarter', 'year']
WEATHER_COL_LOOKUP = {
    'daily': {
        'temp_lo': 'temp_min',
        'temp_hi': 'temp_max',
        'temp_avg': 'temp_avg',
        'precip_amount': 'precip_total',
    },
    'hourly': {
        'temp_lo': 'drybulb_fahrenheit',
        'temp_hi': 'drybulb_fahrenheit',
        'temp_avg': 'drybulb_fahrenheit',
        'precip_amount': 'hourly_precip',
    },
}

api = Blueprint('api', __name__)
cache = Cache(config={'CACHE_TYPE': 'simple'})
dthandler = lambda obj: obj.isoformat() if isinstance(obj, date) else None

def crossdomain(origin=None, methods=None, headers=None,
                max_age=21600, attach_to_all=True,
                automatic_options=True): # pragma: no cover
    if methods is not None:
        methods = ', '.join(sorted(x.upper() for x in methods))
    if headers is not None and not isinstance(headers, basestring):
        headers = ', '.join(x.upper() for x in headers)
    if not isinstance(origin, basestring):
        origin = ', '.join(origin)
    if isinstance(max_age, timedelta):
        max_age = max_age.total_seconds()

    def get_methods():
        if methods is not None:
            return methods

        options_resp = current_app.make_default_options_response()
        return options_resp.headers['allow']

    def decorator(f):
        def wrapped_function(*args, **kwargs):
            if automatic_options and request.method == 'OPTIONS':
                resp = current_app.make_default_options_response()
            else:
                resp = make_response(f(*args, **kwargs))
            if not attach_to_all and request.method != 'OPTIONS':
                return resp

            h = resp.headers

            h['Access-Control-Allow-Origin'] = origin
            h['Access-Control-Allow-Methods'] = get_methods()
            h['Access-Control-Max-Age'] = str(max_age)
            if headers is not None:
                h['Access-Control-Allow-Headers'] = headers
            return resp

        f.provide_automatic_options = False
        return update_wrapper(wrapped_function, f)
    return decorator

def make_cache_key(*args, **kwargs):
    path = request.path
    args = str(hash(frozenset(request.args.items())))
    # print 'cache_key:', (path+args)
    return (path + args).encode('utf-8')

@api.route(API_VERSION + '/api/datasets')
@cache.cached(timeout=60*60)
@crossdomain(origin="*")
def meta():
    status_code = 200
    resp = {
            'meta': {
                'status': 'ok',
                'message': '',
            },
            'objects': []
        }
    dataset_name = request.args.get('dataset_name')
    if dataset_name:
        metas = session.query(MetaTable)\
            .filter(MetaTable.dataset_name == dataset_name).all()
    else:
        metas = session.query(MetaTable).all()
    resp['objects'].extend([m.as_dict() for m in metas])
    resp['meta']['total'] = len(resp['objects'])
    resp = make_response(json.dumps(resp, default=dthandler), status_code)
    resp.headers['Content-Type'] = 'application/json'
    return resp

@api.route(API_VERSION + '/api/fields/<dataset_name>/')
@cache.cached(timeout=60*60)
@crossdomain(origin="*")
def dataset_fields(dataset_name):
    try:
        table = Table('dat_%s' % dataset_name, Base.metadata,
            autoload=True, autoload_with=engine,<commit_msg>added cache key to /api/datasets<commit_after>from flask import make_response, request, render_template, current_app, g, \
    Blueprint, abort
from flask.ext.cache import Cache
from functools import update_wrapper
import os
import re
import math
from datetime import date, datetime, timedelta
from dateutil.parser import parse
from datetime_truncate import truncate
import time
import json
from sqlalchemy import func, distinct, Column, Float, Table
from sqlalchemy.exc import NoSuchTableError
from sqlalchemy.types import NullType
from sqlalchemy.sql.expression import cast
from geoalchemy2 import Geometry
from operator import itemgetter
from itertools import groupby
from cStringIO import StringIO
import csv
from shapely.wkb import loads
from shapely.geometry import box, asShape
from collections import OrderedDict
from urlparse import urlparse
from hashlib import md5

from plenario.models import MasterTable, MetaTable
from plenario.database import session, app_engine as engine, Base
from plenario.utils.helpers import get_socrata_data_info, slugify, increment_datetime_aggregate
from plenario.tasks import add_dataset

API_VERSION = '/v1'
RESPONSE_LIMIT = 500
VALID_DATA_TYPE = ['csv', 'json']
VALID_AGG = ['day', 'week', 'month', 'quarter', 'year']
WEATHER_COL_LOOKUP = {
    'daily': {
        'temp_lo': 'temp_min',
        'temp_hi': 'temp_max',
        'temp_avg': 'temp_avg',
        'precip_amount': 'precip_total',
    },
    'hourly': {
        'temp_lo': 'drybulb_fahrenheit',
        'temp_hi': 'drybulb_fahrenheit',
        'temp_avg': 'drybulb_fahrenheit',
        'precip_amount': 'hourly_precip',
    },
}

api = Blueprint('api', __name__)
cache = Cache(config={'CACHE_TYPE': 'simple'})
dthandler = lambda obj: obj.isoformat() if isinstance(obj, date) else None

def crossdomain(origin=None, methods=None, headers=None,
                max_age=21600, attach_to_all=True,
                automatic_options=True): # pragma: no cover
    if methods is not None:
        methods = ', '.join(sorted(x.upper() for x in methods))
    if headers is not None and not isinstance(headers, basestring):
        headers = ', '.join(x.upper() for x in headers)
    if not isinstance(origin, basestring):
        origin = ', '.join(origin)
    if isinstance(max_age, timedelta):
        max_age = max_age.total_seconds()

    def get_methods():
        if methods is not None:
            return methods

        options_resp = current_app.make_default_options_response()
        return options_resp.headers['allow']

    def decorator(f):
        def wrapped_function(*args, **kwargs):
            if automatic_options and request.method == 'OPTIONS':
                resp = current_app.make_default_options_response()
            else:
                resp = make_response(f(*args, **kwargs))
            if not attach_to_all and request.method != 'OPTIONS':
                return resp

            h = resp.headers

            h['Access-Control-Allow-Origin'] = origin
            h['Access-Control-Allow-Methods'] = get_methods()
            h['Access-Control-Max-Age'] = str(max_age)
            if headers is not None:
                h['Access-Control-Allow-Headers'] = headers
            return resp

        f.provide_automatic_options = False
        return update_wrapper(wrapped_function, f)
    return decorator

def make_cache_key(*args, **kwargs):
    path = request.path
    args = str(hash(frozenset(request.args.items())))
    # print 'cache_key:', (path+args)
    return (path + args).encode('utf-8')

@api.route(API_VERSION + '/api/datasets')
@cache.cached(timeout=60*60, key_prefix=make_cache_key)
@crossdomain(origin="*")
def meta():
    status_code = 200
    resp = {
            'meta': {
                'status': 'ok',
                'message': '',
            },
            'objects': []
        }
    dataset_name = request.args.get('dataset_name')
    if dataset_name:
        metas = session.query(MetaTable)\
            .filter(MetaTable.dataset_name == dataset_name).all()
    else:
        metas = session.query(MetaTable).all()
    resp['objects'].extend([m.as_dict() for m in metas])
    resp['meta']['total'] = len(resp['objects'])
    resp = make_response(json.dumps(resp, default=dthandler), status_code)
    resp.headers['Content-Type'] = 'application/json'
    return resp

@api.route(API_VERSION + '/api/fields/<dataset_name>/')
@cache.cached(timeout=60*60)
@crossdomain(origin="*")
def dataset_fields(dataset_name):
    try:
        table = Table('dat_%s' % dataset_name, Base.metadata,
            autoload=True, autoload_with=engine,
{'fraction_non_alphanumeric': 0.11356673960612691, 'fraction_numerical': 0.0050328227571115975, 'mean_word_length': 3.2202216066481992, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3167280', 'n_tokens_mistral': 2813, 'n_tokens_neox': 2720, 'n_words': 694}